diff options
Diffstat (limited to 'arch')
303 files changed, 6657 insertions, 5133 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 4f55c73..5b448a7 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -47,18 +47,29 @@ config KPROBES If in doubt, say "N". config JUMP_LABEL - bool "Optimize trace point call sites" + bool "Optimize very unlikely/likely branches" depends on HAVE_ARCH_JUMP_LABEL help + This option enables a transparent branch optimization that + makes certain almost-always-true or almost-always-false branch + conditions even cheaper to execute within the kernel. + + Certain performance-sensitive kernel code, such as trace points, + scheduler functionality, networking code and KVM have such + branches and include support for this optimization technique. + If it is detected that the compiler has support for "asm goto", - the kernel will compile trace point locations with just a - nop instruction. When trace points are enabled, the nop will - be converted to a jump to the trace function. This technique - lowers overhead and stress on the branch prediction of the - processor. - - On i386, options added to the compiler flags may increase - the size of the kernel slightly. + the kernel will compile such branches with just a nop + instruction. When the condition flag is toggled to true, the + nop will be converted to a jump instruction to execute the + conditional block of instructions. + + This technique lowers overhead and stress on the branch prediction + of the processor and generally makes the kernel faster. The update + of the condition is slower, but those are always very rare. + + ( On 32-bit x86, the necessary options added to the compiler + flags may increase the size of the kernel slightly. ) config OPTPROBES def_bool y diff --git a/arch/alpha/include/asm/machvec.h b/arch/alpha/include/asm/machvec.h index 13cd427..72dbf23 100644 --- a/arch/alpha/include/asm/machvec.h +++ b/arch/alpha/include/asm/machvec.h @@ -90,7 +90,7 @@ struct alpha_machine_vector void (*kill_arch)(int); u8 (*pci_swizzle)(struct pci_dev *, u8 *); - int (*pci_map_irq)(struct pci_dev *, u8, u8); + int (*pci_map_irq)(const struct pci_dev *, u8, u8); struct pci_ops *pci_ops; struct _alpha_agp_info *(*agp_info)(void); diff --git a/arch/alpha/include/asm/socket.h b/arch/alpha/include/asm/socket.h index 082355f..dcb221a 100644 --- a/arch/alpha/include/asm/socket.h +++ b/arch/alpha/include/asm/socket.h @@ -71,6 +71,10 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 /* O_NONBLOCK clashes with the bits used for socket types. Therefore we * have to define SOCK_NONBLOCK to a different value here. diff --git a/arch/alpha/kernel/perf_event.c b/arch/alpha/kernel/perf_event.c index 8143cd7..0dae252 100644 --- a/arch/alpha/kernel/perf_event.c +++ b/arch/alpha/kernel/perf_event.c @@ -685,6 +685,10 @@ static int alpha_pmu_event_init(struct perf_event *event) { int err; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (event->attr.type) { case PERF_TYPE_RAW: case PERF_TYPE_HARDWARE: diff --git a/arch/alpha/kernel/srmcons.c b/arch/alpha/kernel/srmcons.c index 783f4e5..3ea8094 100644 --- a/arch/alpha/kernel/srmcons.c +++ b/arch/alpha/kernel/srmcons.c @@ -30,10 +30,9 @@ static int srm_is_registered_console = 0; #define MAX_SRM_CONSOLE_DEVICES 1 /* only support 1 console device */ struct srmcons_private { - struct tty_struct *tty; + struct tty_port port; struct timer_list timer; - spinlock_t lock; -}; +} srmcons_singleton; typedef union _srmcons_result { struct { @@ -68,22 +67,21 @@ static void srmcons_receive_chars(unsigned long data) { struct srmcons_private *srmconsp = (struct srmcons_private *)data; + struct tty_port *port = &srmconsp->port; unsigned long flags; int incr = 10; local_irq_save(flags); if (spin_trylock(&srmcons_callback_lock)) { - if (!srmcons_do_receive_chars(srmconsp->tty)) + if (!srmcons_do_receive_chars(port->tty)) incr = 100; spin_unlock(&srmcons_callback_lock); } - spin_lock(&srmconsp->lock); - if (srmconsp->tty) { - srmconsp->timer.expires = jiffies + incr; - add_timer(&srmconsp->timer); - } - spin_unlock(&srmconsp->lock); + spin_lock(&port->lock); + if (port->tty) + mod_timer(&srmconsp->timer, jiffies + incr); + spin_unlock(&port->lock); local_irq_restore(flags); } @@ -156,56 +154,22 @@ srmcons_chars_in_buffer(struct tty_struct *tty) } static int -srmcons_get_private_struct(struct srmcons_private **ps) -{ - static struct srmcons_private *srmconsp = NULL; - static DEFINE_SPINLOCK(srmconsp_lock); - unsigned long flags; - int retval = 0; - - if (srmconsp == NULL) { - srmconsp = kmalloc(sizeof(*srmconsp), GFP_KERNEL); - spin_lock_irqsave(&srmconsp_lock, flags); - - if (srmconsp == NULL) - retval = -ENOMEM; - else { - srmconsp->tty = NULL; - spin_lock_init(&srmconsp->lock); - init_timer(&srmconsp->timer); - } - - spin_unlock_irqrestore(&srmconsp_lock, flags); - } - - *ps = srmconsp; - return retval; -} - -static int srmcons_open(struct tty_struct *tty, struct file *filp) { - struct srmcons_private *srmconsp; + struct srmcons_private *srmconsp = &srmcons_singleton; + struct tty_port *port = &srmconsp->port; unsigned long flags; - int retval; - - retval = srmcons_get_private_struct(&srmconsp); - if (retval) - return retval; - spin_lock_irqsave(&srmconsp->lock, flags); + spin_lock_irqsave(&port->lock, flags); - if (!srmconsp->tty) { + if (!port->tty) { tty->driver_data = srmconsp; - - srmconsp->tty = tty; - srmconsp->timer.function = srmcons_receive_chars; - srmconsp->timer.data = (unsigned long)srmconsp; - srmconsp->timer.expires = jiffies + 10; - add_timer(&srmconsp->timer); + tty->port = port; + port->tty = tty; /* XXX proper refcounting */ + mod_timer(&srmconsp->timer, jiffies + 10); } - spin_unlock_irqrestore(&srmconsp->lock, flags); + spin_unlock_irqrestore(&port->lock, flags); return 0; } @@ -214,16 +178,17 @@ static void srmcons_close(struct tty_struct *tty, struct file *filp) { struct srmcons_private *srmconsp = tty->driver_data; + struct tty_port *port = &srmconsp->port; unsigned long flags; - spin_lock_irqsave(&srmconsp->lock, flags); + spin_lock_irqsave(&port->lock, flags); if (tty->count == 1) { - srmconsp->tty = NULL; + port->tty = NULL; del_timer(&srmconsp->timer); } - spin_unlock_irqrestore(&srmconsp->lock, flags); + spin_unlock_irqrestore(&port->lock, flags); } @@ -240,6 +205,9 @@ static const struct tty_operations srmcons_ops = { static int __init srmcons_init(void) { + tty_port_init(&srmcons_singleton.port); + setup_timer(&srmcons_singleton.timer, srmcons_receive_chars, + (unsigned long)&srmcons_singleton); if (srm_is_registered_console) { struct tty_driver *driver; int err; diff --git a/arch/alpha/kernel/sys_dp264.c b/arch/alpha/kernel/sys_dp264.c index bb7f0c7..13f0717 100644 --- a/arch/alpha/kernel/sys_dp264.c +++ b/arch/alpha/kernel/sys_dp264.c @@ -366,7 +366,7 @@ clipper_init_irq(void) */ static int __init -isa_irq_fixup(struct pci_dev *dev, int irq) +isa_irq_fixup(const struct pci_dev *dev, int irq) { u8 irq8; diff --git a/arch/arm/boot/dts/testcases/tests-phandle.dtsi b/arch/arm/boot/dts/testcases/tests-phandle.dtsi index ec0c4e6..0007d3c 100644 --- a/arch/arm/boot/dts/testcases/tests-phandle.dtsi +++ b/arch/arm/boot/dts/testcases/tests-phandle.dtsi @@ -31,6 +31,8 @@ phandle-list-bad-phandle = <12345678 0 0>; phandle-list-bad-args = <&provider2 1 0>, <&provider3 0>; + empty-property; + unterminated-string = [40 41 42 43]; }; }; }; diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c index c47d619..f0783be 100644 --- a/arch/arm/common/gic.c +++ b/arch/arm/common/gic.c @@ -51,7 +51,6 @@ union gic_base { }; struct gic_chip_data { - unsigned int irq_offset; union gic_base dist_base; union gic_base cpu_base; #ifdef CONFIG_CPU_PM @@ -61,9 +60,7 @@ struct gic_chip_data { u32 __percpu *saved_ppi_enable; u32 __percpu *saved_ppi_conf; #endif -#ifdef CONFIG_IRQ_DOMAIN - struct irq_domain domain; -#endif + struct irq_domain *domain; unsigned int gic_irqs; #ifdef CONFIG_GIC_NON_BANKED void __iomem *(*get_base)(union gic_base *); @@ -282,7 +279,7 @@ asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) irqnr = irqstat & ~0x1c00; if (likely(irqnr > 15 && irqnr < 1021)) { - irqnr = irq_domain_to_irq(&gic->domain, irqnr); + irqnr = irq_find_mapping(gic->domain, irqnr); handle_IRQ(irqnr, regs); continue; } @@ -314,8 +311,8 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) if (gic_irq == 1023) goto out; - cascade_irq = irq_domain_to_irq(&chip_data->domain, gic_irq); - if (unlikely(gic_irq < 32 || gic_irq > 1020 || cascade_irq >= NR_IRQS)) + cascade_irq = irq_find_mapping(chip_data->domain, gic_irq); + if (unlikely(gic_irq < 32 || gic_irq > 1020)) do_bad_IRQ(cascade_irq, desc); else generic_handle_irq(cascade_irq); @@ -348,10 +345,9 @@ void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq) static void __init gic_dist_init(struct gic_chip_data *gic) { - unsigned int i, irq; + unsigned int i; u32 cpumask; unsigned int gic_irqs = gic->gic_irqs; - struct irq_domain *domain = &gic->domain; void __iomem *base = gic_data_dist_base(gic); u32 cpu = cpu_logical_map(smp_processor_id()); @@ -386,23 +382,6 @@ static void __init gic_dist_init(struct gic_chip_data *gic) for (i = 32; i < gic_irqs; i += 32) writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32); - /* - * Setup the Linux IRQ subsystem. - */ - irq_domain_for_each_irq(domain, i, irq) { - if (i < 32) { - irq_set_percpu_devid(irq); - irq_set_chip_and_handler(irq, &gic_chip, - handle_percpu_devid_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); - } else { - irq_set_chip_and_handler(irq, &gic_chip, - handle_fasteoi_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); - } - irq_set_chip_data(irq, gic); - } - writel_relaxed(1, base + GIC_DIST_CTRL); } @@ -618,11 +597,27 @@ static void __init gic_pm_init(struct gic_chip_data *gic) } #endif -#ifdef CONFIG_OF -static int gic_irq_domain_dt_translate(struct irq_domain *d, - struct device_node *controller, - const u32 *intspec, unsigned int intsize, - unsigned long *out_hwirq, unsigned int *out_type) +static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + if (hw < 32) { + irq_set_percpu_devid(irq); + irq_set_chip_and_handler(irq, &gic_chip, + handle_percpu_devid_irq); + set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); + } else { + irq_set_chip_and_handler(irq, &gic_chip, + handle_fasteoi_irq); + set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + } + irq_set_chip_data(irq, d->host_data); + return 0; +} + +static int gic_irq_domain_xlate(struct irq_domain *d, + struct device_node *controller, + const u32 *intspec, unsigned int intsize, + unsigned long *out_hwirq, unsigned int *out_type) { if (d->of_node != controller) return -EINVAL; @@ -639,26 +634,23 @@ static int gic_irq_domain_dt_translate(struct irq_domain *d, *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; return 0; } -#endif const struct irq_domain_ops gic_irq_domain_ops = { -#ifdef CONFIG_OF - .dt_translate = gic_irq_domain_dt_translate, -#endif + .map = gic_irq_domain_map, + .xlate = gic_irq_domain_xlate, }; void __init gic_init_bases(unsigned int gic_nr, int irq_start, void __iomem *dist_base, void __iomem *cpu_base, - u32 percpu_offset) + u32 percpu_offset, struct device_node *node) { + irq_hw_number_t hwirq_base; struct gic_chip_data *gic; - struct irq_domain *domain; - int gic_irqs; + int gic_irqs, irq_base; BUG_ON(gic_nr >= MAX_GIC_NR); gic = &gic_data[gic_nr]; - domain = &gic->domain; #ifdef CONFIG_GIC_NON_BANKED if (percpu_offset) { /* Frankein-GIC without banked registers... */ unsigned int cpu; @@ -694,10 +686,10 @@ void __init gic_init_bases(unsigned int gic_nr, int irq_start, * For primary GICs, skip over SGIs. * For secondary GICs, skip over PPIs, too. */ - domain->hwirq_base = 32; + hwirq_base = 32; if (gic_nr == 0) { if ((irq_start & 31) > 0) { - domain->hwirq_base = 16; + hwirq_base = 16; if (irq_start != -1) irq_start = (irq_start & ~31) + 16; } @@ -713,17 +705,17 @@ void __init gic_init_bases(unsigned int gic_nr, int irq_start, gic_irqs = 1020; gic->gic_irqs = gic_irqs; - domain->nr_irq = gic_irqs - domain->hwirq_base; - domain->irq_base = irq_alloc_descs(irq_start, 16, domain->nr_irq, - numa_node_id()); - if (IS_ERR_VALUE(domain->irq_base)) { + gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */ + irq_base = irq_alloc_descs(irq_start, 16, gic_irqs, numa_node_id()); + if (IS_ERR_VALUE(irq_base)) { WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", irq_start); - domain->irq_base = irq_start; + irq_base = irq_start; } - domain->priv = gic; - domain->ops = &gic_irq_domain_ops; - irq_domain_add(domain); + gic->domain = irq_domain_add_legacy(node, gic_irqs, irq_base, + hwirq_base, &gic_irq_domain_ops, gic); + if (WARN_ON(!gic->domain)) + return; gic_chip.flags |= gic_arch_extn.flags; gic_dist_init(gic); @@ -768,7 +760,6 @@ int __init gic_of_init(struct device_node *node, struct device_node *parent) void __iomem *dist_base; u32 percpu_offset; int irq; - struct irq_domain *domain = &gic_data[gic_cnt].domain; if (WARN_ON(!node)) return -ENODEV; @@ -782,9 +773,7 @@ int __init gic_of_init(struct device_node *node, struct device_node *parent) if (of_property_read_u32(node, "cpu-offset", &percpu_offset)) percpu_offset = 0; - domain->of_node = of_node_get(node); - - gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset); + gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset, node); if (parent) { irq = irq_of_parse_and_map(node, 0); diff --git a/arch/arm/common/vic.c b/arch/arm/common/vic.c index dcb004a..7a66311 100644 --- a/arch/arm/common/vic.c +++ b/arch/arm/common/vic.c @@ -56,7 +56,7 @@ struct vic_device { u32 int_enable; u32 soft_int; u32 protect; - struct irq_domain domain; + struct irq_domain *domain; }; /* we cannot allocate memory when VICs are initially registered */ @@ -192,14 +192,8 @@ static void __init vic_register(void __iomem *base, unsigned int irq, v->resume_sources = resume_sources; v->irq = irq; vic_id++; - - v->domain.irq_base = irq; - v->domain.nr_irq = 32; -#ifdef CONFIG_OF_IRQ - v->domain.of_node = of_node_get(node); -#endif /* CONFIG_OF */ - v->domain.ops = &irq_domain_simple_ops; - irq_domain_add(&v->domain); + v->domain = irq_domain_add_legacy(node, 32, irq, 0, + &irq_domain_simple_ops, v); } static void vic_ack_irq(struct irq_data *d) @@ -348,7 +342,7 @@ static void __init vic_init_st(void __iomem *base, unsigned int irq_start, vic_register(base, irq_start, 0, node); } -static void __init __vic_init(void __iomem *base, unsigned int irq_start, +void __init __vic_init(void __iomem *base, unsigned int irq_start, u32 vic_sources, u32 resume_sources, struct device_node *node) { @@ -444,7 +438,7 @@ static int handle_one_vic(struct vic_device *vic, struct pt_regs *regs) stat = readl_relaxed(vic->base + VIC_IRQ_STATUS); while (stat) { irq = ffs(stat) - 1; - handle_IRQ(irq_domain_to_irq(&vic->domain, irq), regs); + handle_IRQ(irq_find_mapping(vic->domain, irq), regs); stat &= ~(1 << irq); handled = 1; } diff --git a/arch/arm/include/asm/hardware/gic.h b/arch/arm/include/asm/hardware/gic.h index 4bdfe00..4b1ce6c 100644 --- a/arch/arm/include/asm/hardware/gic.h +++ b/arch/arm/include/asm/hardware/gic.h @@ -39,7 +39,7 @@ struct device_node; extern struct irq_chip gic_arch_extn; void gic_init_bases(unsigned int, int, void __iomem *, void __iomem *, - u32 offset); + u32 offset, struct device_node *); int gic_of_init(struct device_node *node, struct device_node *parent); void gic_secondary_init(unsigned int); void gic_handle_irq(struct pt_regs *regs); @@ -49,7 +49,7 @@ void gic_raise_softirq(const struct cpumask *mask, unsigned int irq); static inline void gic_init(unsigned int nr, int start, void __iomem *dist , void __iomem *cpu) { - gic_init_bases(nr, start, dist, cpu, 0); + gic_init_bases(nr, start, dist, cpu, 0, NULL); } #endif diff --git a/arch/arm/include/asm/hardware/vic.h b/arch/arm/include/asm/hardware/vic.h index f42ebd6..e14af1a 100644 --- a/arch/arm/include/asm/hardware/vic.h +++ b/arch/arm/include/asm/hardware/vic.h @@ -47,6 +47,8 @@ struct device_node; struct pt_regs; +void __vic_init(void __iomem *base, unsigned int irq_start, u32 vic_sources, + u32 resume_sources, struct device_node *node); void vic_init(void __iomem *base, unsigned int irq_start, u32 vic_sources, u32 resume_sources); int vic_of_init(struct device_node *node, struct device_node *parent); void vic_handle_irq(struct pt_regs *regs); diff --git a/arch/arm/include/asm/highmem.h b/arch/arm/include/asm/highmem.h index a4edd19..8c5e828 100644 --- a/arch/arm/include/asm/highmem.h +++ b/arch/arm/include/asm/highmem.h @@ -57,7 +57,7 @@ static inline void *kmap_high_get(struct page *page) #ifdef CONFIG_HIGHMEM extern void *kmap(struct page *page); extern void kunmap(struct page *page); -extern void *__kmap_atomic(struct page *page); +extern void *kmap_atomic(struct page *page); extern void __kunmap_atomic(void *kvaddr); extern void *kmap_atomic_pfn(unsigned long pfn); extern struct page *kmap_atomic_to_page(const void *ptr); diff --git a/arch/arm/include/asm/perf_event.h b/arch/arm/include/asm/perf_event.h index 99cfe36..7523340 100644 --- a/arch/arm/include/asm/perf_event.h +++ b/arch/arm/include/asm/perf_event.h @@ -12,10 +12,6 @@ #ifndef __ARM_PERF_EVENT_H__ #define __ARM_PERF_EVENT_H__ -/* ARM performance counters start from 1 (in the cp15 accesses) so use the - * same indexes here for consistency. */ -#define PERF_EVENT_INDEX_OFFSET 1 - /* ARM perf PMU IDs for use by internal perf clients. */ enum arm_perf_pmu_ids { ARM_PERF_PMU_ID_XSCALE1 = 0, diff --git a/arch/arm/include/asm/socket.h b/arch/arm/include/asm/socket.h index dec6f9a..6433cad 100644 --- a/arch/arm/include/asm/socket.h +++ b/arch/arm/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index b2abfa1..8a89d3b 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -539,6 +539,10 @@ static int armpmu_event_init(struct perf_event *event) int err = 0; atomic_t *active_events = &armpmu->active_events; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + if (armpmu->map_event(event) == -ENOENT) return -ENOENT; diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 971d65c..c2ae3cd 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -239,9 +239,7 @@ void cpu_idle(void) leds_event(led_idle_end); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index cdeb727..d616ed5 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -295,13 +295,6 @@ asmlinkage void __cpuinit secondary_start_kernel(void) */ percpu_timer_setup(); - while (!cpu_active(cpu)) - cpu_relax(); - - /* - * cpu_active bit is set, so it's safe to enalbe interrupts - * now. - */ local_irq_enable(); local_fiq_enable(); diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c index 1024396..e5fd241 100644 --- a/arch/arm/mach-bcmring/dma.c +++ b/arch/arm/mach-bcmring/dma.c @@ -35,7 +35,6 @@ #include <linux/pfn.h> #include <linux/atomic.h> -#include <linux/sched.h> #include <mach/dma.h> /* ---- Public Variables ------------------------------------------------- */ diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig index 5d602f68..dfad653 100644 --- a/arch/arm/mach-exynos/Kconfig +++ b/arch/arm/mach-exynos/Kconfig @@ -34,6 +34,7 @@ config CPU_EXYNOS4210 select ARM_CPU_SUSPEND if PM select S5P_PM if PM select S5P_SLEEP if PM + select PM_GENERIC_DOMAINS help Enable EXYNOS4210 CPU support @@ -74,11 +75,6 @@ config EXYNOS4_SETUP_FIMD0 help Common setup code for FIMD0. -config EXYNOS4_DEV_PD - bool - help - Compile in platform device definitions for Power Domain - config EXYNOS4_DEV_SYSMMU bool help @@ -195,7 +191,6 @@ config MACH_SMDKV310 select EXYNOS4_DEV_AHCI select SAMSUNG_DEV_KEYPAD select EXYNOS4_DEV_DMA - select EXYNOS4_DEV_PD select SAMSUNG_DEV_PWM select EXYNOS4_DEV_USB_OHCI select EXYNOS4_DEV_SYSMMU @@ -243,7 +238,6 @@ config MACH_UNIVERSAL_C210 select S5P_DEV_ONENAND select S5P_DEV_TV select EXYNOS4_DEV_DMA - select EXYNOS4_DEV_PD select EXYNOS4_SETUP_FIMD0 select EXYNOS4_SETUP_I2C1 select EXYNOS4_SETUP_I2C3 @@ -277,7 +271,6 @@ config MACH_NURI select S5P_DEV_USB_EHCI select S5P_SETUP_MIPIPHY select EXYNOS4_DEV_DMA - select EXYNOS4_DEV_PD select EXYNOS4_SETUP_FIMC select EXYNOS4_SETUP_FIMD0 select EXYNOS4_SETUP_I2C1 @@ -310,7 +303,6 @@ config MACH_ORIGEN select SAMSUNG_DEV_BACKLIGHT select SAMSUNG_DEV_PWM select EXYNOS4_DEV_DMA - select EXYNOS4_DEV_PD select EXYNOS4_DEV_USB_OHCI select EXYNOS4_SETUP_FIMD0 select EXYNOS4_SETUP_SDHCI diff --git a/arch/arm/mach-exynos/Makefile b/arch/arm/mach-exynos/Makefile index 5fc202c..d9191f9 100644 --- a/arch/arm/mach-exynos/Makefile +++ b/arch/arm/mach-exynos/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_CPU_EXYNOS4210) += clock-exynos4210.o obj-$(CONFIG_SOC_EXYNOS4212) += clock-exynos4212.o obj-$(CONFIG_PM) += pm.o +obj-$(CONFIG_PM_GENERIC_DOMAINS) += pm_domains.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_ARCH_EXYNOS4) += pmu.o @@ -45,7 +46,6 @@ obj-$(CONFIG_MACH_EXYNOS4_DT) += mach-exynos4-dt.o obj-$(CONFIG_ARCH_EXYNOS4) += dev-audio.o obj-$(CONFIG_EXYNOS4_DEV_AHCI) += dev-ahci.o -obj-$(CONFIG_EXYNOS4_DEV_PD) += dev-pd.o obj-$(CONFIG_EXYNOS4_DEV_SYSMMU) += dev-sysmmu.o obj-$(CONFIG_EXYNOS4_DEV_DWMCI) += dev-dwmci.o obj-$(CONFIG_EXYNOS4_DEV_DMA) += dma.o diff --git a/arch/arm/mach-exynos/common.c b/arch/arm/mach-exynos/common.c index c59e188..6de298c 100644 --- a/arch/arm/mach-exynos/common.c +++ b/arch/arm/mach-exynos/common.c @@ -402,7 +402,7 @@ void __init exynos4_init_irq(void) gic_bank_offset = soc_is_exynos4412() ? 0x4000 : 0x8000; if (!of_have_populated_dt()) - gic_init_bases(0, IRQ_PPI(0), S5P_VA_GIC_DIST, S5P_VA_GIC_CPU, gic_bank_offset); + gic_init_bases(0, IRQ_PPI(0), S5P_VA_GIC_DIST, S5P_VA_GIC_CPU, gic_bank_offset, NULL); #ifdef CONFIG_OF else of_irq_init(exynos4_dt_irq_match); diff --git a/arch/arm/mach-exynos/dev-pd.c b/arch/arm/mach-exynos/dev-pd.c deleted file mode 100644 index 3273f25..0000000 --- a/arch/arm/mach-exynos/dev-pd.c +++ /dev/null @@ -1,139 +0,0 @@ -/* linux/arch/arm/mach-exynos4/dev-pd.c - * - * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS4 - Power Domain support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include <linux/io.h> -#include <linux/kernel.h> -#include <linux/platform_device.h> -#include <linux/delay.h> - -#include <mach/regs-pmu.h> - -#include <plat/pd.h> - -static int exynos4_pd_enable(struct device *dev) -{ - struct samsung_pd_info *pdata = dev->platform_data; - u32 timeout; - - __raw_writel(S5P_INT_LOCAL_PWR_EN, pdata->base); - - /* Wait max 1ms */ - timeout = 10; - while ((__raw_readl(pdata->base + 0x4) & S5P_INT_LOCAL_PWR_EN) - != S5P_INT_LOCAL_PWR_EN) { - if (timeout == 0) { - printk(KERN_ERR "Power domain %s enable failed.\n", - dev_name(dev)); - return -ETIMEDOUT; - } - timeout--; - udelay(100); - } - - return 0; -} - -static int exynos4_pd_disable(struct device *dev) -{ - struct samsung_pd_info *pdata = dev->platform_data; - u32 timeout; - - __raw_writel(0, pdata->base); - - /* Wait max 1ms */ - timeout = 10; - while (__raw_readl(pdata->base + 0x4) & S5P_INT_LOCAL_PWR_EN) { - if (timeout == 0) { - printk(KERN_ERR "Power domain %s disable failed.\n", - dev_name(dev)); - return -ETIMEDOUT; - } - timeout--; - udelay(100); - } - - return 0; -} - -struct platform_device exynos4_device_pd[] = { - { - .name = "samsung-pd", - .id = 0, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_MFC_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 1, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_G3D_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 2, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_LCD0_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 3, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_LCD1_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 4, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_TV_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 5, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_CAM_CONF, - }, - }, - }, { - .name = "samsung-pd", - .id = 6, - .dev = { - .platform_data = &(struct samsung_pd_info) { - .enable = exynos4_pd_enable, - .disable = exynos4_pd_disable, - .base = S5P_PMU_GPS_CONF, - }, - }, - }, -}; diff --git a/arch/arm/mach-exynos/mach-nuri.c b/arch/arm/mach-exynos/mach-nuri.c index 435261f..aa37179 100644 --- a/arch/arm/mach-exynos/mach-nuri.c +++ b/arch/arm/mach-exynos/mach-nuri.c @@ -1263,9 +1263,6 @@ static struct platform_device *nuri_devices[] __initdata = { &s5p_device_mfc, &s5p_device_mfc_l, &s5p_device_mfc_r, - &exynos4_device_pd[PD_MFC], - &exynos4_device_pd[PD_LCD0], - &exynos4_device_pd[PD_CAM], &s5p_device_fimc_md, /* NURI Devices */ @@ -1315,14 +1312,6 @@ static void __init nuri_machine_init(void) /* Last */ platform_add_devices(nuri_devices, ARRAY_SIZE(nuri_devices)); - s5p_device_mfc.dev.parent = &exynos4_device_pd[PD_MFC].dev; - s5p_device_fimd0.dev.parent = &exynos4_device_pd[PD_LCD0].dev; - - s5p_device_fimc0.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc1.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc2.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc3.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_mipi_csis0.dev.parent = &exynos4_device_pd[PD_CAM].dev; } MACHINE_START(NURI, "NURI") diff --git a/arch/arm/mach-exynos/mach-origen.c b/arch/arm/mach-exynos/mach-origen.c index 0679b8a..fa5c4a5 100644 --- a/arch/arm/mach-exynos/mach-origen.c +++ b/arch/arm/mach-exynos/mach-origen.c @@ -621,13 +621,6 @@ static struct platform_device *origen_devices[] __initdata = { &s5p_device_mfc_r, &s5p_device_mixer, &exynos4_device_ohci, - &exynos4_device_pd[PD_LCD0], - &exynos4_device_pd[PD_TV], - &exynos4_device_pd[PD_G3D], - &exynos4_device_pd[PD_LCD1], - &exynos4_device_pd[PD_CAM], - &exynos4_device_pd[PD_GPS], - &exynos4_device_pd[PD_MFC], &origen_device_gpiokeys, &origen_lcd_hv070wsa, }; @@ -695,13 +688,6 @@ static void __init origen_machine_init(void) platform_add_devices(origen_devices, ARRAY_SIZE(origen_devices)); - s5p_device_fimd0.dev.parent = &exynos4_device_pd[PD_LCD0].dev; - - s5p_device_hdmi.dev.parent = &exynos4_device_pd[PD_TV].dev; - s5p_device_mixer.dev.parent = &exynos4_device_pd[PD_TV].dev; - - s5p_device_mfc.dev.parent = &exynos4_device_pd[PD_MFC].dev; - samsung_bl_set(&origen_bl_gpio_info, &origen_bl_data); } diff --git a/arch/arm/mach-exynos/mach-smdkv310.c b/arch/arm/mach-exynos/mach-smdkv310.c index b2c5557..5258b85 100644 --- a/arch/arm/mach-exynos/mach-smdkv310.c +++ b/arch/arm/mach-exynos/mach-smdkv310.c @@ -277,13 +277,6 @@ static struct platform_device *smdkv310_devices[] __initdata = { &s5p_device_mfc, &s5p_device_mfc_l, &s5p_device_mfc_r, - &exynos4_device_pd[PD_MFC], - &exynos4_device_pd[PD_G3D], - &exynos4_device_pd[PD_LCD0], - &exynos4_device_pd[PD_LCD1], - &exynos4_device_pd[PD_CAM], - &exynos4_device_pd[PD_TV], - &exynos4_device_pd[PD_GPS], &exynos4_device_spdif, &exynos4_device_sysmmu, &samsung_asoc_dma, @@ -336,10 +329,6 @@ static void s5p_tv_setup(void) WARN_ON(gpio_request_one(EXYNOS4_GPX3(7), GPIOF_IN, "hpd-plug")); s3c_gpio_cfgpin(EXYNOS4_GPX3(7), S3C_GPIO_SFN(0x3)); s3c_gpio_setpull(EXYNOS4_GPX3(7), S3C_GPIO_PULL_NONE); - - /* setup dependencies between TV devices */ - s5p_device_hdmi.dev.parent = &exynos4_device_pd[PD_TV].dev; - s5p_device_mixer.dev.parent = &exynos4_device_pd[PD_TV].dev; } static void __init smdkv310_map_io(void) @@ -379,7 +368,6 @@ static void __init smdkv310_machine_init(void) clk_xusbxti.rate = 24000000; platform_add_devices(smdkv310_devices, ARRAY_SIZE(smdkv310_devices)); - s5p_device_mfc.dev.parent = &exynos4_device_pd[PD_MFC].dev; } MACHINE_START(SMDKV310, "SMDKV310") diff --git a/arch/arm/mach-exynos/mach-universal_c210.c b/arch/arm/mach-exynos/mach-universal_c210.c index 3893995..b2d495b 100644 --- a/arch/arm/mach-exynos/mach-universal_c210.c +++ b/arch/arm/mach-exynos/mach-universal_c210.c @@ -971,7 +971,6 @@ static struct platform_device *universal_devices[] __initdata = { &s3c_device_i2c5, &s5p_device_i2c_hdmiphy, &hdmi_fixed_voltage, - &exynos4_device_pd[PD_TV], &s5p_device_hdmi, &s5p_device_sdo, &s5p_device_mixer, @@ -984,9 +983,6 @@ static struct platform_device *universal_devices[] __initdata = { &s5p_device_mfc, &s5p_device_mfc_l, &s5p_device_mfc_r, - &exynos4_device_pd[PD_MFC], - &exynos4_device_pd[PD_LCD0], - &exynos4_device_pd[PD_CAM], &cam_i_core_fixed_reg_dev, &cam_s_if_fixed_reg_dev, &s5p_device_fimc_md, @@ -1005,10 +1001,6 @@ void s5p_tv_setup(void) gpio_request_one(EXYNOS4_GPX3(7), GPIOF_IN, "hpd-plug"); s3c_gpio_cfgpin(EXYNOS4_GPX3(7), S3C_GPIO_SFN(0x3)); s3c_gpio_setpull(EXYNOS4_GPX3(7), S3C_GPIO_PULL_NONE); - - /* setup dependencies between TV devices */ - s5p_device_hdmi.dev.parent = &exynos4_device_pd[PD_TV].dev; - s5p_device_mixer.dev.parent = &exynos4_device_pd[PD_TV].dev; } static void __init universal_reserve(void) @@ -1042,15 +1034,6 @@ static void __init universal_machine_init(void) /* Last */ platform_add_devices(universal_devices, ARRAY_SIZE(universal_devices)); - - s5p_device_mfc.dev.parent = &exynos4_device_pd[PD_MFC].dev; - s5p_device_fimd0.dev.parent = &exynos4_device_pd[PD_LCD0].dev; - - s5p_device_fimc0.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc1.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc2.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_fimc3.dev.parent = &exynos4_device_pd[PD_CAM].dev; - s5p_device_mipi_csis0.dev.parent = &exynos4_device_pd[PD_CAM].dev; } MACHINE_START(UNIVERSAL_C210, "UNIVERSAL_C210") diff --git a/arch/arm/mach-exynos/pm_domains.c b/arch/arm/mach-exynos/pm_domains.c new file mode 100644 index 0000000..0b04af2 --- /dev/null +++ b/arch/arm/mach-exynos/pm_domains.c @@ -0,0 +1,195 @@ +/* + * Exynos Generic power domain support. + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * Implementation of Exynos specific power domain control which is used in + * conjunction with runtime-pm. Support for both device-tree and non-device-tree + * based power domain support is included. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. +*/ + +#include <linux/io.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/pm_domain.h> +#include <linux/delay.h> +#include <linux/of_address.h> + +#include <mach/regs-pmu.h> +#include <plat/devs.h> + +/* + * Exynos specific wrapper around the generic power domain + */ +struct exynos_pm_domain { + void __iomem *base; + char const *name; + bool is_off; + struct generic_pm_domain pd; +}; + +static int exynos_pd_power(struct generic_pm_domain *domain, bool power_on) +{ + struct exynos_pm_domain *pd; + void __iomem *base; + u32 timeout, pwr; + char *op; + + pd = container_of(domain, struct exynos_pm_domain, pd); + base = pd->base; + + pwr = power_on ? S5P_INT_LOCAL_PWR_EN : 0; + __raw_writel(pwr, base); + + /* Wait max 1ms */ + timeout = 10; + + while ((__raw_readl(base + 0x4) & S5P_INT_LOCAL_PWR_EN) != pwr) { + if (!timeout) { + op = (power_on) ? "enable" : "disable"; + pr_err("Power domain %s %s failed\n", domain->name, op); + return -ETIMEDOUT; + } + timeout--; + cpu_relax(); + usleep_range(80, 100); + } + return 0; +} + +static int exynos_pd_power_on(struct generic_pm_domain *domain) +{ + return exynos_pd_power(domain, true); +} + +static int exynos_pd_power_off(struct generic_pm_domain *domain) +{ + return exynos_pd_power(domain, false); +} + +#define EXYNOS_GPD(PD, BASE, NAME) \ +static struct exynos_pm_domain PD = { \ + .base = (void __iomem *)BASE, \ + .name = NAME, \ + .pd = { \ + .power_off = exynos_pd_power_off, \ + .power_on = exynos_pd_power_on, \ + }, \ +} + +#ifdef CONFIG_OF +static __init int exynos_pm_dt_parse_domains(void) +{ + struct device_node *np; + + for_each_compatible_node(np, NULL, "samsung,exynos4210-pd") { + struct exynos_pm_domain *pd; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) { + pr_err("%s: failed to allocate memory for domain\n", + __func__); + return -ENOMEM; + } + + if (of_get_property(np, "samsung,exynos4210-pd-off", NULL)) + pd->is_off = true; + pd->name = np->name; + pd->base = of_iomap(np, 0); + pd->pd.power_off = exynos_pd_power_off; + pd->pd.power_on = exynos_pd_power_on; + pd->pd.of_node = np; + pm_genpd_init(&pd->pd, NULL, false); + } + return 0; +} +#else +static __init int exynos_pm_dt_parse_domains(void) +{ + return 0; +} +#endif /* CONFIG_OF */ + +static __init void exynos_pm_add_dev_to_genpd(struct platform_device *pdev, + struct exynos_pm_domain *pd) +{ + if (pdev->dev.bus) { + if (pm_genpd_add_device(&pd->pd, &pdev->dev)) + pr_info("%s: error in adding %s device to %s power" + "domain\n", __func__, dev_name(&pdev->dev), + pd->name); + } +} + +EXYNOS_GPD(exynos4_pd_mfc, S5P_PMU_MFC_CONF, "pd-mfc"); +EXYNOS_GPD(exynos4_pd_g3d, S5P_PMU_G3D_CONF, "pd-g3d"); +EXYNOS_GPD(exynos4_pd_lcd0, S5P_PMU_LCD0_CONF, "pd-lcd0"); +EXYNOS_GPD(exynos4_pd_lcd1, S5P_PMU_LCD1_CONF, "pd-lcd1"); +EXYNOS_GPD(exynos4_pd_tv, S5P_PMU_TV_CONF, "pd-tv"); +EXYNOS_GPD(exynos4_pd_cam, S5P_PMU_CAM_CONF, "pd-cam"); +EXYNOS_GPD(exynos4_pd_gps, S5P_PMU_GPS_CONF, "pd-gps"); + +static struct exynos_pm_domain *exynos4_pm_domains[] = { + &exynos4_pd_mfc, + &exynos4_pd_g3d, + &exynos4_pd_lcd0, + &exynos4_pd_lcd1, + &exynos4_pd_tv, + &exynos4_pd_cam, + &exynos4_pd_gps, +}; + +static __init int exynos4_pm_init_power_domain(void) +{ + int idx; + + if (of_have_populated_dt()) + return exynos_pm_dt_parse_domains(); + + for (idx = 0; idx < ARRAY_SIZE(exynos4_pm_domains); idx++) + pm_genpd_init(&exynos4_pm_domains[idx]->pd, NULL, + exynos4_pm_domains[idx]->is_off); + +#ifdef CONFIG_S5P_DEV_FIMD0 + exynos_pm_add_dev_to_genpd(&s5p_device_fimd0, &exynos4_pd_lcd0); +#endif +#ifdef CONFIG_S5P_DEV_TV + exynos_pm_add_dev_to_genpd(&s5p_device_hdmi, &exynos4_pd_tv); + exynos_pm_add_dev_to_genpd(&s5p_device_mixer, &exynos4_pd_tv); +#endif +#ifdef CONFIG_S5P_DEV_MFC + exynos_pm_add_dev_to_genpd(&s5p_device_mfc, &exynos4_pd_mfc); +#endif +#ifdef CONFIG_S5P_DEV_FIMC0 + exynos_pm_add_dev_to_genpd(&s5p_device_fimc0, &exynos4_pd_cam); +#endif +#ifdef CONFIG_S5P_DEV_FIMC1 + exynos_pm_add_dev_to_genpd(&s5p_device_fimc1, &exynos4_pd_cam); +#endif +#ifdef CONFIG_S5P_DEV_FIMC2 + exynos_pm_add_dev_to_genpd(&s5p_device_fimc2, &exynos4_pd_cam); +#endif +#ifdef CONFIG_S5P_DEV_FIMC3 + exynos_pm_add_dev_to_genpd(&s5p_device_fimc3, &exynos4_pd_cam); +#endif +#ifdef CONFIG_S5P_DEV_CSIS0 + exynos_pm_add_dev_to_genpd(&s5p_device_mipi_csis0, &exynos4_pd_cam); +#endif +#ifdef CONFIG_S5P_DEV_CSIS1 + exynos_pm_add_dev_to_genpd(&s5p_device_mipi_csis1, &exynos4_pd_cam); +#endif + return 0; +} +arch_initcall(exynos4_pm_init_power_domain); + +static __init int exynos_pm_late_initcall(void) +{ + pm_genpd_poweroff_unused(); + return 0; +} +late_initcall(exynos_pm_late_initcall); diff --git a/arch/arm/mach-imx/imx51-dt.c b/arch/arm/mach-imx/imx51-dt.c index e6bad17..1e03ef4 100644 --- a/arch/arm/mach-imx/imx51-dt.c +++ b/arch/arm/mach-imx/imx51-dt.c @@ -47,7 +47,7 @@ static const struct of_dev_auxdata imx51_auxdata_lookup[] __initconst = { static int __init imx51_tzic_add_irq_domain(struct device_node *np, struct device_node *interrupt_parent) { - irq_domain_add_simple(np, 0); + irq_domain_add_legacy(np, 128, 0, 0, &irq_domain_simple_ops, NULL); return 0; } @@ -57,7 +57,7 @@ static int __init imx51_gpio_add_irq_domain(struct device_node *np, static int gpio_irq_base = MXC_GPIO_IRQ_START + ARCH_NR_GPIOS; gpio_irq_base -= 32; - irq_domain_add_simple(np, gpio_irq_base); + irq_domain_add_legacy(np, 32, gpio_irq_base, 0, &irq_domain_simple_ops, NULL); return 0; } diff --git a/arch/arm/mach-imx/imx53-dt.c b/arch/arm/mach-imx/imx53-dt.c index 05ebb3e..fd5be0f2 100644 --- a/arch/arm/mach-imx/imx53-dt.c +++ b/arch/arm/mach-imx/imx53-dt.c @@ -51,7 +51,7 @@ static const struct of_dev_auxdata imx53_auxdata_lookup[] __initconst = { static int __init imx53_tzic_add_irq_domain(struct device_node *np, struct device_node *interrupt_parent) { - irq_domain_add_simple(np, 0); + irq_domain_add_legacy(np, 128, 0, 0, &irq_domain_simple_ops, NULL); return 0; } @@ -61,7 +61,7 @@ static int __init imx53_gpio_add_irq_domain(struct device_node *np, static int gpio_irq_base = MXC_GPIO_IRQ_START + ARCH_NR_GPIOS; gpio_irq_base -= 32; - irq_domain_add_simple(np, gpio_irq_base); + irq_domain_add_legacy(np, 32, gpio_irq_base, 0, &irq_domain_simple_ops, NULL); return 0; } diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index c257281..6075d4d 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c @@ -97,7 +97,8 @@ static int __init imx6q_gpio_add_irq_domain(struct device_node *np, static int gpio_irq_base = MXC_GPIO_IRQ_START + ARCH_NR_GPIOS; gpio_irq_base -= 32; - irq_domain_add_simple(np, gpio_irq_base); + irq_domain_add_legacy(np, 32, gpio_irq_base, 0, &irq_domain_simple_ops, + NULL); return 0; } diff --git a/arch/arm/mach-imx/mach-pcm037.c b/arch/arm/mach-imx/mach-pcm037.c index e48854b..5fddf94 100644 --- a/arch/arm/mach-imx/mach-pcm037.c +++ b/arch/arm/mach-imx/mach-pcm037.c @@ -32,6 +32,8 @@ #include <linux/usb/ulpi.h> #include <linux/gfp.h> #include <linux/memblock.h> +#include <linux/regulator/machine.h> +#include <linux/regulator/fixed.h> #include <media/soc_camera.h> @@ -570,6 +572,11 @@ static int __init pcm037_otg_mode(char *options) } __setup("otg_mode=", pcm037_otg_mode); +static struct regulator_consumer_supply dummy_supplies[] = { + REGULATOR_SUPPLY("vdd33a", "smsc911x"), + REGULATOR_SUPPLY("vddvario", "smsc911x"), +}; + /* * Board specific initialization. */ @@ -579,6 +586,8 @@ static void __init pcm037_init(void) imx31_soc_init(); + regulator_register_fixed(0, dummy_supplies, ARRAY_SIZE(dummy_supplies)); + mxc_iomux_set_gpr(MUX_PGP_UH2, 1); mxc_iomux_setup_multiple_pins(pcm037_pins, ARRAY_SIZE(pcm037_pins), diff --git a/arch/arm/mach-imx/mx31moboard-devboard.c b/arch/arm/mach-imx/mx31moboard-devboard.c index 0aa2536..cc285e5 100644 --- a/arch/arm/mach-imx/mx31moboard-devboard.c +++ b/arch/arm/mach-imx/mx31moboard-devboard.c @@ -158,7 +158,7 @@ static int devboard_usbh1_hw_init(struct platform_device *pdev) #define USBH1_VBUSEN_B IOMUX_TO_GPIO(MX31_PIN_NFRE_B) #define USBH1_MODE IOMUX_TO_GPIO(MX31_PIN_NFALE) -static int devboard_isp1105_init(struct otg_transceiver *otg) +static int devboard_isp1105_init(struct usb_phy *otg) { int ret = gpio_request(USBH1_MODE, "usbh1-mode"); if (ret) @@ -177,7 +177,7 @@ static int devboard_isp1105_init(struct otg_transceiver *otg) } -static int devboard_isp1105_set_vbus(struct otg_transceiver *otg, bool on) +static int devboard_isp1105_set_vbus(struct usb_otg *otg, bool on) { if (on) gpio_set_value(USBH1_VBUSEN_B, 0); @@ -194,18 +194,24 @@ static struct mxc_usbh_platform_data usbh1_pdata __initdata = { static int __init devboard_usbh1_init(void) { - struct otg_transceiver *otg; + struct usb_phy *phy; struct platform_device *pdev; - otg = kzalloc(sizeof(*otg), GFP_KERNEL); - if (!otg) + phy = kzalloc(sizeof(*phy), GFP_KERNEL); + if (!phy) return -ENOMEM; - otg->label = "ISP1105"; - otg->init = devboard_isp1105_init; - otg->set_vbus = devboard_isp1105_set_vbus; + phy->otg = kzalloc(sizeof(struct usb_otg), GFP_KERNEL); + if (!phy->otg) { + kfree(phy); + return -ENOMEM; + } + + phy->label = "ISP1105"; + phy->init = devboard_isp1105_init; + phy->otg->set_vbus = devboard_isp1105_set_vbus; - usbh1_pdata.otg = otg; + usbh1_pdata.otg = phy; pdev = imx31_add_mxc_ehci_hs(1, &usbh1_pdata); if (IS_ERR(pdev)) diff --git a/arch/arm/mach-imx/mx31moboard-marxbot.c b/arch/arm/mach-imx/mx31moboard-marxbot.c index bb639cb..135c90e 100644 --- a/arch/arm/mach-imx/mx31moboard-marxbot.c +++ b/arch/arm/mach-imx/mx31moboard-marxbot.c @@ -272,7 +272,7 @@ static int marxbot_usbh1_hw_init(struct platform_device *pdev) #define USBH1_VBUSEN_B IOMUX_TO_GPIO(MX31_PIN_NFRE_B) #define USBH1_MODE IOMUX_TO_GPIO(MX31_PIN_NFALE) -static int marxbot_isp1105_init(struct otg_transceiver *otg) +static int marxbot_isp1105_init(struct usb_phy *otg) { int ret = gpio_request(USBH1_MODE, "usbh1-mode"); if (ret) @@ -291,7 +291,7 @@ static int marxbot_isp1105_init(struct otg_transceiver *otg) } -static int marxbot_isp1105_set_vbus(struct otg_transceiver *otg, bool on) +static int marxbot_isp1105_set_vbus(struct usb_otg *otg, bool on) { if (on) gpio_set_value(USBH1_VBUSEN_B, 0); @@ -308,18 +308,24 @@ static struct mxc_usbh_platform_data usbh1_pdata __initdata = { static int __init marxbot_usbh1_init(void) { - struct otg_transceiver *otg; + struct usb_phy *phy; struct platform_device *pdev; - otg = kzalloc(sizeof(*otg), GFP_KERNEL); - if (!otg) + phy = kzalloc(sizeof(*phy), GFP_KERNEL); + if (!phy) return -ENOMEM; - otg->label = "ISP1105"; - otg->init = marxbot_isp1105_init; - otg->set_vbus = marxbot_isp1105_set_vbus; + phy->otg = kzalloc(sizeof(struct usb_otg), GFP_KERNEL); + if (!phy->otg) { + kfree(phy); + return -ENOMEM; + } + + phy->label = "ISP1105"; + phy->init = marxbot_isp1105_init; + phy->otg->set_vbus = marxbot_isp1105_set_vbus; - usbh1_pdata.otg = otg; + usbh1_pdata.otg = phy; pdev = imx31_add_mxc_ehci_hs(1, &usbh1_pdata); if (IS_ERR(pdev)) diff --git a/arch/arm/mach-ks8695/leds.c b/arch/arm/mach-ks8695/leds.c index d6f6502..4bd7075 100644 --- a/arch/arm/mach-ks8695/leds.c +++ b/arch/arm/mach-ks8695/leds.c @@ -11,7 +11,6 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> -#include <linux/gpio.h> #include <asm/leds.h> #include <mach/devices.h> diff --git a/arch/arm/mach-msm/board-msm8x60.c b/arch/arm/mach-msm/board-msm8x60.c index 0a11342..962e711 100644 --- a/arch/arm/mach-msm/board-msm8x60.c +++ b/arch/arm/mach-msm/board-msm8x60.c @@ -80,12 +80,8 @@ static struct of_device_id msm_dt_gic_match[] __initdata = { static void __init msm8x60_dt_init(void) { - struct device_node *node; - - node = of_find_matching_node_by_address(NULL, msm_dt_gic_match, - MSM8X60_QGIC_DIST_PHYS); - if (node) - irq_domain_add_simple(node, GIC_SPI_START); + irq_domain_generate_simple(msm_dt_gic_match, MSM8X60_QGIC_DIST_PHYS, + GIC_SPI_START); if (of_machine_is_compatible("qcom,msm8660-surf")) { printk(KERN_INFO "Init surf UART registers\n"); diff --git a/arch/arm/mach-omap1/lcd_dma.c b/arch/arm/mach-omap1/lcd_dma.c index 4538093..4c5ce7d 100644 --- a/arch/arm/mach-omap1/lcd_dma.c +++ b/arch/arm/mach-omap1/lcd_dma.c @@ -117,7 +117,7 @@ EXPORT_SYMBOL(omap_set_lcd_dma_b1_mirror); void omap_set_lcd_dma_b1_vxres(unsigned long vxres) { if (cpu_is_omap15xx()) { - printk(KERN_ERR "DMA virtual resulotion is not supported " + printk(KERN_ERR "DMA virtual resolution is not supported " "in 1510 mode\n"); BUG(); } diff --git a/arch/arm/mach-omap2/board-generic.c b/arch/arm/mach-omap2/board-generic.c index ad49762..45fdfe2 100644 --- a/arch/arm/mach-omap2/board-generic.c +++ b/arch/arm/mach-omap2/board-generic.c @@ -68,7 +68,7 @@ static void __init omap_generic_init(void) { struct device_node *node = of_find_matching_node(NULL, intc_match); if (node) - irq_domain_add_simple(node, 0); + irq_domain_add_legacy(node, 32, 0, 0, &irq_domain_simple_ops, NULL); omap_sdrc_init(NULL, NULL); diff --git a/arch/arm/mach-orion5x/db88f5281-setup.c b/arch/arm/mach-orion5x/db88f5281-setup.c index a104d5a..e52108c 100644 --- a/arch/arm/mach-orion5x/db88f5281-setup.c +++ b/arch/arm/mach-orion5x/db88f5281-setup.c @@ -214,7 +214,7 @@ void __init db88f5281_pci_preinit(void) if (gpio_direction_input(pin) == 0) { irq_set_irq_type(gpio_to_irq(pin), IRQ_TYPE_LEVEL_LOW); } else { - printk(KERN_ERR "db88f5281_pci_preinit faield to " + printk(KERN_ERR "db88f5281_pci_preinit failed to " "set_irq_type pin %d\n", pin); gpio_free(pin); } @@ -227,7 +227,7 @@ void __init db88f5281_pci_preinit(void) if (gpio_direction_input(pin) == 0) { irq_set_irq_type(gpio_to_irq(pin), IRQ_TYPE_LEVEL_LOW); } else { - printk(KERN_ERR "db88f5281_pci_preinit faield " + printk(KERN_ERR "db88f5281_pci_preinit failed " "to set_irq_type pin %d\n", pin); gpio_free(pin); } diff --git a/arch/arm/mach-orion5x/rd88f5182-setup.c b/arch/arm/mach-orion5x/rd88f5182-setup.c index 96438b6..e3ce617 100644 --- a/arch/arm/mach-orion5x/rd88f5182-setup.c +++ b/arch/arm/mach-orion5x/rd88f5182-setup.c @@ -149,7 +149,7 @@ void __init rd88f5182_pci_preinit(void) if (gpio_direction_input(pin) == 0) { irq_set_irq_type(gpio_to_irq(pin), IRQ_TYPE_LEVEL_LOW); } else { - printk(KERN_ERR "rd88f5182_pci_preinit faield to " + printk(KERN_ERR "rd88f5182_pci_preinit failed to " "set_irq_type pin %d\n", pin); gpio_free(pin); } @@ -162,7 +162,7 @@ void __init rd88f5182_pci_preinit(void) if (gpio_direction_input(pin) == 0) { irq_set_irq_type(gpio_to_irq(pin), IRQ_TYPE_LEVEL_LOW); } else { - printk(KERN_ERR "rd88f5182_pci_preinit faield to " + printk(KERN_ERR "rd88f5182_pci_preinit failed to " "set_irq_type pin %d\n", pin); gpio_free(pin); } diff --git a/arch/arm/mach-prima2/irq.c b/arch/arm/mach-prima2/irq.c index d93ceef..37c2de9 100644 --- a/arch/arm/mach-prima2/irq.c +++ b/arch/arm/mach-prima2/irq.c @@ -68,7 +68,7 @@ void __init sirfsoc_of_irq_init(void) if (!sirfsoc_intc_base) panic("unable to map intc cpu registers\n"); - irq_domain_add_simple(np, 0); + irq_domain_add_legacy(np, 32, 0, 0, &irq_domain_simple_ops, NULL); of_node_put(np); diff --git a/arch/arm/mach-pxa/pxa3xx-ulpi.c b/arch/arm/mach-pxa/pxa3xx-ulpi.c index e28dfb8..5ead6d4 100644 --- a/arch/arm/mach-pxa/pxa3xx-ulpi.c +++ b/arch/arm/mach-pxa/pxa3xx-ulpi.c @@ -33,7 +33,7 @@ struct pxa3xx_u2d_ulpi { struct clk *clk; void __iomem *mmio_base; - struct otg_transceiver *otg; + struct usb_phy *otg; unsigned int ulpi_mode; }; @@ -79,7 +79,7 @@ static int pxa310_ulpi_poll(void) return -ETIMEDOUT; } -static int pxa310_ulpi_read(struct otg_transceiver *otg, u32 reg) +static int pxa310_ulpi_read(struct usb_phy *otg, u32 reg) { int err; @@ -98,7 +98,7 @@ static int pxa310_ulpi_read(struct otg_transceiver *otg, u32 reg) return u2d_readl(U2DOTGUCR) & U2DOTGUCR_RDATA; } -static int pxa310_ulpi_write(struct otg_transceiver *otg, u32 val, u32 reg) +static int pxa310_ulpi_write(struct usb_phy *otg, u32 val, u32 reg) { if (pxa310_ulpi_get_phymode() != SYNCH) { pr_warning("%s: PHY is not in SYNCH mode!\n", __func__); @@ -111,7 +111,7 @@ static int pxa310_ulpi_write(struct otg_transceiver *otg, u32 val, u32 reg) return pxa310_ulpi_poll(); } -struct otg_io_access_ops pxa310_ulpi_access_ops = { +struct usb_phy_io_ops pxa310_ulpi_access_ops = { .read = pxa310_ulpi_read, .write = pxa310_ulpi_write, }; @@ -139,19 +139,19 @@ static int pxa310_start_otg_host_transcvr(struct usb_bus *host) pxa310_otg_transceiver_rtsm(); - err = otg_init(u2d->otg); + err = usb_phy_init(u2d->otg); if (err) { pr_err("OTG transceiver init failed"); return err; } - err = otg_set_vbus(u2d->otg, 1); + err = otg_set_vbus(u2d->otg->otg, 1); if (err) { pr_err("OTG transceiver VBUS set failed"); return err; } - err = otg_set_host(u2d->otg, host); + err = otg_set_host(u2d->otg->otg, host); if (err) pr_err("OTG transceiver Host mode set failed"); @@ -189,9 +189,9 @@ static void pxa310_stop_otg_hc(void) { pxa310_otg_transceiver_rtsm(); - otg_set_host(u2d->otg, NULL); - otg_set_vbus(u2d->otg, 0); - otg_shutdown(u2d->otg); + otg_set_host(u2d->otg->otg, NULL); + otg_set_vbus(u2d->otg->otg, 0); + usb_phy_shutdown(u2d->otg); } static void pxa310_u2d_setup_otg_hc(void) diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c index a83cf51..cccf91b 100644 --- a/arch/arm/mach-shmobile/setup-sh7372.c +++ b/arch/arm/mach-shmobile/setup-sh7372.c @@ -1043,6 +1043,8 @@ void __init sh7372_add_standard_devices(void) sh7372_add_device_to_domain(&sh7372_a4r, &veu2_device); sh7372_add_device_to_domain(&sh7372_a4r, &veu3_device); sh7372_add_device_to_domain(&sh7372_a4r, &jpu_device); + sh7372_add_device_to_domain(&sh7372_a4r, &tmu00_device); + sh7372_add_device_to_domain(&sh7372_a4r, &tmu01_device); } void __init sh7372_add_early_devices(void) diff --git a/arch/arm/mach-spear3xx/spear300.c b/arch/arm/mach-spear3xx/spear300.c index a5e46b4..4f7f518 100644 --- a/arch/arm/mach-spear3xx/spear300.c +++ b/arch/arm/mach-spear3xx/spear300.c @@ -469,7 +469,7 @@ void __init spear300_init(struct pmx_mode *pmx_mode, struct pmx_dev **pmx_devs, if (pmx_driver.base) { ret = pmx_register(&pmx_driver); if (ret) - printk(KERN_ERR "padmux: registeration failed. err no" + printk(KERN_ERR "padmux: registration failed. err no" ": %d\n", ret); /* Free Mapping, device selection already done */ iounmap(pmx_driver.base); diff --git a/arch/arm/mach-spear3xx/spear310.c b/arch/arm/mach-spear3xx/spear310.c index 9004cf9..febaa6f 100644 --- a/arch/arm/mach-spear3xx/spear310.c +++ b/arch/arm/mach-spear3xx/spear310.c @@ -303,6 +303,6 @@ void __init spear310_init(struct pmx_mode *pmx_mode, struct pmx_dev **pmx_devs, ret = pmx_register(&pmx_driver); if (ret) - printk(KERN_ERR "padmux: registeration failed. err no: %d\n", + printk(KERN_ERR "padmux: registration failed. err no: %d\n", ret); } diff --git a/arch/arm/mach-spear3xx/spear320.c b/arch/arm/mach-spear3xx/spear320.c index ee29bef..deaaf19 100644 --- a/arch/arm/mach-spear3xx/spear320.c +++ b/arch/arm/mach-spear3xx/spear320.c @@ -550,6 +550,6 @@ void __init spear320_init(struct pmx_mode *pmx_mode, struct pmx_dev **pmx_devs, ret = pmx_register(&pmx_driver); if (ret) - printk(KERN_ERR "padmux: registeration failed. err no: %d\n", + printk(KERN_ERR "padmux: registration failed. err no: %d\n", ret); } diff --git a/arch/arm/mach-tegra/fuse.c b/arch/arm/mach-tegra/fuse.c index 1fa26d9..ea49bd9 100644 --- a/arch/arm/mach-tegra/fuse.c +++ b/arch/arm/mach-tegra/fuse.c @@ -19,6 +19,7 @@ #include <linux/kernel.h> #include <linux/io.h> +#include <linux/module.h> #include <mach/iomap.h> @@ -58,6 +59,7 @@ unsigned long long tegra_chip_uid(void) hi = fuse_readl(FUSE_UID_HIGH); return (hi << 32ull) | lo; } +EXPORT_SYMBOL(tegra_chip_uid); int tegra_sku_id(void) { diff --git a/arch/arm/mach-tegra/include/mach/usb_phy.h b/arch/arm/mach-tegra/include/mach/usb_phy.h index d4b8f9e..de1a0f6 100644 --- a/arch/arm/mach-tegra/include/mach/usb_phy.h +++ b/arch/arm/mach-tegra/include/mach/usb_phy.h @@ -58,7 +58,7 @@ struct tegra_usb_phy { struct clk *pad_clk; enum tegra_usb_phy_mode mode; void *config; - struct otg_transceiver *ulpi; + struct usb_phy *ulpi; }; struct tegra_usb_phy *tegra_usb_phy_open(int instance, void __iomem *regs, diff --git a/arch/arm/mach-tegra/usb_phy.c b/arch/arm/mach-tegra/usb_phy.c index 37576a7..ad321f9 100644 --- a/arch/arm/mach-tegra/usb_phy.c +++ b/arch/arm/mach-tegra/usb_phy.c @@ -608,13 +608,13 @@ static int ulpi_phy_power_on(struct tegra_usb_phy *phy) writel(val, base + ULPI_TIMING_CTRL_1); /* Fix VbusInvalid due to floating VBUS */ - ret = otg_io_write(phy->ulpi, 0x40, 0x08); + ret = usb_phy_io_write(phy->ulpi, 0x40, 0x08); if (ret) { pr_err("%s: ulpi write failed\n", __func__); return ret; } - ret = otg_io_write(phy->ulpi, 0x80, 0x0B); + ret = usb_phy_io_write(phy->ulpi, 0x80, 0x0B); if (ret) { pr_err("%s: ulpi write failed\n", __func__); return ret; diff --git a/arch/arm/mach-u300/i2c.c b/arch/arm/mach-u300/i2c.c index 5140dee..a38f802 100644 --- a/arch/arm/mach-u300/i2c.c +++ b/arch/arm/mach-u300/i2c.c @@ -60,7 +60,6 @@ static struct regulator_consumer_supply supply_ldo_c[] = { */ static struct regulator_consumer_supply supply_ldo_d[] = { { - .dev = NULL, .supply = "vana15", /* Powers the SoC (CPU etc) */ }, }; @@ -92,7 +91,6 @@ static struct regulator_consumer_supply supply_ldo_k[] = { */ static struct regulator_consumer_supply supply_ldo_ext[] = { { - .dev = NULL, .supply = "vext", /* External power */ }, }; diff --git a/arch/arm/mach-versatile/core.c b/arch/arm/mach-versatile/core.c index 02b7b93..008ce22 100644 --- a/arch/arm/mach-versatile/core.c +++ b/arch/arm/mach-versatile/core.c @@ -98,8 +98,11 @@ static const struct of_device_id sic_of_match[] __initconst = { void __init versatile_init_irq(void) { - vic_init(VA_VIC_BASE, IRQ_VIC_START, ~0, 0); - irq_domain_generate_simple(vic_of_match, VERSATILE_VIC_BASE, IRQ_VIC_START); + struct device_node *np; + + np = of_find_matching_node_by_address(NULL, vic_of_match, + VERSATILE_VIC_BASE); + __vic_init(VA_VIC_BASE, IRQ_VIC_START, ~0, 0, np); writel(~0, VA_SIC_BASE + SIC_IRQ_ENABLE_CLEAR); diff --git a/arch/arm/mm/copypage-fa.c b/arch/arm/mm/copypage-fa.c index d2852e1..d130a5e 100644 --- a/arch/arm/mm/copypage-fa.c +++ b/arch/arm/mm/copypage-fa.c @@ -44,11 +44,11 @@ void fa_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); fa_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } /* @@ -58,7 +58,7 @@ void fa_copy_user_highpage(struct page *to, struct page *from, */ void fa_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile("\ mov r1, %2 @ 1\n\ mov r2, #0 @ 1\n\ @@ -77,7 +77,7 @@ void fa_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 32) : "r1", "r2", "r3", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns fa_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c index ac163de..49ee0c1 100644 --- a/arch/arm/mm/copypage-feroceon.c +++ b/arch/arm/mm/copypage-feroceon.c @@ -72,17 +72,17 @@ void feroceon_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); flush_cache_page(vma, vaddr, page_to_pfn(from)); feroceon_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile ("\ mov r1, %2 \n\ mov r2, #0 \n\ @@ -102,7 +102,7 @@ void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 32) : "r1", "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns feroceon_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-v3.c b/arch/arm/mm/copypage-v3.c index f72303e..3935bdd 100644 --- a/arch/arm/mm/copypage-v3.c +++ b/arch/arm/mm/copypage-v3.c @@ -42,11 +42,11 @@ void v3_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); v3_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } /* @@ -56,7 +56,7 @@ void v3_copy_user_highpage(struct page *to, struct page *from, */ void v3_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile("\n\ mov r1, %2 @ 1\n\ mov r2, #0 @ 1\n\ @@ -72,7 +72,7 @@ void v3_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 64) : "r1", "r2", "r3", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns v3_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 7d0a8c2..ec8c3be 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -71,7 +71,7 @@ mc_copy_user_page(void *from, void *to) void v4_mc_copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { - void *kto = kmap_atomic(to, KM_USER1); + void *kto = kmap_atomic(to); if (!test_and_set_bit(PG_dcache_clean, &from->flags)) __flush_dcache_page(page_mapping(from), from); @@ -85,7 +85,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from, raw_spin_unlock(&minicache_lock); - kunmap_atomic(kto, KM_USER1); + kunmap_atomic(kto); } /* @@ -93,7 +93,7 @@ void v4_mc_copy_user_highpage(struct page *to, struct page *from, */ void v4_mc_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile("\ mov r1, %2 @ 1\n\ mov r2, #0 @ 1\n\ @@ -111,7 +111,7 @@ void v4_mc_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 64) : "r1", "r2", "r3", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns v4_mc_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c index cb589cb..067d0fd 100644 --- a/arch/arm/mm/copypage-v4wb.c +++ b/arch/arm/mm/copypage-v4wb.c @@ -52,12 +52,12 @@ void v4wb_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); flush_cache_page(vma, vaddr, page_to_pfn(from)); v4wb_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } /* @@ -67,7 +67,7 @@ void v4wb_copy_user_highpage(struct page *to, struct page *from, */ void v4wb_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile("\ mov r1, %2 @ 1\n\ mov r2, #0 @ 1\n\ @@ -86,7 +86,7 @@ void v4wb_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 64) : "r1", "r2", "r3", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns v4wb_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c index 30c7d04..b85c5da 100644 --- a/arch/arm/mm/copypage-v4wt.c +++ b/arch/arm/mm/copypage-v4wt.c @@ -48,11 +48,11 @@ void v4wt_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); v4wt_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } /* @@ -62,7 +62,7 @@ void v4wt_copy_user_highpage(struct page *to, struct page *from, */ void v4wt_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile("\ mov r1, %2 @ 1\n\ mov r2, #0 @ 1\n\ @@ -79,7 +79,7 @@ void v4wt_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 64) : "r1", "r2", "r3", "ip", "lr"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns v4wt_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 3d9a155..8b03a58 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -38,11 +38,11 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, { void *kto, *kfrom; - kfrom = kmap_atomic(from, KM_USER0); - kto = kmap_atomic(to, KM_USER1); + kfrom = kmap_atomic(from); + kto = kmap_atomic(to); copy_page(kto, kfrom); - kunmap_atomic(kto, KM_USER1); - kunmap_atomic(kfrom, KM_USER0); + kunmap_atomic(kto); + kunmap_atomic(kfrom); } /* @@ -51,9 +51,9 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, */ static void v6_clear_user_highpage_nonaliasing(struct page *page, unsigned long vaddr) { - void *kaddr = kmap_atomic(page, KM_USER0); + void *kaddr = kmap_atomic(page); clear_page(kaddr); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } /* diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c index f9cde07..03a2042 100644 --- a/arch/arm/mm/copypage-xsc3.c +++ b/arch/arm/mm/copypage-xsc3.c @@ -75,12 +75,12 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, { void *kto, *kfrom; - kto = kmap_atomic(to, KM_USER0); - kfrom = kmap_atomic(from, KM_USER1); + kto = kmap_atomic(to); + kfrom = kmap_atomic(from); flush_cache_page(vma, vaddr, page_to_pfn(from)); xsc3_mc_copy_user_page(kto, kfrom); - kunmap_atomic(kfrom, KM_USER1); - kunmap_atomic(kto, KM_USER0); + kunmap_atomic(kfrom); + kunmap_atomic(kto); } /* @@ -90,7 +90,7 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, */ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile ("\ mov r1, %2 \n\ mov r2, #0 \n\ @@ -105,7 +105,7 @@ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 32) : "r1", "r2", "r3"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns xsc3_mc_user_fns __initdata = { diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 610c24c..439d106 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -93,7 +93,7 @@ mc_copy_user_page(void *from, void *to) void xscale_mc_copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { - void *kto = kmap_atomic(to, KM_USER1); + void *kto = kmap_atomic(to); if (!test_and_set_bit(PG_dcache_clean, &from->flags)) __flush_dcache_page(page_mapping(from), from); @@ -107,7 +107,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from, raw_spin_unlock(&minicache_lock); - kunmap_atomic(kto, KM_USER1); + kunmap_atomic(kto); } /* @@ -116,7 +116,7 @@ void xscale_mc_copy_user_highpage(struct page *to, struct page *from, void xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr) { - void *ptr, *kaddr = kmap_atomic(page, KM_USER0); + void *ptr, *kaddr = kmap_atomic(page); asm volatile( "mov r1, %2 \n\ mov r2, #0 \n\ @@ -133,7 +133,7 @@ xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr) : "=r" (ptr) : "0" (kaddr), "I" (PAGE_SIZE / 32) : "r1", "r2", "r3", "ip"); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } struct cpu_user_fns xscale_mc_user_fns __initdata = { diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index 807c057..5a21505 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -36,7 +36,7 @@ void kunmap(struct page *page) } EXPORT_SYMBOL(kunmap); -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { unsigned int idx; unsigned long vaddr; @@ -81,7 +81,7 @@ void *__kmap_atomic(struct page *page) return (void *)vaddr; } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 5dc7d12..245a55a 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -32,7 +32,6 @@ #include <asm/mach/arch.h> #include <asm/mach/map.h> -#include <asm/memblock.h> #include "mm.h" diff --git a/arch/arm/plat-mxc/3ds_debugboard.c b/arch/arm/plat-mxc/3ds_debugboard.c index f0ba072..d1e31fa 100644 --- a/arch/arm/plat-mxc/3ds_debugboard.c +++ b/arch/arm/plat-mxc/3ds_debugboard.c @@ -16,6 +16,8 @@ #include <linux/platform_device.h> #include <linux/gpio.h> #include <linux/smsc911x.h> +#include <linux/regulator/machine.h> +#include <linux/regulator/fixed.h> #include <mach/hardware.h> @@ -148,6 +150,11 @@ static struct irq_chip expio_irq_chip = { .irq_unmask = expio_unmask_irq, }; +static struct regulator_consumer_supply dummy_supplies[] = { + REGULATOR_SUPPLY("vdd33a", "smsc911x"), + REGULATOR_SUPPLY("vddvario", "smsc911x"), +}; + int __init mxc_expio_init(u32 base, u32 p_irq) { int i; @@ -188,6 +195,8 @@ int __init mxc_expio_init(u32 base, u32 p_irq) irq_set_chained_handler(p_irq, mxc_expio_irq_handler); /* Register Lan device on the debugboard */ + regulator_register_fixed(0, dummy_supplies, ARRAY_SIZE(dummy_supplies)); + smsc911x_resources[0].start = LAN9217_BASE_ADDR(base); smsc911x_resources[0].end = LAN9217_BASE_ADDR(base) + 0x100 - 1; platform_device_register(&smsc_lan9217_device); diff --git a/arch/arm/plat-mxc/include/mach/mxc_ehci.h b/arch/arm/plat-mxc/include/mach/mxc_ehci.h index 2c159dc..9ffd1bb 100644 --- a/arch/arm/plat-mxc/include/mach/mxc_ehci.h +++ b/arch/arm/plat-mxc/include/mach/mxc_ehci.h @@ -44,7 +44,7 @@ struct mxc_usbh_platform_data { int (*exit)(struct platform_device *pdev); unsigned int portsc; - struct otg_transceiver *otg; + struct usb_phy *otg; }; int mx51_initialize_usb_hw(int port, unsigned int flags); diff --git a/arch/arm/plat-mxc/include/mach/ulpi.h b/arch/arm/plat-mxc/include/mach/ulpi.h index f9161c9..42bdaca 100644 --- a/arch/arm/plat-mxc/include/mach/ulpi.h +++ b/arch/arm/plat-mxc/include/mach/ulpi.h @@ -2,15 +2,15 @@ #define __MACH_ULPI_H #ifdef CONFIG_USB_ULPI -struct otg_transceiver *imx_otg_ulpi_create(unsigned int flags); +struct usb_phy *imx_otg_ulpi_create(unsigned int flags); #else -static inline struct otg_transceiver *imx_otg_ulpi_create(unsigned int flags) +static inline struct usb_phy *imx_otg_ulpi_create(unsigned int flags) { return NULL; } #endif -extern struct otg_io_access_ops mxc_ulpi_access_ops; +extern struct usb_phy_io_ops mxc_ulpi_access_ops; #endif /* __MACH_ULPI_H */ diff --git a/arch/arm/plat-mxc/ulpi.c b/arch/arm/plat-mxc/ulpi.c index 477e45b..d296342 100644 --- a/arch/arm/plat-mxc/ulpi.c +++ b/arch/arm/plat-mxc/ulpi.c @@ -58,7 +58,7 @@ static int ulpi_poll(void __iomem *view, u32 bit) return -ETIMEDOUT; } -static int ulpi_read(struct otg_transceiver *otg, u32 reg) +static int ulpi_read(struct usb_phy *otg, u32 reg) { int ret; void __iomem *view = otg->io_priv; @@ -84,7 +84,7 @@ static int ulpi_read(struct otg_transceiver *otg, u32 reg) return (__raw_readl(view) >> ULPIVW_RDATA_SHIFT) & ULPIVW_RDATA_MASK; } -static int ulpi_write(struct otg_transceiver *otg, u32 val, u32 reg) +static int ulpi_write(struct usb_phy *otg, u32 val, u32 reg) { int ret; void __iomem *view = otg->io_priv; @@ -106,13 +106,13 @@ static int ulpi_write(struct otg_transceiver *otg, u32 val, u32 reg) return ulpi_poll(view, ULPIVW_RUN); } -struct otg_io_access_ops mxc_ulpi_access_ops = { +struct usb_phy_io_ops mxc_ulpi_access_ops = { .read = ulpi_read, .write = ulpi_write, }; EXPORT_SYMBOL_GPL(mxc_ulpi_access_ops); -struct otg_transceiver *imx_otg_ulpi_create(unsigned int flags) +struct usb_phy *imx_otg_ulpi_create(unsigned int flags) { return otg_ulpi_create(&mxc_ulpi_access_ops, flags); } diff --git a/arch/arm/plat-s3c24xx/pm-simtec.c b/arch/arm/plat-s3c24xx/pm-simtec.c index 68296b1..699f931 100644 --- a/arch/arm/plat-s3c24xx/pm-simtec.c +++ b/arch/arm/plat-s3c24xx/pm-simtec.c @@ -52,7 +52,7 @@ static __init int pm_simtec_init(void) !machine_is_aml_m5900()) return 0; - printk(KERN_INFO "Simtec Board Power Manangement" COPYRIGHT "\n"); + printk(KERN_INFO "Simtec Board Power Management" COPYRIGHT "\n"); gstatus4 = (__raw_readl(S3C2410_BANKCON7) & 0x3) << 30; gstatus4 |= (__raw_readl(S3C2410_BANKCON6) & 0x3) << 28; diff --git a/arch/avr32/include/asm/socket.h b/arch/avr32/include/asm/socket.h index 247b88c..a473f8c 100644 --- a/arch/avr32/include/asm/socket.h +++ b/arch/avr32/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* __ASM_AVR32_SOCKET_H */ diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c index ea33957..92c5af9 100644 --- a/arch/avr32/kernel/process.c +++ b/arch/avr32/kernel/process.c @@ -40,9 +40,7 @@ void cpu_idle(void) cpu_idle_sleep(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c index 8dd0416..a80a643 100644 --- a/arch/blackfin/kernel/process.c +++ b/arch/blackfin/kernel/process.c @@ -94,9 +94,7 @@ void cpu_idle(void) idle(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/blackfin/mach-bf518/boards/ezbrd.c b/arch/blackfin/mach-bf518/boards/ezbrd.c index a2d96d3..a173957 100644 --- a/arch/blackfin/mach-bf518/boards/ezbrd.c +++ b/arch/blackfin/mach-bf518/boards/ezbrd.c @@ -821,7 +821,7 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { /* the MAC is stored in OTP memory page 0xDF */ u32 ret; @@ -834,5 +834,6 @@ void bfin_get_ether_addr(char *addr) for (ret = 0; ret < 6; ++ret) addr[ret] = otp_mac_p[5 - ret]; } + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf518/boards/tcm-bf518.c b/arch/blackfin/mach-bf518/boards/tcm-bf518.c index f271310..6eebee4 100644 --- a/arch/blackfin/mach-bf518/boards/tcm-bf518.c +++ b/arch/blackfin/mach-bf518/boards/tcm-bf518.c @@ -730,9 +730,8 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf527/boards/ad7160eval.c b/arch/blackfin/mach-bf527/boards/ad7160eval.c index c8d5d2b..fad7fea 100644 --- a/arch/blackfin/mach-bf527/boards/ad7160eval.c +++ b/arch/blackfin/mach-bf527/boards/ad7160eval.c @@ -846,7 +846,7 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { /* the MAC is stored in OTP memory page 0xDF */ u32 ret; @@ -859,5 +859,6 @@ void bfin_get_ether_addr(char *addr) for (ret = 0; ret < 6; ++ret) addr[ret] = otp_mac_p[5 - ret]; } + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf527/boards/cm_bf527.c b/arch/blackfin/mach-bf527/boards/cm_bf527.c index 7330607..65b7fbd 100644 --- a/arch/blackfin/mach-bf527/boards/cm_bf527.c +++ b/arch/blackfin/mach-bf527/boards/cm_bf527.c @@ -983,9 +983,8 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf527/boards/ezbrd.c b/arch/blackfin/mach-bf527/boards/ezbrd.c index db3ecfc..17c6a24 100644 --- a/arch/blackfin/mach-bf527/boards/ezbrd.c +++ b/arch/blackfin/mach-bf527/boards/ezbrd.c @@ -870,7 +870,7 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { /* the MAC is stored in OTP memory page 0xDF */ u32 ret; @@ -883,5 +883,6 @@ void bfin_get_ether_addr(char *addr) for (ret = 0; ret < 6; ++ret) addr[ret] = otp_mac_p[5 - ret]; } + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf527/boards/ezkit.c b/arch/blackfin/mach-bf527/boards/ezkit.c index dfdd8e6..2f9a2bd 100644 --- a/arch/blackfin/mach-bf527/boards/ezkit.c +++ b/arch/blackfin/mach-bf527/boards/ezkit.c @@ -1311,7 +1311,7 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { /* the MAC is stored in OTP memory page 0xDF */ u32 ret; @@ -1324,5 +1324,6 @@ void bfin_get_ether_addr(char *addr) for (ret = 0; ret < 6; ++ret) addr[ret] = otp_mac_p[5 - ret]; } + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf527/boards/tll6527m.c b/arch/blackfin/mach-bf527/boards/tll6527m.c index 360e97f..d192c0a 100644 --- a/arch/blackfin/mach-bf527/boards/tll6527m.c +++ b/arch/blackfin/mach-bf527/boards/tll6527m.c @@ -931,7 +931,7 @@ void native_machine_restart(char *cmd) bfin_reset_boot_spi_cs(P_DEFAULT_BOOT_SPI_CS); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { /* the MAC is stored in OTP memory page 0xDF */ u32 ret; @@ -945,5 +945,6 @@ void bfin_get_ether_addr(char *addr) for (ret = 0; ret < 6; ++ret) addr[ret] = otp_mac_p[5 - ret]; } + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/cm_bf537e.c b/arch/blackfin/mach-bf537/boards/cm_bf537e.c index 0d4a2f6..27fd2c3 100644 --- a/arch/blackfin/mach-bf537/boards/cm_bf537e.c +++ b/arch/blackfin/mach-bf537/boards/cm_bf537e.c @@ -813,9 +813,8 @@ void __init native_machine_early_platform_add_devices(void) ARRAY_SIZE(cm_bf537e_early_devices)); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/cm_bf537u.c b/arch/blackfin/mach-bf537/boards/cm_bf537u.c index f553698..3f3abad 100644 --- a/arch/blackfin/mach-bf537/boards/cm_bf537u.c +++ b/arch/blackfin/mach-bf537/boards/cm_bf537u.c @@ -790,9 +790,8 @@ void __init native_machine_early_platform_add_devices(void) ARRAY_SIZE(cm_bf537u_early_devices)); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/dnp5370.c b/arch/blackfin/mach-bf537/boards/dnp5370.c index 11dadeb..6f77bf7 100644 --- a/arch/blackfin/mach-bf537/boards/dnp5370.c +++ b/arch/blackfin/mach-bf537/boards/dnp5370.c @@ -399,9 +399,10 @@ arch_initcall(dnp5370_init); /* * Currently the MAC address is saved in Flash by U-Boot */ -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { *(u32 *)(&(addr[0])) = bfin_read32(FLASH_MAC); *(u16 *)(&(addr[4])) = bfin_read16(FLASH_MAC + 4); + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/pnav10.c b/arch/blackfin/mach-bf537/boards/pnav10.c index 6fd8470..e9507fe 100644 --- a/arch/blackfin/mach-bf537/boards/pnav10.c +++ b/arch/blackfin/mach-bf537/boards/pnav10.c @@ -535,9 +535,8 @@ void __init native_machine_early_platform_add_devices(void) ARRAY_SIZE(stamp_early_devices)); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/stamp.c b/arch/blackfin/mach-bf537/boards/stamp.c index 2221173..0b80725 100644 --- a/arch/blackfin/mach-bf537/boards/stamp.c +++ b/arch/blackfin/mach-bf537/boards/stamp.c @@ -2993,9 +2993,10 @@ void native_machine_restart(char *cmd) * Currently the MAC address is saved in Flash by U-Boot */ #define FLASH_MAC 0x203f0000 -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { *(u32 *)(&(addr[0])) = bfin_read32(FLASH_MAC); *(u16 *)(&(addr[4])) = bfin_read16(FLASH_MAC + 4); + return 0; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/blackfin/mach-bf537/boards/tcm_bf537.c b/arch/blackfin/mach-bf537/boards/tcm_bf537.c index 9885176..3fb4218 100644 --- a/arch/blackfin/mach-bf537/boards/tcm_bf537.c +++ b/arch/blackfin/mach-bf537/boards/tcm_bf537.c @@ -780,9 +780,8 @@ void __init native_machine_early_platform_add_devices(void) ARRAY_SIZE(cm_bf537_early_devices)); } -void bfin_get_ether_addr(char *addr) +int bfin_get_ether_addr(char *addr) { - random_ether_addr(addr); - printk(KERN_WARNING "%s:%s: Setting Ethernet MAC to a random one\n", __FILE__, __func__); + return 1; } EXPORT_SYMBOL(bfin_get_ether_addr); diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig index 26e67f0..3c64b28 100644 --- a/arch/c6x/Kconfig +++ b/arch/c6x/Kconfig @@ -12,6 +12,7 @@ config TMS320C6X select HAVE_GENERIC_HARDIRQS select HAVE_MEMBLOCK select HAVE_SPARSE_IRQ + select IRQ_DOMAIN select OF select OF_EARLY_FLATTREE diff --git a/arch/c6x/include/asm/irq.h b/arch/c6x/include/asm/irq.h index a6ae3c9..f13b78d 100644 --- a/arch/c6x/include/asm/irq.h +++ b/arch/c6x/include/asm/irq.h @@ -13,6 +13,7 @@ #ifndef _ASM_C6X_IRQ_H #define _ASM_C6X_IRQ_H +#include <linux/irqdomain.h> #include <linux/threads.h> #include <linux/list.h> #include <linux/radix-tree.h> @@ -41,253 +42,9 @@ /* This number is used when no interrupt has been assigned */ #define NO_IRQ 0 -/* This type is the placeholder for a hardware interrupt number. It has to - * be big enough to enclose whatever representation is used by a given - * platform. - */ -typedef unsigned long irq_hw_number_t; - -/* Interrupt controller "host" data structure. This could be defined as a - * irq domain controller. That is, it handles the mapping between hardware - * and virtual interrupt numbers for a given interrupt domain. The host - * structure is generally created by the PIC code for a given PIC instance - * (though a host can cover more than one PIC if they have a flat number - * model). It's the host callbacks that are responsible for setting the - * irq_chip on a given irq_desc after it's been mapped. - * - * The host code and data structures are fairly agnostic to the fact that - * we use an open firmware device-tree. We do have references to struct - * device_node in two places: in irq_find_host() to find the host matching - * a given interrupt controller node, and of course as an argument to its - * counterpart host->ops->match() callback. However, those are treated as - * generic pointers by the core and the fact that it's actually a device-node - * pointer is purely a convention between callers and implementation. This - * code could thus be used on other architectures by replacing those two - * by some sort of arch-specific void * "token" used to identify interrupt - * controllers. - */ -struct irq_host; -struct radix_tree_root; -struct device_node; - -/* Functions below are provided by the host and called whenever a new mapping - * is created or an old mapping is disposed. The host can then proceed to - * whatever internal data structures management is required. It also needs - * to setup the irq_desc when returning from map(). - */ -struct irq_host_ops { - /* Match an interrupt controller device node to a host, returns - * 1 on a match - */ - int (*match)(struct irq_host *h, struct device_node *node); - - /* Create or update a mapping between a virtual irq number and a hw - * irq number. This is called only once for a given mapping. - */ - int (*map)(struct irq_host *h, unsigned int virq, irq_hw_number_t hw); - - /* Dispose of such a mapping */ - void (*unmap)(struct irq_host *h, unsigned int virq); - - /* Translate device-tree interrupt specifier from raw format coming - * from the firmware to a irq_hw_number_t (interrupt line number) and - * type (sense) that can be passed to set_irq_type(). In the absence - * of this callback, irq_create_of_mapping() and irq_of_parse_and_map() - * will return the hw number in the first cell and IRQ_TYPE_NONE for - * the type (which amount to keeping whatever default value the - * interrupt controller has for that line) - */ - int (*xlate)(struct irq_host *h, struct device_node *ctrler, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type); -}; - -struct irq_host { - struct list_head link; - - /* type of reverse mapping technique */ - unsigned int revmap_type; -#define IRQ_HOST_MAP_PRIORITY 0 /* core priority irqs, get irqs 1..15 */ -#define IRQ_HOST_MAP_NOMAP 1 /* no fast reverse mapping */ -#define IRQ_HOST_MAP_LINEAR 2 /* linear map of interrupts */ -#define IRQ_HOST_MAP_TREE 3 /* radix tree */ - union { - struct { - unsigned int size; - unsigned int *revmap; - } linear; - struct radix_tree_root tree; - } revmap_data; - struct irq_host_ops *ops; - void *host_data; - irq_hw_number_t inval_irq; - - /* Optional device node pointer */ - struct device_node *of_node; -}; - struct irq_data; extern irq_hw_number_t irqd_to_hwirq(struct irq_data *d); extern irq_hw_number_t virq_to_hw(unsigned int virq); -extern bool virq_is_host(unsigned int virq, struct irq_host *host); - -/** - * irq_alloc_host - Allocate a new irq_host data structure - * @of_node: optional device-tree node of the interrupt controller - * @revmap_type: type of reverse mapping to use - * @revmap_arg: for IRQ_HOST_MAP_LINEAR linear only: size of the map - * @ops: map/unmap host callbacks - * @inval_irq: provide a hw number in that host space that is always invalid - * - * Allocates and initialize and irq_host structure. Note that in the case of - * IRQ_HOST_MAP_LEGACY, the map() callback will be called before this returns - * for all legacy interrupts except 0 (which is always the invalid irq for - * a legacy controller). For a IRQ_HOST_MAP_LINEAR, the map is allocated by - * this call as well. For a IRQ_HOST_MAP_TREE, the radix tree will be allocated - * later during boot automatically (the reverse mapping will use the slow path - * until that happens). - */ -extern struct irq_host *irq_alloc_host(struct device_node *of_node, - unsigned int revmap_type, - unsigned int revmap_arg, - struct irq_host_ops *ops, - irq_hw_number_t inval_irq); - - -/** - * irq_find_host - Locates a host for a given device node - * @node: device-tree node of the interrupt controller - */ -extern struct irq_host *irq_find_host(struct device_node *node); - - -/** - * irq_set_default_host - Set a "default" host - * @host: default host pointer - * - * For convenience, it's possible to set a "default" host that will be used - * whenever NULL is passed to irq_create_mapping(). It makes life easier for - * platforms that want to manipulate a few hard coded interrupt numbers that - * aren't properly represented in the device-tree. - */ -extern void irq_set_default_host(struct irq_host *host); - - -/** - * irq_set_virq_count - Set the maximum number of virt irqs - * @count: number of linux virtual irqs, capped with NR_IRQS - * - * This is mainly for use by platforms like iSeries who want to program - * the virtual irq number in the controller to avoid the reverse mapping - */ -extern void irq_set_virq_count(unsigned int count); - - -/** - * irq_create_mapping - Map a hardware interrupt into linux virq space - * @host: host owning this hardware interrupt or NULL for default host - * @hwirq: hardware irq number in that host space - * - * Only one mapping per hardware interrupt is permitted. Returns a linux - * virq number. - * If the sense/trigger is to be specified, set_irq_type() should be called - * on the number returned from that call. - */ -extern unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq); - - -/** - * irq_dispose_mapping - Unmap an interrupt - * @virq: linux virq number of the interrupt to unmap - */ -extern void irq_dispose_mapping(unsigned int virq); - -/** - * irq_find_mapping - Find a linux virq from an hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a slow path, for use by generic code. It's expected that an - * irq controller implementation directly calls the appropriate low level - * mapping function. - */ -extern unsigned int irq_find_mapping(struct irq_host *host, - irq_hw_number_t hwirq); - -/** - * irq_create_direct_mapping - Allocate a virq for direct mapping - * @host: host to allocate the virq for or NULL for default host - * - * This routine is used for irq controllers which can choose the hardware - * interrupt numbers they generate. In such a case it's simplest to use - * the linux virq as the hardware interrupt number. - */ -extern unsigned int irq_create_direct_mapping(struct irq_host *host); - -/** - * irq_radix_revmap_insert - Insert a hw irq to linux virq number mapping. - * @host: host owning this hardware interrupt - * @virq: linux irq number - * @hwirq: hardware irq number in that host space - * - * This is for use by irq controllers that use a radix tree reverse - * mapping for fast lookup. - */ -extern void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq); - -/** - * irq_radix_revmap_lookup - Find a linux virq from a hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a fast path, for use by irq controller code that uses radix tree - * revmaps - */ -extern unsigned int irq_radix_revmap_lookup(struct irq_host *host, - irq_hw_number_t hwirq); - -/** - * irq_linear_revmap - Find a linux virq from a hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a fast path, for use by irq controller code that uses linear - * revmaps. It does fallback to the slow path if the revmap doesn't exist - * yet and will create the revmap entry with appropriate locking - */ - -extern unsigned int irq_linear_revmap(struct irq_host *host, - irq_hw_number_t hwirq); - - - -/** - * irq_alloc_virt - Allocate virtual irq numbers - * @host: host owning these new virtual irqs - * @count: number of consecutive numbers to allocate - * @hint: pass a hint number, the allocator will try to use a 1:1 mapping - * - * This is a low level function that is used internally by irq_create_mapping() - * and that can be used by some irq controllers implementations for things - * like allocating ranges of numbers for MSIs. The revmaps are left untouched. - */ -extern unsigned int irq_alloc_virt(struct irq_host *host, - unsigned int count, - unsigned int hint); - -/** - * irq_free_virt - Free virtual irq numbers - * @virq: virtual irq number of the first interrupt to free - * @count: number of interrupts to free - * - * This function is the opposite of irq_alloc_virt. It will not clear reverse - * maps, this should be done previously by unmap'ing the interrupt. In fact, - * all interrupts covered by the range being freed should have been unmapped - * prior to calling this. - */ -extern void irq_free_virt(unsigned int virq, unsigned int count); extern void __init init_pic_c64xplus(void); diff --git a/arch/c6x/kernel/irq.c b/arch/c6x/kernel/irq.c index 0929e4b..d77bcfd 100644 --- a/arch/c6x/kernel/irq.c +++ b/arch/c6x/kernel/irq.c @@ -73,10 +73,10 @@ asmlinkage void c6x_do_IRQ(unsigned int prio, struct pt_regs *regs) set_irq_regs(old_regs); } -static struct irq_host *core_host; +static struct irq_domain *core_domain; -static int core_host_map(struct irq_host *h, unsigned int virq, - irq_hw_number_t hw) +static int core_domain_map(struct irq_domain *h, unsigned int virq, + irq_hw_number_t hw) { if (hw < 4 || hw >= NR_PRIORITY_IRQS) return -EINVAL; @@ -86,8 +86,9 @@ static int core_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops core_host_ops = { - .map = core_host_map, +static const struct irq_domain_ops core_domain_ops = { + .map = core_domain_map, + .xlate = irq_domain_xlate_onecell, }; void __init init_IRQ(void) @@ -100,10 +101,11 @@ void __init init_IRQ(void) np = of_find_compatible_node(NULL, NULL, "ti,c64x+core-pic"); if (np != NULL) { /* create the core host */ - core_host = irq_alloc_host(np, IRQ_HOST_MAP_PRIORITY, 0, - &core_host_ops, 0); - if (core_host) - irq_set_default_host(core_host); + core_domain = irq_domain_add_legacy(np, NR_PRIORITY_IRQS, + 0, 0, &core_domain_ops, + NULL); + if (core_domain) + irq_set_default_host(core_domain); of_node_put(np); } @@ -128,601 +130,15 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } -/* - * IRQ controller and virtual interrupts - */ - -/* The main irq map itself is an array of NR_IRQ entries containing the - * associate host and irq number. An entry with a host of NULL is free. - * An entry can be allocated if it's free, the allocator always then sets - * hwirq first to the host's invalid irq number and then fills ops. - */ -struct irq_map_entry { - irq_hw_number_t hwirq; - struct irq_host *host; -}; - -static LIST_HEAD(irq_hosts); -static DEFINE_RAW_SPINLOCK(irq_big_lock); -static DEFINE_MUTEX(revmap_trees_mutex); -static struct irq_map_entry irq_map[NR_IRQS]; -static unsigned int irq_virq_count = NR_IRQS; -static struct irq_host *irq_default_host; - irq_hw_number_t irqd_to_hwirq(struct irq_data *d) { - return irq_map[d->irq].hwirq; + return d->hwirq; } EXPORT_SYMBOL_GPL(irqd_to_hwirq); irq_hw_number_t virq_to_hw(unsigned int virq) { - return irq_map[virq].hwirq; + struct irq_data *irq_data = irq_get_irq_data(virq); + return WARN_ON(!irq_data) ? 0 : irq_data->hwirq; } EXPORT_SYMBOL_GPL(virq_to_hw); - -bool virq_is_host(unsigned int virq, struct irq_host *host) -{ - return irq_map[virq].host == host; -} -EXPORT_SYMBOL_GPL(virq_is_host); - -static int default_irq_host_match(struct irq_host *h, struct device_node *np) -{ - return h->of_node != NULL && h->of_node == np; -} - -struct irq_host *irq_alloc_host(struct device_node *of_node, - unsigned int revmap_type, - unsigned int revmap_arg, - struct irq_host_ops *ops, - irq_hw_number_t inval_irq) -{ - struct irq_host *host; - unsigned int size = sizeof(struct irq_host); - unsigned int i; - unsigned int *rmap; - unsigned long flags; - - /* Allocate structure and revmap table if using linear mapping */ - if (revmap_type == IRQ_HOST_MAP_LINEAR) - size += revmap_arg * sizeof(unsigned int); - host = kzalloc(size, GFP_KERNEL); - if (host == NULL) - return NULL; - - /* Fill structure */ - host->revmap_type = revmap_type; - host->inval_irq = inval_irq; - host->ops = ops; - host->of_node = of_node_get(of_node); - - if (host->ops->match == NULL) - host->ops->match = default_irq_host_match; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* Check for the priority controller. */ - if (revmap_type == IRQ_HOST_MAP_PRIORITY) { - if (irq_map[0].host != NULL) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - of_node_put(host->of_node); - kfree(host); - return NULL; - } - irq_map[0].host = host; - } - - list_add(&host->link, &irq_hosts); - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - - /* Additional setups per revmap type */ - switch (revmap_type) { - case IRQ_HOST_MAP_PRIORITY: - /* 0 is always the invalid number for priority */ - host->inval_irq = 0; - /* setup us as the host for all priority interrupts */ - for (i = 1; i < NR_PRIORITY_IRQS; i++) { - irq_map[i].hwirq = i; - smp_wmb(); - irq_map[i].host = host; - smp_wmb(); - - ops->map(host, i, i); - } - break; - case IRQ_HOST_MAP_LINEAR: - rmap = (unsigned int *)(host + 1); - for (i = 0; i < revmap_arg; i++) - rmap[i] = NO_IRQ; - host->revmap_data.linear.size = revmap_arg; - smp_wmb(); - host->revmap_data.linear.revmap = rmap; - break; - case IRQ_HOST_MAP_TREE: - INIT_RADIX_TREE(&host->revmap_data.tree, GFP_KERNEL); - break; - default: - break; - } - - pr_debug("irq: Allocated host of type %d @0x%p\n", revmap_type, host); - - return host; -} - -struct irq_host *irq_find_host(struct device_node *node) -{ - struct irq_host *h, *found = NULL; - unsigned long flags; - - /* We might want to match the legacy controller last since - * it might potentially be set to match all interrupts in - * the absence of a device node. This isn't a problem so far - * yet though... - */ - raw_spin_lock_irqsave(&irq_big_lock, flags); - list_for_each_entry(h, &irq_hosts, link) - if (h->ops->match(h, node)) { - found = h; - break; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} -EXPORT_SYMBOL_GPL(irq_find_host); - -void irq_set_default_host(struct irq_host *host) -{ - pr_debug("irq: Default host set to @0x%p\n", host); - - irq_default_host = host; -} - -void irq_set_virq_count(unsigned int count) -{ - pr_debug("irq: Trying to set virq count to %d\n", count); - - BUG_ON(count < NR_PRIORITY_IRQS); - if (count < NR_IRQS) - irq_virq_count = count; -} - -static int irq_setup_virq(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - int res; - - res = irq_alloc_desc_at(virq, 0); - if (res != virq) { - pr_debug("irq: -> allocating desc failed\n"); - goto error; - } - - /* map it */ - smp_wmb(); - irq_map[virq].hwirq = hwirq; - smp_mb(); - - if (host->ops->map(host, virq, hwirq)) { - pr_debug("irq: -> mapping failed, freeing\n"); - goto errdesc; - } - - irq_clear_status_flags(virq, IRQ_NOREQUEST); - - return 0; - -errdesc: - irq_free_descs(virq, 1); -error: - irq_free_virt(virq, 1); - return -1; -} - -unsigned int irq_create_direct_mapping(struct irq_host *host) -{ - unsigned int virq; - - if (host == NULL) - host = irq_default_host; - - BUG_ON(host == NULL); - WARN_ON(host->revmap_type != IRQ_HOST_MAP_NOMAP); - - virq = irq_alloc_virt(host, 1, 0); - if (virq == NO_IRQ) { - pr_debug("irq: create_direct virq allocation failed\n"); - return NO_IRQ; - } - - pr_debug("irq: create_direct obtained virq %d\n", virq); - - if (irq_setup_virq(host, virq, virq)) - return NO_IRQ; - - return virq; -} - -unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int virq, hint; - - pr_debug("irq: irq_create_mapping(0x%p, 0x%lx)\n", host, hwirq); - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) { - printk(KERN_WARNING "irq_create_mapping called for" - " NULL host, hwirq=%lx\n", hwirq); - WARN_ON(1); - return NO_IRQ; - } - pr_debug("irq: -> using host @%p\n", host); - - /* Check if mapping already exists */ - virq = irq_find_mapping(host, hwirq); - if (virq != NO_IRQ) { - pr_debug("irq: -> existing mapping on virq %d\n", virq); - return virq; - } - - /* Allocate a virtual interrupt number */ - hint = hwirq % irq_virq_count; - virq = irq_alloc_virt(host, 1, hint); - if (virq == NO_IRQ) { - pr_debug("irq: -> virq allocation failed\n"); - return NO_IRQ; - } - - if (irq_setup_virq(host, virq, hwirq)) - return NO_IRQ; - - pr_debug("irq: irq %lu on host %s mapped to virtual irq %u\n", - hwirq, host->of_node ? host->of_node->full_name : "null", virq); - - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_mapping); - -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) -{ - struct irq_host *host; - irq_hw_number_t hwirq; - unsigned int type = IRQ_TYPE_NONE; - unsigned int virq; - - if (controller == NULL) - host = irq_default_host; - else - host = irq_find_host(controller); - if (host == NULL) { - printk(KERN_WARNING "irq: no irq host found for %s !\n", - controller->full_name); - return NO_IRQ; - } - - /* If host has no translation, then we assume interrupt line */ - if (host->ops->xlate == NULL) - hwirq = intspec[0]; - else { - if (host->ops->xlate(host, controller, intspec, intsize, - &hwirq, &type)) - return NO_IRQ; - } - - /* Create mapping */ - virq = irq_create_mapping(host, hwirq); - if (virq == NO_IRQ) - return virq; - - /* Set type if specified and different than the current one */ - if (type != IRQ_TYPE_NONE && - type != (irqd_get_trigger_type(irq_get_irq_data(virq)))) - irq_set_irq_type(virq, type); - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); - -void irq_dispose_mapping(unsigned int virq) -{ - struct irq_host *host; - irq_hw_number_t hwirq; - - if (virq == NO_IRQ) - return; - - /* Never unmap priority interrupts */ - if (virq < NR_PRIORITY_IRQS) - return; - - host = irq_map[virq].host; - if (WARN_ON(host == NULL)) - return; - - irq_set_status_flags(virq, IRQ_NOREQUEST); - - /* remove chip and handler */ - irq_set_chip_and_handler(virq, NULL, NULL); - - /* Make sure it's completed */ - synchronize_irq(virq); - - /* Tell the PIC about it */ - if (host->ops->unmap) - host->ops->unmap(host, virq); - smp_mb(); - - /* Clear reverse map */ - hwirq = irq_map[virq].hwirq; - switch (host->revmap_type) { - case IRQ_HOST_MAP_LINEAR: - if (hwirq < host->revmap_data.linear.size) - host->revmap_data.linear.revmap[hwirq] = NO_IRQ; - break; - case IRQ_HOST_MAP_TREE: - mutex_lock(&revmap_trees_mutex); - radix_tree_delete(&host->revmap_data.tree, hwirq); - mutex_unlock(&revmap_trees_mutex); - break; - } - - /* Destroy map */ - smp_mb(); - irq_map[virq].hwirq = host->inval_irq; - - irq_free_descs(virq, 1); - /* Free it */ - irq_free_virt(virq, 1); -} -EXPORT_SYMBOL_GPL(irq_dispose_mapping); - -unsigned int irq_find_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int i; - unsigned int hint = hwirq % irq_virq_count; - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) - return NO_IRQ; - - /* Slow path does a linear search of the map */ - i = hint; - do { - if (irq_map[i].host == host && - irq_map[i].hwirq == hwirq) - return i; - i++; - if (i >= irq_virq_count) - i = 4; - } while (i != hint); - return NO_IRQ; -} -EXPORT_SYMBOL_GPL(irq_find_mapping); - -unsigned int irq_radix_revmap_lookup(struct irq_host *host, - irq_hw_number_t hwirq) -{ - struct irq_map_entry *ptr; - unsigned int virq; - - if (WARN_ON_ONCE(host->revmap_type != IRQ_HOST_MAP_TREE)) - return irq_find_mapping(host, hwirq); - - /* - * The ptr returned references the static global irq_map. - * but freeing an irq can delete nodes along the path to - * do the lookup via call_rcu. - */ - rcu_read_lock(); - ptr = radix_tree_lookup(&host->revmap_data.tree, hwirq); - rcu_read_unlock(); - - /* - * If found in radix tree, then fine. - * Else fallback to linear lookup - this should not happen in practice - * as it means that we failed to insert the node in the radix tree. - */ - if (ptr) - virq = ptr - irq_map; - else - virq = irq_find_mapping(host, hwirq); - - return virq; -} - -void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - if (WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE)) - return; - - if (virq != NO_IRQ) { - mutex_lock(&revmap_trees_mutex); - radix_tree_insert(&host->revmap_data.tree, hwirq, - &irq_map[virq]); - mutex_unlock(&revmap_trees_mutex); - } -} - -unsigned int irq_linear_revmap(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int *revmap; - - if (WARN_ON_ONCE(host->revmap_type != IRQ_HOST_MAP_LINEAR)) - return irq_find_mapping(host, hwirq); - - /* Check revmap bounds */ - if (unlikely(hwirq >= host->revmap_data.linear.size)) - return irq_find_mapping(host, hwirq); - - /* Check if revmap was allocated */ - revmap = host->revmap_data.linear.revmap; - if (unlikely(revmap == NULL)) - return irq_find_mapping(host, hwirq); - - /* Fill up revmap with slow path if no mapping found */ - if (unlikely(revmap[hwirq] == NO_IRQ)) - revmap[hwirq] = irq_find_mapping(host, hwirq); - - return revmap[hwirq]; -} - -unsigned int irq_alloc_virt(struct irq_host *host, - unsigned int count, - unsigned int hint) -{ - unsigned long flags; - unsigned int i, j, found = NO_IRQ; - - if (count == 0 || count > (irq_virq_count - NR_PRIORITY_IRQS)) - return NO_IRQ; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* Use hint for 1 interrupt if any */ - if (count == 1 && hint >= NR_PRIORITY_IRQS && - hint < irq_virq_count && irq_map[hint].host == NULL) { - found = hint; - goto hint_found; - } - - /* Look for count consecutive numbers in the allocatable - * (non-legacy) space - */ - for (i = NR_PRIORITY_IRQS, j = 0; i < irq_virq_count; i++) { - if (irq_map[i].host != NULL) - j = 0; - else - j++; - - if (j == count) { - found = i - count + 1; - break; - } - } - if (found == NO_IRQ) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return NO_IRQ; - } - hint_found: - for (i = found; i < (found + count); i++) { - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = host; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} - -void irq_free_virt(unsigned int virq, unsigned int count) -{ - unsigned long flags; - unsigned int i; - - WARN_ON(virq < NR_PRIORITY_IRQS); - WARN_ON(count == 0 || (virq + count) > irq_virq_count); - - if (virq < NR_PRIORITY_IRQS) { - if (virq + count < NR_PRIORITY_IRQS) - return; - count -= NR_PRIORITY_IRQS - virq; - virq = NR_PRIORITY_IRQS; - } - - if (count > irq_virq_count || virq > irq_virq_count - count) { - if (virq > irq_virq_count) - return; - count = irq_virq_count - virq; - } - - raw_spin_lock_irqsave(&irq_big_lock, flags); - for (i = virq; i < (virq + count); i++) { - struct irq_host *host; - - host = irq_map[i].host; - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = NULL; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); -} - -#ifdef CONFIG_VIRQ_DEBUG -static int virq_debug_show(struct seq_file *m, void *private) -{ - unsigned long flags; - struct irq_desc *desc; - const char *p; - static const char none[] = "none"; - void *data; - int i; - - seq_printf(m, "%-5s %-7s %-15s %-18s %s\n", "virq", "hwirq", - "chip name", "chip data", "host name"); - - for (i = 1; i < nr_irqs; i++) { - desc = irq_to_desc(i); - if (!desc) - continue; - - raw_spin_lock_irqsave(&desc->lock, flags); - - if (desc->action && desc->action->handler) { - struct irq_chip *chip; - - seq_printf(m, "%5d ", i); - seq_printf(m, "0x%05lx ", irq_map[i].hwirq); - - chip = irq_desc_get_chip(desc); - if (chip && chip->name) - p = chip->name; - else - p = none; - seq_printf(m, "%-15s ", p); - - data = irq_desc_get_chip_data(desc); - seq_printf(m, "0x%16p ", data); - - if (irq_map[i].host && irq_map[i].host->of_node) - p = irq_map[i].host->of_node->full_name; - else - p = none; - seq_printf(m, "%s\n", p); - } - - raw_spin_unlock_irqrestore(&desc->lock, flags); - } - - return 0; -} - -static int virq_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, virq_debug_show, inode->i_private); -} - -static const struct file_operations virq_debug_fops = { - .open = virq_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int __init irq_debugfs_init(void) -{ - if (debugfs_create_file("virq_mapping", S_IRUGO, powerpc_debugfs_root, - NULL, &virq_debug_fops) == NULL) - return -ENOMEM; - - return 0; -} -device_initcall(irq_debugfs_init); -#endif /* CONFIG_VIRQ_DEBUG */ diff --git a/arch/c6x/platforms/megamod-pic.c b/arch/c6x/platforms/megamod-pic.c index 7c37a94..c1c4e2a 100644 --- a/arch/c6x/platforms/megamod-pic.c +++ b/arch/c6x/platforms/megamod-pic.c @@ -48,7 +48,7 @@ struct megamod_regs { }; struct megamod_pic { - struct irq_host *irqhost; + struct irq_domain *irqhost; struct megamod_regs __iomem *regs; raw_spinlock_t lock; @@ -116,7 +116,7 @@ static void megamod_irq_cascade(unsigned int irq, struct irq_desc *desc) } } -static int megamod_map(struct irq_host *h, unsigned int virq, +static int megamod_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct megamod_pic *pic = h->host_data; @@ -136,21 +136,9 @@ static int megamod_map(struct irq_host *h, unsigned int virq, return 0; } -static int megamod_xlate(struct irq_host *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type) - -{ - /* megamod intspecs must have 1 cell */ - BUG_ON(intsize != 1); - *out_hwirq = intspec[0]; - *out_type = IRQ_TYPE_NONE; - return 0; -} - -static struct irq_host_ops megamod_host_ops = { +static const struct irq_domain_ops megamod_domain_ops = { .map = megamod_map, - .xlate = megamod_xlate, + .xlate = irq_domain_xlate_onecell, }; static void __init set_megamod_mux(struct megamod_pic *pic, int src, int output) @@ -223,9 +211,8 @@ static struct megamod_pic * __init init_megamod_pic(struct device_node *np) return NULL; } - pic->irqhost = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - NR_COMBINERS * 32, &megamod_host_ops, - IRQ_UNMAPPED); + pic->irqhost = irq_domain_add_linear(np, NR_COMBINERS * 32, + &megamod_domain_ops, pic); if (!pic->irqhost) { pr_err("%s: Could not alloc host.\n", np->full_name); goto error_free; diff --git a/arch/cris/include/asm/socket.h b/arch/cris/include/asm/socket.h index e269264..ae52825 100644 --- a/arch/cris/include/asm/socket.h +++ b/arch/cris/include/asm/socket.h @@ -66,6 +66,10 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/cris/kernel/process.c b/arch/cris/kernel/process.c index aa585e4..d8f50ff 100644 --- a/arch/cris/kernel/process.c +++ b/arch/cris/kernel/process.c @@ -115,9 +115,7 @@ void cpu_idle (void) idle = default_idle; idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/frv/include/asm/highmem.h b/arch/frv/include/asm/highmem.h index a8d6565..716956a 100644 --- a/arch/frv/include/asm/highmem.h +++ b/arch/frv/include/asm/highmem.h @@ -157,7 +157,7 @@ static inline void kunmap_atomic_primary(void *kvaddr, enum km_type type) pagefault_enable(); } -void *__kmap_atomic(struct page *page); +void *kmap_atomic(struct page *page); void __kunmap_atomic(void *kvaddr); #endif /* !__ASSEMBLY__ */ diff --git a/arch/frv/include/asm/perf_event.h b/arch/frv/include/asm/perf_event.h index a69e015..c52ea55 100644 --- a/arch/frv/include/asm/perf_event.h +++ b/arch/frv/include/asm/perf_event.h @@ -12,6 +12,4 @@ #ifndef _ASM_PERF_EVENT_H #define _ASM_PERF_EVENT_H -#define PERF_EVENT_INDEX_OFFSET 0 - #endif /* _ASM_PERF_EVENT_H */ diff --git a/arch/frv/include/asm/socket.h b/arch/frv/include/asm/socket.h index ce80fda..a5b1d7d 100644 --- a/arch/frv/include/asm/socket.h +++ b/arch/frv/include/asm/socket.h @@ -64,6 +64,10 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/frv/kernel/process.c b/arch/frv/kernel/process.c index 3901df1..29cc497 100644 --- a/arch/frv/kernel/process.c +++ b/arch/frv/kernel/process.c @@ -92,9 +92,7 @@ void cpu_idle(void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/frv/mm/highmem.c b/arch/frv/mm/highmem.c index fd7fcd4..31902c9 100644 --- a/arch/frv/mm/highmem.c +++ b/arch/frv/mm/highmem.c @@ -37,7 +37,7 @@ struct page *kmap_atomic_to_page(void *ptr) return virt_to_page(ptr); } -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { unsigned long paddr; int type; @@ -64,7 +64,7 @@ void *__kmap_atomic(struct page *page) return NULL; } } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { diff --git a/arch/h8300/include/asm/socket.h b/arch/h8300/include/asm/socket.h index cf1daab..ec4554e 100644 --- a/arch/h8300/include/asm/socket.h +++ b/arch/h8300/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/h8300/kernel/process.c b/arch/h8300/kernel/process.c index 933bd38..1a173b3 100644 --- a/arch/h8300/kernel/process.c +++ b/arch/h8300/kernel/process.c @@ -81,9 +81,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/hexagon/include/asm/perf_event.h b/arch/hexagon/include/asm/perf_event.h index 6c2910f..8b8526b 100644 --- a/arch/hexagon/include/asm/perf_event.h +++ b/arch/hexagon/include/asm/perf_event.h @@ -19,6 +19,4 @@ #ifndef _ASM_PERF_EVENT_H #define _ASM_PERF_EVENT_H -#define PERF_EVENT_INDEX_OFFSET 0 - #endif /* _ASM_PERF_EVENT_H */ diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c index c871a2c..0123c63 100644 --- a/arch/hexagon/kernel/smp.c +++ b/arch/hexagon/kernel/smp.c @@ -179,8 +179,6 @@ void __cpuinit start_secondary(void) printk(KERN_INFO "%s cpu %d\n", __func__, current_thread_info()->cpu); set_cpu_online(cpu, true); - while (!cpumask_test_cpu(cpu, cpu_active_mask)) - cpu_relax(); local_irq_enable(); cpu_idle(); diff --git a/arch/ia64/hp/sim/boot/fw-emu.c b/arch/ia64/hp/sim/boot/fw-emu.c index bf6d9d8..0216e28 100644 --- a/arch/ia64/hp/sim/boot/fw-emu.c +++ b/arch/ia64/hp/sim/boot/fw-emu.c @@ -160,28 +160,19 @@ sal_emulator (long index, unsigned long in1, unsigned long in2, */ status = 0; if (index == SAL_FREQ_BASE) { - switch (in1) { - case SAL_FREQ_BASE_PLATFORM: + if (in1 == SAL_FREQ_BASE_PLATFORM) r9 = 200000000; - break; - - case SAL_FREQ_BASE_INTERVAL_TIMER: + else if (in1 == SAL_FREQ_BASE_INTERVAL_TIMER) { /* * Is this supposed to be the cr.itc frequency * or something platform specific? The SAL * doc ain't exactly clear on this... */ r9 = 700000000; - break; - - case SAL_FREQ_BASE_REALTIME_CLOCK: + } else if (in1 == SAL_FREQ_BASE_REALTIME_CLOCK) r9 = 1; - break; - - default: + else status = -1; - break; - } } else if (index == SAL_SET_VECTORS) { ; } else if (index == SAL_GET_STATE_INFO) { diff --git a/arch/ia64/hp/sim/hpsim_irq.c b/arch/ia64/hp/sim/hpsim_irq.c index 4bd9a63..0aa70eb 100644 --- a/arch/ia64/hp/sim/hpsim_irq.c +++ b/arch/ia64/hp/sim/hpsim_irq.c @@ -10,6 +10,8 @@ #include <linux/sched.h> #include <linux/irq.h> +#include "hpsim_ssc.h" + static unsigned int hpsim_irq_startup(struct irq_data *data) { @@ -37,15 +39,37 @@ static struct irq_chip irq_type_hp_sim = { .irq_set_affinity = hpsim_set_affinity_noop, }; +static void hpsim_irq_set_chip(int irq) +{ + struct irq_chip *chip = irq_get_chip(irq); + + if (chip == &no_irq_chip) + irq_set_chip(irq, &irq_type_hp_sim); +} + +static void hpsim_connect_irq(int intr, int irq) +{ + ia64_ssc(intr, irq, 0, 0, SSC_CONNECT_INTERRUPT); +} + +int hpsim_get_irq(int intr) +{ + int irq = assign_irq_vector(AUTO_ASSIGN); + + if (irq >= 0) { + hpsim_irq_set_chip(irq); + irq_set_handler(irq, handle_simple_irq); + hpsim_connect_irq(intr, irq); + } + + return irq; +} + void __init hpsim_irq_init (void) { int i; - for_each_active_irq(i) { - struct irq_chip *chip = irq_get_chip(i); - - if (chip == &no_irq_chip) - irq_set_chip(i, &irq_type_hp_sim); - } + for_each_active_irq(i) + hpsim_irq_set_chip(i); } diff --git a/arch/ia64/hp/sim/hpsim_setup.c b/arch/ia64/hp/sim/hpsim_setup.c index f629e90..664a540 100644 --- a/arch/ia64/hp/sim/hpsim_setup.c +++ b/arch/ia64/hp/sim/hpsim_setup.c @@ -26,12 +26,6 @@ #include "hpsim_ssc.h" void -ia64_ssc_connect_irq (long intr, long irq) -{ - ia64_ssc(intr, irq, 0, 0, SSC_CONNECT_INTERRUPT); -} - -void ia64_ctl_trace (long on) { ia64_ssc(on, 0, 0, 0, SSC_CTL_TRACE); diff --git a/arch/ia64/hp/sim/simeth.c b/arch/ia64/hp/sim/simeth.c index 47afcc6..a63218e 100644 --- a/arch/ia64/hp/sim/simeth.c +++ b/arch/ia64/hp/sim/simeth.c @@ -129,17 +129,6 @@ netdev_probe(char *name, unsigned char *ether) static inline int -netdev_connect(int irq) -{ - /* XXX Fix me - * this does not support multiple cards - * also no return value - */ - ia64_ssc_connect_irq(NETWORK_INTR, irq); - return 0; -} - -static inline int netdev_attach(int fd, int irq, unsigned int ipaddr) { /* this puts the host interface in the right mode (start interrupting) */ @@ -193,7 +182,7 @@ simeth_probe1(void) unsigned char mac_addr[ETH_ALEN]; struct simeth_local *local; struct net_device *dev; - int fd, i, err, rc; + int fd, err, rc; /* * XXX Fix me @@ -226,22 +215,16 @@ simeth_probe1(void) return err; } - if ((rc = assign_irq_vector(AUTO_ASSIGN)) < 0) - panic("%s: out of interrupt vectors!\n", __func__); - dev->irq = rc; - /* * attach the interrupt in the simulator, this does enable interrupts * until a netdev_attach() is called */ - netdev_connect(dev->irq); + if ((rc = hpsim_get_irq(NETWORK_INTR)) < 0) + panic("%s: out of interrupt vectors!\n", __func__); + dev->irq = rc; - printk(KERN_INFO "%s: hosteth=%s simfd=%d, HwAddr", - dev->name, simeth_device, local->simfd); - for(i = 0; i < ETH_ALEN; i++) { - printk(" %2.2x", dev->dev_addr[i]); - } - printk(", IRQ %d\n", dev->irq); + printk(KERN_INFO "%s: hosteth=%s simfd=%d, HwAddr=%pm, IRQ %d\n", + dev->name, simeth_device, local->simfd, dev->dev_addr, dev->irq); return 0; } diff --git a/arch/ia64/hp/sim/simserial.c b/arch/ia64/hp/sim/simserial.c index bff0824..c34785d 100644 --- a/arch/ia64/hp/sim/simserial.c +++ b/arch/ia64/hp/sim/simserial.c @@ -4,16 +4,11 @@ * This driver is mostly used for bringup purposes and will go away. * It has a strong dependency on the system console. All outputs * are rerouted to the same facility as the one used by printk which, in our - * case means sys_sim.c console (goes via the simulator). The code hereafter - * is completely leveraged from the serial.c driver. + * case means sys_sim.c console (goes via the simulator). * * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co * Stephane Eranian <eranian@hpl.hp.com> * David Mosberger-Tang <davidm@hpl.hp.com> - * - * 02/04/00 D. Mosberger Merged in serial.c bug fixes in rs_close(). - * 02/25/00 D. Mosberger Synced up with 2.3.99pre-5 version of serial.c. - * 07/30/02 D. Mosberger Replace sti()/cli() with explicit spinlocks & local irq masking */ #include <linux/init.h> @@ -27,15 +22,17 @@ #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/capability.h> +#include <linux/circ_buf.h> #include <linux/console.h> +#include <linux/irq.h> #include <linux/module.h> #include <linux/serial.h> -#include <linux/serialP.h> #include <linux/sysrq.h> +#include <linux/uaccess.h> + +#include <asm/hpsim.h> -#include <asm/irq.h> -#include <asm/hw_irq.h> -#include <asm/uaccess.h> +#include "hpsim_ssc.h" #undef SIMSERIAL_DEBUG /* define this to get some debug information */ @@ -43,118 +40,44 @@ #define NR_PORTS 1 /* only one port for now */ -#define IRQ_T(info) ((info->flags & ASYNC_SHARE_IRQ) ? IRQF_SHARED : IRQF_DISABLED) - -#define SSC_GETCHAR 21 - -extern long ia64_ssc (long, long, long, long, int); -extern void ia64_ssc_connect_irq (long intr, long irq); - -static char *serial_name = "SimSerial driver"; -static char *serial_version = "0.6"; - -/* - * This has been extracted from asm/serial.h. We need one eventually but - * I don't know exactly what we're going to put in it so just fake one - * for now. - */ -#define BASE_BAUD ( 1843200 / 16 ) - -#define STD_COM_FLAGS (ASYNC_BOOT_AUTOCONF | ASYNC_SKIP_TEST) - -/* - * Most of the values here are meaningless to this particular driver. - * However some values must be preserved for the code (leveraged from serial.c - * to work correctly). - * port must not be 0 - * type must not be UNKNOWN - * So I picked arbitrary (guess from where?) values instead - */ -static struct serial_state rs_table[NR_PORTS]={ - /* UART CLK PORT IRQ FLAGS */ - { 0, BASE_BAUD, 0x3F8, 0, STD_COM_FLAGS,0,PORT_16550 } /* ttyS0 */ +struct serial_state { + struct tty_port port; + struct circ_buf xmit; + int irq; + int x_char; }; -/* - * Just for the fun of it ! - */ -static struct serial_uart_config uart_config[] = { - { "unknown", 1, 0 }, - { "8250", 1, 0 }, - { "16450", 1, 0 }, - { "16550", 1, 0 }, - { "16550A", 16, UART_CLEAR_FIFO | UART_USE_FIFO }, - { "cirrus", 1, 0 }, - { "ST16650", 1, UART_CLEAR_FIFO | UART_STARTECH }, - { "ST16650V2", 32, UART_CLEAR_FIFO | UART_USE_FIFO | - UART_STARTECH }, - { "TI16750", 64, UART_CLEAR_FIFO | UART_USE_FIFO}, - { NULL, 0} -}; +static struct serial_state rs_table[NR_PORTS]; struct tty_driver *hp_simserial_driver; -static struct async_struct *IRQ_ports[NR_IRQS]; - static struct console *console; -static unsigned char *tmp_buf; - -extern struct console *console_drivers; /* from kernel/printk.c */ - -/* - * ------------------------------------------------------------ - * rs_stop() and rs_start() - * - * This routines are called before setting or resetting tty->stopped. - * They enable or disable transmitter interrupts, as necessary. - * ------------------------------------------------------------ - */ -static void rs_stop(struct tty_struct *tty) -{ -#ifdef SIMSERIAL_DEBUG - printk("rs_stop: tty->stopped=%d tty->hw_stopped=%d tty->flow_stopped=%d\n", - tty->stopped, tty->hw_stopped, tty->flow_stopped); -#endif - -} - -static void rs_start(struct tty_struct *tty) -{ -#ifdef SIMSERIAL_DEBUG - printk("rs_start: tty->stopped=%d tty->hw_stopped=%d tty->flow_stopped=%d\n", - tty->stopped, tty->hw_stopped, tty->flow_stopped); -#endif -} - -static void receive_chars(struct tty_struct *tty) +static void receive_chars(struct tty_struct *tty) { unsigned char ch; static unsigned char seen_esc = 0; while ( (ch = ia64_ssc(0, 0, 0, 0, SSC_GETCHAR)) ) { - if ( ch == 27 && seen_esc == 0 ) { + if (ch == 27 && seen_esc == 0) { seen_esc = 1; continue; - } else { - if ( seen_esc==1 && ch == 'O' ) { - seen_esc = 2; - continue; - } else if ( seen_esc == 2 ) { - if ( ch == 'P' ) /* F1 */ - show_state(); + } else if (seen_esc == 1 && ch == 'O') { + seen_esc = 2; + continue; + } else if (seen_esc == 2) { + if (ch == 'P') /* F1 */ + show_state(); #ifdef CONFIG_MAGIC_SYSRQ - if ( ch == 'S' ) { /* F4 */ - do - ch = ia64_ssc(0, 0, 0, 0, - SSC_GETCHAR); - while (!ch); - handle_sysrq(ch); - } -#endif - seen_esc = 0; - continue; + if (ch == 'S') { /* F4 */ + do { + ch = ia64_ssc(0, 0, 0, 0, SSC_GETCHAR); + } while (!ch); + handle_sysrq(ch); } +#endif + seen_esc = 0; + continue; } seen_esc = 0; @@ -169,22 +92,19 @@ static void receive_chars(struct tty_struct *tty) */ static irqreturn_t rs_interrupt_single(int irq, void *dev_id) { - struct async_struct * info; + struct serial_state *info = dev_id; + struct tty_struct *tty = tty_port_tty_get(&info->port); - /* - * I don't know exactly why they don't use the dev_id opaque data - * pointer instead of this extra lookup table - */ - info = IRQ_ports[irq]; - if (!info || !info->tty) { - printk(KERN_INFO "simrs_interrupt_single: info|tty=0 info=%p problem\n", info); + if (!tty) { + printk(KERN_INFO "%s: tty=0 problem\n", __func__); return IRQ_NONE; } /* * pretty simple in our case, because we only get interrupts * on inbound traffic */ - receive_chars(info->tty); + receive_chars(tty); + tty_kref_put(tty); return IRQ_HANDLED; } @@ -194,17 +114,12 @@ static irqreturn_t rs_interrupt_single(int irq, void *dev_id) * ------------------------------------------------------------------- */ -static void do_softint(struct work_struct *private_) -{ - printk(KERN_ERR "simserial: do_softint called\n"); -} - static int rs_put_char(struct tty_struct *tty, unsigned char ch) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; unsigned long flags; - if (!tty || !info->xmit.buf) + if (!info->xmit.buf) return 0; local_irq_save(flags); @@ -218,12 +133,12 @@ static int rs_put_char(struct tty_struct *tty, unsigned char ch) return 1; } -static void transmit_chars(struct async_struct *info, int *intr_done) +static void transmit_chars(struct tty_struct *tty, struct serial_state *info, + int *intr_done) { int count; unsigned long flags; - local_irq_save(flags); if (info->x_char) { @@ -231,16 +146,16 @@ static void transmit_chars(struct async_struct *info, int *intr_done) console->write(console, &c, 1); - info->state->icount.tx++; info->x_char = 0; goto out; } - if (info->xmit.head == info->xmit.tail || info->tty->stopped || info->tty->hw_stopped) { + if (info->xmit.head == info->xmit.tail || tty->stopped || + tty->hw_stopped) { #ifdef SIMSERIAL_DEBUG printk("transmit_chars: head=%d, tail=%d, stopped=%d\n", - info->xmit.head, info->xmit.tail, info->tty->stopped); + info->xmit.head, info->xmit.tail, tty->stopped); #endif goto out; } @@ -272,24 +187,24 @@ out: static void rs_flush_chars(struct tty_struct *tty) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; - if (info->xmit.head == info->xmit.tail || tty->stopped || tty->hw_stopped || - !info->xmit.buf) + if (info->xmit.head == info->xmit.tail || tty->stopped || + tty->hw_stopped || !info->xmit.buf) return; - transmit_chars(info, NULL); + transmit_chars(tty, info, NULL); } - static int rs_write(struct tty_struct * tty, const unsigned char *buf, int count) { + struct serial_state *info = tty->driver_data; int c, ret = 0; - struct async_struct *info = (struct async_struct *)tty->driver_data; unsigned long flags; - if (!tty || !info->xmit.buf || !tmp_buf) return 0; + if (!info->xmit.buf) + return 0; local_irq_save(flags); while (1) { @@ -310,30 +225,30 @@ static int rs_write(struct tty_struct * tty, /* * Hey, we transmit directly from here in our case */ - if (CIRC_CNT(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE) - && !tty->stopped && !tty->hw_stopped) { - transmit_chars(info, NULL); - } + if (CIRC_CNT(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE) && + !tty->stopped && !tty->hw_stopped) + transmit_chars(tty, info, NULL); + return ret; } static int rs_write_room(struct tty_struct *tty) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; return CIRC_SPACE(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); } static int rs_chars_in_buffer(struct tty_struct *tty) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; return CIRC_CNT(info->xmit.head, info->xmit.tail, SERIAL_XMIT_SIZE); } static void rs_flush_buffer(struct tty_struct *tty) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; unsigned long flags; local_irq_save(flags); @@ -349,7 +264,7 @@ static void rs_flush_buffer(struct tty_struct *tty) */ static void rs_send_xchar(struct tty_struct *tty, char ch) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; info->x_char = ch; if (ch) { @@ -357,7 +272,7 @@ static void rs_send_xchar(struct tty_struct *tty, char ch) * I guess we could call console->write() directly but * let's do that for now. */ - transmit_chars(info, NULL); + transmit_chars(tty, info, NULL); } } @@ -371,14 +286,15 @@ static void rs_send_xchar(struct tty_struct *tty, char ch) */ static void rs_throttle(struct tty_struct * tty) { - if (I_IXOFF(tty)) rs_send_xchar(tty, STOP_CHAR(tty)); + if (I_IXOFF(tty)) + rs_send_xchar(tty, STOP_CHAR(tty)); printk(KERN_INFO "simrs_throttle called\n"); } static void rs_unthrottle(struct tty_struct * tty) { - struct async_struct *info = (struct async_struct *)tty->driver_data; + struct serial_state *info = tty->driver_data; if (I_IXOFF(tty)) { if (info->x_char) @@ -389,7 +305,6 @@ static void rs_unthrottle(struct tty_struct * tty) printk(KERN_INFO "simrs_unthrottle called\n"); } - static int rs_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { if ((cmd != TIOCGSERIAL) && (cmd != TIOCSSERIAL) && @@ -400,48 +315,21 @@ static int rs_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) } switch (cmd) { - case TIOCGSERIAL: - printk(KERN_INFO "simrs_ioctl TIOCGSERIAL called\n"); - return 0; - case TIOCSSERIAL: - printk(KERN_INFO "simrs_ioctl TIOCSSERIAL called\n"); - return 0; - case TIOCSERCONFIG: - printk(KERN_INFO "rs_ioctl: TIOCSERCONFIG called\n"); - return -EINVAL; - - case TIOCSERGETLSR: /* Get line status register */ - printk(KERN_INFO "rs_ioctl: TIOCSERGETLSR called\n"); - return -EINVAL; - - case TIOCSERGSTRUCT: - printk(KERN_INFO "rs_ioctl: TIOCSERGSTRUCT called\n"); -#if 0 - if (copy_to_user((struct async_struct *) arg, - info, sizeof(struct async_struct))) - return -EFAULT; -#endif - return 0; - - /* - * Wait for any of the 4 modem inputs (DCD,RI,DSR,CTS) to change - * - mask passed in arg for lines of interest - * (use |'ed TIOCM_RNG/DSR/CD/CTS for masking) - * Caller should use TIOCGICOUNT to see which one it was - */ - case TIOCMIWAIT: - printk(KERN_INFO "rs_ioctl: TIOCMIWAIT: called\n"); - return 0; - case TIOCSERGWILD: - case TIOCSERSWILD: - /* "setserial -W" is called in Debian boot */ - printk (KERN_INFO "TIOCSER?WILD ioctl obsolete, ignored.\n"); - return 0; - - default: - return -ENOIOCTLCMD; - } - return 0; + case TIOCGSERIAL: + case TIOCSSERIAL: + case TIOCSERGSTRUCT: + case TIOCMIWAIT: + return 0; + case TIOCSERCONFIG: + case TIOCSERGETLSR: /* Get line status register */ + return -EINVAL; + case TIOCSERGWILD: + case TIOCSERSWILD: + /* "setserial -W" is called in Debian boot */ + printk (KERN_INFO "TIOCSER?WILD ioctl obsolete, ignored.\n"); + return 0; + } + return -ENOIOCTLCMD; } #define RELEVANT_IFLAG(iflag) (iflag & (IGNBRK|BRKINT|IGNPAR|PARMRK|INPCK)) @@ -452,220 +340,50 @@ static void rs_set_termios(struct tty_struct *tty, struct ktermios *old_termios) if ((old_termios->c_cflag & CRTSCTS) && !(tty->termios->c_cflag & CRTSCTS)) { tty->hw_stopped = 0; - rs_start(tty); } } /* * This routine will shutdown a serial port; interrupts are disabled, and * DTR is dropped if the hangup on close termio flag is on. */ -static void shutdown(struct async_struct * info) +static void shutdown(struct tty_port *port) { - unsigned long flags; - struct serial_state *state; - int retval; - - if (!(info->flags & ASYNC_INITIALIZED)) return; - - state = info->state; - -#ifdef SIMSERIAL_DEBUG - printk("Shutting down serial port %d (irq %d)....", info->line, - state->irq); -#endif + struct serial_state *info = container_of(port, struct serial_state, + port); + unsigned long flags; local_irq_save(flags); - { - /* - * First unlink the serial port from the IRQ chain... - */ - if (info->next_port) - info->next_port->prev_port = info->prev_port; - if (info->prev_port) - info->prev_port->next_port = info->next_port; - else - IRQ_ports[state->irq] = info->next_port; - - /* - * Free the IRQ, if necessary - */ - if (state->irq && (!IRQ_ports[state->irq] || - !IRQ_ports[state->irq]->next_port)) { - if (IRQ_ports[state->irq]) { - free_irq(state->irq, NULL); - retval = request_irq(state->irq, rs_interrupt_single, - IRQ_T(info), "serial", NULL); - - if (retval) - printk(KERN_ERR "serial shutdown: request_irq: error %d" - " Couldn't reacquire IRQ.\n", retval); - } else - free_irq(state->irq, NULL); - } - - if (info->xmit.buf) { - free_page((unsigned long) info->xmit.buf); - info->xmit.buf = NULL; - } - - if (info->tty) set_bit(TTY_IO_ERROR, &info->tty->flags); + if (info->irq) + free_irq(info->irq, info); - info->flags &= ~ASYNC_INITIALIZED; + if (info->xmit.buf) { + free_page((unsigned long) info->xmit.buf); + info->xmit.buf = NULL; } local_irq_restore(flags); } -/* - * ------------------------------------------------------------ - * rs_close() - * - * This routine is called when the serial port gets closed. First, we - * wait for the last remaining data to be sent. Then, we unlink its - * async structure from the interrupt chain if necessary, and we free - * that IRQ if nothing is left in the chain. - * ------------------------------------------------------------ - */ static void rs_close(struct tty_struct *tty, struct file * filp) { - struct async_struct * info = (struct async_struct *)tty->driver_data; - struct serial_state *state; - unsigned long flags; - - if (!info ) return; - - state = info->state; - - local_irq_save(flags); - if (tty_hung_up_p(filp)) { -#ifdef SIMSERIAL_DEBUG - printk("rs_close: hung_up\n"); -#endif - local_irq_restore(flags); - return; - } -#ifdef SIMSERIAL_DEBUG - printk("rs_close ttys%d, count = %d\n", info->line, state->count); -#endif - if ((tty->count == 1) && (state->count != 1)) { - /* - * Uh, oh. tty->count is 1, which means that the tty - * structure will be freed. state->count should always - * be one in these conditions. If it's greater than - * one, we've got real problems, since it means the - * serial port won't be shutdown. - */ - printk(KERN_ERR "rs_close: bad serial port count; tty->count is 1, " - "state->count is %d\n", state->count); - state->count = 1; - } - if (--state->count < 0) { - printk(KERN_ERR "rs_close: bad serial port count for ttys%d: %d\n", - info->line, state->count); - state->count = 0; - } - if (state->count) { - local_irq_restore(flags); - return; - } - info->flags |= ASYNC_CLOSING; - local_irq_restore(flags); + struct serial_state *info = tty->driver_data; - /* - * Now we wait for the transmit buffer to clear; and we notify - * the line discipline to only process XON/XOFF characters. - */ - shutdown(info); - rs_flush_buffer(tty); - tty_ldisc_flush(tty); - info->event = 0; - info->tty = NULL; - if (info->blocked_open) { - if (info->close_delay) - schedule_timeout_interruptible(info->close_delay); - wake_up_interruptible(&info->open_wait); - } - info->flags &= ~(ASYNC_NORMAL_ACTIVE|ASYNC_CLOSING); - wake_up_interruptible(&info->close_wait); + tty_port_close(&info->port, tty, filp); } -/* - * rs_wait_until_sent() --- wait until the transmitter is empty - */ -static void rs_wait_until_sent(struct tty_struct *tty, int timeout) -{ -} - - -/* - * rs_hangup() --- called by tty_hangup() when a hangup is signaled. - */ static void rs_hangup(struct tty_struct *tty) { - struct async_struct * info = (struct async_struct *)tty->driver_data; - struct serial_state *state = info->state; - -#ifdef SIMSERIAL_DEBUG - printk("rs_hangup: called\n"); -#endif - - state = info->state; + struct serial_state *info = tty->driver_data; rs_flush_buffer(tty); - if (info->flags & ASYNC_CLOSING) - return; - shutdown(info); - - info->event = 0; - state->count = 0; - info->flags &= ~ASYNC_NORMAL_ACTIVE; - info->tty = NULL; - wake_up_interruptible(&info->open_wait); + tty_port_hangup(&info->port); } - -static int get_async_struct(int line, struct async_struct **ret_info) +static int activate(struct tty_port *port, struct tty_struct *tty) { - struct async_struct *info; - struct serial_state *sstate; - - sstate = rs_table + line; - sstate->count++; - if (sstate->info) { - *ret_info = sstate->info; - return 0; - } - info = kzalloc(sizeof(struct async_struct), GFP_KERNEL); - if (!info) { - sstate->count--; - return -ENOMEM; - } - init_waitqueue_head(&info->open_wait); - init_waitqueue_head(&info->close_wait); - init_waitqueue_head(&info->delta_msr_wait); - info->magic = SERIAL_MAGIC; - info->port = sstate->port; - info->flags = sstate->flags; - info->xmit_fifo_size = sstate->xmit_fifo_size; - info->line = line; - INIT_WORK(&info->work, do_softint); - info->state = sstate; - if (sstate->info) { - kfree(info); - *ret_info = sstate->info; - return 0; - } - *ret_info = sstate->info = info; - return 0; -} - -static int -startup(struct async_struct *info) -{ - unsigned long flags; - int retval=0; - irq_handler_t handler; - struct serial_state *state= info->state; - unsigned long page; + struct serial_state *state = container_of(port, struct serial_state, + port); + unsigned long flags, page; + int retval = 0; page = get_zeroed_page(GFP_KERNEL); if (!page) @@ -673,86 +391,31 @@ startup(struct async_struct *info) local_irq_save(flags); - if (info->flags & ASYNC_INITIALIZED) { - free_page(page); - goto errout; - } - - if (!state->port || !state->type) { - if (info->tty) set_bit(TTY_IO_ERROR, &info->tty->flags); - free_page(page); - goto errout; - } - if (info->xmit.buf) + if (state->xmit.buf) free_page(page); else - info->xmit.buf = (unsigned char *) page; - -#ifdef SIMSERIAL_DEBUG - printk("startup: ttys%d (irq %d)...", info->line, state->irq); -#endif + state->xmit.buf = (unsigned char *) page; - /* - * Allocate the IRQ if necessary - */ - if (state->irq && (!IRQ_ports[state->irq] || - !IRQ_ports[state->irq]->next_port)) { - if (IRQ_ports[state->irq]) { - retval = -EBUSY; - goto errout; - } else - handler = rs_interrupt_single; - - retval = request_irq(state->irq, handler, IRQ_T(info), "simserial", NULL); - if (retval) { - if (capable(CAP_SYS_ADMIN)) { - if (info->tty) - set_bit(TTY_IO_ERROR, - &info->tty->flags); - retval = 0; - } + if (state->irq) { + retval = request_irq(state->irq, rs_interrupt_single, 0, + "simserial", state); + if (retval) goto errout; - } } - /* - * Insert serial port into IRQ chain. - */ - info->prev_port = NULL; - info->next_port = IRQ_ports[state->irq]; - if (info->next_port) - info->next_port->prev_port = info; - IRQ_ports[state->irq] = info; - - if (info->tty) clear_bit(TTY_IO_ERROR, &info->tty->flags); - - info->xmit.head = info->xmit.tail = 0; - -#if 0 - /* - * Set up serial timers... - */ - timer_table[RS_TIMER].expires = jiffies + 2*HZ/100; - timer_active |= 1 << RS_TIMER; -#endif + state->xmit.head = state->xmit.tail = 0; /* * Set up the tty->alt_speed kludge */ - if (info->tty) { - if ((info->flags & ASYNC_SPD_MASK) == ASYNC_SPD_HI) - info->tty->alt_speed = 57600; - if ((info->flags & ASYNC_SPD_MASK) == ASYNC_SPD_VHI) - info->tty->alt_speed = 115200; - if ((info->flags & ASYNC_SPD_MASK) == ASYNC_SPD_SHI) - info->tty->alt_speed = 230400; - if ((info->flags & ASYNC_SPD_MASK) == ASYNC_SPD_WARP) - info->tty->alt_speed = 460800; - } - - info->flags |= ASYNC_INITIALIZED; - local_irq_restore(flags); - return 0; + if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_HI) + tty->alt_speed = 57600; + if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_VHI) + tty->alt_speed = 115200; + if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_SHI) + tty->alt_speed = 230400; + if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_WARP) + tty->alt_speed = 460800; errout: local_irq_restore(flags); @@ -768,56 +431,11 @@ errout: */ static int rs_open(struct tty_struct *tty, struct file * filp) { - struct async_struct *info; - int retval, line; - unsigned long page; + struct serial_state *info = rs_table + tty->index; + struct tty_port *port = &info->port; - line = tty->index; - if ((line < 0) || (line >= NR_PORTS)) - return -ENODEV; - retval = get_async_struct(line, &info); - if (retval) - return retval; tty->driver_data = info; - info->tty = tty; - -#ifdef SIMSERIAL_DEBUG - printk("rs_open %s, count = %d\n", tty->name, info->state->count); -#endif - info->tty->low_latency = (info->flags & ASYNC_LOW_LATENCY) ? 1 : 0; - - if (!tmp_buf) { - page = get_zeroed_page(GFP_KERNEL); - if (!page) - return -ENOMEM; - if (tmp_buf) - free_page(page); - else - tmp_buf = (unsigned char *) page; - } - - /* - * If the port is the middle of closing, bail out now - */ - if (tty_hung_up_p(filp) || - (info->flags & ASYNC_CLOSING)) { - if (info->flags & ASYNC_CLOSING) - interruptible_sleep_on(&info->close_wait); -#ifdef SERIAL_DO_RESTART - return ((info->flags & ASYNC_HUP_NOTIFY) ? - -EAGAIN : -ERESTARTSYS); -#else - return -EAGAIN; -#endif - } - - /* - * Start up serial port - */ - retval = startup(info); - if (retval) { - return retval; - } + tty->low_latency = (port->flags & ASYNC_LOW_LATENCY) ? 1 : 0; /* * figure out which console to use (should be one already) @@ -828,30 +446,21 @@ static int rs_open(struct tty_struct *tty, struct file * filp) console = console->next; } -#ifdef SIMSERIAL_DEBUG - printk("rs_open ttys%d successful\n", info->line); -#endif - return 0; + return tty_port_open(port, tty, filp); } /* * /proc fs routines.... */ -static inline void line_info(struct seq_file *m, struct serial_state *state) -{ - seq_printf(m, "%d: uart:%s port:%lX irq:%d\n", - state->line, uart_config[state->type].name, - state->port, state->irq); -} - static int rs_proc_show(struct seq_file *m, void *v) { int i; - seq_printf(m, "simserinfo:1.0 driver:%s\n", serial_version); + seq_printf(m, "simserinfo:1.0\n"); for (i = 0; i < NR_PORTS; i++) - line_info(m, &rs_table[i]); + seq_printf(m, "%d: uart:16550 port:3F8 irq:%d\n", + i, rs_table[i].irq); return 0; } @@ -868,25 +477,6 @@ static const struct file_operations rs_proc_fops = { .release = single_release, }; -/* - * --------------------------------------------------------------------- - * rs_init() and friends - * - * rs_init() is called at boot-time to initialize the serial driver. - * --------------------------------------------------------------------- - */ - -/* - * This routine prints out the appropriate serial driver version - * number, and identifies which options were configured into this - * driver. - */ -static inline void show_serial_version(void) -{ - printk(KERN_INFO "%s version %s with", serial_name, serial_version); - printk(KERN_INFO " no serial options enabled\n"); -} - static const struct tty_operations hp_ops = { .open = rs_open, .close = rs_close, @@ -901,34 +491,31 @@ static const struct tty_operations hp_ops = { .unthrottle = rs_unthrottle, .send_xchar = rs_send_xchar, .set_termios = rs_set_termios, - .stop = rs_stop, - .start = rs_start, .hangup = rs_hangup, - .wait_until_sent = rs_wait_until_sent, .proc_fops = &rs_proc_fops, }; -/* - * The serial driver boot-time initialization code! - */ -static int __init -simrs_init (void) +static const struct tty_port_operations hp_port_ops = { + .activate = activate, + .shutdown = shutdown, +}; + +static int __init simrs_init(void) { - int i, rc; - struct serial_state *state; + struct serial_state *state; + int retval; if (!ia64_platform_is("hpsim")) return -ENODEV; - hp_simserial_driver = alloc_tty_driver(1); + hp_simserial_driver = alloc_tty_driver(NR_PORTS); if (!hp_simserial_driver) return -ENOMEM; - show_serial_version(); + printk(KERN_INFO "SimSerial driver with no serial options enabled\n"); /* Initialize the tty_driver structure */ - hp_simserial_driver->owner = THIS_MODULE; hp_simserial_driver->driver_name = "simserial"; hp_simserial_driver->name = "ttyS"; hp_simserial_driver->major = TTY_MAJOR; @@ -941,31 +528,33 @@ simrs_init (void) hp_simserial_driver->flags = TTY_DRIVER_REAL_RAW; tty_set_operations(hp_simserial_driver, &hp_ops); - /* - * Let's have a little bit of fun ! - */ - for (i = 0, state = rs_table; i < NR_PORTS; i++,state++) { + state = rs_table; + tty_port_init(&state->port); + state->port.ops = &hp_port_ops; + state->port.close_delay = 0; /* XXX really 0? */ - if (state->type == PORT_UNKNOWN) continue; + retval = hpsim_get_irq(KEYBOARD_INTR); + if (retval < 0) { + printk(KERN_ERR "%s: out of interrupt vectors!\n", + __func__); + goto err_free_tty; + } - if (!state->irq) { - if ((rc = assign_irq_vector(AUTO_ASSIGN)) < 0) - panic("%s: out of interrupt vectors!\n", - __func__); - state->irq = rc; - ia64_ssc_connect_irq(KEYBOARD_INTR, state->irq); - } + state->irq = retval; - printk(KERN_INFO "ttyS%d at 0x%04lx (irq = %d) is a %s\n", - state->line, - state->port, state->irq, - uart_config[state->type].name); - } + /* the port is imaginary */ + printk(KERN_INFO "ttyS0 at 0x03f8 (irq = %d) is a 16550\n", state->irq); - if (tty_register_driver(hp_simserial_driver)) - panic("Couldn't register simserial driver\n"); + retval = tty_register_driver(hp_simserial_driver); + if (retval) { + printk(KERN_ERR "Couldn't register simserial driver\n"); + goto err_free_tty; + } return 0; +err_free_tty: + put_tty_driver(hp_simserial_driver); + return retval; } #ifndef MODULE diff --git a/arch/ia64/include/asm/hpsim.h b/arch/ia64/include/asm/hpsim.h index 892ab19..0fe5022 100644 --- a/arch/ia64/include/asm/hpsim.h +++ b/arch/ia64/include/asm/hpsim.h @@ -10,7 +10,7 @@ int simcons_register(void); struct tty_driver; extern struct tty_driver *hp_simserial_driver; -void ia64_ssc_connect_irq(long intr, long irq); +extern int hpsim_get_irq(int intr); void ia64_ctl_trace(long on); #endif diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h index 32551d3..b149b88 100644 --- a/arch/ia64/include/asm/paravirt.h +++ b/arch/ia64/include/asm/paravirt.h @@ -281,9 +281,9 @@ paravirt_init_missing_ticks_accounting(int cpu) pv_time_ops.init_missing_ticks_accounting(cpu); } -struct jump_label_key; -extern struct jump_label_key paravirt_steal_enabled; -extern struct jump_label_key paravirt_steal_rq_enabled; +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; static inline int paravirt_do_steal_accounting(unsigned long *new_itm) diff --git a/arch/ia64/include/asm/socket.h b/arch/ia64/include/asm/socket.h index 4b03664..41fc28a 100644 --- a/arch/ia64/include/asm/socket.h +++ b/arch/ia64/include/asm/socket.h @@ -73,5 +73,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_IA64_SOCKET_H */ diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c index 1008682..1b22f6d 100644 --- a/arch/ia64/kernel/paravirt.c +++ b/arch/ia64/kernel/paravirt.c @@ -634,8 +634,8 @@ struct pv_irq_ops pv_irq_ops = { * pv_time_ops * time operations */ -struct jump_label_key paravirt_steal_enabled; -struct jump_label_key paravirt_steal_rq_enabled; +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; static int ia64_native_do_steal_accounting(unsigned long *new_itm) diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 6d33c5c..9dc52b6 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -330,9 +330,7 @@ cpu_idle (void) normal_xtp(); #endif } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); if (cpu_is_offline(cpu)) play_dead(); diff --git a/arch/ia64/xen/irq_xen.c b/arch/ia64/xen/irq_xen.c index b279e14..3bb1223 100644 --- a/arch/ia64/xen/irq_xen.c +++ b/arch/ia64/xen/irq_xen.c @@ -58,7 +58,7 @@ xen_free_irq_vector(int vector) irq_op.vector = vector; if (HYPERVISOR_physdev_op(PHYSDEVOP_free_irq_vector, &irq_op)) - printk(KERN_WARNING "%s: xen_free_irq_vecotr fail vector=%d\n", + printk(KERN_WARNING "%s: xen_free_irq_vector fail vector=%d\n", __func__, vector); } diff --git a/arch/m32r/include/asm/socket.h b/arch/m32r/include/asm/socket.h index e8b8c5b..a15f40b 100644 --- a/arch/m32r/include/asm/socket.h +++ b/arch/m32r/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_M32R_SOCKET_H */ diff --git a/arch/m32r/kernel/process.c b/arch/m32r/kernel/process.c index 422bea9..3a4a32b 100644 --- a/arch/m32r/kernel/process.c +++ b/arch/m32r/kernel/process.c @@ -90,9 +90,7 @@ void cpu_idle (void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/m68k/emu/nfcon.c b/arch/m68k/emu/nfcon.c index ab20dc0..8db25e8 100644 --- a/arch/m68k/emu/nfcon.c +++ b/arch/m68k/emu/nfcon.c @@ -127,7 +127,6 @@ static int __init nfcon_init(void) if (!nfcon_tty_driver) return -ENOMEM; - nfcon_tty_driver->owner = THIS_MODULE; nfcon_tty_driver->driver_name = "nfcon"; nfcon_tty_driver->name = "nfcon"; nfcon_tty_driver->type = TTY_DRIVER_TYPE_SYSTEM; diff --git a/arch/m68k/include/asm/socket.h b/arch/m68k/include/asm/socket.h index d4708ce..d1be684 100644 --- a/arch/m68k/include/asm/socket.h +++ b/arch/m68k/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/m68k/kernel/process_mm.c b/arch/m68k/kernel/process_mm.c index 099283e..fe4186b 100644 --- a/arch/m68k/kernel/process_mm.c +++ b/arch/m68k/kernel/process_mm.c @@ -78,9 +78,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/m68k/kernel/process_no.c b/arch/m68k/kernel/process_no.c index 5e1078c..f7fe6c3 100644 --- a/arch/m68k/kernel/process_no.c +++ b/arch/m68k/kernel/process_no.c @@ -73,9 +73,7 @@ void cpu_idle(void) /* endless idle loop with no priority at all */ while (1) { idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index c8d6efb..11060fa 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -14,6 +14,7 @@ config MICROBLAZE select TRACING_SUPPORT select OF select OF_EARLY_FLATTREE + select IRQ_DOMAIN select HAVE_GENERIC_HARDIRQS select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW diff --git a/arch/microblaze/include/asm/hardirq.h b/arch/microblaze/include/asm/hardirq.h index cd1ac9a..fb3c05a 100644 --- a/arch/microblaze/include/asm/hardirq.h +++ b/arch/microblaze/include/asm/hardirq.h @@ -1,17 +1 @@ -/* - * Copyright (C) 2006 Atmark Techno, Inc. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ - -#ifndef _ASM_MICROBLAZE_HARDIRQ_H -#define _ASM_MICROBLAZE_HARDIRQ_H - -/* should be defined in each interrupt controller driver */ -extern unsigned int get_irq(struct pt_regs *regs); - #include <asm-generic/hardirq.h> - -#endif /* _ASM_MICROBLAZE_HARDIRQ_H */ diff --git a/arch/microblaze/include/asm/irq.h b/arch/microblaze/include/asm/irq.h index a175132..bab3b13 100644 --- a/arch/microblaze/include/asm/irq.h +++ b/arch/microblaze/include/asm/irq.h @@ -9,49 +9,13 @@ #ifndef _ASM_MICROBLAZE_IRQ_H #define _ASM_MICROBLAZE_IRQ_H - -/* - * Linux IRQ# is currently offset by one to map to the hardware - * irq number. So hardware IRQ0 maps to Linux irq 1. - */ -#define NO_IRQ_OFFSET 1 -#define IRQ_OFFSET NO_IRQ_OFFSET -#define NR_IRQS (32 + IRQ_OFFSET) +#define NR_IRQS (32 + 1) #include <asm-generic/irq.h> -/* This type is the placeholder for a hardware interrupt number. It has to - * be big enough to enclose whatever representation is used by a given - * platform. - */ -typedef unsigned long irq_hw_number_t; - -extern unsigned int nr_irq; - struct pt_regs; extern void do_IRQ(struct pt_regs *regs); -/** FIXME - not implement - * irq_dispose_mapping - Unmap an interrupt - * @virq: linux virq number of the interrupt to unmap - */ -static inline void irq_dispose_mapping(unsigned int virq) -{ - return; -} - -struct irq_host; - -/** - * irq_create_mapping - Map a hardware interrupt into linux virq space - * @host: host owning this hardware interrupt or NULL for default host - * @hwirq: hardware irq number in that host space - * - * Only one mapping per hardware interrupt is permitted. Returns a linux - * virq number. - * If the sense/trigger is to be specified, set_irq_type() should be called - * on the number returned from that call. - */ -extern unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq); +/* should be defined in each interrupt controller driver */ +extern unsigned int get_irq(void); #endif /* _ASM_MICROBLAZE_IRQ_H */ diff --git a/arch/microblaze/kernel/intc.c b/arch/microblaze/kernel/intc.c index 44b177e..ad12067 100644 --- a/arch/microblaze/kernel/intc.c +++ b/arch/microblaze/kernel/intc.c @@ -9,6 +9,7 @@ */ #include <linux/init.h> +#include <linux/irqdomain.h> #include <linux/irq.h> #include <asm/page.h> #include <linux/io.h> @@ -25,8 +26,6 @@ static unsigned int intc_baseaddr; #define INTC_BASE intc_baseaddr #endif -unsigned int nr_irq; - /* No one else should require these constants, so define them locally here. */ #define ISR 0x00 /* Interrupt Status Register */ #define IPR 0x04 /* Interrupt Pending Register */ @@ -84,24 +83,45 @@ static struct irq_chip intc_dev = { .irq_mask_ack = intc_mask_ack, }; -unsigned int get_irq(struct pt_regs *regs) +static struct irq_domain *root_domain; + +unsigned int get_irq(void) { - int irq; + unsigned int hwirq, irq = -1; - /* - * NOTE: This function is the one that needs to be improved in - * order to handle multiple interrupt controllers. It currently - * is hardcoded to check for interrupts only on the first INTC. - */ - irq = in_be32(INTC_BASE + IVR) + NO_IRQ_OFFSET; - pr_debug("get_irq: %d\n", irq); + hwirq = in_be32(INTC_BASE + IVR); + if (hwirq != -1U) + irq = irq_find_mapping(root_domain, hwirq); + + pr_debug("get_irq: hwirq=%d, irq=%d\n", hwirq, irq); return irq; } +int xintc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) +{ + u32 intr_mask = (u32)d->host_data; + + if (intr_mask & (1 << hw)) { + irq_set_chip_and_handler_name(irq, &intc_dev, + handle_edge_irq, "edge"); + irq_clear_status_flags(irq, IRQ_LEVEL); + } else { + irq_set_chip_and_handler_name(irq, &intc_dev, + handle_level_irq, "level"); + irq_set_status_flags(irq, IRQ_LEVEL); + } + return 0; +} + +static const struct irq_domain_ops xintc_irq_domain_ops = { + .xlate = irq_domain_xlate_onetwocell, + .map = xintc_map, +}; + void __init init_IRQ(void) { - u32 i, intr_mask; + u32 nr_irq, intr_mask; struct device_node *intc = NULL; #ifdef CONFIG_SELFMOD_INTC unsigned int intc_baseaddr = 0; @@ -146,16 +166,9 @@ void __init init_IRQ(void) /* Turn on the Master Enable. */ out_be32(intc_baseaddr + MER, MER_HIE | MER_ME); - for (i = IRQ_OFFSET; i < (nr_irq + IRQ_OFFSET); ++i) { - if (intr_mask & (0x00000001 << (i - IRQ_OFFSET))) { - irq_set_chip_and_handler_name(i, &intc_dev, - handle_edge_irq, "edge"); - irq_clear_status_flags(i, IRQ_LEVEL); - } else { - irq_set_chip_and_handler_name(i, &intc_dev, - handle_level_irq, "level"); - irq_set_status_flags(i, IRQ_LEVEL); - } - irq_get_irq_data(i)->hwirq = i - IRQ_OFFSET; - } + /* Yeah, okay, casting the intr_mask to a void* is butt-ugly, but I'm + * lazy and Michal can clean it up to something nicer when he tests + * and commits this patch. ~~gcl */ + root_domain = irq_domain_add_linear(intc, nr_irq, &xintc_irq_domain_ops, + (void *)intr_mask); } diff --git a/arch/microblaze/kernel/irq.c b/arch/microblaze/kernel/irq.c index bbebcae..ace700a 100644 --- a/arch/microblaze/kernel/irq.c +++ b/arch/microblaze/kernel/irq.c @@ -31,14 +31,13 @@ void __irq_entry do_IRQ(struct pt_regs *regs) trace_hardirqs_off(); irq_enter(); - irq = get_irq(regs); + irq = get_irq(); next_irq: BUG_ON(!irq); - /* Substract 1 because of get_irq */ - generic_handle_irq(irq + IRQ_OFFSET - NO_IRQ_OFFSET); + generic_handle_irq(irq); - irq = get_irq(regs); - if (irq) { + irq = get_irq(); + if (irq != -1U) { pr_debug("next irq: %d\n", irq); ++concurrent_irq; goto next_irq; @@ -48,18 +47,3 @@ next_irq: set_irq_regs(old_regs); trace_hardirqs_on(); } - -/* MS: There is no any advance mapping mechanism. We are using simple 32bit - intc without any cascades or any connection that's why mapping is 1:1 */ -unsigned int irq_create_mapping(struct irq_host *host, irq_hw_number_t hwirq) -{ - return hwirq + IRQ_OFFSET; -} -EXPORT_SYMBOL_GPL(irq_create_mapping); - -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) -{ - return intspec[0] + IRQ_OFFSET; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 7dcb5bf..9155f7d 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -110,9 +110,7 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/microblaze/kernel/setup.c b/arch/microblaze/kernel/setup.c index 604cd9d..70e6d0b 100644 --- a/arch/microblaze/kernel/setup.c +++ b/arch/microblaze/kernel/setup.c @@ -51,8 +51,6 @@ void __init setup_arch(char **cmdline_p) unflatten_device_tree(); - /* NOTE I think that this function is not necessary to call */ - /* irq_early_init(); */ setup_cpuinfo(); microblaze_cache_init(); diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 5ab6e89..edbbae1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2327,6 +2327,7 @@ config USE_OF bool "Flattened Device Tree support" select OF select OF_EARLY_FLATTREE + select IRQ_DOMAIN help Include support for flattened device tree machine descriptions. diff --git a/arch/mips/ath79/dev-usb.c b/arch/mips/ath79/dev-usb.c index 002d6d2..36e9570 100644 --- a/arch/mips/ath79/dev-usb.c +++ b/arch/mips/ath79/dev-usb.c @@ -17,6 +17,8 @@ #include <linux/irq.h> #include <linux/dma-mapping.h> #include <linux/platform_device.h> +#include <linux/usb/ehci_pdriver.h> +#include <linux/usb/ohci_pdriver.h> #include <asm/mach-ath79/ath79.h> #include <asm/mach-ath79/ar71xx_regs.h> @@ -36,14 +38,19 @@ static struct resource ath79_ohci_resources[] = { }; static u64 ath79_ohci_dmamask = DMA_BIT_MASK(32); + +static struct usb_ohci_pdata ath79_ohci_pdata = { +}; + static struct platform_device ath79_ohci_device = { - .name = "ath79-ohci", + .name = "ohci-platform", .id = -1, .resource = ath79_ohci_resources, .num_resources = ARRAY_SIZE(ath79_ohci_resources), .dev = { .dma_mask = &ath79_ohci_dmamask, .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &ath79_ohci_pdata, }, }; @@ -60,8 +67,20 @@ static struct resource ath79_ehci_resources[] = { }; static u64 ath79_ehci_dmamask = DMA_BIT_MASK(32); + +static struct usb_ehci_pdata ath79_ehci_pdata_v1 = { + .has_synopsys_hc_bug = 1, + .port_power_off = 1, +}; + +static struct usb_ehci_pdata ath79_ehci_pdata_v2 = { + .caps_offset = 0x100, + .has_tt = 1, + .port_power_off = 1, +}; + static struct platform_device ath79_ehci_device = { - .name = "ath79-ehci", + .name = "ehci-platform", .id = -1, .resource = ath79_ehci_resources, .num_resources = ARRAY_SIZE(ath79_ehci_resources), @@ -101,7 +120,7 @@ static void __init ath79_usb_setup(void) ath79_ehci_resources[0].start = AR71XX_EHCI_BASE; ath79_ehci_resources[0].end = AR71XX_EHCI_BASE + AR71XX_EHCI_SIZE - 1; - ath79_ehci_device.name = "ar71xx-ehci"; + ath79_ehci_device.dev.platform_data = &ath79_ehci_pdata_v1; platform_device_register(&ath79_ehci_device); } @@ -142,7 +161,7 @@ static void __init ar724x_usb_setup(void) ath79_ehci_resources[0].start = AR724X_EHCI_BASE; ath79_ehci_resources[0].end = AR724X_EHCI_BASE + AR724X_EHCI_SIZE - 1; - ath79_ehci_device.name = "ar724x-ehci"; + ath79_ehci_device.dev.platform_data = &ath79_ehci_pdata_v2; platform_device_register(&ath79_ehci_device); } @@ -159,7 +178,7 @@ static void __init ar913x_usb_setup(void) ath79_ehci_resources[0].start = AR913X_EHCI_BASE; ath79_ehci_resources[0].end = AR913X_EHCI_BASE + AR913X_EHCI_SIZE - 1; - ath79_ehci_device.name = "ar913x-ehci"; + ath79_ehci_device.dev.platform_data = &ath79_ehci_pdata_v2; platform_device_register(&ath79_ehci_device); } @@ -176,7 +195,7 @@ static void __init ar933x_usb_setup(void) ath79_ehci_resources[0].start = AR933X_EHCI_BASE; ath79_ehci_resources[0].end = AR933X_EHCI_BASE + AR933X_EHCI_SIZE - 1; - ath79_ehci_device.name = "ar933x-ehci"; + ath79_ehci_device.dev.platform_data = &ath79_ehci_pdata_v2; platform_device_register(&ath79_ehci_device); } diff --git a/arch/mips/bcm47xx/Makefile b/arch/mips/bcm47xx/Makefile index 4add173..4389de1 100644 --- a/arch/mips/bcm47xx/Makefile +++ b/arch/mips/bcm47xx/Makefile @@ -3,5 +3,5 @@ # under Linux. # -obj-y += gpio.o irq.o nvram.o prom.o serial.o setup.o time.o +obj-y += gpio.o irq.o nvram.o prom.o serial.o setup.o time.o sprom.o obj-$(CONFIG_BCM47XX_SSB) += wgt634u.o diff --git a/arch/mips/bcm47xx/nvram.c b/arch/mips/bcm47xx/nvram.c index a84e3bb..d43ceff 100644 --- a/arch/mips/bcm47xx/nvram.c +++ b/arch/mips/bcm47xx/nvram.c @@ -107,8 +107,7 @@ int nvram_getenv(char *name, char *val, size_t val_len) value = eq + 1; if ((eq - var) == strlen(name) && strncmp(var, name, (eq - var)) == 0) { - snprintf(val, val_len, "%s", value); - return 0; + return snprintf(val, val_len, "%s", value); } } return NVRAM_ERR_ENVNOTFOUND; diff --git a/arch/mips/bcm47xx/setup.c b/arch/mips/bcm47xx/setup.c index aab6b0c..19780aa9 100644 --- a/arch/mips/bcm47xx/setup.c +++ b/arch/mips/bcm47xx/setup.c @@ -3,7 +3,7 @@ * Copyright (C) 2006 Felix Fietkau <nbd@openwrt.org> * Copyright (C) 2006 Michael Buesch <m@bues.ch> * Copyright (C) 2010 Waldemar Brodkorb <wbx@openadk.org> - * Copyright (C) 2010-2011 Hauke Mehrtens <hauke@hauke-m.de> + * Copyright (C) 2010-2012 Hauke Mehrtens <hauke@hauke-m.de> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -85,156 +85,7 @@ static void bcm47xx_machine_halt(void) } #ifdef CONFIG_BCM47XX_SSB -#define READ_FROM_NVRAM(_outvar, name, buf) \ - if (nvram_getprefix(prefix, name, buf, sizeof(buf)) >= 0)\ - sprom->_outvar = simple_strtoul(buf, NULL, 0); - -#define READ_FROM_NVRAM2(_outvar, name1, name2, buf) \ - if (nvram_getprefix(prefix, name1, buf, sizeof(buf)) >= 0 || \ - nvram_getprefix(prefix, name2, buf, sizeof(buf)) >= 0)\ - sprom->_outvar = simple_strtoul(buf, NULL, 0); - -static inline int nvram_getprefix(const char *prefix, char *name, - char *buf, int len) -{ - if (prefix) { - char key[100]; - - snprintf(key, sizeof(key), "%s%s", prefix, name); - return nvram_getenv(key, buf, len); - } - - return nvram_getenv(name, buf, len); -} - -static u32 nvram_getu32(const char *name, char *buf, int len) -{ - int rv; - char key[100]; - u16 var0, var1; - - snprintf(key, sizeof(key), "%s0", name); - rv = nvram_getenv(key, buf, len); - /* return 0 here so this looks like unset */ - if (rv < 0) - return 0; - var0 = simple_strtoul(buf, NULL, 0); - - snprintf(key, sizeof(key), "%s1", name); - rv = nvram_getenv(key, buf, len); - if (rv < 0) - return 0; - var1 = simple_strtoul(buf, NULL, 0); - return var1 << 16 | var0; -} - -static void bcm47xx_fill_sprom(struct ssb_sprom *sprom, const char *prefix) -{ - char buf[100]; - u32 boardflags; - - memset(sprom, 0, sizeof(struct ssb_sprom)); - - sprom->revision = 1; /* Fallback: Old hardware does not define this. */ - READ_FROM_NVRAM(revision, "sromrev", buf); - if (nvram_getprefix(prefix, "il0macaddr", buf, sizeof(buf)) >= 0 || - nvram_getprefix(prefix, "macaddr", buf, sizeof(buf)) >= 0) - nvram_parse_macaddr(buf, sprom->il0mac); - if (nvram_getprefix(prefix, "et0macaddr", buf, sizeof(buf)) >= 0) - nvram_parse_macaddr(buf, sprom->et0mac); - if (nvram_getprefix(prefix, "et1macaddr", buf, sizeof(buf)) >= 0) - nvram_parse_macaddr(buf, sprom->et1mac); - READ_FROM_NVRAM(et0phyaddr, "et0phyaddr", buf); - READ_FROM_NVRAM(et1phyaddr, "et1phyaddr", buf); - READ_FROM_NVRAM(et0mdcport, "et0mdcport", buf); - READ_FROM_NVRAM(et1mdcport, "et1mdcport", buf); - READ_FROM_NVRAM(board_rev, "boardrev", buf); - READ_FROM_NVRAM(country_code, "ccode", buf); - READ_FROM_NVRAM(ant_available_a, "aa5g", buf); - READ_FROM_NVRAM(ant_available_bg, "aa2g", buf); - READ_FROM_NVRAM(pa0b0, "pa0b0", buf); - READ_FROM_NVRAM(pa0b1, "pa0b1", buf); - READ_FROM_NVRAM(pa0b2, "pa0b2", buf); - READ_FROM_NVRAM(pa1b0, "pa1b0", buf); - READ_FROM_NVRAM(pa1b1, "pa1b1", buf); - READ_FROM_NVRAM(pa1b2, "pa1b2", buf); - READ_FROM_NVRAM(pa1lob0, "pa1lob0", buf); - READ_FROM_NVRAM(pa1lob2, "pa1lob1", buf); - READ_FROM_NVRAM(pa1lob1, "pa1lob2", buf); - READ_FROM_NVRAM(pa1hib0, "pa1hib0", buf); - READ_FROM_NVRAM(pa1hib2, "pa1hib1", buf); - READ_FROM_NVRAM(pa1hib1, "pa1hib2", buf); - READ_FROM_NVRAM2(gpio0, "ledbh0", "wl0gpio0", buf); - READ_FROM_NVRAM2(gpio1, "ledbh1", "wl0gpio1", buf); - READ_FROM_NVRAM2(gpio2, "ledbh2", "wl0gpio2", buf); - READ_FROM_NVRAM2(gpio3, "ledbh3", "wl0gpio3", buf); - READ_FROM_NVRAM2(maxpwr_bg, "maxp2ga0", "pa0maxpwr", buf); - READ_FROM_NVRAM2(maxpwr_al, "maxp5gla0", "pa1lomaxpwr", buf); - READ_FROM_NVRAM2(maxpwr_a, "maxp5ga0", "pa1maxpwr", buf); - READ_FROM_NVRAM2(maxpwr_ah, "maxp5gha0", "pa1himaxpwr", buf); - READ_FROM_NVRAM2(itssi_bg, "itt5ga0", "pa0itssit", buf); - READ_FROM_NVRAM2(itssi_a, "itt2ga0", "pa1itssit", buf); - READ_FROM_NVRAM(tri2g, "tri2g", buf); - READ_FROM_NVRAM(tri5gl, "tri5gl", buf); - READ_FROM_NVRAM(tri5g, "tri5g", buf); - READ_FROM_NVRAM(tri5gh, "tri5gh", buf); - READ_FROM_NVRAM(txpid2g[0], "txpid2ga0", buf); - READ_FROM_NVRAM(txpid2g[1], "txpid2ga1", buf); - READ_FROM_NVRAM(txpid2g[2], "txpid2ga2", buf); - READ_FROM_NVRAM(txpid2g[3], "txpid2ga3", buf); - READ_FROM_NVRAM(txpid5g[0], "txpid5ga0", buf); - READ_FROM_NVRAM(txpid5g[1], "txpid5ga1", buf); - READ_FROM_NVRAM(txpid5g[2], "txpid5ga2", buf); - READ_FROM_NVRAM(txpid5g[3], "txpid5ga3", buf); - READ_FROM_NVRAM(txpid5gl[0], "txpid5gla0", buf); - READ_FROM_NVRAM(txpid5gl[1], "txpid5gla1", buf); - READ_FROM_NVRAM(txpid5gl[2], "txpid5gla2", buf); - READ_FROM_NVRAM(txpid5gl[3], "txpid5gla3", buf); - READ_FROM_NVRAM(txpid5gh[0], "txpid5gha0", buf); - READ_FROM_NVRAM(txpid5gh[1], "txpid5gha1", buf); - READ_FROM_NVRAM(txpid5gh[2], "txpid5gha2", buf); - READ_FROM_NVRAM(txpid5gh[3], "txpid5gha3", buf); - READ_FROM_NVRAM(rxpo2g, "rxpo2g", buf); - READ_FROM_NVRAM(rxpo5g, "rxpo5g", buf); - READ_FROM_NVRAM(rssisav2g, "rssisav2g", buf); - READ_FROM_NVRAM(rssismc2g, "rssismc2g", buf); - READ_FROM_NVRAM(rssismf2g, "rssismf2g", buf); - READ_FROM_NVRAM(bxa2g, "bxa2g", buf); - READ_FROM_NVRAM(rssisav5g, "rssisav5g", buf); - READ_FROM_NVRAM(rssismc5g, "rssismc5g", buf); - READ_FROM_NVRAM(rssismf5g, "rssismf5g", buf); - READ_FROM_NVRAM(bxa5g, "bxa5g", buf); - READ_FROM_NVRAM(cck2gpo, "cck2gpo", buf); - - sprom->ofdm2gpo = nvram_getu32("ofdm2gpo", buf, sizeof(buf)); - sprom->ofdm5glpo = nvram_getu32("ofdm5glpo", buf, sizeof(buf)); - sprom->ofdm5gpo = nvram_getu32("ofdm5gpo", buf, sizeof(buf)); - sprom->ofdm5ghpo = nvram_getu32("ofdm5ghpo", buf, sizeof(buf)); - - READ_FROM_NVRAM(antenna_gain.ghz24.a0, "ag0", buf); - READ_FROM_NVRAM(antenna_gain.ghz24.a1, "ag1", buf); - READ_FROM_NVRAM(antenna_gain.ghz24.a2, "ag2", buf); - READ_FROM_NVRAM(antenna_gain.ghz24.a3, "ag3", buf); - memcpy(&sprom->antenna_gain.ghz5, &sprom->antenna_gain.ghz24, - sizeof(sprom->antenna_gain.ghz5)); - - if (nvram_getprefix(prefix, "boardflags", buf, sizeof(buf)) >= 0) { - boardflags = simple_strtoul(buf, NULL, 0); - if (boardflags) { - sprom->boardflags_lo = (boardflags & 0x0000FFFFU); - sprom->boardflags_hi = (boardflags & 0xFFFF0000U) >> 16; - } - } - if (nvram_getprefix(prefix, "boardflags2", buf, sizeof(buf)) >= 0) { - boardflags = simple_strtoul(buf, NULL, 0); - if (boardflags) { - sprom->boardflags2_lo = (boardflags & 0x0000FFFFU); - sprom->boardflags2_hi = (boardflags & 0xFFFF0000U) >> 16; - } - } -} - -int bcm47xx_get_sprom(struct ssb_bus *bus, struct ssb_sprom *out) +static int bcm47xx_get_sprom_ssb(struct ssb_bus *bus, struct ssb_sprom *out) { char prefix[10]; @@ -251,7 +102,7 @@ int bcm47xx_get_sprom(struct ssb_bus *bus, struct ssb_sprom *out) } static int bcm47xx_get_invariants(struct ssb_bus *bus, - struct ssb_init_invariants *iv) + struct ssb_init_invariants *iv) { char buf[20]; @@ -281,7 +132,7 @@ static void __init bcm47xx_register_ssb(void) char buf[100]; struct ssb_mipscore *mcore; - err = ssb_arch_register_fallback_sprom(&bcm47xx_get_sprom); + err = ssb_arch_register_fallback_sprom(&bcm47xx_get_sprom_ssb); if (err) printk(KERN_WARNING "bcm47xx: someone else already registered" " a ssb SPROM callback handler (err %d)\n", err); @@ -308,10 +159,41 @@ static void __init bcm47xx_register_ssb(void) #endif #ifdef CONFIG_BCM47XX_BCMA +static int bcm47xx_get_sprom_bcma(struct bcma_bus *bus, struct ssb_sprom *out) +{ + char prefix[10]; + struct bcma_device *core; + + switch (bus->hosttype) { + case BCMA_HOSTTYPE_PCI: + snprintf(prefix, sizeof(prefix), "pci/%u/%u/", + bus->host_pci->bus->number + 1, + PCI_SLOT(bus->host_pci->devfn)); + bcm47xx_fill_sprom(out, prefix); + return 0; + case BCMA_HOSTTYPE_SOC: + bcm47xx_fill_sprom_ethernet(out, NULL); + core = bcma_find_core(bus, BCMA_CORE_80211); + if (core) { + snprintf(prefix, sizeof(prefix), "sb/%u/", + core->core_index); + bcm47xx_fill_sprom(out, prefix); + } + return 0; + default: + pr_warn("bcm47xx: unable to fill SPROM for given bustype.\n"); + return -EINVAL; + } +} + static void __init bcm47xx_register_bcma(void) { int err; + err = bcma_arch_register_fallback_sprom(&bcm47xx_get_sprom_bcma); + if (err) + pr_warn("bcm47xx: someone else already registered a bcma SPROM callback handler (err %d)\n", err); + err = bcma_host_soc_register(&bcm47xx_bus.bcma); if (err) panic("Failed to initialize BCMA bus (err %d)", err); diff --git a/arch/mips/bcm47xx/sprom.c b/arch/mips/bcm47xx/sprom.c new file mode 100644 index 0000000..5c8dcd2 --- /dev/null +++ b/arch/mips/bcm47xx/sprom.c @@ -0,0 +1,620 @@ +/* + * Copyright (C) 2004 Florian Schirmer <jolt@tuxbox.org> + * Copyright (C) 2006 Felix Fietkau <nbd@openwrt.org> + * Copyright (C) 2006 Michael Buesch <m@bues.ch> + * Copyright (C) 2010 Waldemar Brodkorb <wbx@openadk.org> + * Copyright (C) 2010-2012 Hauke Mehrtens <hauke@hauke-m.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN + * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include <bcm47xx.h> +#include <nvram.h> + +static void create_key(const char *prefix, const char *postfix, + const char *name, char *buf, int len) +{ + if (prefix && postfix) + snprintf(buf, len, "%s%s%s", prefix, name, postfix); + else if (prefix) + snprintf(buf, len, "%s%s", prefix, name); + else if (postfix) + snprintf(buf, len, "%s%s", name, postfix); + else + snprintf(buf, len, "%s", name); +} + +#define NVRAM_READ_VAL(type) \ +static void nvram_read_ ## type (const char *prefix, \ + const char *postfix, const char *name, \ + type *val, type allset) \ +{ \ + char buf[100]; \ + char key[40]; \ + int err; \ + type var; \ + \ + create_key(prefix, postfix, name, key, sizeof(key)); \ + \ + err = nvram_getenv(key, buf, sizeof(buf)); \ + if (err < 0) \ + return; \ + err = kstrto ## type (buf, 0, &var); \ + if (err) { \ + pr_warn("can not parse nvram name %s with value %s" \ + " got %i", key, buf, err); \ + return; \ + } \ + if (allset && var == allset) \ + return; \ + *val = var; \ +} + +NVRAM_READ_VAL(u8) +NVRAM_READ_VAL(s8) +NVRAM_READ_VAL(u16) +NVRAM_READ_VAL(u32) + +#undef NVRAM_READ_VAL + +static void nvram_read_u32_2(const char *prefix, const char *name, + u16 *val_lo, u16 *val_hi) +{ + char buf[100]; + char key[40]; + int err; + u32 val; + + create_key(prefix, NULL, name, key, sizeof(key)); + + err = nvram_getenv(key, buf, sizeof(buf)); + if (err < 0) + return; + err = kstrtou32(buf, 0, &val); + if (err) { + pr_warn("can not parse nvram name %s with value %s got %i", + key, buf, err); + return; + } + *val_lo = (val & 0x0000FFFFU); + *val_hi = (val & 0xFFFF0000U) >> 16; +} + +static void nvram_read_leddc(const char *prefix, const char *name, + u8 *leddc_on_time, u8 *leddc_off_time) +{ + char buf[100]; + char key[40]; + int err; + u32 val; + + create_key(prefix, NULL, name, key, sizeof(key)); + + err = nvram_getenv(key, buf, sizeof(buf)); + if (err < 0) + return; + err = kstrtou32(buf, 0, &val); + if (err) { + pr_warn("can not parse nvram name %s with value %s got %i", + key, buf, err); + return; + } + + if (val == 0xffff || val == 0xffffffff) + return; + + *leddc_on_time = val & 0xff; + *leddc_off_time = (val >> 16) & 0xff; +} + +static void nvram_read_macaddr(const char *prefix, const char *name, + u8 (*val)[6]) +{ + char buf[100]; + char key[40]; + int err; + + create_key(prefix, NULL, name, key, sizeof(key)); + + err = nvram_getenv(key, buf, sizeof(buf)); + if (err < 0) + return; + nvram_parse_macaddr(buf, *val); +} + +static void nvram_read_alpha2(const char *prefix, const char *name, + char (*val)[2]) +{ + char buf[10]; + char key[40]; + int err; + + create_key(prefix, NULL, name, key, sizeof(key)); + + err = nvram_getenv(key, buf, sizeof(buf)); + if (err < 0) + return; + if (buf[0] == '0') + return; + if (strlen(buf) > 2) { + pr_warn("alpha2 is too long %s", buf); + return; + } + memcpy(val, buf, sizeof(val)); +} + +static void bcm47xx_fill_sprom_r1234589(struct ssb_sprom *sprom, + const char *prefix) +{ + nvram_read_u16(prefix, NULL, "boardrev", &sprom->board_rev, 0); + nvram_read_u16(prefix, NULL, "boardnum", &sprom->board_num, 0); + nvram_read_u8(prefix, NULL, "ledbh0", &sprom->gpio0, 0xff); + nvram_read_u8(prefix, NULL, "ledbh1", &sprom->gpio1, 0xff); + nvram_read_u8(prefix, NULL, "ledbh2", &sprom->gpio2, 0xff); + nvram_read_u8(prefix, NULL, "ledbh3", &sprom->gpio3, 0xff); + nvram_read_u8(prefix, NULL, "aa2g", &sprom->ant_available_bg, 0); + nvram_read_u8(prefix, NULL, "aa5g", &sprom->ant_available_a, 0); + nvram_read_s8(prefix, NULL, "ag0", &sprom->antenna_gain.a0, 0); + nvram_read_s8(prefix, NULL, "ag1", &sprom->antenna_gain.a1, 0); + nvram_read_alpha2(prefix, "ccode", &sprom->alpha2); +} + +static void bcm47xx_fill_sprom_r12389(struct ssb_sprom *sprom, + const char *prefix) +{ + nvram_read_u16(prefix, NULL, "pa0b0", &sprom->pa0b0, 0); + nvram_read_u16(prefix, NULL, "pa0b1", &sprom->pa0b1, 0); + nvram_read_u16(prefix, NULL, "pa0b2", &sprom->pa0b2, 0); + nvram_read_u8(prefix, NULL, "pa0itssit", &sprom->itssi_bg, 0); + nvram_read_u8(prefix, NULL, "pa0maxpwr", &sprom->maxpwr_bg, 0); + nvram_read_u16(prefix, NULL, "pa1b0", &sprom->pa1b0, 0); + nvram_read_u16(prefix, NULL, "pa1b1", &sprom->pa1b1, 0); + nvram_read_u16(prefix, NULL, "pa1b2", &sprom->pa1b2, 0); + nvram_read_u8(prefix, NULL, "pa1itssit", &sprom->itssi_a, 0); + nvram_read_u8(prefix, NULL, "pa1maxpwr", &sprom->maxpwr_a, 0); +} + +static void bcm47xx_fill_sprom_r1(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u16(prefix, NULL, "boardflags", &sprom->boardflags_lo, 0); + nvram_read_u8(prefix, NULL, "cc", &sprom->country_code, 0); +} + +static void bcm47xx_fill_sprom_r2389(struct ssb_sprom *sprom, + const char *prefix) +{ + nvram_read_u8(prefix, NULL, "opo", &sprom->opo, 0); + nvram_read_u16(prefix, NULL, "pa1lob0", &sprom->pa1lob0, 0); + nvram_read_u16(prefix, NULL, "pa1lob1", &sprom->pa1lob1, 0); + nvram_read_u16(prefix, NULL, "pa1lob2", &sprom->pa1lob2, 0); + nvram_read_u16(prefix, NULL, "pa1hib0", &sprom->pa1hib0, 0); + nvram_read_u16(prefix, NULL, "pa1hib1", &sprom->pa1hib1, 0); + nvram_read_u16(prefix, NULL, "pa1hib2", &sprom->pa1hib2, 0); + nvram_read_u8(prefix, NULL, "pa1lomaxpwr", &sprom->maxpwr_al, 0); + nvram_read_u8(prefix, NULL, "pa1himaxpwr", &sprom->maxpwr_ah, 0); +} + +static void bcm47xx_fill_sprom_r2(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u32_2(prefix, "boardflags", &sprom->boardflags_lo, + &sprom->boardflags_hi); + nvram_read_u16(prefix, NULL, "boardtype", &sprom->board_type, 0); +} + +static void bcm47xx_fill_sprom_r389(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u8(prefix, NULL, "bxa2g", &sprom->bxa2g, 0); + nvram_read_u8(prefix, NULL, "rssisav2g", &sprom->rssisav2g, 0); + nvram_read_u8(prefix, NULL, "rssismc2g", &sprom->rssismc2g, 0); + nvram_read_u8(prefix, NULL, "rssismf2g", &sprom->rssismf2g, 0); + nvram_read_u8(prefix, NULL, "bxa5g", &sprom->bxa5g, 0); + nvram_read_u8(prefix, NULL, "rssisav5g", &sprom->rssisav5g, 0); + nvram_read_u8(prefix, NULL, "rssismc5g", &sprom->rssismc5g, 0); + nvram_read_u8(prefix, NULL, "rssismf5g", &sprom->rssismf5g, 0); + nvram_read_u8(prefix, NULL, "tri2g", &sprom->tri2g, 0); + nvram_read_u8(prefix, NULL, "tri5g", &sprom->tri5g, 0); + nvram_read_u8(prefix, NULL, "tri5gl", &sprom->tri5gl, 0); + nvram_read_u8(prefix, NULL, "tri5gh", &sprom->tri5gh, 0); + nvram_read_s8(prefix, NULL, "rxpo2g", &sprom->rxpo2g, 0); + nvram_read_s8(prefix, NULL, "rxpo5g", &sprom->rxpo5g, 0); +} + +static void bcm47xx_fill_sprom_r3(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u32_2(prefix, "boardflags", &sprom->boardflags_lo, + &sprom->boardflags_hi); + nvram_read_u16(prefix, NULL, "boardtype", &sprom->board_type, 0); + nvram_read_u8(prefix, NULL, "regrev", &sprom->regrev, 0); + nvram_read_leddc(prefix, "leddc", &sprom->leddc_on_time, + &sprom->leddc_off_time); +} + +static void bcm47xx_fill_sprom_r4589(struct ssb_sprom *sprom, + const char *prefix) +{ + nvram_read_u32_2(prefix, "boardflags", &sprom->boardflags_lo, + &sprom->boardflags_hi); + nvram_read_u32_2(prefix, "boardflags2", &sprom->boardflags2_lo, + &sprom->boardflags2_hi); + nvram_read_u16(prefix, NULL, "boardtype", &sprom->board_type, 0); + nvram_read_u8(prefix, NULL, "regrev", &sprom->regrev, 0); + nvram_read_s8(prefix, NULL, "ag2", &sprom->antenna_gain.a2, 0); + nvram_read_s8(prefix, NULL, "ag3", &sprom->antenna_gain.a3, 0); + nvram_read_u8(prefix, NULL, "txchain", &sprom->txchain, 0xf); + nvram_read_u8(prefix, NULL, "rxchain", &sprom->rxchain, 0xf); + nvram_read_u8(prefix, NULL, "antswitch", &sprom->antswitch, 0xff); + nvram_read_leddc(prefix, "leddc", &sprom->leddc_on_time, + &sprom->leddc_off_time); +} + +static void bcm47xx_fill_sprom_r458(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u16(prefix, NULL, "cck2gpo", &sprom->cck2gpo, 0); + nvram_read_u32(prefix, NULL, "ofdm2gpo", &sprom->ofdm2gpo, 0); + nvram_read_u32(prefix, NULL, "ofdm5gpo", &sprom->ofdm5gpo, 0); + nvram_read_u32(prefix, NULL, "ofdm5glpo", &sprom->ofdm5glpo, 0); + nvram_read_u32(prefix, NULL, "ofdm5ghpo", &sprom->ofdm5ghpo, 0); + nvram_read_u16(prefix, NULL, "cddpo", &sprom->cddpo, 0); + nvram_read_u16(prefix, NULL, "stbcpo", &sprom->stbcpo, 0); + nvram_read_u16(prefix, NULL, "bw40po", &sprom->bw40po, 0); + nvram_read_u16(prefix, NULL, "bwduppo", &sprom->bwduppo, 0); + nvram_read_u16(prefix, NULL, "mcs2gpo0", &sprom->mcs2gpo[0], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo1", &sprom->mcs2gpo[1], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo2", &sprom->mcs2gpo[2], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo3", &sprom->mcs2gpo[3], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo4", &sprom->mcs2gpo[4], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo5", &sprom->mcs2gpo[5], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo6", &sprom->mcs2gpo[6], 0); + nvram_read_u16(prefix, NULL, "mcs2gpo7", &sprom->mcs2gpo[7], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo0", &sprom->mcs5gpo[0], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo1", &sprom->mcs5gpo[1], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo2", &sprom->mcs5gpo[2], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo3", &sprom->mcs5gpo[3], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo4", &sprom->mcs5gpo[4], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo5", &sprom->mcs5gpo[5], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo6", &sprom->mcs5gpo[6], 0); + nvram_read_u16(prefix, NULL, "mcs5gpo7", &sprom->mcs5gpo[7], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo0", &sprom->mcs5glpo[0], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo1", &sprom->mcs5glpo[1], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo2", &sprom->mcs5glpo[2], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo3", &sprom->mcs5glpo[3], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo4", &sprom->mcs5glpo[4], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo5", &sprom->mcs5glpo[5], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo6", &sprom->mcs5glpo[6], 0); + nvram_read_u16(prefix, NULL, "mcs5glpo7", &sprom->mcs5glpo[7], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo0", &sprom->mcs5ghpo[0], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo1", &sprom->mcs5ghpo[1], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo2", &sprom->mcs5ghpo[2], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo3", &sprom->mcs5ghpo[3], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo4", &sprom->mcs5ghpo[4], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo5", &sprom->mcs5ghpo[5], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo6", &sprom->mcs5ghpo[6], 0); + nvram_read_u16(prefix, NULL, "mcs5ghpo7", &sprom->mcs5ghpo[7], 0); +} + +static void bcm47xx_fill_sprom_r45(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u8(prefix, NULL, "txpid2ga0", &sprom->txpid2g[0], 0); + nvram_read_u8(prefix, NULL, "txpid2ga1", &sprom->txpid2g[1], 0); + nvram_read_u8(prefix, NULL, "txpid2ga2", &sprom->txpid2g[2], 0); + nvram_read_u8(prefix, NULL, "txpid2ga3", &sprom->txpid2g[3], 0); + nvram_read_u8(prefix, NULL, "txpid5ga0", &sprom->txpid5g[0], 0); + nvram_read_u8(prefix, NULL, "txpid5ga1", &sprom->txpid5g[1], 0); + nvram_read_u8(prefix, NULL, "txpid5ga2", &sprom->txpid5g[2], 0); + nvram_read_u8(prefix, NULL, "txpid5ga3", &sprom->txpid5g[3], 0); + nvram_read_u8(prefix, NULL, "txpid5gla0", &sprom->txpid5gl[0], 0); + nvram_read_u8(prefix, NULL, "txpid5gla1", &sprom->txpid5gl[1], 0); + nvram_read_u8(prefix, NULL, "txpid5gla2", &sprom->txpid5gl[2], 0); + nvram_read_u8(prefix, NULL, "txpid5gla3", &sprom->txpid5gl[3], 0); + nvram_read_u8(prefix, NULL, "txpid5gha0", &sprom->txpid5gh[0], 0); + nvram_read_u8(prefix, NULL, "txpid5gha1", &sprom->txpid5gh[1], 0); + nvram_read_u8(prefix, NULL, "txpid5gha2", &sprom->txpid5gh[2], 0); + nvram_read_u8(prefix, NULL, "txpid5gha3", &sprom->txpid5gh[3], 0); +} + +static void bcm47xx_fill_sprom_r89(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u8(prefix, NULL, "tssipos2g", &sprom->fem.ghz2.tssipos, 0); + nvram_read_u8(prefix, NULL, "extpagain2g", + &sprom->fem.ghz2.extpa_gain, 0); + nvram_read_u8(prefix, NULL, "pdetrange2g", + &sprom->fem.ghz2.pdet_range, 0); + nvram_read_u8(prefix, NULL, "triso2g", &sprom->fem.ghz2.tr_iso, 0); + nvram_read_u8(prefix, NULL, "antswctl2g", &sprom->fem.ghz2.antswlut, 0); + nvram_read_u8(prefix, NULL, "tssipos5g", &sprom->fem.ghz5.tssipos, 0); + nvram_read_u8(prefix, NULL, "extpagain5g", + &sprom->fem.ghz5.extpa_gain, 0); + nvram_read_u8(prefix, NULL, "pdetrange5g", + &sprom->fem.ghz5.pdet_range, 0); + nvram_read_u8(prefix, NULL, "triso5g", &sprom->fem.ghz5.tr_iso, 0); + nvram_read_u8(prefix, NULL, "antswctl5g", &sprom->fem.ghz5.antswlut, 0); + nvram_read_u8(prefix, NULL, "tempthresh", &sprom->tempthresh, 0); + nvram_read_u8(prefix, NULL, "tempoffset", &sprom->tempoffset, 0); + nvram_read_u16(prefix, NULL, "rawtempsense", &sprom->rawtempsense, 0); + nvram_read_u8(prefix, NULL, "measpower", &sprom->measpower, 0); + nvram_read_u8(prefix, NULL, "tempsense_slope", + &sprom->tempsense_slope, 0); + nvram_read_u8(prefix, NULL, "tempcorrx", &sprom->tempcorrx, 0); + nvram_read_u8(prefix, NULL, "tempsense_option", + &sprom->tempsense_option, 0); + nvram_read_u8(prefix, NULL, "freqoffset_corr", + &sprom->freqoffset_corr, 0); + nvram_read_u8(prefix, NULL, "iqcal_swp_dis", &sprom->iqcal_swp_dis, 0); + nvram_read_u8(prefix, NULL, "hw_iqcal_en", &sprom->hw_iqcal_en, 0); + nvram_read_u8(prefix, NULL, "elna2g", &sprom->elna2g, 0); + nvram_read_u8(prefix, NULL, "elna5g", &sprom->elna5g, 0); + nvram_read_u8(prefix, NULL, "phycal_tempdelta", + &sprom->phycal_tempdelta, 0); + nvram_read_u8(prefix, NULL, "temps_period", &sprom->temps_period, 0); + nvram_read_u8(prefix, NULL, "temps_hysteresis", + &sprom->temps_hysteresis, 0); + nvram_read_u8(prefix, NULL, "measpower1", &sprom->measpower1, 0); + nvram_read_u8(prefix, NULL, "measpower2", &sprom->measpower2, 0); + nvram_read_u8(prefix, NULL, "rxgainerr2ga0", + &sprom->rxgainerr2ga[0], 0); + nvram_read_u8(prefix, NULL, "rxgainerr2ga1", + &sprom->rxgainerr2ga[1], 0); + nvram_read_u8(prefix, NULL, "rxgainerr2ga2", + &sprom->rxgainerr2ga[2], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gla0", + &sprom->rxgainerr5gla[0], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gla1", + &sprom->rxgainerr5gla[1], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gla2", + &sprom->rxgainerr5gla[2], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gma0", + &sprom->rxgainerr5gma[0], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gma1", + &sprom->rxgainerr5gma[1], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gma2", + &sprom->rxgainerr5gma[2], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gha0", + &sprom->rxgainerr5gha[0], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gha1", + &sprom->rxgainerr5gha[1], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gha2", + &sprom->rxgainerr5gha[2], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gua0", + &sprom->rxgainerr5gua[0], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gua1", + &sprom->rxgainerr5gua[1], 0); + nvram_read_u8(prefix, NULL, "rxgainerr5gua2", + &sprom->rxgainerr5gua[2], 0); + nvram_read_u8(prefix, NULL, "noiselvl2ga0", &sprom->noiselvl2ga[0], 0); + nvram_read_u8(prefix, NULL, "noiselvl2ga1", &sprom->noiselvl2ga[1], 0); + nvram_read_u8(prefix, NULL, "noiselvl2ga2", &sprom->noiselvl2ga[2], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gla0", + &sprom->noiselvl5gla[0], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gla1", + &sprom->noiselvl5gla[1], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gla2", + &sprom->noiselvl5gla[2], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gma0", + &sprom->noiselvl5gma[0], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gma1", + &sprom->noiselvl5gma[1], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gma2", + &sprom->noiselvl5gma[2], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gha0", + &sprom->noiselvl5gha[0], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gha1", + &sprom->noiselvl5gha[1], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gha2", + &sprom->noiselvl5gha[2], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gua0", + &sprom->noiselvl5gua[0], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gua1", + &sprom->noiselvl5gua[1], 0); + nvram_read_u8(prefix, NULL, "noiselvl5gua2", + &sprom->noiselvl5gua[2], 0); + nvram_read_u8(prefix, NULL, "pcieingress_war", + &sprom->pcieingress_war, 0); +} + +static void bcm47xx_fill_sprom_r9(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_u16(prefix, NULL, "cckbw202gpo", &sprom->cckbw202gpo, 0); + nvram_read_u16(prefix, NULL, "cckbw20ul2gpo", &sprom->cckbw20ul2gpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw202gpo", + &sprom->legofdmbw202gpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw20ul2gpo", + &sprom->legofdmbw20ul2gpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw205glpo", + &sprom->legofdmbw205glpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw20ul5glpo", + &sprom->legofdmbw20ul5glpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw205gmpo", + &sprom->legofdmbw205gmpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw20ul5gmpo", + &sprom->legofdmbw20ul5gmpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw205ghpo", + &sprom->legofdmbw205ghpo, 0); + nvram_read_u32(prefix, NULL, "legofdmbw20ul5ghpo", + &sprom->legofdmbw20ul5ghpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw202gpo", &sprom->mcsbw202gpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw20ul2gpo", &sprom->mcsbw20ul2gpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw402gpo", &sprom->mcsbw402gpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw205glpo", &sprom->mcsbw205glpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw20ul5glpo", + &sprom->mcsbw20ul5glpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw405glpo", &sprom->mcsbw405glpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw205gmpo", &sprom->mcsbw205gmpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw20ul5gmpo", + &sprom->mcsbw20ul5gmpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw405gmpo", &sprom->mcsbw405gmpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw205ghpo", &sprom->mcsbw205ghpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw20ul5ghpo", + &sprom->mcsbw20ul5ghpo, 0); + nvram_read_u32(prefix, NULL, "mcsbw405ghpo", &sprom->mcsbw405ghpo, 0); + nvram_read_u16(prefix, NULL, "mcs32po", &sprom->mcs32po, 0); + nvram_read_u16(prefix, NULL, "legofdm40duppo", + &sprom->legofdm40duppo, 0); + nvram_read_u8(prefix, NULL, "sar2g", &sprom->sar2g, 0); + nvram_read_u8(prefix, NULL, "sar5g", &sprom->sar5g, 0); +} + +static void bcm47xx_fill_sprom_path_r4589(struct ssb_sprom *sprom, + const char *prefix) +{ + char postfix[2]; + int i; + + for (i = 0; i < ARRAY_SIZE(sprom->core_pwr_info); i++) { + struct ssb_sprom_core_pwr_info *pwr_info = &sprom->core_pwr_info[i]; + snprintf(postfix, sizeof(postfix), "%i", i); + nvram_read_u8(prefix, postfix, "maxp2ga", + &pwr_info->maxpwr_2g, 0); + nvram_read_u8(prefix, postfix, "itt2ga", + &pwr_info->itssi_2g, 0); + nvram_read_u8(prefix, postfix, "itt5ga", + &pwr_info->itssi_5g, 0); + nvram_read_u16(prefix, postfix, "pa2gw0a", + &pwr_info->pa_2g[0], 0); + nvram_read_u16(prefix, postfix, "pa2gw1a", + &pwr_info->pa_2g[1], 0); + nvram_read_u16(prefix, postfix, "pa2gw2a", + &pwr_info->pa_2g[2], 0); + nvram_read_u8(prefix, postfix, "maxp5ga", + &pwr_info->maxpwr_5g, 0); + nvram_read_u8(prefix, postfix, "maxp5gha", + &pwr_info->maxpwr_5gh, 0); + nvram_read_u8(prefix, postfix, "maxp5gla", + &pwr_info->maxpwr_5gl, 0); + nvram_read_u16(prefix, postfix, "pa5gw0a", + &pwr_info->pa_5g[0], 0); + nvram_read_u16(prefix, postfix, "pa5gw1a", + &pwr_info->pa_5g[1], 0); + nvram_read_u16(prefix, postfix, "pa5gw2a", + &pwr_info->pa_5g[2], 0); + nvram_read_u16(prefix, postfix, "pa5glw0a", + &pwr_info->pa_5gl[0], 0); + nvram_read_u16(prefix, postfix, "pa5glw1a", + &pwr_info->pa_5gl[1], 0); + nvram_read_u16(prefix, postfix, "pa5glw2a", + &pwr_info->pa_5gl[2], 0); + nvram_read_u16(prefix, postfix, "pa5ghw0a", + &pwr_info->pa_5gh[0], 0); + nvram_read_u16(prefix, postfix, "pa5ghw1a", + &pwr_info->pa_5gh[1], 0); + nvram_read_u16(prefix, postfix, "pa5ghw2a", + &pwr_info->pa_5gh[2], 0); + } +} + +static void bcm47xx_fill_sprom_path_r45(struct ssb_sprom *sprom, + const char *prefix) +{ + char postfix[2]; + int i; + + for (i = 0; i < ARRAY_SIZE(sprom->core_pwr_info); i++) { + struct ssb_sprom_core_pwr_info *pwr_info = &sprom->core_pwr_info[i]; + snprintf(postfix, sizeof(postfix), "%i", i); + nvram_read_u16(prefix, postfix, "pa2gw3a", + &pwr_info->pa_2g[3], 0); + nvram_read_u16(prefix, postfix, "pa5gw3a", + &pwr_info->pa_5g[3], 0); + nvram_read_u16(prefix, postfix, "pa5glw3a", + &pwr_info->pa_5gl[3], 0); + nvram_read_u16(prefix, postfix, "pa5ghw3a", + &pwr_info->pa_5gh[3], 0); + } +} + +void bcm47xx_fill_sprom_ethernet(struct ssb_sprom *sprom, const char *prefix) +{ + nvram_read_macaddr(prefix, "et0macaddr", &sprom->et0mac); + nvram_read_u8(prefix, NULL, "et0mdcport", &sprom->et0mdcport, 0); + nvram_read_u8(prefix, NULL, "et0phyaddr", &sprom->et0phyaddr, 0); + + nvram_read_macaddr(prefix, "et1macaddr", &sprom->et1mac); + nvram_read_u8(prefix, NULL, "et1mdcport", &sprom->et1mdcport, 0); + nvram_read_u8(prefix, NULL, "et1phyaddr", &sprom->et1phyaddr, 0); + + nvram_read_macaddr(prefix, "macaddr", &sprom->il0mac); + nvram_read_macaddr(prefix, "il0macaddr", &sprom->il0mac); +} + +void bcm47xx_fill_sprom(struct ssb_sprom *sprom, const char *prefix) +{ + memset(sprom, 0, sizeof(struct ssb_sprom)); + + bcm47xx_fill_sprom_ethernet(sprom, prefix); + + nvram_read_u8(prefix, NULL, "sromrev", &sprom->revision, 0); + + switch (sprom->revision) { + case 1: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r1(sprom, prefix); + break; + case 2: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r2389(sprom, prefix); + bcm47xx_fill_sprom_r2(sprom, prefix); + break; + case 3: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r2389(sprom, prefix); + bcm47xx_fill_sprom_r389(sprom, prefix); + bcm47xx_fill_sprom_r3(sprom, prefix); + break; + case 4: + case 5: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r4589(sprom, prefix); + bcm47xx_fill_sprom_r458(sprom, prefix); + bcm47xx_fill_sprom_r45(sprom, prefix); + bcm47xx_fill_sprom_path_r4589(sprom, prefix); + bcm47xx_fill_sprom_path_r45(sprom, prefix); + break; + case 8: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r2389(sprom, prefix); + bcm47xx_fill_sprom_r389(sprom, prefix); + bcm47xx_fill_sprom_r4589(sprom, prefix); + bcm47xx_fill_sprom_r458(sprom, prefix); + bcm47xx_fill_sprom_r89(sprom, prefix); + bcm47xx_fill_sprom_path_r4589(sprom, prefix); + break; + case 9: + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r2389(sprom, prefix); + bcm47xx_fill_sprom_r389(sprom, prefix); + bcm47xx_fill_sprom_r4589(sprom, prefix); + bcm47xx_fill_sprom_r89(sprom, prefix); + bcm47xx_fill_sprom_r9(sprom, prefix); + bcm47xx_fill_sprom_path_r4589(sprom, prefix); + break; + default: + pr_warn("Unsupported SPROM revision %d detected. Will extract" + " v1\n", sprom->revision); + sprom->revision = 1; + bcm47xx_fill_sprom_r1234589(sprom, prefix); + bcm47xx_fill_sprom_r12389(sprom, prefix); + bcm47xx_fill_sprom_r1(sprom, prefix); + } +} diff --git a/arch/mips/bcm63xx/setup.c b/arch/mips/bcm63xx/setup.c index d209f85..356b055 100644 --- a/arch/mips/bcm63xx/setup.c +++ b/arch/mips/bcm63xx/setup.c @@ -33,7 +33,7 @@ static void bcm6348_a1_reboot(void) u32 reg; /* soft reset all blocks */ - printk(KERN_INFO "soft-reseting all blocks ...\n"); + printk(KERN_INFO "soft-resetting all blocks ...\n"); reg = bcm_perf_readl(PERF_SOFTRESET_REG); reg &= ~SOFTRESET_6348_ALL; bcm_perf_writel(reg, PERF_SOFTRESET_REG); diff --git a/arch/mips/include/asm/highmem.h b/arch/mips/include/asm/highmem.h index 77e6440..2d91888 100644 --- a/arch/mips/include/asm/highmem.h +++ b/arch/mips/include/asm/highmem.h @@ -47,7 +47,7 @@ extern void kunmap_high(struct page *page); extern void *kmap(struct page *page); extern void kunmap(struct page *page); -extern void *__kmap_atomic(struct page *page); +extern void *kmap_atomic(struct page *page); extern void __kunmap_atomic(void *kvaddr); extern void *kmap_atomic_pfn(unsigned long pfn); extern struct page *kmap_atomic_to_page(void *ptr); diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index 2354c87..fb698dc 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -11,15 +11,12 @@ #include <linux/linkage.h> #include <linux/smp.h> +#include <linux/irqdomain.h> #include <asm/mipsmtregs.h> #include <irq.h> -static inline void irq_dispose_mapping(unsigned int virq) -{ -} - #ifdef CONFIG_I8259 static inline int irq_canonicalize(int irq) { diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h index 1881b31..4d6d77e 100644 --- a/arch/mips/include/asm/jump_label.h +++ b/arch/mips/include/asm/jump_label.h @@ -20,7 +20,7 @@ #define WORD_INSN ".word" #endif -static __always_inline bool arch_static_branch(struct jump_label_key *key) +static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("1:\tnop\n\t" "nop\n\t" diff --git a/arch/mips/include/asm/mach-bcm47xx/bcm47xx.h b/arch/mips/include/asm/mach-bcm47xx/bcm47xx.h index de95e07..5ecaf47 100644 --- a/arch/mips/include/asm/mach-bcm47xx/bcm47xx.h +++ b/arch/mips/include/asm/mach-bcm47xx/bcm47xx.h @@ -44,4 +44,7 @@ union bcm47xx_bus { extern union bcm47xx_bus bcm47xx_bus; extern enum bcm47xx_bus_type bcm47xx_bus_type; +void bcm47xx_fill_sprom(struct ssb_sprom *sprom, const char *prefix); +void bcm47xx_fill_sprom_ethernet(struct ssb_sprom *sprom, const char *prefix); + #endif /* __ASM_BCM47XX_H */ diff --git a/arch/mips/include/asm/mach-bcm47xx/nvram.h b/arch/mips/include/asm/mach-bcm47xx/nvram.h index 184d5ec..69ef3ef 100644 --- a/arch/mips/include/asm/mach-bcm47xx/nvram.h +++ b/arch/mips/include/asm/mach-bcm47xx/nvram.h @@ -37,7 +37,7 @@ struct nvram_header { extern int nvram_getenv(char *name, char *val, size_t val_len); -static inline void nvram_parse_macaddr(char *buf, u8 *macaddr) +static inline void nvram_parse_macaddr(char *buf, u8 macaddr[6]) { if (strchr(buf, ':')) sscanf(buf, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", &macaddr[0], diff --git a/arch/mips/include/asm/socket.h b/arch/mips/include/asm/socket.h index ad5c0a7..a2ed6fd 100644 --- a/arch/mips/include/asm/socket.h +++ b/arch/mips/include/asm/socket.h @@ -84,6 +84,10 @@ To add: #define SO_REUSEPORT 0x0200 /* Allow local address and port reuse. */ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #ifdef __KERNEL__ diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index e3b897a..811084f 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -606,6 +606,10 @@ static int mipspmu_event_init(struct perf_event *event) { int err = 0; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (event->attr.type) { case PERF_TYPE_RAW: case PERF_TYPE_HARDWARE: diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 7955409..61f1cb4 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -80,9 +80,7 @@ void __noreturn cpu_idle(void) #endif rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c index 6b8b420..558b539 100644 --- a/arch/mips/kernel/prom.c +++ b/arch/mips/kernel/prom.c @@ -60,20 +60,6 @@ void __init early_init_dt_setup_initrd_arch(unsigned long start, } #endif -/* - * irq_create_of_mapping - Hook to resolve OF irq specifier into a Linux irq# - * - * Currently the mapping mechanism is trivial; simple flat hwirq numbers are - * mapped 1:1 onto Linux irq numbers. Cascaded irq controllers are not - * supported. - */ -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) -{ - return intspec[0]; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); - void __init early_init_devtree(void *params) { /* Setup flat device-tree pointer */ diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 4f9eb0b..c97087d 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -498,7 +498,7 @@ static inline void local_r4k_flush_cache_page(void *args) if (map_coherent) vaddr = kmap_coherent(page, addr); else - vaddr = kmap_atomic(page, KM_USER0); + vaddr = kmap_atomic(page); addr = (unsigned long)vaddr; } @@ -521,7 +521,7 @@ static inline void local_r4k_flush_cache_page(void *args) if (map_coherent) kunmap_coherent(); else - kunmap_atomic(vaddr, KM_USER0); + kunmap_atomic(vaddr); } } diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c index 3634c7ea..aff5705 100644 --- a/arch/mips/mm/highmem.c +++ b/arch/mips/mm/highmem.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL(kunmap); * kmaps are appropriate for short, tight code paths only. */ -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { unsigned long vaddr; int idx, type; @@ -62,7 +62,7 @@ void *__kmap_atomic(struct page *page) return (void*) vaddr; } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 3b3ffd4..1a85ba9 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -207,21 +207,21 @@ void copy_user_highpage(struct page *to, struct page *from, { void *vfrom, *vto; - vto = kmap_atomic(to, KM_USER1); + vto = kmap_atomic(to); if (cpu_has_dc_aliases && page_mapped(from) && !Page_dcache_dirty(from)) { vfrom = kmap_coherent(from, vaddr); copy_page(vto, vfrom); kunmap_coherent(); } else { - vfrom = kmap_atomic(from, KM_USER0); + vfrom = kmap_atomic(from); copy_page(vto, vfrom); - kunmap_atomic(vfrom, KM_USER0); + kunmap_atomic(vfrom); } if ((!cpu_has_ic_fills_f_dc) || pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK)) flush_data_cache_page((unsigned long)vto); - kunmap_atomic(vto, KM_USER1); + kunmap_atomic(vto); /* Make sure this page is cleared on other CPU's too before using it */ smp_wmb(); } diff --git a/arch/mips/pci/pci-bcm47xx.c b/arch/mips/pci/pci-bcm47xx.c index 400535a..c682468 100644 --- a/arch/mips/pci/pci-bcm47xx.c +++ b/arch/mips/pci/pci-bcm47xx.c @@ -25,6 +25,7 @@ #include <linux/types.h> #include <linux/pci.h> #include <linux/ssb/ssb.h> +#include <linux/bcma/bcma.h> #include <bcm47xx.h> int __init pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) @@ -32,15 +33,12 @@ int __init pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return 0; } -int pcibios_plat_dev_init(struct pci_dev *dev) -{ #ifdef CONFIG_BCM47XX_SSB +static int bcm47xx_pcibios_plat_dev_init_ssb(struct pci_dev *dev) +{ int res; u8 slot, pin; - if (bcm47xx_bus_type != BCM47XX_BUS_TYPE_SSB) - return 0; - res = ssb_pcibios_plat_dev_init(dev); if (res < 0) { printk(KERN_ALERT "PCI: Failed to init device %s\n", @@ -60,6 +58,47 @@ int pcibios_plat_dev_init(struct pci_dev *dev) } dev->irq = res; + return 0; +} #endif + +#ifdef CONFIG_BCM47XX_BCMA +static int bcm47xx_pcibios_plat_dev_init_bcma(struct pci_dev *dev) +{ + int res; + + res = bcma_core_pci_plat_dev_init(dev); + if (res < 0) { + printk(KERN_ALERT "PCI: Failed to init device %s\n", + pci_name(dev)); + return res; + } + + res = bcma_core_pci_pcibios_map_irq(dev); + + /* IRQ-0 and IRQ-1 are software interrupts. */ + if (res < 2) { + printk(KERN_ALERT "PCI: Failed to map IRQ of device %s\n", + pci_name(dev)); + return res; + } + + dev->irq = res; return 0; } +#endif + +int pcibios_plat_dev_init(struct pci_dev *dev) +{ +#ifdef CONFIG_BCM47XX_SSB + if (bcm47xx_bus_type == BCM47XX_BUS_TYPE_SSB) + return bcm47xx_pcibios_plat_dev_init_ssb(dev); + else +#endif +#ifdef CONFIG_BCM47XX_BCMA + if (bcm47xx_bus_type == BCM47XX_BUS_TYPE_BCMA) + return bcm47xx_pcibios_plat_dev_init_bcma(dev); + else +#endif + return 0; +} diff --git a/arch/mn10300/include/asm/highmem.h b/arch/mn10300/include/asm/highmem.h index bfe2d88..7c137cd 100644 --- a/arch/mn10300/include/asm/highmem.h +++ b/arch/mn10300/include/asm/highmem.h @@ -70,7 +70,7 @@ static inline void kunmap(struct page *page) * be used in IRQ contexts, so in some (very limited) cases we need * it. */ -static inline unsigned long __kmap_atomic(struct page *page) +static inline unsigned long kmap_atomic(struct page *page) { unsigned long vaddr; int idx, type; diff --git a/arch/mn10300/include/asm/socket.h b/arch/mn10300/include/asm/socket.h index 876356d..820463a 100644 --- a/arch/mn10300/include/asm/socket.h +++ b/arch/mn10300/include/asm/socket.h @@ -64,5 +64,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/mn10300/kernel/process.c b/arch/mn10300/kernel/process.c index 28eec31..cac401d 100644 --- a/arch/mn10300/kernel/process.c +++ b/arch/mn10300/kernel/process.c @@ -123,9 +123,7 @@ void cpu_idle(void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/openrisc/include/asm/prom.h b/arch/openrisc/include/asm/prom.h index e1f3fe2..bbb34e5 100644 --- a/arch/openrisc/include/asm/prom.h +++ b/arch/openrisc/include/asm/prom.h @@ -24,6 +24,7 @@ #include <linux/types.h> #include <asm/irq.h> +#include <linux/irqdomain.h> #include <linux/atomic.h> #include <linux/of_irq.h> #include <linux/of_fdt.h> @@ -63,15 +64,6 @@ extern const void *of_get_mac_address(struct device_node *np); struct pci_dev; extern int of_irq_map_pci(struct pci_dev *pdev, struct of_irq *out_irq); -/* This routine is here to provide compatibility with how powerpc - * handles IRQ mapping for OF device nodes. We precompute and permanently - * register them in the platform_device objects, whereas powerpc computes them - * on request. - */ -static inline void irq_dispose_mapping(unsigned int virq) -{ -} - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_OPENRISC_PROM_H */ diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index da601dd..9f21ab0 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -140,7 +140,7 @@ static inline void *kmap(struct page *page) #define kunmap(page) kunmap_parisc(page_address(page)) -static inline void *__kmap_atomic(struct page *page) +static inline void *kmap_atomic(struct page *page) { pagefault_disable(); return page_address(page); diff --git a/arch/parisc/include/asm/socket.h b/arch/parisc/include/asm/socket.h index d28c51b..1b52c2c 100644 --- a/arch/parisc/include/asm/socket.h +++ b/arch/parisc/include/asm/socket.h @@ -63,6 +63,11 @@ #define SO_WIFI_STATUS 0x4022 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 0x4023 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 0x4024 + /* O_NONBLOCK clashes with the bits used for socket types. Therefore we * have to define SOCK_NONBLOCK to a different value here. diff --git a/arch/parisc/kernel/pdc_cons.c b/arch/parisc/kernel/pdc_cons.c index fc770be..4f00459 100644 --- a/arch/parisc/kernel/pdc_cons.c +++ b/arch/parisc/kernel/pdc_cons.c @@ -90,11 +90,13 @@ static int pdc_console_setup(struct console *co, char *options) #define PDC_CONS_POLL_DELAY (30 * HZ / 1000) -static struct timer_list pdc_console_timer; +static void pdc_console_poll(unsigned long unused); +static DEFINE_TIMER(pdc_console_timer, pdc_console_poll, 0, 0); +static struct tty_port tty_port; static int pdc_console_tty_open(struct tty_struct *tty, struct file *filp) { - + tty_port_tty_set(&tty_port, tty); mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY); return 0; @@ -102,8 +104,10 @@ static int pdc_console_tty_open(struct tty_struct *tty, struct file *filp) static void pdc_console_tty_close(struct tty_struct *tty, struct file *filp) { - if (!tty->count) - del_timer(&pdc_console_timer); + if (!tty->count) { + del_timer_sync(&pdc_console_timer); + tty_port_tty_set(&tty_port, NULL); + } } static int pdc_console_tty_write(struct tty_struct *tty, const unsigned char *buf, int count) @@ -122,8 +126,6 @@ static int pdc_console_tty_chars_in_buffer(struct tty_struct *tty) return 0; /* no buffer */ } -static struct tty_driver *pdc_console_tty_driver; - static const struct tty_operations pdc_console_tty_ops = { .open = pdc_console_tty_open, .close = pdc_console_tty_close, @@ -134,10 +136,8 @@ static const struct tty_operations pdc_console_tty_ops = { static void pdc_console_poll(unsigned long unused) { - int data, count = 0; - - struct tty_struct *tty = pdc_console_tty_driver->ttys[0]; + struct tty_struct *tty = tty_port_tty_get(&tty_port); if (!tty) return; @@ -153,15 +153,17 @@ static void pdc_console_poll(unsigned long unused) if (count) tty_flip_buffer_push(tty); - if (tty->count && (pdc_cons.flags & CON_ENABLED)) + tty_kref_put(tty); + + if (pdc_cons.flags & CON_ENABLED) mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY); } +static struct tty_driver *pdc_console_tty_driver; + static int __init pdc_console_tty_driver_init(void) { - int err; - struct tty_driver *drv; /* Check if the console driver is still registered. * It is unregistered if the pdc console was not selected as the @@ -183,32 +185,29 @@ static int __init pdc_console_tty_driver_init(void) printk(KERN_INFO "The PDC console driver is still registered, removing CON_BOOT flag\n"); pdc_cons.flags &= ~CON_BOOT; - drv = alloc_tty_driver(1); + tty_port_init(&tty_port); - if (!drv) - return -ENOMEM; + pdc_console_tty_driver = alloc_tty_driver(1); - drv->driver_name = "pdc_cons"; - drv->name = "ttyB"; - drv->major = MUX_MAJOR; - drv->minor_start = 0; - drv->type = TTY_DRIVER_TYPE_SYSTEM; - drv->init_termios = tty_std_termios; - drv->flags = TTY_DRIVER_REAL_RAW | TTY_DRIVER_RESET_TERMIOS; - tty_set_operations(drv, &pdc_console_tty_ops); + if (!pdc_console_tty_driver) + return -ENOMEM; - err = tty_register_driver(drv); + pdc_console_tty_driver->driver_name = "pdc_cons"; + pdc_console_tty_driver->name = "ttyB"; + pdc_console_tty_driver->major = MUX_MAJOR; + pdc_console_tty_driver->minor_start = 0; + pdc_console_tty_driver->type = TTY_DRIVER_TYPE_SYSTEM; + pdc_console_tty_driver->init_termios = tty_std_termios; + pdc_console_tty_driver->flags = TTY_DRIVER_REAL_RAW | + TTY_DRIVER_RESET_TERMIOS; + tty_set_operations(pdc_console_tty_driver, &pdc_console_tty_ops); + + err = tty_register_driver(pdc_console_tty_driver); if (err) { printk(KERN_ERR "Unable to register the PDC console TTY driver\n"); return err; } - pdc_console_tty_driver = drv; - - /* No need to initialize the pdc_console_timer if tty isn't allocated */ - init_timer(&pdc_console_timer); - pdc_console_timer.function = pdc_console_poll; - return 0; } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index 62c60b8..d4b94b3 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -71,9 +71,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) barrier(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 1919634..303703d 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -135,6 +135,7 @@ config PPC select HAVE_GENERIC_HARDIRQS select HAVE_SPARSE_IRQ select IRQ_PER_CPU + select IRQ_DOMAIN select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL select IRQ_FORCED_THREADING diff --git a/arch/powerpc/boot/dts/bluestone.dts b/arch/powerpc/boot/dts/bluestone.dts index 2a56a0d..74876f7 100644 --- a/arch/powerpc/boot/dts/bluestone.dts +++ b/arch/powerpc/boot/dts/bluestone.dts @@ -222,7 +222,7 @@ EMAC0: ethernet@ef600c00 { device_type = "network"; - compatible = "ibm,emac4sync"; + compatible = "ibm,emac-apm821xx", "ibm,emac4sync"; interrupt-parent = <&EMAC0>; interrupts = <0x0 0x1>; #interrupt-cells = <1>; diff --git a/arch/powerpc/include/asm/ehv_pic.h b/arch/powerpc/include/asm/ehv_pic.h index a9e1f4f..dc7d48e 100644 --- a/arch/powerpc/include/asm/ehv_pic.h +++ b/arch/powerpc/include/asm/ehv_pic.h @@ -25,7 +25,7 @@ struct ehv_pic { /* The remapper for this EHV_PIC */ - struct irq_host *irqhost; + struct irq_domain *irqhost; /* The "linux" controller struct */ struct irq_chip hc_irq; diff --git a/arch/powerpc/include/asm/highmem.h b/arch/powerpc/include/asm/highmem.h index dbc2640..caaf6e0 100644 --- a/arch/powerpc/include/asm/highmem.h +++ b/arch/powerpc/include/asm/highmem.h @@ -79,7 +79,7 @@ static inline void kunmap(struct page *page) kunmap_high(page); } -static inline void *__kmap_atomic(struct page *page) +static inline void *kmap_atomic(struct page *page) { return kmap_atomic_prot(page, kmap_prot); } diff --git a/arch/powerpc/include/asm/i8259.h b/arch/powerpc/include/asm/i8259.h index 105ade2..c3fdfbd 100644 --- a/arch/powerpc/include/asm/i8259.h +++ b/arch/powerpc/include/asm/i8259.h @@ -6,7 +6,7 @@ extern void i8259_init(struct device_node *node, unsigned long intack_addr); extern unsigned int i8259_irq(void); -extern struct irq_host *i8259_get_host(void); +extern struct irq_domain *i8259_get_host(void); #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_I8259_H */ diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index c0e1bc3..fe0b09d 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -9,6 +9,7 @@ * 2 of the License, or (at your option) any later version. */ +#include <linux/irqdomain.h> #include <linux/threads.h> #include <linux/list.h> #include <linux/radix-tree.h> @@ -35,258 +36,12 @@ extern atomic_t ppc_n_lost_interrupts; /* Total number of virq in the platform */ #define NR_IRQS CONFIG_NR_IRQS -/* Number of irqs reserved for the legacy controller */ -#define NUM_ISA_INTERRUPTS 16 - /* Same thing, used by the generic IRQ code */ #define NR_IRQS_LEGACY NUM_ISA_INTERRUPTS -/* This type is the placeholder for a hardware interrupt number. It has to - * be big enough to enclose whatever representation is used by a given - * platform. - */ -typedef unsigned long irq_hw_number_t; - -/* Interrupt controller "host" data structure. This could be defined as a - * irq domain controller. That is, it handles the mapping between hardware - * and virtual interrupt numbers for a given interrupt domain. The host - * structure is generally created by the PIC code for a given PIC instance - * (though a host can cover more than one PIC if they have a flat number - * model). It's the host callbacks that are responsible for setting the - * irq_chip on a given irq_desc after it's been mapped. - * - * The host code and data structures are fairly agnostic to the fact that - * we use an open firmware device-tree. We do have references to struct - * device_node in two places: in irq_find_host() to find the host matching - * a given interrupt controller node, and of course as an argument to its - * counterpart host->ops->match() callback. However, those are treated as - * generic pointers by the core and the fact that it's actually a device-node - * pointer is purely a convention between callers and implementation. This - * code could thus be used on other architectures by replacing those two - * by some sort of arch-specific void * "token" used to identify interrupt - * controllers. - */ -struct irq_host; -struct radix_tree_root; - -/* Functions below are provided by the host and called whenever a new mapping - * is created or an old mapping is disposed. The host can then proceed to - * whatever internal data structures management is required. It also needs - * to setup the irq_desc when returning from map(). - */ -struct irq_host_ops { - /* Match an interrupt controller device node to a host, returns - * 1 on a match - */ - int (*match)(struct irq_host *h, struct device_node *node); - - /* Create or update a mapping between a virtual irq number and a hw - * irq number. This is called only once for a given mapping. - */ - int (*map)(struct irq_host *h, unsigned int virq, irq_hw_number_t hw); - - /* Dispose of such a mapping */ - void (*unmap)(struct irq_host *h, unsigned int virq); - - /* Translate device-tree interrupt specifier from raw format coming - * from the firmware to a irq_hw_number_t (interrupt line number) and - * type (sense) that can be passed to set_irq_type(). In the absence - * of this callback, irq_create_of_mapping() and irq_of_parse_and_map() - * will return the hw number in the first cell and IRQ_TYPE_NONE for - * the type (which amount to keeping whatever default value the - * interrupt controller has for that line) - */ - int (*xlate)(struct irq_host *h, struct device_node *ctrler, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type); -}; - -struct irq_host { - struct list_head link; - - /* type of reverse mapping technique */ - unsigned int revmap_type; -#define IRQ_HOST_MAP_LEGACY 0 /* legacy 8259, gets irqs 1..15 */ -#define IRQ_HOST_MAP_NOMAP 1 /* no fast reverse mapping */ -#define IRQ_HOST_MAP_LINEAR 2 /* linear map of interrupts */ -#define IRQ_HOST_MAP_TREE 3 /* radix tree */ - union { - struct { - unsigned int size; - unsigned int *revmap; - } linear; - struct radix_tree_root tree; - } revmap_data; - struct irq_host_ops *ops; - void *host_data; - irq_hw_number_t inval_irq; - - /* Optional device node pointer */ - struct device_node *of_node; -}; - struct irq_data; extern irq_hw_number_t irqd_to_hwirq(struct irq_data *d); extern irq_hw_number_t virq_to_hw(unsigned int virq); -extern bool virq_is_host(unsigned int virq, struct irq_host *host); - -/** - * irq_alloc_host - Allocate a new irq_host data structure - * @of_node: optional device-tree node of the interrupt controller - * @revmap_type: type of reverse mapping to use - * @revmap_arg: for IRQ_HOST_MAP_LINEAR linear only: size of the map - * @ops: map/unmap host callbacks - * @inval_irq: provide a hw number in that host space that is always invalid - * - * Allocates and initialize and irq_host structure. Note that in the case of - * IRQ_HOST_MAP_LEGACY, the map() callback will be called before this returns - * for all legacy interrupts except 0 (which is always the invalid irq for - * a legacy controller). For a IRQ_HOST_MAP_LINEAR, the map is allocated by - * this call as well. For a IRQ_HOST_MAP_TREE, the radix tree will be allocated - * later during boot automatically (the reverse mapping will use the slow path - * until that happens). - */ -extern struct irq_host *irq_alloc_host(struct device_node *of_node, - unsigned int revmap_type, - unsigned int revmap_arg, - struct irq_host_ops *ops, - irq_hw_number_t inval_irq); - - -/** - * irq_find_host - Locates a host for a given device node - * @node: device-tree node of the interrupt controller - */ -extern struct irq_host *irq_find_host(struct device_node *node); - - -/** - * irq_set_default_host - Set a "default" host - * @host: default host pointer - * - * For convenience, it's possible to set a "default" host that will be used - * whenever NULL is passed to irq_create_mapping(). It makes life easier for - * platforms that want to manipulate a few hard coded interrupt numbers that - * aren't properly represented in the device-tree. - */ -extern void irq_set_default_host(struct irq_host *host); - - -/** - * irq_set_virq_count - Set the maximum number of virt irqs - * @count: number of linux virtual irqs, capped with NR_IRQS - * - * This is mainly for use by platforms like iSeries who want to program - * the virtual irq number in the controller to avoid the reverse mapping - */ -extern void irq_set_virq_count(unsigned int count); - - -/** - * irq_create_mapping - Map a hardware interrupt into linux virq space - * @host: host owning this hardware interrupt or NULL for default host - * @hwirq: hardware irq number in that host space - * - * Only one mapping per hardware interrupt is permitted. Returns a linux - * virq number. - * If the sense/trigger is to be specified, set_irq_type() should be called - * on the number returned from that call. - */ -extern unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq); - - -/** - * irq_dispose_mapping - Unmap an interrupt - * @virq: linux virq number of the interrupt to unmap - */ -extern void irq_dispose_mapping(unsigned int virq); - -/** - * irq_find_mapping - Find a linux virq from an hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a slow path, for use by generic code. It's expected that an - * irq controller implementation directly calls the appropriate low level - * mapping function. - */ -extern unsigned int irq_find_mapping(struct irq_host *host, - irq_hw_number_t hwirq); - -/** - * irq_create_direct_mapping - Allocate a virq for direct mapping - * @host: host to allocate the virq for or NULL for default host - * - * This routine is used for irq controllers which can choose the hardware - * interrupt numbers they generate. In such a case it's simplest to use - * the linux virq as the hardware interrupt number. - */ -extern unsigned int irq_create_direct_mapping(struct irq_host *host); - -/** - * irq_radix_revmap_insert - Insert a hw irq to linux virq number mapping. - * @host: host owning this hardware interrupt - * @virq: linux irq number - * @hwirq: hardware irq number in that host space - * - * This is for use by irq controllers that use a radix tree reverse - * mapping for fast lookup. - */ -extern void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq); - -/** - * irq_radix_revmap_lookup - Find a linux virq from a hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a fast path, for use by irq controller code that uses radix tree - * revmaps - */ -extern unsigned int irq_radix_revmap_lookup(struct irq_host *host, - irq_hw_number_t hwirq); - -/** - * irq_linear_revmap - Find a linux virq from a hw irq number. - * @host: host owning this hardware interrupt - * @hwirq: hardware irq number in that host space - * - * This is a fast path, for use by irq controller code that uses linear - * revmaps. It does fallback to the slow path if the revmap doesn't exist - * yet and will create the revmap entry with appropriate locking - */ - -extern unsigned int irq_linear_revmap(struct irq_host *host, - irq_hw_number_t hwirq); - - - -/** - * irq_alloc_virt - Allocate virtual irq numbers - * @host: host owning these new virtual irqs - * @count: number of consecutive numbers to allocate - * @hint: pass a hint number, the allocator will try to use a 1:1 mapping - * - * This is a low level function that is used internally by irq_create_mapping() - * and that can be used by some irq controllers implementations for things - * like allocating ranges of numbers for MSIs. The revmaps are left untouched. - */ -extern unsigned int irq_alloc_virt(struct irq_host *host, - unsigned int count, - unsigned int hint); - -/** - * irq_free_virt - Free virtual irq numbers - * @virq: virtual irq number of the first interrupt to free - * @count: number of interrupts to free - * - * This function is the opposite of irq_alloc_virt. It will not clear reverse - * maps, this should be done previously by unmap'ing the interrupt. In fact, - * all interrupts covered by the range being freed should have been unmapped - * prior to calling this. - */ -extern void irq_free_virt(unsigned int virq, unsigned int count); /** * irq_early_init - Init irq remapping subsystem diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h index 938986e..ae098c4 100644 --- a/arch/powerpc/include/asm/jump_label.h +++ b/arch/powerpc/include/asm/jump_label.h @@ -17,7 +17,7 @@ #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct jump_label_key *key) +static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("1:\n\t" "nop\n\t" diff --git a/arch/powerpc/include/asm/keylargo.h b/arch/powerpc/include/asm/keylargo.h index fc195d0..2156315 100644 --- a/arch/powerpc/include/asm/keylargo.h +++ b/arch/powerpc/include/asm/keylargo.h @@ -21,7 +21,7 @@ #define KEYLARGO_FCR4 0x48 #define KEYLARGO_FCR5 0x4c /* Pangea only */ -/* K2 aditional FCRs */ +/* K2 additional FCRs */ #define K2_FCR6 0x34 #define K2_FCR7 0x30 #define K2_FCR8 0x2c diff --git a/arch/powerpc/include/asm/mpic.h b/arch/powerpc/include/asm/mpic.h index 67b4d98..a5b7c56 100644 --- a/arch/powerpc/include/asm/mpic.h +++ b/arch/powerpc/include/asm/mpic.h @@ -255,7 +255,7 @@ struct mpic struct device_node *node; /* The remapper for this MPIC */ - struct irq_host *irqhost; + struct irq_domain *irqhost; /* The "linux" controller struct */ struct irq_chip hc_irq; diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h index 8f1df12..1a8093f 100644 --- a/arch/powerpc/include/asm/perf_event_server.h +++ b/arch/powerpc/include/asm/perf_event_server.h @@ -61,8 +61,6 @@ struct pt_regs; extern unsigned long perf_misc_flags(struct pt_regs *regs); extern unsigned long perf_instruction_pointer(struct pt_regs *regs); -#define PERF_EVENT_INDEX_OFFSET 1 - /* * Only override the default definitions in include/linux/perf_event.h * if we have hardware PMU support. diff --git a/arch/powerpc/include/asm/socket.h b/arch/powerpc/include/asm/socket.h index 2fc2af8..3d5179b 100644 --- a/arch/powerpc/include/asm/socket.h +++ b/arch/powerpc/include/asm/socket.h @@ -71,5 +71,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_POWERPC_SOCKET_H */ diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index c48de98..4ae9a09 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -86,7 +86,7 @@ struct ics { extern unsigned int xics_default_server; extern unsigned int xics_default_distrib_server; extern unsigned int xics_interrupt_server_size; -extern struct irq_host *xics_host; +extern struct irq_domain *xics_host; struct xics_cppr { unsigned char stack[MAX_NUM_PRIORITIES]; diff --git a/arch/powerpc/kernel/ibmebus.c b/arch/powerpc/kernel/ibmebus.c index d39ae60..79bb282 100644 --- a/arch/powerpc/kernel/ibmebus.c +++ b/arch/powerpc/kernel/ibmebus.c @@ -713,7 +713,7 @@ static struct dev_pm_ops ibmebus_bus_dev_pm_ops = { struct bus_type ibmebus_bus_type = { .name = "ibmebus", - .uevent = of_device_uevent, + .uevent = of_device_uevent_modalias, .bus_attrs = ibmebus_bus_attrs, .match = ibmebus_bus_bus_match, .probe = ibmebus_bus_device_probe, diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c index 0a48bf5..c97fc60 100644 --- a/arch/powerpc/kernel/idle.c +++ b/arch/powerpc/kernel/idle.c @@ -101,11 +101,11 @@ void cpu_idle(void) ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - if (cpu_should_die()) + if (cpu_should_die()) { + sched_preempt_enable_no_resched(); cpu_die(); - schedule(); - preempt_disable(); + } + schedule_preempt_disabled(); } } diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 01e2877..bdfb3eee 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -490,409 +490,19 @@ void do_softirq(void) local_irq_restore(flags); } - -/* - * IRQ controller and virtual interrupts - */ - -/* The main irq map itself is an array of NR_IRQ entries containing the - * associate host and irq number. An entry with a host of NULL is free. - * An entry can be allocated if it's free, the allocator always then sets - * hwirq first to the host's invalid irq number and then fills ops. - */ -struct irq_map_entry { - irq_hw_number_t hwirq; - struct irq_host *host; -}; - -static LIST_HEAD(irq_hosts); -static DEFINE_RAW_SPINLOCK(irq_big_lock); -static DEFINE_MUTEX(revmap_trees_mutex); -static struct irq_map_entry irq_map[NR_IRQS]; -static unsigned int irq_virq_count = NR_IRQS; -static struct irq_host *irq_default_host; - irq_hw_number_t irqd_to_hwirq(struct irq_data *d) { - return irq_map[d->irq].hwirq; + return d->hwirq; } EXPORT_SYMBOL_GPL(irqd_to_hwirq); irq_hw_number_t virq_to_hw(unsigned int virq) { - return irq_map[virq].hwirq; + struct irq_data *irq_data = irq_get_irq_data(virq); + return WARN_ON(!irq_data) ? 0 : irq_data->hwirq; } EXPORT_SYMBOL_GPL(virq_to_hw); -bool virq_is_host(unsigned int virq, struct irq_host *host) -{ - return irq_map[virq].host == host; -} -EXPORT_SYMBOL_GPL(virq_is_host); - -static int default_irq_host_match(struct irq_host *h, struct device_node *np) -{ - return h->of_node != NULL && h->of_node == np; -} - -struct irq_host *irq_alloc_host(struct device_node *of_node, - unsigned int revmap_type, - unsigned int revmap_arg, - struct irq_host_ops *ops, - irq_hw_number_t inval_irq) -{ - struct irq_host *host; - unsigned int size = sizeof(struct irq_host); - unsigned int i; - unsigned int *rmap; - unsigned long flags; - - /* Allocate structure and revmap table if using linear mapping */ - if (revmap_type == IRQ_HOST_MAP_LINEAR) - size += revmap_arg * sizeof(unsigned int); - host = kzalloc(size, GFP_KERNEL); - if (host == NULL) - return NULL; - - /* Fill structure */ - host->revmap_type = revmap_type; - host->inval_irq = inval_irq; - host->ops = ops; - host->of_node = of_node_get(of_node); - - if (host->ops->match == NULL) - host->ops->match = default_irq_host_match; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* If it's a legacy controller, check for duplicates and - * mark it as allocated (we use irq 0 host pointer for that - */ - if (revmap_type == IRQ_HOST_MAP_LEGACY) { - if (irq_map[0].host != NULL) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - of_node_put(host->of_node); - kfree(host); - return NULL; - } - irq_map[0].host = host; - } - - list_add(&host->link, &irq_hosts); - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - - /* Additional setups per revmap type */ - switch(revmap_type) { - case IRQ_HOST_MAP_LEGACY: - /* 0 is always the invalid number for legacy */ - host->inval_irq = 0; - /* setup us as the host for all legacy interrupts */ - for (i = 1; i < NUM_ISA_INTERRUPTS; i++) { - irq_map[i].hwirq = i; - smp_wmb(); - irq_map[i].host = host; - smp_wmb(); - - /* Legacy flags are left to default at this point, - * one can then use irq_create_mapping() to - * explicitly change them - */ - ops->map(host, i, i); - - /* Clear norequest flags */ - irq_clear_status_flags(i, IRQ_NOREQUEST); - } - break; - case IRQ_HOST_MAP_LINEAR: - rmap = (unsigned int *)(host + 1); - for (i = 0; i < revmap_arg; i++) - rmap[i] = NO_IRQ; - host->revmap_data.linear.size = revmap_arg; - smp_wmb(); - host->revmap_data.linear.revmap = rmap; - break; - case IRQ_HOST_MAP_TREE: - INIT_RADIX_TREE(&host->revmap_data.tree, GFP_KERNEL); - break; - default: - break; - } - - pr_debug("irq: Allocated host of type %d @0x%p\n", revmap_type, host); - - return host; -} - -struct irq_host *irq_find_host(struct device_node *node) -{ - struct irq_host *h, *found = NULL; - unsigned long flags; - - /* We might want to match the legacy controller last since - * it might potentially be set to match all interrupts in - * the absence of a device node. This isn't a problem so far - * yet though... - */ - raw_spin_lock_irqsave(&irq_big_lock, flags); - list_for_each_entry(h, &irq_hosts, link) - if (h->ops->match(h, node)) { - found = h; - break; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} -EXPORT_SYMBOL_GPL(irq_find_host); - -void irq_set_default_host(struct irq_host *host) -{ - pr_debug("irq: Default host set to @0x%p\n", host); - - irq_default_host = host; -} - -void irq_set_virq_count(unsigned int count) -{ - pr_debug("irq: Trying to set virq count to %d\n", count); - - BUG_ON(count < NUM_ISA_INTERRUPTS); - if (count < NR_IRQS) - irq_virq_count = count; -} - -static int irq_setup_virq(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - int res; - - res = irq_alloc_desc_at(virq, 0); - if (res != virq) { - pr_debug("irq: -> allocating desc failed\n"); - goto error; - } - - /* map it */ - smp_wmb(); - irq_map[virq].hwirq = hwirq; - smp_mb(); - - if (host->ops->map(host, virq, hwirq)) { - pr_debug("irq: -> mapping failed, freeing\n"); - goto errdesc; - } - - irq_clear_status_flags(virq, IRQ_NOREQUEST); - - return 0; - -errdesc: - irq_free_descs(virq, 1); -error: - irq_free_virt(virq, 1); - return -1; -} - -unsigned int irq_create_direct_mapping(struct irq_host *host) -{ - unsigned int virq; - - if (host == NULL) - host = irq_default_host; - - BUG_ON(host == NULL); - WARN_ON(host->revmap_type != IRQ_HOST_MAP_NOMAP); - - virq = irq_alloc_virt(host, 1, 0); - if (virq == NO_IRQ) { - pr_debug("irq: create_direct virq allocation failed\n"); - return NO_IRQ; - } - - pr_debug("irq: create_direct obtained virq %d\n", virq); - - if (irq_setup_virq(host, virq, virq)) - return NO_IRQ; - - return virq; -} - -unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int virq, hint; - - pr_debug("irq: irq_create_mapping(0x%p, 0x%lx)\n", host, hwirq); - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) { - printk(KERN_WARNING "irq_create_mapping called for" - " NULL host, hwirq=%lx\n", hwirq); - WARN_ON(1); - return NO_IRQ; - } - pr_debug("irq: -> using host @%p\n", host); - - /* Check if mapping already exists */ - virq = irq_find_mapping(host, hwirq); - if (virq != NO_IRQ) { - pr_debug("irq: -> existing mapping on virq %d\n", virq); - return virq; - } - - /* Get a virtual interrupt number */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) { - /* Handle legacy */ - virq = (unsigned int)hwirq; - if (virq == 0 || virq >= NUM_ISA_INTERRUPTS) - return NO_IRQ; - return virq; - } else { - /* Allocate a virtual interrupt number */ - hint = hwirq % irq_virq_count; - virq = irq_alloc_virt(host, 1, hint); - if (virq == NO_IRQ) { - pr_debug("irq: -> virq allocation failed\n"); - return NO_IRQ; - } - } - - if (irq_setup_virq(host, virq, hwirq)) - return NO_IRQ; - - pr_debug("irq: irq %lu on host %s mapped to virtual irq %u\n", - hwirq, host->of_node ? host->of_node->full_name : "null", virq); - - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_mapping); - -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) -{ - struct irq_host *host; - irq_hw_number_t hwirq; - unsigned int type = IRQ_TYPE_NONE; - unsigned int virq; - - if (controller == NULL) - host = irq_default_host; - else - host = irq_find_host(controller); - if (host == NULL) { - printk(KERN_WARNING "irq: no irq host found for %s !\n", - controller->full_name); - return NO_IRQ; - } - - /* If host has no translation, then we assume interrupt line */ - if (host->ops->xlate == NULL) - hwirq = intspec[0]; - else { - if (host->ops->xlate(host, controller, intspec, intsize, - &hwirq, &type)) - return NO_IRQ; - } - - /* Create mapping */ - virq = irq_create_mapping(host, hwirq); - if (virq == NO_IRQ) - return virq; - - /* Set type if specified and different than the current one */ - if (type != IRQ_TYPE_NONE && - type != (irqd_get_trigger_type(irq_get_irq_data(virq)))) - irq_set_irq_type(virq, type); - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); - -void irq_dispose_mapping(unsigned int virq) -{ - struct irq_host *host; - irq_hw_number_t hwirq; - - if (virq == NO_IRQ) - return; - - host = irq_map[virq].host; - if (WARN_ON(host == NULL)) - return; - - /* Never unmap legacy interrupts */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) - return; - - irq_set_status_flags(virq, IRQ_NOREQUEST); - - /* remove chip and handler */ - irq_set_chip_and_handler(virq, NULL, NULL); - - /* Make sure it's completed */ - synchronize_irq(virq); - - /* Tell the PIC about it */ - if (host->ops->unmap) - host->ops->unmap(host, virq); - smp_mb(); - - /* Clear reverse map */ - hwirq = irq_map[virq].hwirq; - switch(host->revmap_type) { - case IRQ_HOST_MAP_LINEAR: - if (hwirq < host->revmap_data.linear.size) - host->revmap_data.linear.revmap[hwirq] = NO_IRQ; - break; - case IRQ_HOST_MAP_TREE: - mutex_lock(&revmap_trees_mutex); - radix_tree_delete(&host->revmap_data.tree, hwirq); - mutex_unlock(&revmap_trees_mutex); - break; - } - - /* Destroy map */ - smp_mb(); - irq_map[virq].hwirq = host->inval_irq; - - irq_free_descs(virq, 1); - /* Free it */ - irq_free_virt(virq, 1); -} -EXPORT_SYMBOL_GPL(irq_dispose_mapping); - -unsigned int irq_find_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int i; - unsigned int hint = hwirq % irq_virq_count; - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) - return NO_IRQ; - - /* legacy -> bail early */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) - return hwirq; - - /* Slow path does a linear search of the map */ - if (hint < NUM_ISA_INTERRUPTS) - hint = NUM_ISA_INTERRUPTS; - i = hint; - do { - if (irq_map[i].host == host && - irq_map[i].hwirq == hwirq) - return i; - i++; - if (i >= irq_virq_count) - i = NUM_ISA_INTERRUPTS; - } while(i != hint); - return NO_IRQ; -} -EXPORT_SYMBOL_GPL(irq_find_mapping); - #ifdef CONFIG_SMP int irq_choose_cpu(const struct cpumask *mask) { @@ -929,232 +539,11 @@ int irq_choose_cpu(const struct cpumask *mask) } #endif -unsigned int irq_radix_revmap_lookup(struct irq_host *host, - irq_hw_number_t hwirq) -{ - struct irq_map_entry *ptr; - unsigned int virq; - - if (WARN_ON_ONCE(host->revmap_type != IRQ_HOST_MAP_TREE)) - return irq_find_mapping(host, hwirq); - - /* - * The ptr returned references the static global irq_map. - * but freeing an irq can delete nodes along the path to - * do the lookup via call_rcu. - */ - rcu_read_lock(); - ptr = radix_tree_lookup(&host->revmap_data.tree, hwirq); - rcu_read_unlock(); - - /* - * If found in radix tree, then fine. - * Else fallback to linear lookup - this should not happen in practice - * as it means that we failed to insert the node in the radix tree. - */ - if (ptr) - virq = ptr - irq_map; - else - virq = irq_find_mapping(host, hwirq); - - return virq; -} - -void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - if (WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE)) - return; - - if (virq != NO_IRQ) { - mutex_lock(&revmap_trees_mutex); - radix_tree_insert(&host->revmap_data.tree, hwirq, - &irq_map[virq]); - mutex_unlock(&revmap_trees_mutex); - } -} - -unsigned int irq_linear_revmap(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int *revmap; - - if (WARN_ON_ONCE(host->revmap_type != IRQ_HOST_MAP_LINEAR)) - return irq_find_mapping(host, hwirq); - - /* Check revmap bounds */ - if (unlikely(hwirq >= host->revmap_data.linear.size)) - return irq_find_mapping(host, hwirq); - - /* Check if revmap was allocated */ - revmap = host->revmap_data.linear.revmap; - if (unlikely(revmap == NULL)) - return irq_find_mapping(host, hwirq); - - /* Fill up revmap with slow path if no mapping found */ - if (unlikely(revmap[hwirq] == NO_IRQ)) - revmap[hwirq] = irq_find_mapping(host, hwirq); - - return revmap[hwirq]; -} - -unsigned int irq_alloc_virt(struct irq_host *host, - unsigned int count, - unsigned int hint) -{ - unsigned long flags; - unsigned int i, j, found = NO_IRQ; - - if (count == 0 || count > (irq_virq_count - NUM_ISA_INTERRUPTS)) - return NO_IRQ; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* Use hint for 1 interrupt if any */ - if (count == 1 && hint >= NUM_ISA_INTERRUPTS && - hint < irq_virq_count && irq_map[hint].host == NULL) { - found = hint; - goto hint_found; - } - - /* Look for count consecutive numbers in the allocatable - * (non-legacy) space - */ - for (i = NUM_ISA_INTERRUPTS, j = 0; i < irq_virq_count; i++) { - if (irq_map[i].host != NULL) - j = 0; - else - j++; - - if (j == count) { - found = i - count + 1; - break; - } - } - if (found == NO_IRQ) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return NO_IRQ; - } - hint_found: - for (i = found; i < (found + count); i++) { - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = host; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} - -void irq_free_virt(unsigned int virq, unsigned int count) -{ - unsigned long flags; - unsigned int i; - - WARN_ON (virq < NUM_ISA_INTERRUPTS); - WARN_ON (count == 0 || (virq + count) > irq_virq_count); - - if (virq < NUM_ISA_INTERRUPTS) { - if (virq + count < NUM_ISA_INTERRUPTS) - return; - count =- NUM_ISA_INTERRUPTS - virq; - virq = NUM_ISA_INTERRUPTS; - } - - if (count > irq_virq_count || virq > irq_virq_count - count) { - if (virq > irq_virq_count) - return; - count = irq_virq_count - virq; - } - - raw_spin_lock_irqsave(&irq_big_lock, flags); - for (i = virq; i < (virq + count); i++) { - struct irq_host *host; - - host = irq_map[i].host; - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = NULL; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); -} - int arch_early_irq_init(void) { return 0; } -#ifdef CONFIG_VIRQ_DEBUG -static int virq_debug_show(struct seq_file *m, void *private) -{ - unsigned long flags; - struct irq_desc *desc; - const char *p; - static const char none[] = "none"; - void *data; - int i; - - seq_printf(m, "%-5s %-7s %-15s %-18s %s\n", "virq", "hwirq", - "chip name", "chip data", "host name"); - - for (i = 1; i < nr_irqs; i++) { - desc = irq_to_desc(i); - if (!desc) - continue; - - raw_spin_lock_irqsave(&desc->lock, flags); - - if (desc->action && desc->action->handler) { - struct irq_chip *chip; - - seq_printf(m, "%5d ", i); - seq_printf(m, "0x%05lx ", irq_map[i].hwirq); - - chip = irq_desc_get_chip(desc); - if (chip && chip->name) - p = chip->name; - else - p = none; - seq_printf(m, "%-15s ", p); - - data = irq_desc_get_chip_data(desc); - seq_printf(m, "0x%16p ", data); - - if (irq_map[i].host && irq_map[i].host->of_node) - p = irq_map[i].host->of_node->full_name; - else - p = none; - seq_printf(m, "%s\n", p); - } - - raw_spin_unlock_irqrestore(&desc->lock, flags); - } - - return 0; -} - -static int virq_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, virq_debug_show, inode->i_private); -} - -static const struct file_operations virq_debug_fops = { - .open = virq_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int __init irq_debugfs_init(void) -{ - if (debugfs_create_file("virq_mapping", S_IRUGO, powerpc_debugfs_root, - NULL, &virq_debug_fops) == NULL) - return -ENOMEM; - - return 0; -} -__initcall(irq_debugfs_init); -#endif /* CONFIG_VIRQ_DEBUG */ - #ifdef CONFIG_PPC64 static int __init setup_noirqdistrib(char *str) { diff --git a/arch/powerpc/kernel/perf_event.c b/arch/powerpc/kernel/perf_event.c index 64483fd..c2e27ed 100644 --- a/arch/powerpc/kernel/perf_event.c +++ b/arch/powerpc/kernel/perf_event.c @@ -1084,6 +1084,10 @@ static int power_pmu_event_init(struct perf_event *event) if (!ppmu) return -ENOENT; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (event->attr.type) { case PERF_TYPE_HARDWARE: ev = event->attr.config; @@ -1193,6 +1197,11 @@ static int power_pmu_event_init(struct perf_event *event) return err; } +static int power_pmu_event_idx(struct perf_event *event) +{ + return event->hw.idx; +} + struct pmu power_pmu = { .pmu_enable = power_pmu_enable, .pmu_disable = power_pmu_disable, @@ -1205,6 +1214,7 @@ struct pmu power_pmu = { .start_txn = power_pmu_start_txn, .cancel_txn = power_pmu_cancel_txn, .commit_txn = power_pmu_commit_txn, + .event_idx = power_pmu_event_idx, }; /* diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index e2cfb9e..220fcdf 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -227,14 +227,14 @@ static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) hpage_offset /= 4; get_page(hpage); - page = kmap_atomic(hpage, KM_USER0); + page = kmap_atomic(hpage); /* patch dcbz into reserved instruction, so we trap */ for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) if ((page[i] & 0xff0007ff) == INS_DCBZ) page[i] &= 0xfffffff7; - kunmap_atomic(page, KM_USER0); + kunmap_atomic(page); put_page(hpage); } diff --git a/arch/powerpc/mm/dma-noncoherent.c b/arch/powerpc/mm/dma-noncoherent.c index 329be36..6747eec 100644 --- a/arch/powerpc/mm/dma-noncoherent.c +++ b/arch/powerpc/mm/dma-noncoherent.c @@ -365,12 +365,11 @@ static inline void __dma_sync_page_highmem(struct page *page, local_irq_save(flags); do { - start = (unsigned long)kmap_atomic(page + seg_nr, - KM_PPC_SYNC_PAGE) + seg_offset; + start = (unsigned long)kmap_atomic(page + seg_nr) + seg_offset; /* Sync this buffer segment */ __dma_sync((void *)start, seg_size, direction); - kunmap_atomic((void *)start, KM_PPC_SYNC_PAGE); + kunmap_atomic((void *)start); seg_nr++; /* Calculate next buffer segment size */ diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index a8b3cc7..57c7465 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -910,9 +910,9 @@ void flush_dcache_icache_hugepage(struct page *page) if (!PageHighMem(page)) { __flush_dcache_icache(page_address(page+i)); } else { - start = kmap_atomic(page+i, KM_PPC_SYNC_ICACHE); + start = kmap_atomic(page+i); __flush_dcache_icache(start); - kunmap_atomic(start, KM_PPC_SYNC_ICACHE); + kunmap_atomic(start); } } } diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index d974b79..baaafde 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -458,9 +458,9 @@ void flush_dcache_icache_page(struct page *page) #endif #ifdef CONFIG_BOOKE { - void *start = kmap_atomic(page, KM_PPC_SYNC_ICACHE); + void *start = kmap_atomic(page); __flush_dcache_icache(start); - kunmap_atomic(start, KM_PPC_SYNC_ICACHE); + kunmap_atomic(start); } #elif defined(CONFIG_8xx) || defined(CONFIG_PPC64) /* On 8xx there is no need to kmap since highmem is not supported */ diff --git a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c index 9f09319..ca3a062 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c @@ -21,7 +21,7 @@ #include <asm/prom.h> static struct device_node *cpld_pic_node; -static struct irq_host *cpld_pic_host; +static struct irq_domain *cpld_pic_host; /* * Bits to ignore in the misc_status register @@ -123,13 +123,13 @@ cpld_pic_cascade(unsigned int irq, struct irq_desc *desc) } static int -cpld_pic_host_match(struct irq_host *h, struct device_node *node) +cpld_pic_host_match(struct irq_domain *h, struct device_node *node) { return cpld_pic_node == node; } static int -cpld_pic_host_map(struct irq_host *h, unsigned int virq, +cpld_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_status_flags(virq, IRQ_LEVEL); @@ -137,8 +137,7 @@ cpld_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct -irq_host_ops cpld_pic_host_ops = { +static const struct irq_domain_ops cpld_pic_host_ops = { .match = cpld_pic_host_match, .map = cpld_pic_host_map, }; @@ -191,8 +190,7 @@ mpc5121_ads_cpld_pic_init(void) cpld_pic_node = of_node_get(np); - cpld_pic_host = - irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, 16, &cpld_pic_host_ops, 16); + cpld_pic_host = irq_domain_add_linear(np, 16, &cpld_pic_host_ops, NULL); if (!cpld_pic_host) { printk(KERN_ERR "CPLD PIC: failed to allocate irq host!\n"); goto end; diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index 96f85e5..17d91b7 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -45,7 +45,7 @@ static struct of_device_id mpc5200_gpio_ids[] __initdata = { struct media5200_irq { void __iomem *regs; spinlock_t lock; - struct irq_host *irqhost; + struct irq_domain *irqhost; }; struct media5200_irq media5200_irq; @@ -112,7 +112,7 @@ void media5200_irq_cascade(unsigned int virq, struct irq_desc *desc) raw_spin_unlock(&desc->lock); } -static int media5200_irq_map(struct irq_host *h, unsigned int virq, +static int media5200_irq_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { pr_debug("%s: h=%p, virq=%i, hwirq=%i\n", __func__, h, virq, (int)hw); @@ -122,7 +122,7 @@ static int media5200_irq_map(struct irq_host *h, unsigned int virq, return 0; } -static int media5200_irq_xlate(struct irq_host *h, struct device_node *ct, +static int media5200_irq_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -136,7 +136,7 @@ static int media5200_irq_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops media5200_irq_ops = { +static const struct irq_domain_ops media5200_irq_ops = { .map = media5200_irq_map, .xlate = media5200_irq_xlate, }; @@ -173,15 +173,12 @@ static void __init media5200_init_irq(void) spin_lock_init(&media5200_irq.lock); - media5200_irq.irqhost = irq_alloc_host(fpga_np, IRQ_HOST_MAP_LINEAR, - MEDIA5200_NUM_IRQS, - &media5200_irq_ops, -1); + media5200_irq.irqhost = irq_domain_add_linear(fpga_np, + MEDIA5200_NUM_IRQS, &media5200_irq_ops, &media5200_irq); if (!media5200_irq.irqhost) goto out; pr_debug("%s: allocated irqhost\n", __func__); - media5200_irq.irqhost->host_data = &media5200_irq; - irq_set_handler_data(cascade_virq, &media5200_irq); irq_set_chained_handler(cascade_virq, media5200_irq_cascade); diff --git a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c index f94f06e..028470b 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c @@ -81,7 +81,7 @@ MODULE_LICENSE("GPL"); * @regs: virtual address of GPT registers * @lock: spinlock to coordinate between different functions. * @gc: gpio_chip instance structure; used when GPIO is enabled - * @irqhost: Pointer to irq_host instance; used when IRQ mode is supported + * @irqhost: Pointer to irq_domain instance; used when IRQ mode is supported * @wdt_mode: only relevant for gpt0: bit 0 (MPC52xx_GPT_CAN_WDT) indicates * if the gpt may be used as wdt, bit 1 (MPC52xx_GPT_IS_WDT) indicates * if the timer is actively used as wdt which blocks gpt functions @@ -91,7 +91,7 @@ struct mpc52xx_gpt_priv { struct device *dev; struct mpc52xx_gpt __iomem *regs; spinlock_t lock; - struct irq_host *irqhost; + struct irq_domain *irqhost; u32 ipb_freq; u8 wdt_mode; @@ -204,7 +204,7 @@ void mpc52xx_gpt_irq_cascade(unsigned int virq, struct irq_desc *desc) } } -static int mpc52xx_gpt_irq_map(struct irq_host *h, unsigned int virq, +static int mpc52xx_gpt_irq_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct mpc52xx_gpt_priv *gpt = h->host_data; @@ -216,7 +216,7 @@ static int mpc52xx_gpt_irq_map(struct irq_host *h, unsigned int virq, return 0; } -static int mpc52xx_gpt_irq_xlate(struct irq_host *h, struct device_node *ct, +static int mpc52xx_gpt_irq_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -236,7 +236,7 @@ static int mpc52xx_gpt_irq_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops mpc52xx_gpt_irq_ops = { +static const struct irq_domain_ops mpc52xx_gpt_irq_ops = { .map = mpc52xx_gpt_irq_map, .xlate = mpc52xx_gpt_irq_xlate, }; @@ -252,14 +252,12 @@ mpc52xx_gpt_irq_setup(struct mpc52xx_gpt_priv *gpt, struct device_node *node) if (!cascade_virq) return; - gpt->irqhost = irq_alloc_host(node, IRQ_HOST_MAP_LINEAR, 1, - &mpc52xx_gpt_irq_ops, -1); + gpt->irqhost = irq_domain_add_linear(node, 1, &mpc52xx_gpt_irq_ops, gpt); if (!gpt->irqhost) { - dev_err(gpt->dev, "irq_alloc_host() failed\n"); + dev_err(gpt->dev, "irq_domain_add_linear() failed\n"); return; } - gpt->irqhost->host_data = gpt; irq_set_handler_data(cascade_virq, gpt); irq_set_chained_handler(cascade_virq, mpc52xx_gpt_irq_cascade); diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pic.c b/arch/powerpc/platforms/52xx/mpc52xx_pic.c index 1a9a495..8520b58 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pic.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pic.c @@ -132,7 +132,7 @@ static struct of_device_id mpc52xx_sdma_ids[] __initdata = { static struct mpc52xx_intr __iomem *intr; static struct mpc52xx_sdma __iomem *sdma; -static struct irq_host *mpc52xx_irqhost = NULL; +static struct irq_domain *mpc52xx_irqhost = NULL; static unsigned char mpc52xx_map_senses[4] = { IRQ_TYPE_LEVEL_HIGH, @@ -301,7 +301,7 @@ static int mpc52xx_is_extirq(int l1, int l2) /** * mpc52xx_irqhost_xlate - translate virq# from device tree interrupts property */ -static int mpc52xx_irqhost_xlate(struct irq_host *h, struct device_node *ct, +static int mpc52xx_irqhost_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -335,7 +335,7 @@ static int mpc52xx_irqhost_xlate(struct irq_host *h, struct device_node *ct, /** * mpc52xx_irqhost_map - Hook to map from virq to an irq_chip structure */ -static int mpc52xx_irqhost_map(struct irq_host *h, unsigned int virq, +static int mpc52xx_irqhost_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t irq) { int l1irq; @@ -384,7 +384,7 @@ static int mpc52xx_irqhost_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops mpc52xx_irqhost_ops = { +static const struct irq_domain_ops mpc52xx_irqhost_ops = { .xlate = mpc52xx_irqhost_xlate, .map = mpc52xx_irqhost_map, }; @@ -444,9 +444,9 @@ void __init mpc52xx_init_irq(void) * As last step, add an irq host to translate the real * hw irq information provided by the ofw to linux virq */ - mpc52xx_irqhost = irq_alloc_host(picnode, IRQ_HOST_MAP_LINEAR, + mpc52xx_irqhost = irq_domain_add_linear(picnode, MPC52xx_IRQ_HIGHTESTHWIRQ, - &mpc52xx_irqhost_ops, -1); + &mpc52xx_irqhost_ops, NULL); if (!mpc52xx_irqhost) panic(__FILE__ ": Cannot allocate the IRQ host\n"); diff --git a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c index 8ccf9ed..328d221 100644 --- a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c +++ b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c @@ -29,7 +29,7 @@ static DEFINE_RAW_SPINLOCK(pci_pic_lock); struct pq2ads_pci_pic { struct device_node *node; - struct irq_host *host; + struct irq_domain *host; struct { u32 stat; @@ -103,7 +103,7 @@ static void pq2ads_pci_irq_demux(unsigned int irq, struct irq_desc *desc) } } -static int pci_pic_host_map(struct irq_host *h, unsigned int virq, +static int pci_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_status_flags(virq, IRQ_LEVEL); @@ -112,14 +112,14 @@ static int pci_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops pci_pic_host_ops = { +static const struct irq_domain_ops pci_pic_host_ops = { .map = pci_pic_host_map, }; int __init pq2ads_pci_init_irq(void) { struct pq2ads_pci_pic *priv; - struct irq_host *host; + struct irq_domain *host; struct device_node *np; int ret = -ENODEV; int irq; @@ -156,17 +156,13 @@ int __init pq2ads_pci_init_irq(void) out_be32(&priv->regs->mask, ~0); mb(); - host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, NUM_IRQS, - &pci_pic_host_ops, NUM_IRQS); + host = irq_domain_add_linear(np, NUM_IRQS, &pci_pic_host_ops, priv); if (!host) { ret = -ENOMEM; goto out_unmap_regs; } - host->host_data = priv; - priv->host = host; - host->host_data = priv; irq_set_handler_data(irq, priv); irq_set_chained_handler(irq, pq2ads_pci_irq_demux); diff --git a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c index 12cb9bb..3bbbf74 100644 --- a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c +++ b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c @@ -51,7 +51,7 @@ static struct socrates_fpga_irq_info fpga_irqs[SOCRATES_FPGA_NUM_IRQS] = { static DEFINE_RAW_SPINLOCK(socrates_fpga_pic_lock); static void __iomem *socrates_fpga_pic_iobase; -static struct irq_host *socrates_fpga_pic_irq_host; +static struct irq_domain *socrates_fpga_pic_irq_host; static unsigned int socrates_fpga_irqs[3]; static inline uint32_t socrates_fpga_pic_read(int reg) @@ -227,7 +227,7 @@ static struct irq_chip socrates_fpga_pic_chip = { .irq_set_type = socrates_fpga_pic_set_type, }; -static int socrates_fpga_pic_host_map(struct irq_host *h, unsigned int virq, +static int socrates_fpga_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { /* All interrupts are LEVEL sensitive */ @@ -238,7 +238,7 @@ static int socrates_fpga_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int socrates_fpga_pic_host_xlate(struct irq_host *h, +static int socrates_fpga_pic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) { @@ -269,7 +269,7 @@ static int socrates_fpga_pic_host_xlate(struct irq_host *h, return 0; } -static struct irq_host_ops socrates_fpga_pic_host_ops = { +static const struct irq_domain_ops socrates_fpga_pic_host_ops = { .map = socrates_fpga_pic_host_map, .xlate = socrates_fpga_pic_host_xlate, }; @@ -279,10 +279,9 @@ void socrates_fpga_pic_init(struct device_node *pic) unsigned long flags; int i; - /* Setup an irq_host structure */ - socrates_fpga_pic_irq_host = irq_alloc_host(pic, IRQ_HOST_MAP_LINEAR, - SOCRATES_FPGA_NUM_IRQS, &socrates_fpga_pic_host_ops, - SOCRATES_FPGA_NUM_IRQS); + /* Setup an irq_domain structure */ + socrates_fpga_pic_irq_host = irq_domain_add_linear(pic, + SOCRATES_FPGA_NUM_IRQS, &socrates_fpga_pic_host_ops, NULL); if (socrates_fpga_pic_irq_host == NULL) { pr_err("FPGA PIC: Unable to allocate host\n"); return; diff --git a/arch/powerpc/platforms/86xx/gef_pic.c b/arch/powerpc/platforms/86xx/gef_pic.c index 94594e5..af3fd69 100644 --- a/arch/powerpc/platforms/86xx/gef_pic.c +++ b/arch/powerpc/platforms/86xx/gef_pic.c @@ -50,7 +50,7 @@ static DEFINE_RAW_SPINLOCK(gef_pic_lock); static void __iomem *gef_pic_irq_reg_base; -static struct irq_host *gef_pic_irq_host; +static struct irq_domain *gef_pic_irq_host; static int gef_pic_cascade_irq; /* @@ -153,7 +153,7 @@ static struct irq_chip gef_pic_chip = { /* When an interrupt is being configured, this call allows some flexibilty * in deciding which irq_chip structure is used */ -static int gef_pic_host_map(struct irq_host *h, unsigned int virq, +static int gef_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { /* All interrupts are LEVEL sensitive */ @@ -163,7 +163,7 @@ static int gef_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int gef_pic_host_xlate(struct irq_host *h, struct device_node *ct, +static int gef_pic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) { @@ -177,7 +177,7 @@ static int gef_pic_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops gef_pic_host_ops = { +static const struct irq_domain_ops gef_pic_host_ops = { .map = gef_pic_host_map, .xlate = gef_pic_host_xlate, }; @@ -211,10 +211,9 @@ void __init gef_pic_init(struct device_node *np) return; } - /* Setup an irq_host structure */ - gef_pic_irq_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - GEF_PIC_NUM_IRQS, - &gef_pic_host_ops, NO_IRQ); + /* Setup an irq_domain structure */ + gef_pic_irq_host = irq_domain_add_linear(np, GEF_PIC_NUM_IRQS, + &gef_pic_host_ops, NULL); if (gef_pic_irq_host == NULL) return; diff --git a/arch/powerpc/platforms/8xx/Kconfig b/arch/powerpc/platforms/8xx/Kconfig index ee56a9e..1fb0b3c 100644 --- a/arch/powerpc/platforms/8xx/Kconfig +++ b/arch/powerpc/platforms/8xx/Kconfig @@ -26,6 +26,7 @@ config MPC86XADS config MPC885ADS bool "MPC885ADS" select CPM1 + select OF_DYNAMIC help Freescale Semiconductor MPC885 Application Development System (ADS). Also known as DUET. diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c index 40a6e34..db360fc 100644 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ b/arch/powerpc/platforms/cell/axon_msi.c @@ -67,7 +67,7 @@ struct axon_msic { - struct irq_host *irq_host; + struct irq_domain *irq_domain; __le32 *fifo_virt; dma_addr_t fifo_phys; dcr_host_t dcr_host; @@ -152,7 +152,7 @@ static void axon_msi_cascade(unsigned int irq, struct irq_desc *desc) static struct axon_msic *find_msi_translator(struct pci_dev *dev) { - struct irq_host *irq_host; + struct irq_domain *irq_domain; struct device_node *dn, *tmp; const phandle *ph; struct axon_msic *msic = NULL; @@ -184,14 +184,14 @@ static struct axon_msic *find_msi_translator(struct pci_dev *dev) goto out_error; } - irq_host = irq_find_host(dn); - if (!irq_host) { - dev_dbg(&dev->dev, "axon_msi: no irq_host found for node %s\n", + irq_domain = irq_find_host(dn); + if (!irq_domain) { + dev_dbg(&dev->dev, "axon_msi: no irq_domain found for node %s\n", dn->full_name); goto out_error; } - msic = irq_host->host_data; + msic = irq_domain->host_data; out_error: of_node_put(dn); @@ -280,7 +280,7 @@ static int axon_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) BUILD_BUG_ON(NR_IRQS > 65536); list_for_each_entry(entry, &dev->msi_list, list) { - virq = irq_create_direct_mapping(msic->irq_host); + virq = irq_create_direct_mapping(msic->irq_domain); if (virq == NO_IRQ) { dev_warn(&dev->dev, "axon_msi: virq allocation failed!\n"); @@ -318,7 +318,7 @@ static struct irq_chip msic_irq_chip = { .name = "AXON-MSI", }; -static int msic_host_map(struct irq_host *h, unsigned int virq, +static int msic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_chip_data(virq, h->host_data); @@ -327,7 +327,7 @@ static int msic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops msic_host_ops = { +static const struct irq_domain_ops msic_host_ops = { .map = msic_host_map, }; @@ -337,7 +337,7 @@ static void axon_msi_shutdown(struct platform_device *device) u32 tmp; pr_devel("axon_msi: disabling %s\n", - msic->irq_host->of_node->full_name); + msic->irq_domain->of_node->full_name); tmp = dcr_read(msic->dcr_host, MSIC_CTRL_REG); tmp &= ~MSIC_CTRL_ENABLE & ~MSIC_CTRL_IRQ_ENABLE; msic_dcr_write(msic, MSIC_CTRL_REG, tmp); @@ -392,16 +392,13 @@ static int axon_msi_probe(struct platform_device *device) } memset(msic->fifo_virt, 0xff, MSIC_FIFO_SIZE_BYTES); - msic->irq_host = irq_alloc_host(dn, IRQ_HOST_MAP_NOMAP, - NR_IRQS, &msic_host_ops, 0); - if (!msic->irq_host) { - printk(KERN_ERR "axon_msi: couldn't allocate irq_host for %s\n", + msic->irq_domain = irq_domain_add_nomap(dn, &msic_host_ops, msic); + if (!msic->irq_domain) { + printk(KERN_ERR "axon_msi: couldn't allocate irq_domain for %s\n", dn->full_name); goto out_free_fifo; } - msic->irq_host->host_data = msic; - irq_set_handler_data(virq, msic); irq_set_chained_handler(virq, axon_msi_cascade); pr_devel("axon_msi: irq 0x%x setup for axon_msi\n", virq); diff --git a/arch/powerpc/platforms/cell/beat_interrupt.c b/arch/powerpc/platforms/cell/beat_interrupt.c index 55015e1..e5c3a2c 100644 --- a/arch/powerpc/platforms/cell/beat_interrupt.c +++ b/arch/powerpc/platforms/cell/beat_interrupt.c @@ -34,7 +34,7 @@ static DEFINE_RAW_SPINLOCK(beatic_irq_mask_lock); static uint64_t beatic_irq_mask_enable[(MAX_IRQS+255)/64]; static uint64_t beatic_irq_mask_ack[(MAX_IRQS+255)/64]; -static struct irq_host *beatic_host; +static struct irq_domain *beatic_host; /* * In this implementation, "virq" == "IRQ plug number", @@ -122,7 +122,7 @@ static struct irq_chip beatic_pic = { * * Note that the number (virq) is already assigned at upper layer. */ -static void beatic_pic_host_unmap(struct irq_host *h, unsigned int virq) +static void beatic_pic_host_unmap(struct irq_domain *h, unsigned int virq) { beat_destruct_irq_plug(virq); } @@ -133,7 +133,7 @@ static void beatic_pic_host_unmap(struct irq_host *h, unsigned int virq) * * Note that the number (virq) is already assigned at upper layer. */ -static int beatic_pic_host_map(struct irq_host *h, unsigned int virq, +static int beatic_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { int64_t err; @@ -154,7 +154,7 @@ static int beatic_pic_host_map(struct irq_host *h, unsigned int virq, * Called from irq_create_of_mapping() only. * Note: We have only 1 entry to translate. */ -static int beatic_pic_host_xlate(struct irq_host *h, struct device_node *ct, +static int beatic_pic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -166,13 +166,13 @@ static int beatic_pic_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static int beatic_pic_host_match(struct irq_host *h, struct device_node *np) +static int beatic_pic_host_match(struct irq_domain *h, struct device_node *np) { /* Match all */ return 1; } -static struct irq_host_ops beatic_pic_host_ops = { +static const struct irq_domain_ops beatic_pic_host_ops = { .map = beatic_pic_host_map, .unmap = beatic_pic_host_unmap, .xlate = beatic_pic_host_xlate, @@ -239,9 +239,7 @@ void __init beatic_init_IRQ(void) ppc_md.get_irq = beatic_get_irq; /* Allocate an irq host */ - beatic_host = irq_alloc_host(NULL, IRQ_HOST_MAP_NOMAP, 0, - &beatic_pic_host_ops, - 0); + beatic_host = irq_domain_add_nomap(NULL, &beatic_pic_host_ops, NULL); BUG_ON(beatic_host == NULL); irq_set_default_host(beatic_host); } diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 96a433d..2d42f3b 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -56,7 +56,7 @@ struct iic { static DEFINE_PER_CPU(struct iic, cpu_iic); #define IIC_NODE_COUNT 2 -static struct irq_host *iic_host; +static struct irq_domain *iic_host; /* Convert between "pending" bits and hw irq number */ static irq_hw_number_t iic_pending_to_hwnum(struct cbe_iic_pending_bits bits) @@ -186,7 +186,7 @@ void iic_message_pass(int cpu, int msg) out_be64(&per_cpu(cpu_iic, cpu).regs->generate, (0xf - msg) << 4); } -struct irq_host *iic_get_irq_host(int node) +struct irq_domain *iic_get_irq_host(int node) { return iic_host; } @@ -222,13 +222,13 @@ void iic_request_IPIs(void) #endif /* CONFIG_SMP */ -static int iic_host_match(struct irq_host *h, struct device_node *node) +static int iic_host_match(struct irq_domain *h, struct device_node *node) { return of_device_is_compatible(node, "IBM,CBEA-Internal-Interrupt-Controller"); } -static int iic_host_map(struct irq_host *h, unsigned int virq, +static int iic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { switch (hw & IIC_IRQ_TYPE_MASK) { @@ -245,7 +245,7 @@ static int iic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int iic_host_xlate(struct irq_host *h, struct device_node *ct, +static int iic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -285,7 +285,7 @@ static int iic_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops iic_host_ops = { +static const struct irq_domain_ops iic_host_ops = { .match = iic_host_match, .map = iic_host_map, .xlate = iic_host_xlate, @@ -378,8 +378,8 @@ static int __init setup_iic(void) void __init iic_init_IRQ(void) { /* Setup an irq host data structure */ - iic_host = irq_alloc_host(NULL, IRQ_HOST_MAP_LINEAR, IIC_SOURCE_COUNT, - &iic_host_ops, IIC_IRQ_INVALID); + iic_host = irq_domain_add_linear(NULL, IIC_SOURCE_COUNT, &iic_host_ops, + NULL); BUG_ON(iic_host == NULL); irq_set_default_host(iic_host); diff --git a/arch/powerpc/platforms/cell/spider-pic.c b/arch/powerpc/platforms/cell/spider-pic.c index 442c28c..d8b7cc8 100644 --- a/arch/powerpc/platforms/cell/spider-pic.c +++ b/arch/powerpc/platforms/cell/spider-pic.c @@ -62,7 +62,7 @@ enum { #define SPIDER_IRQ_INVALID 63 struct spider_pic { - struct irq_host *host; + struct irq_domain *host; void __iomem *regs; unsigned int node_id; }; @@ -168,7 +168,7 @@ static struct irq_chip spider_pic = { .irq_set_type = spider_set_irq_type, }; -static int spider_host_map(struct irq_host *h, unsigned int virq, +static int spider_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_chip_data(virq, h->host_data); @@ -180,7 +180,7 @@ static int spider_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int spider_host_xlate(struct irq_host *h, struct device_node *ct, +static int spider_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -194,7 +194,7 @@ static int spider_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops spider_host_ops = { +static const struct irq_domain_ops spider_host_ops = { .map = spider_host_map, .xlate = spider_host_xlate, }; @@ -299,12 +299,10 @@ static void __init spider_init_one(struct device_node *of_node, int chip, panic("spider_pic: can't map registers !"); /* Allocate a host */ - pic->host = irq_alloc_host(of_node, IRQ_HOST_MAP_LINEAR, - SPIDER_SRC_COUNT, &spider_host_ops, - SPIDER_IRQ_INVALID); + pic->host = irq_domain_add_linear(of_node, SPIDER_SRC_COUNT, + &spider_host_ops, pic); if (pic->host == NULL) panic("spider_pic: can't allocate irq host !"); - pic->host->host_data = pic; /* Go through all sources and disable them */ for (i = 0; i < SPIDER_SRC_COUNT; i++) { diff --git a/arch/powerpc/platforms/embedded6xx/flipper-pic.c b/arch/powerpc/platforms/embedded6xx/flipper-pic.c index f61a2dd..53d6eee0 100644 --- a/arch/powerpc/platforms/embedded6xx/flipper-pic.c +++ b/arch/powerpc/platforms/embedded6xx/flipper-pic.c @@ -96,9 +96,9 @@ static struct irq_chip flipper_pic = { * */ -static struct irq_host *flipper_irq_host; +static struct irq_domain *flipper_irq_host; -static int flipper_pic_map(struct irq_host *h, unsigned int virq, +static int flipper_pic_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { irq_set_chip_data(virq, h->host_data); @@ -107,13 +107,13 @@ static int flipper_pic_map(struct irq_host *h, unsigned int virq, return 0; } -static int flipper_pic_match(struct irq_host *h, struct device_node *np) +static int flipper_pic_match(struct irq_domain *h, struct device_node *np) { return 1; } -static struct irq_host_ops flipper_irq_host_ops = { +static const struct irq_domain_ops flipper_irq_domain_ops = { .map = flipper_pic_map, .match = flipper_pic_match, }; @@ -130,10 +130,10 @@ static void __flipper_quiesce(void __iomem *io_base) out_be32(io_base + FLIPPER_ICR, 0xffffffff); } -struct irq_host * __init flipper_pic_init(struct device_node *np) +struct irq_domain * __init flipper_pic_init(struct device_node *np) { struct device_node *pi; - struct irq_host *irq_host = NULL; + struct irq_domain *irq_domain = NULL; struct resource res; void __iomem *io_base; int retval; @@ -159,17 +159,15 @@ struct irq_host * __init flipper_pic_init(struct device_node *np) __flipper_quiesce(io_base); - irq_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, FLIPPER_NR_IRQS, - &flipper_irq_host_ops, -1); - if (!irq_host) { - pr_err("failed to allocate irq_host\n"); + irq_domain = irq_domain_add_linear(np, FLIPPER_NR_IRQS, + &flipper_irq_domain_ops, io_base); + if (!irq_domain) { + pr_err("failed to allocate irq_domain\n"); return NULL; } - irq_host->host_data = io_base; - out: - return irq_host; + return irq_domain; } unsigned int flipper_pic_get_irq(void) diff --git a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c index e491917..3006b51 100644 --- a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c +++ b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c @@ -89,9 +89,9 @@ static struct irq_chip hlwd_pic = { * */ -static struct irq_host *hlwd_irq_host; +static struct irq_domain *hlwd_irq_host; -static int hlwd_pic_map(struct irq_host *h, unsigned int virq, +static int hlwd_pic_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { irq_set_chip_data(virq, h->host_data); @@ -100,11 +100,11 @@ static int hlwd_pic_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops hlwd_irq_host_ops = { +static const struct irq_domain_ops hlwd_irq_domain_ops = { .map = hlwd_pic_map, }; -static unsigned int __hlwd_pic_get_irq(struct irq_host *h) +static unsigned int __hlwd_pic_get_irq(struct irq_domain *h) { void __iomem *io_base = h->host_data; int irq; @@ -123,14 +123,14 @@ static void hlwd_pic_irq_cascade(unsigned int cascade_virq, struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - struct irq_host *irq_host = irq_get_handler_data(cascade_virq); + struct irq_domain *irq_domain = irq_get_handler_data(cascade_virq); unsigned int virq; raw_spin_lock(&desc->lock); chip->irq_mask(&desc->irq_data); /* IRQ_LEVEL */ raw_spin_unlock(&desc->lock); - virq = __hlwd_pic_get_irq(irq_host); + virq = __hlwd_pic_get_irq(irq_domain); if (virq != NO_IRQ) generic_handle_irq(virq); else @@ -155,9 +155,9 @@ static void __hlwd_quiesce(void __iomem *io_base) out_be32(io_base + HW_BROADWAY_ICR, 0xffffffff); } -struct irq_host *hlwd_pic_init(struct device_node *np) +struct irq_domain *hlwd_pic_init(struct device_node *np) { - struct irq_host *irq_host; + struct irq_domain *irq_domain; struct resource res; void __iomem *io_base; int retval; @@ -177,15 +177,14 @@ struct irq_host *hlwd_pic_init(struct device_node *np) __hlwd_quiesce(io_base); - irq_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, HLWD_NR_IRQS, - &hlwd_irq_host_ops, -1); - if (!irq_host) { - pr_err("failed to allocate irq_host\n"); + irq_domain = irq_domain_add_linear(np, HLWD_NR_IRQS, + &hlwd_irq_domain_ops, io_base); + if (!irq_domain) { + pr_err("failed to allocate irq_domain\n"); return NULL; } - irq_host->host_data = io_base; - return irq_host; + return irq_domain; } unsigned int hlwd_pic_get_irq(void) @@ -200,7 +199,7 @@ unsigned int hlwd_pic_get_irq(void) void hlwd_pic_probe(void) { - struct irq_host *host; + struct irq_domain *host; struct device_node *np; const u32 *interrupts; int cascade_virq; diff --git a/arch/powerpc/platforms/iseries/Kconfig b/arch/powerpc/platforms/iseries/Kconfig index b57cda3..63835e0 100644 --- a/arch/powerpc/platforms/iseries/Kconfig +++ b/arch/powerpc/platforms/iseries/Kconfig @@ -1,6 +1,7 @@ config PPC_ISERIES bool "IBM Legacy iSeries" depends on PPC64 && PPC_BOOK3S + select OF_DYNAMIC select PPC_SMP_MUXED_IPI select PPC_INDIRECT_PIO select PPC_INDIRECT_MMIO diff --git a/arch/powerpc/platforms/iseries/irq.c b/arch/powerpc/platforms/iseries/irq.c index b210345..05ce516 100644 --- a/arch/powerpc/platforms/iseries/irq.c +++ b/arch/powerpc/platforms/iseries/irq.c @@ -342,7 +342,7 @@ unsigned int iSeries_get_irq(void) #ifdef CONFIG_PCI -static int iseries_irq_host_map(struct irq_host *h, unsigned int virq, +static int iseries_irq_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_chip_and_handler(virq, &iseries_pic, handle_fasteoi_irq); @@ -350,13 +350,13 @@ static int iseries_irq_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int iseries_irq_host_match(struct irq_host *h, struct device_node *np) +static int iseries_irq_host_match(struct irq_domain *h, struct device_node *np) { /* Match all */ return 1; } -static struct irq_host_ops iseries_irq_host_ops = { +static const struct irq_domain_ops iseries_irq_domain_ops = { .map = iseries_irq_host_map, .match = iseries_irq_host_match, }; @@ -368,7 +368,7 @@ static struct irq_host_ops iseries_irq_host_ops = { void __init iSeries_init_IRQ(void) { /* Register PCI event handler and open an event path */ - struct irq_host *host; + struct irq_domain *host; int ret; /* @@ -380,8 +380,7 @@ void __init iSeries_init_IRQ(void) /* Create irq host. No need for a revmap since HV will give us * back our virtual irq number */ - host = irq_alloc_host(NULL, IRQ_HOST_MAP_NOMAP, 0, - &iseries_irq_host_ops, 0); + host = irq_domain_add_nomap(NULL, &iseries_irq_domain_ops, NULL); BUG_ON(host == NULL); irq_set_default_host(host); diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c index 8fc6258..a5fbf4c 100644 --- a/arch/powerpc/platforms/iseries/setup.c +++ b/arch/powerpc/platforms/iseries/setup.c @@ -584,9 +584,7 @@ static void iseries_shared_idle(void) if (hvlpevent_is_pending()) process_iSeries_events(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } @@ -615,9 +613,7 @@ static void iseries_dedicated_idle(void) ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index 7761aab..92afc38 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -61,7 +61,7 @@ static DEFINE_RAW_SPINLOCK(pmac_pic_lock); static unsigned long ppc_lost_interrupts[NR_MASK_WORDS]; static unsigned long ppc_cached_irq_mask[NR_MASK_WORDS]; static int pmac_irq_cascade = -1; -static struct irq_host *pmac_pic_host; +static struct irq_domain *pmac_pic_host; static void __pmac_retrigger(unsigned int irq_nr) { @@ -268,13 +268,13 @@ static struct irqaction gatwick_cascade_action = { .name = "cascade", }; -static int pmac_pic_host_match(struct irq_host *h, struct device_node *node) +static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node) { /* We match all, we don't always have a node anyway */ return 1; } -static int pmac_pic_host_map(struct irq_host *h, unsigned int virq, +static int pmac_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { if (hw >= max_irqs) @@ -288,21 +288,10 @@ static int pmac_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int pmac_pic_host_xlate(struct irq_host *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, - unsigned int *out_flags) - -{ - *out_flags = IRQ_TYPE_NONE; - *out_hwirq = *intspec; - return 0; -} - -static struct irq_host_ops pmac_pic_host_ops = { +static const struct irq_domain_ops pmac_pic_host_ops = { .match = pmac_pic_host_match, .map = pmac_pic_host_map, - .xlate = pmac_pic_host_xlate, + .xlate = irq_domain_xlate_onecell, }; static void __init pmac_pic_probe_oldstyle(void) @@ -352,9 +341,8 @@ static void __init pmac_pic_probe_oldstyle(void) /* * Allocate an irq host */ - pmac_pic_host = irq_alloc_host(master, IRQ_HOST_MAP_LINEAR, max_irqs, - &pmac_pic_host_ops, - max_irqs); + pmac_pic_host = irq_domain_add_linear(master, max_irqs, + &pmac_pic_host_ops, NULL); BUG_ON(pmac_pic_host == NULL); irq_set_default_host(pmac_pic_host); diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index 44d7692..a81e5a8 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -125,7 +125,7 @@ static volatile u32 __iomem *psurge_start; static int psurge_type = PSURGE_NONE; /* irq for secondary cpus to report */ -static struct irq_host *psurge_host; +static struct irq_domain *psurge_host; int psurge_secondary_virq; /* @@ -176,7 +176,7 @@ static void smp_psurge_cause_ipi(int cpu, unsigned long data) psurge_set_ipi(cpu); } -static int psurge_host_map(struct irq_host *h, unsigned int virq, +static int psurge_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { irq_set_chip_and_handler(virq, &dummy_irq_chip, handle_percpu_irq); @@ -184,7 +184,7 @@ static int psurge_host_map(struct irq_host *h, unsigned int virq, return 0; } -struct irq_host_ops psurge_host_ops = { +static const struct irq_domain_ops psurge_host_ops = { .map = psurge_host_map, }; @@ -192,8 +192,7 @@ static int psurge_secondary_ipi_init(void) { int rc = -ENOMEM; - psurge_host = irq_alloc_host(NULL, IRQ_HOST_MAP_NOMAP, 0, - &psurge_host_ops, 0); + psurge_host = irq_domain_add_nomap(NULL, &psurge_host_ops, NULL); if (psurge_host) psurge_secondary_virq = irq_create_direct_mapping(psurge_host); diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index 617efa1..2a4ff86 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -667,7 +667,7 @@ static void __maybe_unused _dump_mask(struct ps3_private *pd, static void dump_bmp(struct ps3_private* pd) {}; #endif /* defined(DEBUG) */ -static int ps3_host_map(struct irq_host *h, unsigned int virq, +static int ps3_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { DBG("%s:%d: hwirq %lu, virq %u\n", __func__, __LINE__, hwirq, @@ -678,13 +678,13 @@ static int ps3_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int ps3_host_match(struct irq_host *h, struct device_node *np) +static int ps3_host_match(struct irq_domain *h, struct device_node *np) { /* Match all */ return 1; } -static struct irq_host_ops ps3_host_ops = { +static const struct irq_domain_ops ps3_host_ops = { .map = ps3_host_map, .match = ps3_host_match, }; @@ -751,10 +751,9 @@ void __init ps3_init_IRQ(void) { int result; unsigned cpu; - struct irq_host *host; + struct irq_domain *host; - host = irq_alloc_host(NULL, IRQ_HOST_MAP_NOMAP, 0, &ps3_host_ops, - PS3_INVALID_OUTLET); + host = irq_domain_add_nomap(NULL, &ps3_host_ops, NULL); irq_set_default_host(host); irq_set_virq_count(PS3_PLUG_MAX + 1); diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index 31f22c1..f255625 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -3,6 +3,7 @@ config PPC_PSERIES bool "IBM pSeries & new (POWER5-based) iSeries" select HAVE_PCSPKR_PLATFORM select MPIC + select OF_DYNAMIC select PCI_MSI select PPC_XICS select PPC_ICP_NATIVE diff --git a/arch/powerpc/platforms/wsp/Kconfig b/arch/powerpc/platforms/wsp/Kconfig index 57d22a2..79d2225 100644 --- a/arch/powerpc/platforms/wsp/Kconfig +++ b/arch/powerpc/platforms/wsp/Kconfig @@ -25,6 +25,7 @@ config PPC_CHROMA bool "PowerEN PCIe Chroma Card" select EPAPR_BOOT select PPC_WSP + select OF_DYNAMIC default y endmenu diff --git a/arch/powerpc/platforms/wsp/opb_pic.c b/arch/powerpc/platforms/wsp/opb_pic.c index 19f353d..cb565bf 100644 --- a/arch/powerpc/platforms/wsp/opb_pic.c +++ b/arch/powerpc/platforms/wsp/opb_pic.c @@ -30,7 +30,7 @@ static int opb_index = 0; struct opb_pic { - struct irq_host *host; + struct irq_domain *host; void *regs; int index; spinlock_t lock; @@ -179,7 +179,7 @@ static struct irq_chip opb_irq_chip = { .irq_set_type = opb_set_irq_type }; -static int opb_host_map(struct irq_host *host, unsigned int virq, +static int opb_host_map(struct irq_domain *host, unsigned int virq, irq_hw_number_t hwirq) { struct opb_pic *opb; @@ -196,20 +196,9 @@ static int opb_host_map(struct irq_host *host, unsigned int virq, return 0; } -static int opb_host_xlate(struct irq_host *host, struct device_node *dn, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type) -{ - /* Interrupt size must == 2 */ - BUG_ON(intsize != 2); - *out_hwirq = intspec[0]; - *out_type = intspec[1]; - return 0; -} - -static struct irq_host_ops opb_host_ops = { +static const struct irq_domain_ops opb_host_ops = { .map = opb_host_map, - .xlate = opb_host_xlate, + .xlate = irq_domain_xlate_twocell, }; irqreturn_t opb_irq_handler(int irq, void *private) @@ -263,13 +252,11 @@ struct opb_pic *opb_pic_init_one(struct device_node *dn) goto free_opb; } - /* Allocate an irq host so that Linux knows that despite only + /* Allocate an irq domain so that Linux knows that despite only * having one interrupt to issue, we're the controller for multiple * hardware IRQs, so later we can lookup their virtual IRQs. */ - opb->host = irq_alloc_host(dn, IRQ_HOST_MAP_LINEAR, - OPB_NR_IRQS, &opb_host_ops, -1); - + opb->host = irq_domain_add_linear(dn, OPB_NR_IRQS, &opb_host_ops, opb); if (!opb->host) { printk(KERN_ERR "opb: Failed to allocate IRQ host!\n"); goto free_regs; @@ -277,7 +264,6 @@ struct opb_pic *opb_pic_init_one(struct device_node *dn) opb->index = opb_index++; spin_lock_init(&opb->lock); - opb->host->host_data = opb; /* Disable all interrupts by default */ opb_out(opb, OPB_MLSASIER, 0); diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c index 5d7d59a..d4fa03f 100644 --- a/arch/powerpc/sysdev/cpm1.c +++ b/arch/powerpc/sysdev/cpm1.c @@ -54,7 +54,7 @@ cpm8xx_t __iomem *cpmp; /* Pointer to comm processor space */ immap_t __iomem *mpc8xx_immr; static cpic8xx_t __iomem *cpic_reg; -static struct irq_host *cpm_pic_host; +static struct irq_domain *cpm_pic_host; static void cpm_mask_irq(struct irq_data *d) { @@ -98,7 +98,7 @@ int cpm_get_irq(void) return irq_linear_revmap(cpm_pic_host, cpm_vec); } -static int cpm_pic_host_map(struct irq_host *h, unsigned int virq, +static int cpm_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { pr_debug("cpm_pic_host_map(%d, 0x%lx)\n", virq, hw); @@ -123,7 +123,7 @@ static struct irqaction cpm_error_irqaction = { .name = "error", }; -static struct irq_host_ops cpm_pic_host_ops = { +static const struct irq_domain_ops cpm_pic_host_ops = { .map = cpm_pic_host_map, }; @@ -164,8 +164,7 @@ unsigned int cpm_pic_init(void) out_be32(&cpic_reg->cpic_cimr, 0); - cpm_pic_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - 64, &cpm_pic_host_ops, 64); + cpm_pic_host = irq_domain_add_linear(np, 64, &cpm_pic_host_ops, NULL); if (cpm_pic_host == NULL) { printk(KERN_ERR "CPM2 PIC: failed to allocate irq host!\n"); sirq = NO_IRQ; diff --git a/arch/powerpc/sysdev/cpm2_pic.c b/arch/powerpc/sysdev/cpm2_pic.c index bcab50e..d3be961 100644 --- a/arch/powerpc/sysdev/cpm2_pic.c +++ b/arch/powerpc/sysdev/cpm2_pic.c @@ -50,7 +50,7 @@ static intctl_cpm2_t __iomem *cpm2_intctl; -static struct irq_host *cpm2_pic_host; +static struct irq_domain *cpm2_pic_host; #define NR_MASK_WORDS ((NR_IRQS + 31) / 32) static unsigned long ppc_cached_irq_mask[NR_MASK_WORDS]; @@ -214,7 +214,7 @@ unsigned int cpm2_get_irq(void) return irq_linear_revmap(cpm2_pic_host, irq); } -static int cpm2_pic_host_map(struct irq_host *h, unsigned int virq, +static int cpm2_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { pr_debug("cpm2_pic_host_map(%d, 0x%lx)\n", virq, hw); @@ -224,21 +224,9 @@ static int cpm2_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int cpm2_pic_host_xlate(struct irq_host *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_flags) -{ - *out_hwirq = intspec[0]; - if (intsize > 1) - *out_flags = intspec[1]; - else - *out_flags = IRQ_TYPE_NONE; - return 0; -} - -static struct irq_host_ops cpm2_pic_host_ops = { +static const struct irq_domain_ops cpm2_pic_host_ops = { .map = cpm2_pic_host_map, - .xlate = cpm2_pic_host_xlate, + .xlate = irq_domain_xlate_onetwocell, }; void cpm2_pic_init(struct device_node *node) @@ -275,8 +263,7 @@ void cpm2_pic_init(struct device_node *node) out_be32(&cpm2_intctl->ic_scprrl, 0x05309770); /* create a legacy host */ - cpm2_pic_host = irq_alloc_host(node, IRQ_HOST_MAP_LINEAR, - 64, &cpm2_pic_host_ops, 64); + cpm2_pic_host = irq_domain_add_linear(node, 64, &cpm2_pic_host_ops, NULL); if (cpm2_pic_host == NULL) { printk(KERN_ERR "CPM2 PIC: failed to allocate irq host!\n"); return; diff --git a/arch/powerpc/sysdev/ehv_pic.c b/arch/powerpc/sysdev/ehv_pic.c index b6731e4..6e0e100 100644 --- a/arch/powerpc/sysdev/ehv_pic.c +++ b/arch/powerpc/sysdev/ehv_pic.c @@ -182,13 +182,13 @@ unsigned int ehv_pic_get_irq(void) return irq_linear_revmap(global_ehv_pic->irqhost, irq); } -static int ehv_pic_host_match(struct irq_host *h, struct device_node *node) +static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node) { /* Exact match, unless ehv_pic node is NULL */ return h->of_node == NULL || h->of_node == node; } -static int ehv_pic_host_map(struct irq_host *h, unsigned int virq, +static int ehv_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct ehv_pic *ehv_pic = h->host_data; @@ -217,7 +217,7 @@ static int ehv_pic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int ehv_pic_host_xlate(struct irq_host *h, struct device_node *ct, +static int ehv_pic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -248,7 +248,7 @@ static int ehv_pic_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops ehv_pic_host_ops = { +static const struct irq_domain_ops ehv_pic_host_ops = { .match = ehv_pic_host_match, .map = ehv_pic_host_map, .xlate = ehv_pic_host_xlate, @@ -275,9 +275,8 @@ void __init ehv_pic_init(void) return; } - ehv_pic->irqhost = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - NR_EHV_PIC_INTS, &ehv_pic_host_ops, 0); - + ehv_pic->irqhost = irq_domain_add_linear(np, NR_EHV_PIC_INTS, + &ehv_pic_host_ops, ehv_pic); if (!ehv_pic->irqhost) { of_node_put(np); kfree(ehv_pic); @@ -293,7 +292,6 @@ void __init ehv_pic_init(void) of_node_put(np2); } - ehv_pic->irqhost->host_data = ehv_pic; ehv_pic->hc_irq = ehv_pic_irq_chip; ehv_pic->hc_irq.irq_set_affinity = ehv_pic_set_affinity; ehv_pic->coreint_flag = coreint_flag; diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index ecb5c19..0c01deb 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -60,7 +60,7 @@ static struct irq_chip fsl_msi_chip = { .name = "FSL-MSI", }; -static int fsl_msi_host_map(struct irq_host *h, unsigned int virq, +static int fsl_msi_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct fsl_msi *msi_data = h->host_data; @@ -74,7 +74,7 @@ static int fsl_msi_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops fsl_msi_host_ops = { +static const struct irq_domain_ops fsl_msi_host_ops = { .map = fsl_msi_host_map, }; @@ -387,8 +387,8 @@ static int __devinit fsl_of_msi_probe(struct platform_device *dev) } platform_set_drvdata(dev, msi); - msi->irqhost = irq_alloc_host(dev->dev.of_node, IRQ_HOST_MAP_LINEAR, - NR_MSI_IRQS, &fsl_msi_host_ops, 0); + msi->irqhost = irq_domain_add_linear(dev->dev.of_node, + NR_MSI_IRQS, &fsl_msi_host_ops, msi); if (msi->irqhost == NULL) { dev_err(&dev->dev, "No memory for MSI irqhost\n"); @@ -420,8 +420,6 @@ static int __devinit fsl_of_msi_probe(struct platform_device *dev) msi->feature = features->fsl_pic_ip; - msi->irqhost->host_data = msi; - /* * Remember the phandle, so that we can match with any PCI nodes * that have an "fsl,msi" property. diff --git a/arch/powerpc/sysdev/fsl_msi.h b/arch/powerpc/sysdev/fsl_msi.h index f6c646a..8225f86 100644 --- a/arch/powerpc/sysdev/fsl_msi.h +++ b/arch/powerpc/sysdev/fsl_msi.h @@ -26,7 +26,7 @@ #define FSL_PIC_IP_VMPIC 0x00000003 struct fsl_msi { - struct irq_host *irqhost; + struct irq_domain *irqhost; unsigned long cascade_irq; diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c index d18bb27..997df6a 100644 --- a/arch/powerpc/sysdev/i8259.c +++ b/arch/powerpc/sysdev/i8259.c @@ -25,7 +25,7 @@ static unsigned char cached_8259[2] = { 0xff, 0xff }; static DEFINE_RAW_SPINLOCK(i8259_lock); -static struct irq_host *i8259_host; +static struct irq_domain *i8259_host; /* * Acknowledge the IRQ using either the PCI host bridge's interrupt @@ -163,12 +163,12 @@ static struct resource pic_edgectrl_iores = { .flags = IORESOURCE_BUSY, }; -static int i8259_host_match(struct irq_host *h, struct device_node *node) +static int i8259_host_match(struct irq_domain *h, struct device_node *node) { return h->of_node == NULL || h->of_node == node; } -static int i8259_host_map(struct irq_host *h, unsigned int virq, +static int i8259_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { pr_debug("i8259_host_map(%d, 0x%lx)\n", virq, hw); @@ -185,7 +185,7 @@ static int i8259_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int i8259_host_xlate(struct irq_host *h, struct device_node *ct, +static int i8259_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) { @@ -205,13 +205,13 @@ static int i8259_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops i8259_host_ops = { +static struct irq_domain_ops i8259_host_ops = { .match = i8259_host_match, .map = i8259_host_map, .xlate = i8259_host_xlate, }; -struct irq_host *i8259_get_host(void) +struct irq_domain *i8259_get_host(void) { return i8259_host; } @@ -263,8 +263,7 @@ void i8259_init(struct device_node *node, unsigned long intack_addr) raw_spin_unlock_irqrestore(&i8259_lock, flags); /* create a legacy host */ - i8259_host = irq_alloc_host(node, IRQ_HOST_MAP_LEGACY, - 0, &i8259_host_ops, 0); + i8259_host = irq_domain_add_legacy_isa(node, &i8259_host_ops, NULL); if (i8259_host == NULL) { printk(KERN_ERR "i8259: failed to allocate irq host !\n"); return; diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index 95da897..b50f978 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -672,13 +672,13 @@ static struct irq_chip ipic_edge_irq_chip = { .irq_set_type = ipic_set_irq_type, }; -static int ipic_host_match(struct irq_host *h, struct device_node *node) +static int ipic_host_match(struct irq_domain *h, struct device_node *node) { /* Exact match, unless ipic node is NULL */ return h->of_node == NULL || h->of_node == node; } -static int ipic_host_map(struct irq_host *h, unsigned int virq, +static int ipic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct ipic *ipic = h->host_data; @@ -692,26 +692,10 @@ static int ipic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int ipic_host_xlate(struct irq_host *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_flags) - -{ - /* interrupt sense values coming from the device tree equal either - * LEVEL_LOW (low assertion) or EDGE_FALLING (high-to-low change) - */ - *out_hwirq = intspec[0]; - if (intsize > 1) - *out_flags = intspec[1]; - else - *out_flags = IRQ_TYPE_NONE; - return 0; -} - -static struct irq_host_ops ipic_host_ops = { +static struct irq_domain_ops ipic_host_ops = { .match = ipic_host_match, .map = ipic_host_map, - .xlate = ipic_host_xlate, + .xlate = irq_domain_xlate_onetwocell, }; struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) @@ -728,9 +712,8 @@ struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) if (ipic == NULL) return NULL; - ipic->irqhost = irq_alloc_host(node, IRQ_HOST_MAP_LINEAR, - NR_IPIC_INTS, - &ipic_host_ops, 0); + ipic->irqhost = irq_domain_add_linear(node, NR_IPIC_INTS, + &ipic_host_ops, ipic); if (ipic->irqhost == NULL) { kfree(ipic); return NULL; @@ -738,8 +721,6 @@ struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) ipic->regs = ioremap(res.start, resource_size(&res)); - ipic->irqhost->host_data = ipic; - /* init hw */ ipic_write(ipic->regs, IPIC_SICNR, 0x0); diff --git a/arch/powerpc/sysdev/ipic.h b/arch/powerpc/sysdev/ipic.h index 9391c57..90031d1 100644 --- a/arch/powerpc/sysdev/ipic.h +++ b/arch/powerpc/sysdev/ipic.h @@ -43,7 +43,7 @@ struct ipic { volatile u32 __iomem *regs; /* The remapper for this IPIC */ - struct irq_host *irqhost; + struct irq_domain *irqhost; }; struct ipic_info { diff --git a/arch/powerpc/sysdev/mpc8xx_pic.c b/arch/powerpc/sysdev/mpc8xx_pic.c index 2ca0a85..d5f5416 100644 --- a/arch/powerpc/sysdev/mpc8xx_pic.c +++ b/arch/powerpc/sysdev/mpc8xx_pic.c @@ -17,7 +17,7 @@ extern int cpm_get_irq(struct pt_regs *regs); -static struct irq_host *mpc8xx_pic_host; +static struct irq_domain *mpc8xx_pic_host; #define NR_MASK_WORDS ((NR_IRQS + 31) / 32) static unsigned long ppc_cached_irq_mask[NR_MASK_WORDS]; static sysconf8xx_t __iomem *siu_reg; @@ -110,7 +110,7 @@ unsigned int mpc8xx_get_irq(void) } -static int mpc8xx_pic_host_map(struct irq_host *h, unsigned int virq, +static int mpc8xx_pic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { pr_debug("mpc8xx_pic_host_map(%d, 0x%lx)\n", virq, hw); @@ -121,7 +121,7 @@ static int mpc8xx_pic_host_map(struct irq_host *h, unsigned int virq, } -static int mpc8xx_pic_host_xlate(struct irq_host *h, struct device_node *ct, +static int mpc8xx_pic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) { @@ -142,7 +142,7 @@ static int mpc8xx_pic_host_xlate(struct irq_host *h, struct device_node *ct, } -static struct irq_host_ops mpc8xx_pic_host_ops = { +static struct irq_domain_ops mpc8xx_pic_host_ops = { .map = mpc8xx_pic_host_map, .xlate = mpc8xx_pic_host_xlate, }; @@ -171,8 +171,7 @@ int mpc8xx_pic_init(void) goto out; } - mpc8xx_pic_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - 64, &mpc8xx_pic_host_ops, 64); + mpc8xx_pic_host = irq_domain_add_linear(np, 64, &mpc8xx_pic_host_ops, NULL); if (mpc8xx_pic_host == NULL) { printk(KERN_ERR "MPC8xx PIC: failed to allocate irq host!\n"); ret = -ENOMEM; diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index 4e9ccb1..c83a512 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -965,13 +965,13 @@ static struct irq_chip mpic_irq_ht_chip = { #endif /* CONFIG_MPIC_U3_HT_IRQS */ -static int mpic_host_match(struct irq_host *h, struct device_node *node) +static int mpic_host_match(struct irq_domain *h, struct device_node *node) { /* Exact match, unless mpic node is NULL */ return h->of_node == NULL || h->of_node == node; } -static int mpic_host_map(struct irq_host *h, unsigned int virq, +static int mpic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct mpic *mpic = h->host_data; @@ -1041,7 +1041,7 @@ static int mpic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int mpic_host_xlate(struct irq_host *h, struct device_node *ct, +static int mpic_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -1121,13 +1121,13 @@ static void mpic_cascade(unsigned int irq, struct irq_desc *desc) BUG_ON(!(mpic->flags & MPIC_SECONDARY)); virq = mpic_get_one_irq(mpic); - if (virq != NO_IRQ) + if (virq) generic_handle_irq(virq); chip->irq_eoi(&desc->irq_data); } -static struct irq_host_ops mpic_host_ops = { +static struct irq_domain_ops mpic_host_ops = { .match = mpic_host_match, .map = mpic_host_map, .xlate = mpic_host_xlate, @@ -1345,10 +1345,9 @@ struct mpic * __init mpic_alloc(struct device_node *node, mpic->isu_shift = 1 + __ilog2(mpic->isu_size - 1); mpic->isu_mask = (1 << mpic->isu_shift) - 1; - mpic->irqhost = irq_alloc_host(mpic->node, IRQ_HOST_MAP_LINEAR, + mpic->irqhost = irq_domain_add_linear(mpic->node, isu_size ? isu_size : mpic->num_sources, - &mpic_host_ops, - flags & MPIC_LARGE_VECTORS ? 2048 : 256); + &mpic_host_ops, mpic); /* * FIXME: The code leaks the MPIC object and mappings here; this @@ -1357,8 +1356,6 @@ struct mpic * __init mpic_alloc(struct device_node *node, if (mpic->irqhost == NULL) return NULL; - mpic->irqhost->host_data = mpic; - /* Display version */ switch (greg_feature & MPIC_GREG_FEATURE_VERSION_MASK) { case 1: diff --git a/arch/powerpc/sysdev/mpic_msi.c b/arch/powerpc/sysdev/mpic_msi.c index 0f67cd7..0622aa9 100644 --- a/arch/powerpc/sysdev/mpic_msi.c +++ b/arch/powerpc/sysdev/mpic_msi.c @@ -32,7 +32,7 @@ void mpic_msi_reserve_hwirq(struct mpic *mpic, irq_hw_number_t hwirq) static int mpic_msi_reserve_u3_hwirqs(struct mpic *mpic) { irq_hw_number_t hwirq; - struct irq_host_ops *ops = mpic->irqhost->ops; + const struct irq_domain_ops *ops = mpic->irqhost->ops; struct device_node *np; int flags, index, i; struct of_irq oirq; diff --git a/arch/powerpc/sysdev/mv64x60_pic.c b/arch/powerpc/sysdev/mv64x60_pic.c index 14d1302..8848e99 100644 --- a/arch/powerpc/sysdev/mv64x60_pic.c +++ b/arch/powerpc/sysdev/mv64x60_pic.c @@ -70,7 +70,7 @@ static u32 mv64x60_cached_low_mask; static u32 mv64x60_cached_high_mask = MV64X60_HIGH_GPP_GROUPS; static u32 mv64x60_cached_gpp_mask; -static struct irq_host *mv64x60_irq_host; +static struct irq_domain *mv64x60_irq_host; /* * mv64x60_chip_low functions @@ -208,7 +208,7 @@ static struct irq_chip *mv64x60_chips[] = { [MV64x60_LEVEL1_GPP] = &mv64x60_chip_gpp, }; -static int mv64x60_host_map(struct irq_host *h, unsigned int virq, +static int mv64x60_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hwirq) { int level1; @@ -223,7 +223,7 @@ static int mv64x60_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops mv64x60_host_ops = { +static struct irq_domain_ops mv64x60_host_ops = { .map = mv64x60_host_map, }; @@ -250,9 +250,8 @@ void __init mv64x60_init_irq(void) paddr = of_translate_address(np, reg); mv64x60_irq_reg_base = ioremap(paddr, reg[1]); - mv64x60_irq_host = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, - MV64x60_NUM_IRQS, - &mv64x60_host_ops, MV64x60_NUM_IRQS); + mv64x60_irq_host = irq_domain_add_linear(np, MV64x60_NUM_IRQS, + &mv64x60_host_ops, NULL); spin_lock_irqsave(&mv64x60_lock, flags); out_le32(mv64x60_gpp_reg_base + MV64x60_GPP_INTR_MASK, diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.c b/arch/powerpc/sysdev/qe_lib/qe_ic.c index 73034bd..2fba6ef2 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_ic.c +++ b/arch/powerpc/sysdev/qe_lib/qe_ic.c @@ -245,13 +245,13 @@ static struct irq_chip qe_ic_irq_chip = { .irq_mask_ack = qe_ic_mask_irq, }; -static int qe_ic_host_match(struct irq_host *h, struct device_node *node) +static int qe_ic_host_match(struct irq_domain *h, struct device_node *node) { /* Exact match, unless qe_ic node is NULL */ return h->of_node == NULL || h->of_node == node; } -static int qe_ic_host_map(struct irq_host *h, unsigned int virq, +static int qe_ic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct qe_ic *qe_ic = h->host_data; @@ -272,23 +272,10 @@ static int qe_ic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int qe_ic_host_xlate(struct irq_host *h, struct device_node *ct, - const u32 * intspec, unsigned int intsize, - irq_hw_number_t * out_hwirq, - unsigned int *out_flags) -{ - *out_hwirq = intspec[0]; - if (intsize > 1) - *out_flags = intspec[1]; - else - *out_flags = IRQ_TYPE_NONE; - return 0; -} - -static struct irq_host_ops qe_ic_host_ops = { +static struct irq_domain_ops qe_ic_host_ops = { .match = qe_ic_host_match, .map = qe_ic_host_map, - .xlate = qe_ic_host_xlate, + .xlate = irq_domain_xlate_onetwocell, }; /* Return an interrupt vector or NO_IRQ if no interrupt is pending. */ @@ -339,8 +326,8 @@ void __init qe_ic_init(struct device_node *node, unsigned int flags, if (qe_ic == NULL) return; - qe_ic->irqhost = irq_alloc_host(node, IRQ_HOST_MAP_LINEAR, - NR_QE_IC_INTS, &qe_ic_host_ops, 0); + qe_ic->irqhost = irq_domain_add_linear(node, NR_QE_IC_INTS, + &qe_ic_host_ops, qe_ic); if (qe_ic->irqhost == NULL) { kfree(qe_ic); return; @@ -348,7 +335,6 @@ void __init qe_ic_init(struct device_node *node, unsigned int flags, qe_ic->regs = ioremap(res.start, resource_size(&res)); - qe_ic->irqhost->host_data = qe_ic; qe_ic->hc_irq = qe_ic_irq_chip; qe_ic->virq_high = irq_of_parse_and_map(node, 0); diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.h b/arch/powerpc/sysdev/qe_lib/qe_ic.h index c1361d0..c327872 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_ic.h +++ b/arch/powerpc/sysdev/qe_lib/qe_ic.h @@ -79,7 +79,7 @@ struct qe_ic { volatile u32 __iomem *regs; /* The remapper for this QEIC */ - struct irq_host *irqhost; + struct irq_domain *irqhost; /* The "linux" controller struct */ struct irq_chip hc_irq; diff --git a/arch/powerpc/sysdev/tsi108_pci.c b/arch/powerpc/sysdev/tsi108_pci.c index 4d18658..188012c 100644 --- a/arch/powerpc/sysdev/tsi108_pci.c +++ b/arch/powerpc/sysdev/tsi108_pci.c @@ -51,7 +51,7 @@ u32 tsi108_pci_cfg_base; static u32 tsi108_pci_cfg_phys; u32 tsi108_csr_vir_base; -static struct irq_host *pci_irq_host; +static struct irq_domain *pci_irq_host; extern u32 get_vir_csrbase(void); extern u32 tsi108_read_reg(u32 reg_offset); @@ -376,7 +376,7 @@ static struct irq_chip tsi108_pci_irq = { .irq_unmask = tsi108_pci_irq_unmask, }; -static int pci_irq_host_xlate(struct irq_host *h, struct device_node *ct, +static int pci_irq_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) { @@ -385,7 +385,7 @@ static int pci_irq_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static int pci_irq_host_map(struct irq_host *h, unsigned int virq, +static int pci_irq_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { unsigned int irq; DBG("%s(%d, 0x%lx)\n", __func__, virq, hw); @@ -397,7 +397,7 @@ static int pci_irq_host_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops pci_irq_host_ops = { +static struct irq_domain_ops pci_irq_domain_ops = { .map = pci_irq_host_map, .xlate = pci_irq_host_xlate, }; @@ -419,10 +419,9 @@ void __init tsi108_pci_int_init(struct device_node *node) { DBG("Tsi108_pci_int_init: initializing PCI interrupts\n"); - pci_irq_host = irq_alloc_host(node, IRQ_HOST_MAP_LEGACY, - 0, &pci_irq_host_ops, 0); + pci_irq_host = irq_domain_add_legacy_isa(node, &pci_irq_domain_ops, NULL); if (pci_irq_host == NULL) { - printk(KERN_ERR "pci_irq_host: failed to allocate irq host !\n"); + printk(KERN_ERR "pci_irq_host: failed to allocate irq domain!\n"); return; } diff --git a/arch/powerpc/sysdev/uic.c b/arch/powerpc/sysdev/uic.c index 063c901..9203393 100644 --- a/arch/powerpc/sysdev/uic.c +++ b/arch/powerpc/sysdev/uic.c @@ -49,7 +49,7 @@ struct uic { raw_spinlock_t lock; /* The remapper for this UIC */ - struct irq_host *irqhost; + struct irq_domain *irqhost; }; static void uic_unmask_irq(struct irq_data *d) @@ -174,7 +174,7 @@ static struct irq_chip uic_irq_chip = { .irq_set_type = uic_set_irq_type, }; -static int uic_host_map(struct irq_host *h, unsigned int virq, +static int uic_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct uic *uic = h->host_data; @@ -190,21 +190,9 @@ static int uic_host_map(struct irq_host *h, unsigned int virq, return 0; } -static int uic_host_xlate(struct irq_host *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type) - -{ - /* UIC intspecs must have 2 cells */ - BUG_ON(intsize != 2); - *out_hwirq = intspec[0]; - *out_type = intspec[1]; - return 0; -} - -static struct irq_host_ops uic_host_ops = { +static struct irq_domain_ops uic_host_ops = { .map = uic_host_map, - .xlate = uic_host_xlate, + .xlate = irq_domain_xlate_twocell, }; void uic_irq_cascade(unsigned int virq, struct irq_desc *desc) @@ -270,13 +258,11 @@ static struct uic * __init uic_init_one(struct device_node *node) } uic->dcrbase = *dcrreg; - uic->irqhost = irq_alloc_host(node, IRQ_HOST_MAP_LINEAR, - NR_UIC_INTS, &uic_host_ops, -1); + uic->irqhost = irq_domain_add_linear(node, NR_UIC_INTS, &uic_host_ops, + uic); if (! uic->irqhost) return NULL; /* FIXME: panic? */ - uic->irqhost->host_data = uic; - /* Start with all interrupts disabled, level and non-critical */ mtdcr(uic->dcrbase + UIC_ER, 0); mtdcr(uic->dcrbase + UIC_CR, 0); diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c index d72eda6..ea5e204 100644 --- a/arch/powerpc/sysdev/xics/xics-common.c +++ b/arch/powerpc/sysdev/xics/xics-common.c @@ -40,7 +40,7 @@ unsigned int xics_interrupt_server_size = 8; DEFINE_PER_CPU(struct xics_cppr, xics_cppr); -struct irq_host *xics_host; +struct irq_domain *xics_host; static LIST_HEAD(ics_list); @@ -212,16 +212,16 @@ void xics_migrate_irqs_away(void) /* We can't set affinity on ISA interrupts */ if (virq < NUM_ISA_INTERRUPTS) continue; - if (!virq_is_host(virq, xics_host)) - continue; - irq = (unsigned int)virq_to_hw(virq); - /* We need to get IPIs still. */ - if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) - continue; desc = irq_to_desc(virq); /* We only need to migrate enabled IRQS */ if (!desc || !desc->action) continue; + if (desc->irq_data.domain != xics_host) + continue; + irq = desc->irq_data.hwirq; + /* We need to get IPIs still. */ + if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) + continue; chip = irq_desc_get_chip(desc); if (!chip || !chip->irq_set_affinity) continue; @@ -301,7 +301,7 @@ int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, } #endif /* CONFIG_SMP */ -static int xics_host_match(struct irq_host *h, struct device_node *node) +static int xics_host_match(struct irq_domain *h, struct device_node *node) { struct ics *ics; @@ -323,7 +323,7 @@ static struct irq_chip xics_ipi_chip = { .irq_unmask = xics_ipi_unmask, }; -static int xics_host_map(struct irq_host *h, unsigned int virq, +static int xics_host_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t hw) { struct ics *ics; @@ -351,7 +351,7 @@ static int xics_host_map(struct irq_host *h, unsigned int virq, return -EINVAL; } -static int xics_host_xlate(struct irq_host *h, struct device_node *ct, +static int xics_host_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -366,7 +366,7 @@ static int xics_host_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static struct irq_host_ops xics_host_ops = { +static struct irq_domain_ops xics_host_ops = { .match = xics_host_match, .map = xics_host_map, .xlate = xics_host_xlate, @@ -374,8 +374,7 @@ static struct irq_host_ops xics_host_ops = { static void __init xics_init_host(void) { - xics_host = irq_alloc_host(NULL, IRQ_HOST_MAP_TREE, 0, &xics_host_ops, - XICS_IRQ_SPURIOUS); + xics_host = irq_domain_add_tree(NULL, &xics_host_ops, NULL); BUG_ON(xics_host == NULL); irq_set_default_host(xics_host); } diff --git a/arch/powerpc/sysdev/xilinx_intc.c b/arch/powerpc/sysdev/xilinx_intc.c index 6183799..8d73c3c 100644 --- a/arch/powerpc/sysdev/xilinx_intc.c +++ b/arch/powerpc/sysdev/xilinx_intc.c @@ -40,7 +40,7 @@ #define XINTC_IVR 24 /* Interrupt Vector */ #define XINTC_MER 28 /* Master Enable */ -static struct irq_host *master_irqhost; +static struct irq_domain *master_irqhost; #define XILINX_INTC_MAXIRQS (32) @@ -141,7 +141,7 @@ static struct irq_chip xilinx_intc_edge_irqchip = { /** * xilinx_intc_xlate - translate virq# from device tree interrupts property */ -static int xilinx_intc_xlate(struct irq_host *h, struct device_node *ct, +static int xilinx_intc_xlate(struct irq_domain *h, struct device_node *ct, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_flags) @@ -161,7 +161,7 @@ static int xilinx_intc_xlate(struct irq_host *h, struct device_node *ct, return 0; } -static int xilinx_intc_map(struct irq_host *h, unsigned int virq, +static int xilinx_intc_map(struct irq_domain *h, unsigned int virq, irq_hw_number_t irq) { irq_set_chip_data(virq, h->host_data); @@ -177,15 +177,15 @@ static int xilinx_intc_map(struct irq_host *h, unsigned int virq, return 0; } -static struct irq_host_ops xilinx_intc_ops = { +static struct irq_domain_ops xilinx_intc_ops = { .map = xilinx_intc_map, .xlate = xilinx_intc_xlate, }; -struct irq_host * __init +struct irq_domain * __init xilinx_intc_init(struct device_node *np) { - struct irq_host * irq; + struct irq_domain * irq; void * regs; /* Find and map the intc registers */ @@ -200,12 +200,11 @@ xilinx_intc_init(struct device_node *np) out_be32(regs + XINTC_IAR, ~(u32) 0); /* Acknowledge pending irqs */ out_be32(regs + XINTC_MER, 0x3UL); /* Turn on the Master Enable. */ - /* Allocate and initialize an irq_host structure. */ - irq = irq_alloc_host(np, IRQ_HOST_MAP_LINEAR, XILINX_INTC_MAXIRQS, - &xilinx_intc_ops, -1); + /* Allocate and initialize an irq_domain structure. */ + irq = irq_domain_add_linear(np, XILINX_INTC_MAXIRQS, &xilinx_intc_ops, + regs); if (!irq) panic(__FILE__ ": Cannot allocate IRQ host\n"); - irq->host_data = regs; return irq; } diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index 95a6cf2..6c32190 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -13,7 +13,7 @@ #define ASM_ALIGN ".balign 4" #endif -static __always_inline bool arch_static_branch(struct jump_label_key *key) +static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("0: brcl 0,0\n" ".pushsection __jump_table, \"aw\"\n" diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index a75f168..4eb444e 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -6,4 +6,3 @@ /* Empty, just to avoid compiling error */ -#define PERF_EVENT_INDEX_OFFSET 0 diff --git a/arch/s390/include/asm/qeth.h b/arch/s390/include/asm/qeth.h index 90efda0..2c7c898 100644 --- a/arch/s390/include/asm/qeth.h +++ b/arch/s390/include/asm/qeth.h @@ -20,6 +20,7 @@ #define SIOC_QETH_ARP_FLUSH_CACHE (SIOCDEVPRIVATE + 4) #define SIOC_QETH_ADP_SET_SNMP_CONTROL (SIOCDEVPRIVATE + 5) #define SIOC_QETH_GET_CARD_TYPE (SIOCDEVPRIVATE + 6) +#define SIOC_QETH_QUERY_OAT (SIOCDEVPRIVATE + 7) struct qeth_arp_cache_entry { __u8 macaddr[6]; @@ -107,4 +108,10 @@ struct qeth_arp_query_user_data { char *entries; } __attribute__((packed)); +struct qeth_query_oat_data { + __u32 command; + __u32 buffer_len; + __u32 response_len; + __u64 ptr; +}; #endif /* __ASM_S390_QETH_IOCTL_H__ */ diff --git a/arch/s390/include/asm/socket.h b/arch/s390/include/asm/socket.h index 67b5c1b..c91b720 100644 --- a/arch/s390/include/asm/socket.h +++ b/arch/s390/include/asm/socket.h @@ -72,5 +72,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _ASM_SOCKET_H */ diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index b9a7fdd..e30b2df 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -165,13 +165,6 @@ static inline int ext_hash(u16 code) return (code + (code >> 9)) & 0xff; } -static void ext_int_hash_update(struct rcu_head *head) -{ - struct ext_int_info *p = container_of(head, struct ext_int_info, rcu); - - kfree(p); -} - int register_external_interrupt(u16 code, ext_int_handler_t handler) { struct ext_int_info *p; @@ -202,7 +195,7 @@ int unregister_external_interrupt(u16 code, ext_int_handler_t handler) list_for_each_entry_rcu(p, &ext_int_hash[index], entry) if (p->code == code && p->handler == handler) { list_del_rcu(&p->entry); - call_rcu(&p->rcu, ext_int_hash_update); + kfree_rcu(p, rcu); } spin_unlock_irqrestore(&ext_int_hash_lock, flags); return 0; diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index e795933..7618085 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -97,9 +97,7 @@ void cpu_idle(void) tick_nohz_idle_exit(); if (test_thread_flag(TIF_MCCK_PENDING)) s390_handle_mcck(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 2398ce6..b0e28c4 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -550,12 +550,6 @@ int __cpuinit start_secondary(void *cpuvoid) S390_lowcore.restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler; __ctl_set_bit(0, 28); /* Enable lowcore protection */ - /* - * Wait until the cpu which brought this one up marked it - * active before enabling interrupts. - */ - while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask)) - cpu_relax(); local_irq_enable(); /* cpu_idle will call schedule for us */ cpu_idle(); diff --git a/arch/score/Kconfig.debug b/arch/score/Kconfig.debug index a1f346d..d8a9b2d 100644 --- a/arch/score/Kconfig.debug +++ b/arch/score/Kconfig.debug @@ -22,7 +22,7 @@ config RUNTIME_DEBUG help If you say Y here, some debugging macros will do run-time checking. If you say N here, those macros will mostly turn to no-ops. See - include/asm-score/debug.h for debuging macros. + include/asm-score/debug.h for debugging macros. If unsure, say N. endmenu diff --git a/arch/score/kernel/process.c b/arch/score/kernel/process.c index 25d0803..2707023 100644 --- a/arch/score/kernel/process.c +++ b/arch/score/kernel/process.c @@ -53,9 +53,7 @@ void __noreturn cpu_idle(void) while (!need_resched()) barrier(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 406508d..7e48928 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -114,9 +114,7 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/sh/kernel/perf_event.c b/arch/sh/kernel/perf_event.c index 10b14e3..068b8a2 100644 --- a/arch/sh/kernel/perf_event.c +++ b/arch/sh/kernel/perf_event.c @@ -310,6 +310,10 @@ static int sh_pmu_event_init(struct perf_event *event) { int err; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (event->attr.type) { case PERF_TYPE_RAW: case PERF_TYPE_HW_CACHE: diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index 92eb986..112fea1 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -244,7 +244,7 @@ static void sh4_flush_cache_page(void *args) if (map_coherent) vaddr = kmap_coherent(page, address); else - vaddr = kmap_atomic(page, KM_USER0); + vaddr = kmap_atomic(page); address = (unsigned long)vaddr; } @@ -259,7 +259,7 @@ static void sh4_flush_cache_page(void *args) if (map_coherent) kunmap_coherent(vaddr); else - kunmap_atomic(vaddr, KM_USER0); + kunmap_atomic(vaddr); } } diff --git a/arch/sh/mm/cache.c b/arch/sh/mm/cache.c index 5a580ea..616966a 100644 --- a/arch/sh/mm/cache.c +++ b/arch/sh/mm/cache.c @@ -95,7 +95,7 @@ void copy_user_highpage(struct page *to, struct page *from, { void *vfrom, *vto; - vto = kmap_atomic(to, KM_USER1); + vto = kmap_atomic(to); if (boot_cpu_data.dcache.n_aliases && page_mapped(from) && test_bit(PG_dcache_clean, &from->flags)) { @@ -103,16 +103,16 @@ void copy_user_highpage(struct page *to, struct page *from, copy_page(vto, vfrom); kunmap_coherent(vfrom); } else { - vfrom = kmap_atomic(from, KM_USER0); + vfrom = kmap_atomic(from); copy_page(vto, vfrom); - kunmap_atomic(vfrom, KM_USER0); + kunmap_atomic(vfrom); } if (pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK) || (vma->vm_flags & VM_EXEC)) __flush_purge_region(vto, PAGE_SIZE); - kunmap_atomic(vto, KM_USER1); + kunmap_atomic(vto); /* Make sure this page is cleared on other CPU's too before using it */ smp_wmb(); } @@ -120,14 +120,14 @@ EXPORT_SYMBOL(copy_user_highpage); void clear_user_highpage(struct page *page, unsigned long vaddr) { - void *kaddr = kmap_atomic(page, KM_USER0); + void *kaddr = kmap_atomic(page); clear_page(kaddr); if (pages_do_alias((unsigned long)kaddr, vaddr & PAGE_MASK)) __flush_purge_region(kaddr, PAGE_SIZE); - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); } EXPORT_SYMBOL(clear_user_highpage); diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h index 3d7afbb..3b6e00d 100644 --- a/arch/sparc/include/asm/highmem.h +++ b/arch/sparc/include/asm/highmem.h @@ -70,7 +70,7 @@ static inline void kunmap(struct page *page) kunmap_high(page); } -extern void *__kmap_atomic(struct page *page); +extern void *kmap_atomic(struct page *page); extern void __kunmap_atomic(void *kvaddr); extern struct page *kmap_atomic_to_page(void *vaddr); diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index fc73a82..5080d16 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h @@ -7,7 +7,7 @@ #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct jump_label_key *key) +static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("1:\n\t" "nop\n\t" diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h index edd3d3cde..c287651 100644 --- a/arch/sparc/include/asm/prom.h +++ b/arch/sparc/include/asm/prom.h @@ -22,6 +22,7 @@ #include <linux/proc_fs.h> #include <linux/mutex.h> #include <linux/atomic.h> +#include <linux/irqdomain.h> #define OF_ROOT_NODE_ADDR_CELLS_DEFAULT 2 #define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1 @@ -55,15 +56,6 @@ struct resource; extern void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); extern void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); -/* These routines are here to provide compatibility with how powerpc - * handles IRQ mapping for OF device nodes. We precompute and permanently - * register them in the platform_device objects, whereas powerpc computes them - * on request. - */ -static inline void irq_dispose_mapping(unsigned int virq) -{ -} - extern struct device_node *of_console_device; extern char *of_console_path; extern char *of_console_options; diff --git a/arch/sparc/include/asm/socket.h b/arch/sparc/include/asm/socket.h index 8af1b64..bea1568 100644 --- a/arch/sparc/include/asm/socket.h +++ b/arch/sparc/include/asm/socket.h @@ -60,6 +60,11 @@ #define SO_WIFI_STATUS 0x0025 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 0x0026 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 0x0027 + /* Security levels - as per NRL IPv6 - don't actually do anything */ #define SO_SECURITY_AUTHENTICATION 0x5001 diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 614da62..8e16a4a 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -1105,6 +1105,10 @@ static int sparc_pmu_event_init(struct perf_event *event) if (atomic_read(&nmi_active) < 0) return -ENODEV; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (attr->type) { case PERF_TYPE_HARDWARE: if (attr->config >= sparc_pmu->max_events) diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index f793742..935fdbc 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -113,9 +113,7 @@ void cpu_idle(void) while (!need_resched()) cpu_relax(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } @@ -138,9 +136,7 @@ void cpu_idle(void) while (!need_resched()) cpu_relax(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 39d8b05..06b5b5f 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -104,15 +104,13 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - #ifdef CONFIG_HOTPLUG_CPU - if (cpu_is_offline(cpu)) + if (cpu_is_offline(cpu)) { + sched_preempt_enable_no_resched(); cpu_play_dead(); + } #endif - - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 77140a0..055c66c 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -30,7 +30,7 @@ #include <asm/tlbflush.h> #include <asm/fixmap.h> -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { unsigned long vaddr; long idx, type; @@ -64,7 +64,7 @@ void *__kmap_atomic(struct page *page) return (void*) vaddr; } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { diff --git a/arch/tile/include/asm/highmem.h b/arch/tile/include/asm/highmem.h index b2a6c5d..fc8429a 100644 --- a/arch/tile/include/asm/highmem.h +++ b/arch/tile/include/asm/highmem.h @@ -59,7 +59,7 @@ void *kmap_fix_kpte(struct page *page, int finished); /* This macro is used only in map_new_virtual() to map "page". */ #define kmap_prot page_to_kpgprot(page) -void *__kmap_atomic(struct page *page); +void *kmap_atomic(struct page *page); void __kunmap_atomic(void *kvaddr); void *kmap_atomic_pfn(unsigned long pfn); void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c index 4c1ac6e..6ae495e 100644 --- a/arch/tile/kernel/process.c +++ b/arch/tile/kernel/process.c @@ -108,9 +108,7 @@ void cpu_idle(void) } rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/tile/mm/highmem.c b/arch/tile/mm/highmem.c index 31dbbd9..ef8e5a6 100644 --- a/arch/tile/mm/highmem.c +++ b/arch/tile/mm/highmem.c @@ -224,12 +224,12 @@ void *kmap_atomic_prot(struct page *page, pgprot_t prot) } EXPORT_SYMBOL(kmap_atomic_prot); -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { /* PAGE_NONE is a magic value that tells us to check immutability. */ return kmap_atomic_prot(page, PAGE_NONE); } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { diff --git a/arch/um/drivers/net_kern.c b/arch/um/drivers/net_kern.c index a492e59..d299618 100644 --- a/arch/um/drivers/net_kern.c +++ b/arch/um/drivers/net_kern.c @@ -293,7 +293,7 @@ static void uml_net_user_timer_expire(unsigned long _conn) #endif } -static void setup_etheraddr(char *str, unsigned char *addr, char *name) +static int setup_etheraddr(char *str, unsigned char *addr, char *name) { char *end; int i; @@ -334,12 +334,13 @@ static void setup_etheraddr(char *str, unsigned char *addr, char *name) addr[0] | 0x02, addr[1], addr[2], addr[3], addr[4], addr[5]); } - return; + return 0; random: printk(KERN_INFO "Choosing a random ethernet address for device %s\n", name); random_ether_addr(addr); + return 1; } static DEFINE_SPINLOCK(devices_lock); @@ -391,6 +392,7 @@ static void eth_configure(int n, void *init, char *mac, struct net_device *dev; struct uml_net_private *lp; int err, size; + int random_mac; size = transport->private_size + sizeof(struct uml_net_private); @@ -417,7 +419,7 @@ static void eth_configure(int n, void *init, char *mac, */ snprintf(dev->name, sizeof(dev->name), "eth%d", n); - setup_etheraddr(mac, device->mac, dev->name); + random_mac = setup_etheraddr(mac, device->mac, dev->name); printk(KERN_INFO "Netdevice %d (%pM) : ", n, device->mac); @@ -474,6 +476,9 @@ static void eth_configure(int n, void *init, char *mac, /* don't use eth_mac_addr, it will not work here */ memcpy(dev->dev_addr, device->mac, ETH_ALEN); + if (random_mac) + dev->addr_assign_type |= NET_ADDR_RANDOM; + dev->mtu = transport->user->mtu; dev->netdev_ops = ¨_netdev_ops; dev->ethtool_ops = ¨_net_ethtool_ops; diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index 9fefd92..cd7df79 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -69,7 +69,7 @@ static int do_op_one_page(unsigned long addr, int len, int is_write, return -1; page = pte_page(*pte); - addr = (unsigned long) kmap_atomic(page, KM_UML_USERCOPY) + + addr = (unsigned long) kmap_atomic(page) + (addr & ~PAGE_MASK); current->thread.fault_catcher = &buf; @@ -82,7 +82,7 @@ static int do_op_one_page(unsigned long addr, int len, int is_write, current->thread.fault_catcher = NULL; - kunmap_atomic((void *)addr, KM_UML_USERCOPY); + kunmap_atomic((void *)addr); return n; } diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5bed94e1..6c29256a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -82,6 +82,7 @@ config X86 select CLKEVT_I8253 select ARCH_HAVE_NMI_SAFE_CMPXCHG select GENERIC_IOMAP + select DCACHE_WORD_ACCESS if !DEBUG_PAGEALLOC config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) @@ -179,6 +180,9 @@ config ARCH_HAS_DEFAULT_IDLE config ARCH_HAS_CACHE_LINE_SIZE def_bool y +config ARCH_HAS_CPU_AUTOPROBE + def_bool y + config HAVE_SETUP_PER_CPU_AREA def_bool y @@ -398,6 +402,7 @@ config X86_INTEL_CE select X86_REBOOTFIXUPS select OF select OF_EARLY_FLATTREE + select IRQ_DOMAIN ---help--- Select for the Intel CE media processor (CE4100) SOC. This option compiles in support for the CE4100 SOC for settop @@ -2076,6 +2081,7 @@ config OLPC select GPIOLIB select OF select OF_PROMTREE + select IRQ_DOMAIN ---help--- Add support for detecting the unique features of the OLPC XO hardware. diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 2b0b963..e191ac0 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o +obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o @@ -25,6 +26,7 @@ salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o +camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 545d0ce..c799352 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -28,6 +28,7 @@ #include <crypto/aes.h> #include <crypto/cryptd.h> #include <crypto/ctr.h> +#include <asm/cpu_device_id.h> #include <asm/i387.h> #include <asm/aes.h> #include <crypto/scatterwalk.h> @@ -1107,12 +1108,12 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) one_entry_in_sg = 1; scatterwalk_start(&src_sg_walk, req->src); scatterwalk_start(&assoc_sg_walk, req->assoc); - src = scatterwalk_map(&src_sg_walk, 0); - assoc = scatterwalk_map(&assoc_sg_walk, 0); + src = scatterwalk_map(&src_sg_walk); + assoc = scatterwalk_map(&assoc_sg_walk); dst = src; if (unlikely(req->src != req->dst)) { scatterwalk_start(&dst_sg_walk, req->dst); - dst = scatterwalk_map(&dst_sg_walk, 0); + dst = scatterwalk_map(&dst_sg_walk); } } else { @@ -1136,11 +1137,11 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) * back to the packet. */ if (one_entry_in_sg) { if (unlikely(req->src != req->dst)) { - scatterwalk_unmap(dst, 0); + scatterwalk_unmap(dst); scatterwalk_done(&dst_sg_walk, 0, 0); } - scatterwalk_unmap(src, 0); - scatterwalk_unmap(assoc, 0); + scatterwalk_unmap(src); + scatterwalk_unmap(assoc); scatterwalk_done(&src_sg_walk, 0, 0); scatterwalk_done(&assoc_sg_walk, 0, 0); } else { @@ -1189,12 +1190,12 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) one_entry_in_sg = 1; scatterwalk_start(&src_sg_walk, req->src); scatterwalk_start(&assoc_sg_walk, req->assoc); - src = scatterwalk_map(&src_sg_walk, 0); - assoc = scatterwalk_map(&assoc_sg_walk, 0); + src = scatterwalk_map(&src_sg_walk); + assoc = scatterwalk_map(&assoc_sg_walk); dst = src; if (unlikely(req->src != req->dst)) { scatterwalk_start(&dst_sg_walk, req->dst); - dst = scatterwalk_map(&dst_sg_walk, 0); + dst = scatterwalk_map(&dst_sg_walk); } } else { @@ -1219,11 +1220,11 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) if (one_entry_in_sg) { if (unlikely(req->src != req->dst)) { - scatterwalk_unmap(dst, 0); + scatterwalk_unmap(dst); scatterwalk_done(&dst_sg_walk, 0, 0); } - scatterwalk_unmap(src, 0); - scatterwalk_unmap(assoc, 0); + scatterwalk_unmap(src); + scatterwalk_unmap(assoc); scatterwalk_done(&src_sg_walk, 0, 0); scatterwalk_done(&assoc_sg_walk, 0, 0); } else { @@ -1253,14 +1254,19 @@ static struct crypto_alg __rfc4106_alg = { }; #endif + +static const struct x86_cpu_id aesni_cpu_id[] = { + X86_FEATURE_MATCH(X86_FEATURE_AES), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); + static int __init aesni_init(void) { int err; - if (!cpu_has_aes) { - printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); + if (!x86_match_cpu(aesni_cpu_id)) return -ENODEV; - } if ((err = crypto_fpu_init())) goto fpu_err; diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index b05aa16..7967474 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -25,6 +25,7 @@ * */ +#include <asm/processor.h> #include <crypto/blowfish.h> #include <linux/crypto.h> #include <linux/init.h> @@ -76,27 +77,6 @@ static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src); } -static struct crypto_alg bf_alg = { - .cra_name = "blowfish", - .cra_driver_name = "blowfish-asm", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = BF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct bf_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(bf_alg.cra_list), - .cra_u = { - .cipher = { - .cia_min_keysize = BF_MIN_KEY_SIZE, - .cia_max_keysize = BF_MAX_KEY_SIZE, - .cia_setkey = blowfish_setkey, - .cia_encrypt = blowfish_encrypt, - .cia_decrypt = blowfish_decrypt, - } - } -}; - static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, void (*fn)(struct bf_ctx *, u8 *, const u8 *), void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *)) @@ -160,28 +140,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way); } -static struct crypto_alg blk_ecb_alg = { - .cra_name = "ecb(blowfish)", - .cra_driver_name = "ecb-blowfish-asm", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = BF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct bf_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = BF_MIN_KEY_SIZE, - .max_keysize = BF_MAX_KEY_SIZE, - .setkey = blowfish_setkey, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}; - static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { @@ -307,29 +265,6 @@ static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_cbc_alg = { - .cra_name = "cbc(blowfish)", - .cra_driver_name = "cbc-blowfish-asm", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = BF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct bf_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = BF_MIN_KEY_SIZE, - .max_keysize = BF_MAX_KEY_SIZE, - .ivsize = BF_BLOCK_SIZE, - .setkey = blowfish_setkey, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -}; - static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk) { u8 *ctrblk = walk->iv; @@ -423,7 +358,67 @@ static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_ctr_alg = { +static struct crypto_alg bf_algs[4] = { { + .cra_name = "blowfish", + .cra_driver_name = "blowfish-asm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(bf_algs[0].cra_list), + .cra_u = { + .cipher = { + .cia_min_keysize = BF_MIN_KEY_SIZE, + .cia_max_keysize = BF_MAX_KEY_SIZE, + .cia_setkey = blowfish_setkey, + .cia_encrypt = blowfish_encrypt, + .cia_decrypt = blowfish_decrypt, + } + } +}, { + .cra_name = "ecb(blowfish)", + .cra_driver_name = "ecb-blowfish-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(bf_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .setkey = blowfish_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(blowfish)", + .cra_driver_name = "cbc-blowfish-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = BF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct bf_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(bf_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = BF_MIN_KEY_SIZE, + .max_keysize = BF_MAX_KEY_SIZE, + .ivsize = BF_BLOCK_SIZE, + .setkey = blowfish_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { .cra_name = "ctr(blowfish)", .cra_driver_name = "ctr-blowfish-asm", .cra_priority = 300, @@ -433,7 +428,7 @@ static struct crypto_alg blk_ctr_alg = { .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), + .cra_list = LIST_HEAD_INIT(bf_algs[3].cra_list), .cra_u = { .blkcipher = { .min_keysize = BF_MIN_KEY_SIZE, @@ -444,43 +439,45 @@ static struct crypto_alg blk_ctr_alg = { .decrypt = ctr_crypt, }, }, -}; +} }; + +static bool is_blacklisted_cpu(void) +{ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return false; + + if (boot_cpu_data.x86 == 0x0f) { + /* + * On Pentium 4, blowfish-x86_64 is slower than generic C + * implementation because use of 64bit rotates (which are really + * slow on P4). Therefore blacklist P4s. + */ + return true; + } + + return false; +} + +static int force; +module_param(force, int, 0); +MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); static int __init init(void) { - int err; + if (!force && is_blacklisted_cpu()) { + printk(KERN_INFO + "blowfish-x86_64: performance on this CPU " + "would be suboptimal: disabling " + "blowfish-x86_64.\n"); + return -ENODEV; + } - err = crypto_register_alg(&bf_alg); - if (err) - goto bf_err; - err = crypto_register_alg(&blk_ecb_alg); - if (err) - goto ecb_err; - err = crypto_register_alg(&blk_cbc_alg); - if (err) - goto cbc_err; - err = crypto_register_alg(&blk_ctr_alg); - if (err) - goto ctr_err; - - return 0; - -ctr_err: - crypto_unregister_alg(&blk_cbc_alg); -cbc_err: - crypto_unregister_alg(&blk_ecb_alg); -ecb_err: - crypto_unregister_alg(&bf_alg); -bf_err: - return err; + return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs)); } static void __exit fini(void) { - crypto_unregister_alg(&blk_ctr_alg); - crypto_unregister_alg(&blk_cbc_alg); - crypto_unregister_alg(&blk_ecb_alg); - crypto_unregister_alg(&bf_alg); + crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs)); } module_init(init); diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S new file mode 100644 index 0000000..0b33743 --- /dev/null +++ b/arch/x86/crypto/camellia-x86_64-asm_64.S @@ -0,0 +1,520 @@ +/* + * Camellia Cipher Algorithm (x86_64) + * + * Copyright (C) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +.file "camellia-x86_64-asm_64.S" +.text + +.extern camellia_sp10011110; +.extern camellia_sp22000222; +.extern camellia_sp03303033; +.extern camellia_sp00444404; +.extern camellia_sp02220222; +.extern camellia_sp30333033; +.extern camellia_sp44044404; +.extern camellia_sp11101110; + +#define sp10011110 camellia_sp10011110 +#define sp22000222 camellia_sp22000222 +#define sp03303033 camellia_sp03303033 +#define sp00444404 camellia_sp00444404 +#define sp02220222 camellia_sp02220222 +#define sp30333033 camellia_sp30333033 +#define sp44044404 camellia_sp44044404 +#define sp11101110 camellia_sp11101110 + +#define CAMELLIA_TABLE_BYTE_LEN 272 + +/* struct camellia_ctx: */ +#define key_table 0 +#define key_length CAMELLIA_TABLE_BYTE_LEN + +/* register macros */ +#define CTX %rdi +#define RIO %rsi +#define RIOd %esi + +#define RAB0 %rax +#define RCD0 %rcx +#define RAB1 %rbx +#define RCD1 %rdx + +#define RAB0d %eax +#define RCD0d %ecx +#define RAB1d %ebx +#define RCD1d %edx + +#define RAB0bl %al +#define RCD0bl %cl +#define RAB1bl %bl +#define RCD1bl %dl + +#define RAB0bh %ah +#define RCD0bh %ch +#define RAB1bh %bh +#define RCD1bh %dh + +#define RT0 %rsi +#define RT1 %rbp +#define RT2 %r8 + +#define RT0d %esi +#define RT1d %ebp +#define RT2d %r8d + +#define RT2bl %r8b + +#define RXOR %r9 +#define RRBP %r10 +#define RDST %r11 + +#define RXORd %r9d +#define RXORbl %r9b + +#define xor2ror16(T0, T1, tmp1, tmp2, ab, dst) \ + movzbl ab ## bl, tmp2 ## d; \ + movzbl ab ## bh, tmp1 ## d; \ + rorq $16, ab; \ + xorq T0(, tmp2, 8), dst; \ + xorq T1(, tmp1, 8), dst; + +/********************************************************************** + 1-way camellia + **********************************************************************/ +#define roundsm(ab, subkey, cd) \ + movq (key_table + ((subkey) * 2) * 4)(CTX), RT2; \ + \ + xor2ror16(sp00444404, sp03303033, RT0, RT1, ab ## 0, cd ## 0); \ + xor2ror16(sp22000222, sp10011110, RT0, RT1, ab ## 0, RT2); \ + xor2ror16(sp11101110, sp44044404, RT0, RT1, ab ## 0, cd ## 0); \ + xor2ror16(sp30333033, sp02220222, RT0, RT1, ab ## 0, RT2); \ + \ + xorq RT2, cd ## 0; + +#define fls(l, r, kl, kr) \ + movl (key_table + ((kl) * 2) * 4)(CTX), RT0d; \ + andl l ## 0d, RT0d; \ + roll $1, RT0d; \ + shlq $32, RT0; \ + xorq RT0, l ## 0; \ + movq (key_table + ((kr) * 2) * 4)(CTX), RT1; \ + orq r ## 0, RT1; \ + shrq $32, RT1; \ + xorq RT1, r ## 0; \ + \ + movq (key_table + ((kl) * 2) * 4)(CTX), RT2; \ + orq l ## 0, RT2; \ + shrq $32, RT2; \ + xorq RT2, l ## 0; \ + movl (key_table + ((kr) * 2) * 4)(CTX), RT0d; \ + andl r ## 0d, RT0d; \ + roll $1, RT0d; \ + shlq $32, RT0; \ + xorq RT0, r ## 0; + +#define enc_rounds(i) \ + roundsm(RAB, i + 2, RCD); \ + roundsm(RCD, i + 3, RAB); \ + roundsm(RAB, i + 4, RCD); \ + roundsm(RCD, i + 5, RAB); \ + roundsm(RAB, i + 6, RCD); \ + roundsm(RCD, i + 7, RAB); + +#define enc_fls(i) \ + fls(RAB, RCD, i + 0, i + 1); + +#define enc_inpack() \ + movq (RIO), RAB0; \ + bswapq RAB0; \ + rolq $32, RAB0; \ + movq 4*2(RIO), RCD0; \ + bswapq RCD0; \ + rorq $32, RCD0; \ + xorq key_table(CTX), RAB0; + +#define enc_outunpack(op, max) \ + xorq key_table(CTX, max, 8), RCD0; \ + rorq $32, RCD0; \ + bswapq RCD0; \ + op ## q RCD0, (RIO); \ + rolq $32, RAB0; \ + bswapq RAB0; \ + op ## q RAB0, 4*2(RIO); + +#define dec_rounds(i) \ + roundsm(RAB, i + 7, RCD); \ + roundsm(RCD, i + 6, RAB); \ + roundsm(RAB, i + 5, RCD); \ + roundsm(RCD, i + 4, RAB); \ + roundsm(RAB, i + 3, RCD); \ + roundsm(RCD, i + 2, RAB); + +#define dec_fls(i) \ + fls(RAB, RCD, i + 1, i + 0); + +#define dec_inpack(max) \ + movq (RIO), RAB0; \ + bswapq RAB0; \ + rolq $32, RAB0; \ + movq 4*2(RIO), RCD0; \ + bswapq RCD0; \ + rorq $32, RCD0; \ + xorq key_table(CTX, max, 8), RAB0; + +#define dec_outunpack() \ + xorq key_table(CTX), RCD0; \ + rorq $32, RCD0; \ + bswapq RCD0; \ + movq RCD0, (RIO); \ + rolq $32, RAB0; \ + bswapq RAB0; \ + movq RAB0, 4*2(RIO); + +.global __camellia_enc_blk; +.type __camellia_enc_blk,@function; + +__camellia_enc_blk: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: bool xor + */ + movq %rbp, RRBP; + + movq %rcx, RXOR; + movq %rsi, RDST; + movq %rdx, RIO; + + enc_inpack(); + + enc_rounds(0); + enc_fls(8); + enc_rounds(8); + enc_fls(16); + enc_rounds(16); + movl $24, RT1d; /* max */ + + cmpb $16, key_length(CTX); + je __enc_done; + + enc_fls(24); + enc_rounds(24); + movl $32, RT1d; /* max */ + +__enc_done: + testb RXORbl, RXORbl; + movq RDST, RIO; + + jnz __enc_xor; + + enc_outunpack(mov, RT1); + + movq RRBP, %rbp; + ret; + +__enc_xor: + enc_outunpack(xor, RT1); + + movq RRBP, %rbp; + ret; + +.global camellia_dec_blk; +.type camellia_dec_blk,@function; + +camellia_dec_blk: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + cmpl $16, key_length(CTX); + movl $32, RT2d; + movl $24, RXORd; + cmovel RXORd, RT2d; /* max */ + + movq %rbp, RRBP; + movq %rsi, RDST; + movq %rdx, RIO; + + dec_inpack(RT2); + + cmpb $24, RT2bl; + je __dec_rounds16; + + dec_rounds(24); + dec_fls(24); + +__dec_rounds16: + dec_rounds(16); + dec_fls(16); + dec_rounds(8); + dec_fls(8); + dec_rounds(0); + + movq RDST, RIO; + + dec_outunpack(); + + movq RRBP, %rbp; + ret; + +/********************************************************************** + 2-way camellia + **********************************************************************/ +#define roundsm2(ab, subkey, cd) \ + movq (key_table + ((subkey) * 2) * 4)(CTX), RT2; \ + xorq RT2, cd ## 1; \ + \ + xor2ror16(sp00444404, sp03303033, RT0, RT1, ab ## 0, cd ## 0); \ + xor2ror16(sp22000222, sp10011110, RT0, RT1, ab ## 0, RT2); \ + xor2ror16(sp11101110, sp44044404, RT0, RT1, ab ## 0, cd ## 0); \ + xor2ror16(sp30333033, sp02220222, RT0, RT1, ab ## 0, RT2); \ + \ + xor2ror16(sp00444404, sp03303033, RT0, RT1, ab ## 1, cd ## 1); \ + xorq RT2, cd ## 0; \ + xor2ror16(sp22000222, sp10011110, RT0, RT1, ab ## 1, cd ## 1); \ + xor2ror16(sp11101110, sp44044404, RT0, RT1, ab ## 1, cd ## 1); \ + xor2ror16(sp30333033, sp02220222, RT0, RT1, ab ## 1, cd ## 1); + +#define fls2(l, r, kl, kr) \ + movl (key_table + ((kl) * 2) * 4)(CTX), RT0d; \ + andl l ## 0d, RT0d; \ + roll $1, RT0d; \ + shlq $32, RT0; \ + xorq RT0, l ## 0; \ + movq (key_table + ((kr) * 2) * 4)(CTX), RT1; \ + orq r ## 0, RT1; \ + shrq $32, RT1; \ + xorq RT1, r ## 0; \ + \ + movl (key_table + ((kl) * 2) * 4)(CTX), RT2d; \ + andl l ## 1d, RT2d; \ + roll $1, RT2d; \ + shlq $32, RT2; \ + xorq RT2, l ## 1; \ + movq (key_table + ((kr) * 2) * 4)(CTX), RT0; \ + orq r ## 1, RT0; \ + shrq $32, RT0; \ + xorq RT0, r ## 1; \ + \ + movq (key_table + ((kl) * 2) * 4)(CTX), RT1; \ + orq l ## 0, RT1; \ + shrq $32, RT1; \ + xorq RT1, l ## 0; \ + movl (key_table + ((kr) * 2) * 4)(CTX), RT2d; \ + andl r ## 0d, RT2d; \ + roll $1, RT2d; \ + shlq $32, RT2; \ + xorq RT2, r ## 0; \ + \ + movq (key_table + ((kl) * 2) * 4)(CTX), RT0; \ + orq l ## 1, RT0; \ + shrq $32, RT0; \ + xorq RT0, l ## 1; \ + movl (key_table + ((kr) * 2) * 4)(CTX), RT1d; \ + andl r ## 1d, RT1d; \ + roll $1, RT1d; \ + shlq $32, RT1; \ + xorq RT1, r ## 1; + +#define enc_rounds2(i) \ + roundsm2(RAB, i + 2, RCD); \ + roundsm2(RCD, i + 3, RAB); \ + roundsm2(RAB, i + 4, RCD); \ + roundsm2(RCD, i + 5, RAB); \ + roundsm2(RAB, i + 6, RCD); \ + roundsm2(RCD, i + 7, RAB); + +#define enc_fls2(i) \ + fls2(RAB, RCD, i + 0, i + 1); + +#define enc_inpack2() \ + movq (RIO), RAB0; \ + bswapq RAB0; \ + rorq $32, RAB0; \ + movq 4*2(RIO), RCD0; \ + bswapq RCD0; \ + rolq $32, RCD0; \ + xorq key_table(CTX), RAB0; \ + \ + movq 8*2(RIO), RAB1; \ + bswapq RAB1; \ + rorq $32, RAB1; \ + movq 12*2(RIO), RCD1; \ + bswapq RCD1; \ + rolq $32, RCD1; \ + xorq key_table(CTX), RAB1; + +#define enc_outunpack2(op, max) \ + xorq key_table(CTX, max, 8), RCD0; \ + rolq $32, RCD0; \ + bswapq RCD0; \ + op ## q RCD0, (RIO); \ + rorq $32, RAB0; \ + bswapq RAB0; \ + op ## q RAB0, 4*2(RIO); \ + \ + xorq key_table(CTX, max, 8), RCD1; \ + rolq $32, RCD1; \ + bswapq RCD1; \ + op ## q RCD1, 8*2(RIO); \ + rorq $32, RAB1; \ + bswapq RAB1; \ + op ## q RAB1, 12*2(RIO); + +#define dec_rounds2(i) \ + roundsm2(RAB, i + 7, RCD); \ + roundsm2(RCD, i + 6, RAB); \ + roundsm2(RAB, i + 5, RCD); \ + roundsm2(RCD, i + 4, RAB); \ + roundsm2(RAB, i + 3, RCD); \ + roundsm2(RCD, i + 2, RAB); + +#define dec_fls2(i) \ + fls2(RAB, RCD, i + 1, i + 0); + +#define dec_inpack2(max) \ + movq (RIO), RAB0; \ + bswapq RAB0; \ + rorq $32, RAB0; \ + movq 4*2(RIO), RCD0; \ + bswapq RCD0; \ + rolq $32, RCD0; \ + xorq key_table(CTX, max, 8), RAB0; \ + \ + movq 8*2(RIO), RAB1; \ + bswapq RAB1; \ + rorq $32, RAB1; \ + movq 12*2(RIO), RCD1; \ + bswapq RCD1; \ + rolq $32, RCD1; \ + xorq key_table(CTX, max, 8), RAB1; + +#define dec_outunpack2() \ + xorq key_table(CTX), RCD0; \ + rolq $32, RCD0; \ + bswapq RCD0; \ + movq RCD0, (RIO); \ + rorq $32, RAB0; \ + bswapq RAB0; \ + movq RAB0, 4*2(RIO); \ + \ + xorq key_table(CTX), RCD1; \ + rolq $32, RCD1; \ + bswapq RCD1; \ + movq RCD1, 8*2(RIO); \ + rorq $32, RAB1; \ + bswapq RAB1; \ + movq RAB1, 12*2(RIO); + +.global __camellia_enc_blk_2way; +.type __camellia_enc_blk_2way,@function; + +__camellia_enc_blk_2way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: bool xor + */ + pushq %rbx; + + movq %rbp, RRBP; + movq %rcx, RXOR; + movq %rsi, RDST; + movq %rdx, RIO; + + enc_inpack2(); + + enc_rounds2(0); + enc_fls2(8); + enc_rounds2(8); + enc_fls2(16); + enc_rounds2(16); + movl $24, RT2d; /* max */ + + cmpb $16, key_length(CTX); + je __enc2_done; + + enc_fls2(24); + enc_rounds2(24); + movl $32, RT2d; /* max */ + +__enc2_done: + test RXORbl, RXORbl; + movq RDST, RIO; + jnz __enc2_xor; + + enc_outunpack2(mov, RT2); + + movq RRBP, %rbp; + popq %rbx; + ret; + +__enc2_xor: + enc_outunpack2(xor, RT2); + + movq RRBP, %rbp; + popq %rbx; + ret; + +.global camellia_dec_blk_2way; +.type camellia_dec_blk_2way,@function; + +camellia_dec_blk_2way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + cmpl $16, key_length(CTX); + movl $32, RT2d; + movl $24, RXORd; + cmovel RXORd, RT2d; /* max */ + + movq %rbx, RXOR; + movq %rbp, RRBP; + movq %rsi, RDST; + movq %rdx, RIO; + + dec_inpack2(RT2); + + cmpb $24, RT2bl; + je __dec2_rounds16; + + dec_rounds2(24); + dec_fls2(24); + +__dec2_rounds16: + dec_rounds2(16); + dec_fls2(16); + dec_rounds2(8); + dec_fls2(8); + dec_rounds2(0); + + movq RDST, RIO; + + dec_outunpack2(); + + movq RRBP, %rbp; + movq RXOR, %rbx; + ret; diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c new file mode 100644 index 0000000..1ca36a9 --- /dev/null +++ b/arch/x86/crypto/camellia_glue.c @@ -0,0 +1,1952 @@ +/* + * Glue Code for assembler optimized version of Camellia + * + * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * Camellia parts based on code by: + * Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation) + * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: + * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> + * CTR part based on code (crypto/ctr.c) by: + * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include <asm/processor.h> +#include <asm/unaligned.h> +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/types.h> +#include <crypto/algapi.h> +#include <crypto/b128ops.h> +#include <crypto/lrw.h> +#include <crypto/xts.h> + +#define CAMELLIA_MIN_KEY_SIZE 16 +#define CAMELLIA_MAX_KEY_SIZE 32 +#define CAMELLIA_BLOCK_SIZE 16 +#define CAMELLIA_TABLE_BYTE_LEN 272 + +struct camellia_ctx { + u64 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)]; + u32 key_length; +}; + +/* regular block cipher functions */ +asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); + +/* 2-way parallel cipher functions */ +asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src); + +static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst, + const u8 *src) +{ + __camellia_enc_blk(ctx, dst, src, false); +} + +static inline void camellia_enc_blk_xor(struct camellia_ctx *ctx, u8 *dst, + const u8 *src) +{ + __camellia_enc_blk(ctx, dst, src, true); +} + +static inline void camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src) +{ + __camellia_enc_blk_2way(ctx, dst, src, false); +} + +static inline void camellia_enc_blk_xor_2way(struct camellia_ctx *ctx, u8 *dst, + const u8 *src) +{ + __camellia_enc_blk_2way(ctx, dst, src, true); +} + +static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + camellia_enc_blk(crypto_tfm_ctx(tfm), dst, src); +} + +static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + camellia_dec_blk(crypto_tfm_ctx(tfm), dst, src); +} + +/* camellia sboxes */ +const u64 camellia_sp10011110[256] = { + 0x7000007070707000, 0x8200008282828200, 0x2c00002c2c2c2c00, + 0xec0000ecececec00, 0xb30000b3b3b3b300, 0x2700002727272700, + 0xc00000c0c0c0c000, 0xe50000e5e5e5e500, 0xe40000e4e4e4e400, + 0x8500008585858500, 0x5700005757575700, 0x3500003535353500, + 0xea0000eaeaeaea00, 0x0c00000c0c0c0c00, 0xae0000aeaeaeae00, + 0x4100004141414100, 0x2300002323232300, 0xef0000efefefef00, + 0x6b00006b6b6b6b00, 0x9300009393939300, 0x4500004545454500, + 0x1900001919191900, 0xa50000a5a5a5a500, 0x2100002121212100, + 0xed0000edededed00, 0x0e00000e0e0e0e00, 0x4f00004f4f4f4f00, + 0x4e00004e4e4e4e00, 0x1d00001d1d1d1d00, 0x6500006565656500, + 0x9200009292929200, 0xbd0000bdbdbdbd00, 0x8600008686868600, + 0xb80000b8b8b8b800, 0xaf0000afafafaf00, 0x8f00008f8f8f8f00, + 0x7c00007c7c7c7c00, 0xeb0000ebebebeb00, 0x1f00001f1f1f1f00, + 0xce0000cececece00, 0x3e00003e3e3e3e00, 0x3000003030303000, + 0xdc0000dcdcdcdc00, 0x5f00005f5f5f5f00, 0x5e00005e5e5e5e00, + 0xc50000c5c5c5c500, 0x0b00000b0b0b0b00, 0x1a00001a1a1a1a00, + 0xa60000a6a6a6a600, 0xe10000e1e1e1e100, 0x3900003939393900, + 0xca0000cacacaca00, 0xd50000d5d5d5d500, 0x4700004747474700, + 0x5d00005d5d5d5d00, 0x3d00003d3d3d3d00, 0xd90000d9d9d9d900, + 0x0100000101010100, 0x5a00005a5a5a5a00, 0xd60000d6d6d6d600, + 0x5100005151515100, 0x5600005656565600, 0x6c00006c6c6c6c00, + 0x4d00004d4d4d4d00, 0x8b00008b8b8b8b00, 0x0d00000d0d0d0d00, + 0x9a00009a9a9a9a00, 0x6600006666666600, 0xfb0000fbfbfbfb00, + 0xcc0000cccccccc00, 0xb00000b0b0b0b000, 0x2d00002d2d2d2d00, + 0x7400007474747400, 0x1200001212121200, 0x2b00002b2b2b2b00, + 0x2000002020202000, 0xf00000f0f0f0f000, 0xb10000b1b1b1b100, + 0x8400008484848400, 0x9900009999999900, 0xdf0000dfdfdfdf00, + 0x4c00004c4c4c4c00, 0xcb0000cbcbcbcb00, 0xc20000c2c2c2c200, + 0x3400003434343400, 0x7e00007e7e7e7e00, 0x7600007676767600, + 0x0500000505050500, 0x6d00006d6d6d6d00, 0xb70000b7b7b7b700, + 0xa90000a9a9a9a900, 0x3100003131313100, 0xd10000d1d1d1d100, + 0x1700001717171700, 0x0400000404040400, 0xd70000d7d7d7d700, + 0x1400001414141400, 0x5800005858585800, 0x3a00003a3a3a3a00, + 0x6100006161616100, 0xde0000dededede00, 0x1b00001b1b1b1b00, + 0x1100001111111100, 0x1c00001c1c1c1c00, 0x3200003232323200, + 0x0f00000f0f0f0f00, 0x9c00009c9c9c9c00, 0x1600001616161600, + 0x5300005353535300, 0x1800001818181800, 0xf20000f2f2f2f200, + 0x2200002222222200, 0xfe0000fefefefe00, 0x4400004444444400, + 0xcf0000cfcfcfcf00, 0xb20000b2b2b2b200, 0xc30000c3c3c3c300, + 0xb50000b5b5b5b500, 0x7a00007a7a7a7a00, 0x9100009191919100, + 0x2400002424242400, 0x0800000808080800, 0xe80000e8e8e8e800, + 0xa80000a8a8a8a800, 0x6000006060606000, 0xfc0000fcfcfcfc00, + 0x6900006969696900, 0x5000005050505000, 0xaa0000aaaaaaaa00, + 0xd00000d0d0d0d000, 0xa00000a0a0a0a000, 0x7d00007d7d7d7d00, + 0xa10000a1a1a1a100, 0x8900008989898900, 0x6200006262626200, + 0x9700009797979700, 0x5400005454545400, 0x5b00005b5b5b5b00, + 0x1e00001e1e1e1e00, 0x9500009595959500, 0xe00000e0e0e0e000, + 0xff0000ffffffff00, 0x6400006464646400, 0xd20000d2d2d2d200, + 0x1000001010101000, 0xc40000c4c4c4c400, 0x0000000000000000, + 0x4800004848484800, 0xa30000a3a3a3a300, 0xf70000f7f7f7f700, + 0x7500007575757500, 0xdb0000dbdbdbdb00, 0x8a00008a8a8a8a00, + 0x0300000303030300, 0xe60000e6e6e6e600, 0xda0000dadadada00, + 0x0900000909090900, 0x3f00003f3f3f3f00, 0xdd0000dddddddd00, + 0x9400009494949400, 0x8700008787878700, 0x5c00005c5c5c5c00, + 0x8300008383838300, 0x0200000202020200, 0xcd0000cdcdcdcd00, + 0x4a00004a4a4a4a00, 0x9000009090909000, 0x3300003333333300, + 0x7300007373737300, 0x6700006767676700, 0xf60000f6f6f6f600, + 0xf30000f3f3f3f300, 0x9d00009d9d9d9d00, 0x7f00007f7f7f7f00, + 0xbf0000bfbfbfbf00, 0xe20000e2e2e2e200, 0x5200005252525200, + 0x9b00009b9b9b9b00, 0xd80000d8d8d8d800, 0x2600002626262600, + 0xc80000c8c8c8c800, 0x3700003737373700, 0xc60000c6c6c6c600, + 0x3b00003b3b3b3b00, 0x8100008181818100, 0x9600009696969600, + 0x6f00006f6f6f6f00, 0x4b00004b4b4b4b00, 0x1300001313131300, + 0xbe0000bebebebe00, 0x6300006363636300, 0x2e00002e2e2e2e00, + 0xe90000e9e9e9e900, 0x7900007979797900, 0xa70000a7a7a7a700, + 0x8c00008c8c8c8c00, 0x9f00009f9f9f9f00, 0x6e00006e6e6e6e00, + 0xbc0000bcbcbcbc00, 0x8e00008e8e8e8e00, 0x2900002929292900, + 0xf50000f5f5f5f500, 0xf90000f9f9f9f900, 0xb60000b6b6b6b600, + 0x2f00002f2f2f2f00, 0xfd0000fdfdfdfd00, 0xb40000b4b4b4b400, + 0x5900005959595900, 0x7800007878787800, 0x9800009898989800, + 0x0600000606060600, 0x6a00006a6a6a6a00, 0xe70000e7e7e7e700, + 0x4600004646464600, 0x7100007171717100, 0xba0000babababa00, + 0xd40000d4d4d4d400, 0x2500002525252500, 0xab0000abababab00, + 0x4200004242424200, 0x8800008888888800, 0xa20000a2a2a2a200, + 0x8d00008d8d8d8d00, 0xfa0000fafafafa00, 0x7200007272727200, + 0x0700000707070700, 0xb90000b9b9b9b900, 0x5500005555555500, + 0xf80000f8f8f8f800, 0xee0000eeeeeeee00, 0xac0000acacacac00, + 0x0a00000a0a0a0a00, 0x3600003636363600, 0x4900004949494900, + 0x2a00002a2a2a2a00, 0x6800006868686800, 0x3c00003c3c3c3c00, + 0x3800003838383800, 0xf10000f1f1f1f100, 0xa40000a4a4a4a400, + 0x4000004040404000, 0x2800002828282800, 0xd30000d3d3d3d300, + 0x7b00007b7b7b7b00, 0xbb0000bbbbbbbb00, 0xc90000c9c9c9c900, + 0x4300004343434300, 0xc10000c1c1c1c100, 0x1500001515151500, + 0xe30000e3e3e3e300, 0xad0000adadadad00, 0xf40000f4f4f4f400, + 0x7700007777777700, 0xc70000c7c7c7c700, 0x8000008080808000, + 0x9e00009e9e9e9e00, +}; + +const u64 camellia_sp22000222[256] = { + 0xe0e0000000e0e0e0, 0x0505000000050505, 0x5858000000585858, + 0xd9d9000000d9d9d9, 0x6767000000676767, 0x4e4e0000004e4e4e, + 0x8181000000818181, 0xcbcb000000cbcbcb, 0xc9c9000000c9c9c9, + 0x0b0b0000000b0b0b, 0xaeae000000aeaeae, 0x6a6a0000006a6a6a, + 0xd5d5000000d5d5d5, 0x1818000000181818, 0x5d5d0000005d5d5d, + 0x8282000000828282, 0x4646000000464646, 0xdfdf000000dfdfdf, + 0xd6d6000000d6d6d6, 0x2727000000272727, 0x8a8a0000008a8a8a, + 0x3232000000323232, 0x4b4b0000004b4b4b, 0x4242000000424242, + 0xdbdb000000dbdbdb, 0x1c1c0000001c1c1c, 0x9e9e0000009e9e9e, + 0x9c9c0000009c9c9c, 0x3a3a0000003a3a3a, 0xcaca000000cacaca, + 0x2525000000252525, 0x7b7b0000007b7b7b, 0x0d0d0000000d0d0d, + 0x7171000000717171, 0x5f5f0000005f5f5f, 0x1f1f0000001f1f1f, + 0xf8f8000000f8f8f8, 0xd7d7000000d7d7d7, 0x3e3e0000003e3e3e, + 0x9d9d0000009d9d9d, 0x7c7c0000007c7c7c, 0x6060000000606060, + 0xb9b9000000b9b9b9, 0xbebe000000bebebe, 0xbcbc000000bcbcbc, + 0x8b8b0000008b8b8b, 0x1616000000161616, 0x3434000000343434, + 0x4d4d0000004d4d4d, 0xc3c3000000c3c3c3, 0x7272000000727272, + 0x9595000000959595, 0xabab000000ababab, 0x8e8e0000008e8e8e, + 0xbaba000000bababa, 0x7a7a0000007a7a7a, 0xb3b3000000b3b3b3, + 0x0202000000020202, 0xb4b4000000b4b4b4, 0xadad000000adadad, + 0xa2a2000000a2a2a2, 0xacac000000acacac, 0xd8d8000000d8d8d8, + 0x9a9a0000009a9a9a, 0x1717000000171717, 0x1a1a0000001a1a1a, + 0x3535000000353535, 0xcccc000000cccccc, 0xf7f7000000f7f7f7, + 0x9999000000999999, 0x6161000000616161, 0x5a5a0000005a5a5a, + 0xe8e8000000e8e8e8, 0x2424000000242424, 0x5656000000565656, + 0x4040000000404040, 0xe1e1000000e1e1e1, 0x6363000000636363, + 0x0909000000090909, 0x3333000000333333, 0xbfbf000000bfbfbf, + 0x9898000000989898, 0x9797000000979797, 0x8585000000858585, + 0x6868000000686868, 0xfcfc000000fcfcfc, 0xecec000000ececec, + 0x0a0a0000000a0a0a, 0xdada000000dadada, 0x6f6f0000006f6f6f, + 0x5353000000535353, 0x6262000000626262, 0xa3a3000000a3a3a3, + 0x2e2e0000002e2e2e, 0x0808000000080808, 0xafaf000000afafaf, + 0x2828000000282828, 0xb0b0000000b0b0b0, 0x7474000000747474, + 0xc2c2000000c2c2c2, 0xbdbd000000bdbdbd, 0x3636000000363636, + 0x2222000000222222, 0x3838000000383838, 0x6464000000646464, + 0x1e1e0000001e1e1e, 0x3939000000393939, 0x2c2c0000002c2c2c, + 0xa6a6000000a6a6a6, 0x3030000000303030, 0xe5e5000000e5e5e5, + 0x4444000000444444, 0xfdfd000000fdfdfd, 0x8888000000888888, + 0x9f9f0000009f9f9f, 0x6565000000656565, 0x8787000000878787, + 0x6b6b0000006b6b6b, 0xf4f4000000f4f4f4, 0x2323000000232323, + 0x4848000000484848, 0x1010000000101010, 0xd1d1000000d1d1d1, + 0x5151000000515151, 0xc0c0000000c0c0c0, 0xf9f9000000f9f9f9, + 0xd2d2000000d2d2d2, 0xa0a0000000a0a0a0, 0x5555000000555555, + 0xa1a1000000a1a1a1, 0x4141000000414141, 0xfafa000000fafafa, + 0x4343000000434343, 0x1313000000131313, 0xc4c4000000c4c4c4, + 0x2f2f0000002f2f2f, 0xa8a8000000a8a8a8, 0xb6b6000000b6b6b6, + 0x3c3c0000003c3c3c, 0x2b2b0000002b2b2b, 0xc1c1000000c1c1c1, + 0xffff000000ffffff, 0xc8c8000000c8c8c8, 0xa5a5000000a5a5a5, + 0x2020000000202020, 0x8989000000898989, 0x0000000000000000, + 0x9090000000909090, 0x4747000000474747, 0xefef000000efefef, + 0xeaea000000eaeaea, 0xb7b7000000b7b7b7, 0x1515000000151515, + 0x0606000000060606, 0xcdcd000000cdcdcd, 0xb5b5000000b5b5b5, + 0x1212000000121212, 0x7e7e0000007e7e7e, 0xbbbb000000bbbbbb, + 0x2929000000292929, 0x0f0f0000000f0f0f, 0xb8b8000000b8b8b8, + 0x0707000000070707, 0x0404000000040404, 0x9b9b0000009b9b9b, + 0x9494000000949494, 0x2121000000212121, 0x6666000000666666, + 0xe6e6000000e6e6e6, 0xcece000000cecece, 0xeded000000ededed, + 0xe7e7000000e7e7e7, 0x3b3b0000003b3b3b, 0xfefe000000fefefe, + 0x7f7f0000007f7f7f, 0xc5c5000000c5c5c5, 0xa4a4000000a4a4a4, + 0x3737000000373737, 0xb1b1000000b1b1b1, 0x4c4c0000004c4c4c, + 0x9191000000919191, 0x6e6e0000006e6e6e, 0x8d8d0000008d8d8d, + 0x7676000000767676, 0x0303000000030303, 0x2d2d0000002d2d2d, + 0xdede000000dedede, 0x9696000000969696, 0x2626000000262626, + 0x7d7d0000007d7d7d, 0xc6c6000000c6c6c6, 0x5c5c0000005c5c5c, + 0xd3d3000000d3d3d3, 0xf2f2000000f2f2f2, 0x4f4f0000004f4f4f, + 0x1919000000191919, 0x3f3f0000003f3f3f, 0xdcdc000000dcdcdc, + 0x7979000000797979, 0x1d1d0000001d1d1d, 0x5252000000525252, + 0xebeb000000ebebeb, 0xf3f3000000f3f3f3, 0x6d6d0000006d6d6d, + 0x5e5e0000005e5e5e, 0xfbfb000000fbfbfb, 0x6969000000696969, + 0xb2b2000000b2b2b2, 0xf0f0000000f0f0f0, 0x3131000000313131, + 0x0c0c0000000c0c0c, 0xd4d4000000d4d4d4, 0xcfcf000000cfcfcf, + 0x8c8c0000008c8c8c, 0xe2e2000000e2e2e2, 0x7575000000757575, + 0xa9a9000000a9a9a9, 0x4a4a0000004a4a4a, 0x5757000000575757, + 0x8484000000848484, 0x1111000000111111, 0x4545000000454545, + 0x1b1b0000001b1b1b, 0xf5f5000000f5f5f5, 0xe4e4000000e4e4e4, + 0x0e0e0000000e0e0e, 0x7373000000737373, 0xaaaa000000aaaaaa, + 0xf1f1000000f1f1f1, 0xdddd000000dddddd, 0x5959000000595959, + 0x1414000000141414, 0x6c6c0000006c6c6c, 0x9292000000929292, + 0x5454000000545454, 0xd0d0000000d0d0d0, 0x7878000000787878, + 0x7070000000707070, 0xe3e3000000e3e3e3, 0x4949000000494949, + 0x8080000000808080, 0x5050000000505050, 0xa7a7000000a7a7a7, + 0xf6f6000000f6f6f6, 0x7777000000777777, 0x9393000000939393, + 0x8686000000868686, 0x8383000000838383, 0x2a2a0000002a2a2a, + 0xc7c7000000c7c7c7, 0x5b5b0000005b5b5b, 0xe9e9000000e9e9e9, + 0xeeee000000eeeeee, 0x8f8f0000008f8f8f, 0x0101000000010101, + 0x3d3d0000003d3d3d, +}; + +const u64 camellia_sp03303033[256] = { + 0x0038380038003838, 0x0041410041004141, 0x0016160016001616, + 0x0076760076007676, 0x00d9d900d900d9d9, 0x0093930093009393, + 0x0060600060006060, 0x00f2f200f200f2f2, 0x0072720072007272, + 0x00c2c200c200c2c2, 0x00abab00ab00abab, 0x009a9a009a009a9a, + 0x0075750075007575, 0x0006060006000606, 0x0057570057005757, + 0x00a0a000a000a0a0, 0x0091910091009191, 0x00f7f700f700f7f7, + 0x00b5b500b500b5b5, 0x00c9c900c900c9c9, 0x00a2a200a200a2a2, + 0x008c8c008c008c8c, 0x00d2d200d200d2d2, 0x0090900090009090, + 0x00f6f600f600f6f6, 0x0007070007000707, 0x00a7a700a700a7a7, + 0x0027270027002727, 0x008e8e008e008e8e, 0x00b2b200b200b2b2, + 0x0049490049004949, 0x00dede00de00dede, 0x0043430043004343, + 0x005c5c005c005c5c, 0x00d7d700d700d7d7, 0x00c7c700c700c7c7, + 0x003e3e003e003e3e, 0x00f5f500f500f5f5, 0x008f8f008f008f8f, + 0x0067670067006767, 0x001f1f001f001f1f, 0x0018180018001818, + 0x006e6e006e006e6e, 0x00afaf00af00afaf, 0x002f2f002f002f2f, + 0x00e2e200e200e2e2, 0x0085850085008585, 0x000d0d000d000d0d, + 0x0053530053005353, 0x00f0f000f000f0f0, 0x009c9c009c009c9c, + 0x0065650065006565, 0x00eaea00ea00eaea, 0x00a3a300a300a3a3, + 0x00aeae00ae00aeae, 0x009e9e009e009e9e, 0x00ecec00ec00ecec, + 0x0080800080008080, 0x002d2d002d002d2d, 0x006b6b006b006b6b, + 0x00a8a800a800a8a8, 0x002b2b002b002b2b, 0x0036360036003636, + 0x00a6a600a600a6a6, 0x00c5c500c500c5c5, 0x0086860086008686, + 0x004d4d004d004d4d, 0x0033330033003333, 0x00fdfd00fd00fdfd, + 0x0066660066006666, 0x0058580058005858, 0x0096960096009696, + 0x003a3a003a003a3a, 0x0009090009000909, 0x0095950095009595, + 0x0010100010001010, 0x0078780078007878, 0x00d8d800d800d8d8, + 0x0042420042004242, 0x00cccc00cc00cccc, 0x00efef00ef00efef, + 0x0026260026002626, 0x00e5e500e500e5e5, 0x0061610061006161, + 0x001a1a001a001a1a, 0x003f3f003f003f3f, 0x003b3b003b003b3b, + 0x0082820082008282, 0x00b6b600b600b6b6, 0x00dbdb00db00dbdb, + 0x00d4d400d400d4d4, 0x0098980098009898, 0x00e8e800e800e8e8, + 0x008b8b008b008b8b, 0x0002020002000202, 0x00ebeb00eb00ebeb, + 0x000a0a000a000a0a, 0x002c2c002c002c2c, 0x001d1d001d001d1d, + 0x00b0b000b000b0b0, 0x006f6f006f006f6f, 0x008d8d008d008d8d, + 0x0088880088008888, 0x000e0e000e000e0e, 0x0019190019001919, + 0x0087870087008787, 0x004e4e004e004e4e, 0x000b0b000b000b0b, + 0x00a9a900a900a9a9, 0x000c0c000c000c0c, 0x0079790079007979, + 0x0011110011001111, 0x007f7f007f007f7f, 0x0022220022002222, + 0x00e7e700e700e7e7, 0x0059590059005959, 0x00e1e100e100e1e1, + 0x00dada00da00dada, 0x003d3d003d003d3d, 0x00c8c800c800c8c8, + 0x0012120012001212, 0x0004040004000404, 0x0074740074007474, + 0x0054540054005454, 0x0030300030003030, 0x007e7e007e007e7e, + 0x00b4b400b400b4b4, 0x0028280028002828, 0x0055550055005555, + 0x0068680068006868, 0x0050500050005050, 0x00bebe00be00bebe, + 0x00d0d000d000d0d0, 0x00c4c400c400c4c4, 0x0031310031003131, + 0x00cbcb00cb00cbcb, 0x002a2a002a002a2a, 0x00adad00ad00adad, + 0x000f0f000f000f0f, 0x00caca00ca00caca, 0x0070700070007070, + 0x00ffff00ff00ffff, 0x0032320032003232, 0x0069690069006969, + 0x0008080008000808, 0x0062620062006262, 0x0000000000000000, + 0x0024240024002424, 0x00d1d100d100d1d1, 0x00fbfb00fb00fbfb, + 0x00baba00ba00baba, 0x00eded00ed00eded, 0x0045450045004545, + 0x0081810081008181, 0x0073730073007373, 0x006d6d006d006d6d, + 0x0084840084008484, 0x009f9f009f009f9f, 0x00eeee00ee00eeee, + 0x004a4a004a004a4a, 0x00c3c300c300c3c3, 0x002e2e002e002e2e, + 0x00c1c100c100c1c1, 0x0001010001000101, 0x00e6e600e600e6e6, + 0x0025250025002525, 0x0048480048004848, 0x0099990099009999, + 0x00b9b900b900b9b9, 0x00b3b300b300b3b3, 0x007b7b007b007b7b, + 0x00f9f900f900f9f9, 0x00cece00ce00cece, 0x00bfbf00bf00bfbf, + 0x00dfdf00df00dfdf, 0x0071710071007171, 0x0029290029002929, + 0x00cdcd00cd00cdcd, 0x006c6c006c006c6c, 0x0013130013001313, + 0x0064640064006464, 0x009b9b009b009b9b, 0x0063630063006363, + 0x009d9d009d009d9d, 0x00c0c000c000c0c0, 0x004b4b004b004b4b, + 0x00b7b700b700b7b7, 0x00a5a500a500a5a5, 0x0089890089008989, + 0x005f5f005f005f5f, 0x00b1b100b100b1b1, 0x0017170017001717, + 0x00f4f400f400f4f4, 0x00bcbc00bc00bcbc, 0x00d3d300d300d3d3, + 0x0046460046004646, 0x00cfcf00cf00cfcf, 0x0037370037003737, + 0x005e5e005e005e5e, 0x0047470047004747, 0x0094940094009494, + 0x00fafa00fa00fafa, 0x00fcfc00fc00fcfc, 0x005b5b005b005b5b, + 0x0097970097009797, 0x00fefe00fe00fefe, 0x005a5a005a005a5a, + 0x00acac00ac00acac, 0x003c3c003c003c3c, 0x004c4c004c004c4c, + 0x0003030003000303, 0x0035350035003535, 0x00f3f300f300f3f3, + 0x0023230023002323, 0x00b8b800b800b8b8, 0x005d5d005d005d5d, + 0x006a6a006a006a6a, 0x0092920092009292, 0x00d5d500d500d5d5, + 0x0021210021002121, 0x0044440044004444, 0x0051510051005151, + 0x00c6c600c600c6c6, 0x007d7d007d007d7d, 0x0039390039003939, + 0x0083830083008383, 0x00dcdc00dc00dcdc, 0x00aaaa00aa00aaaa, + 0x007c7c007c007c7c, 0x0077770077007777, 0x0056560056005656, + 0x0005050005000505, 0x001b1b001b001b1b, 0x00a4a400a400a4a4, + 0x0015150015001515, 0x0034340034003434, 0x001e1e001e001e1e, + 0x001c1c001c001c1c, 0x00f8f800f800f8f8, 0x0052520052005252, + 0x0020200020002020, 0x0014140014001414, 0x00e9e900e900e9e9, + 0x00bdbd00bd00bdbd, 0x00dddd00dd00dddd, 0x00e4e400e400e4e4, + 0x00a1a100a100a1a1, 0x00e0e000e000e0e0, 0x008a8a008a008a8a, + 0x00f1f100f100f1f1, 0x00d6d600d600d6d6, 0x007a7a007a007a7a, + 0x00bbbb00bb00bbbb, 0x00e3e300e300e3e3, 0x0040400040004040, + 0x004f4f004f004f4f, +}; + +const u64 camellia_sp00444404[256] = { + 0x0000707070700070, 0x00002c2c2c2c002c, 0x0000b3b3b3b300b3, + 0x0000c0c0c0c000c0, 0x0000e4e4e4e400e4, 0x0000575757570057, + 0x0000eaeaeaea00ea, 0x0000aeaeaeae00ae, 0x0000232323230023, + 0x00006b6b6b6b006b, 0x0000454545450045, 0x0000a5a5a5a500a5, + 0x0000edededed00ed, 0x00004f4f4f4f004f, 0x00001d1d1d1d001d, + 0x0000929292920092, 0x0000868686860086, 0x0000afafafaf00af, + 0x00007c7c7c7c007c, 0x00001f1f1f1f001f, 0x00003e3e3e3e003e, + 0x0000dcdcdcdc00dc, 0x00005e5e5e5e005e, 0x00000b0b0b0b000b, + 0x0000a6a6a6a600a6, 0x0000393939390039, 0x0000d5d5d5d500d5, + 0x00005d5d5d5d005d, 0x0000d9d9d9d900d9, 0x00005a5a5a5a005a, + 0x0000515151510051, 0x00006c6c6c6c006c, 0x00008b8b8b8b008b, + 0x00009a9a9a9a009a, 0x0000fbfbfbfb00fb, 0x0000b0b0b0b000b0, + 0x0000747474740074, 0x00002b2b2b2b002b, 0x0000f0f0f0f000f0, + 0x0000848484840084, 0x0000dfdfdfdf00df, 0x0000cbcbcbcb00cb, + 0x0000343434340034, 0x0000767676760076, 0x00006d6d6d6d006d, + 0x0000a9a9a9a900a9, 0x0000d1d1d1d100d1, 0x0000040404040004, + 0x0000141414140014, 0x00003a3a3a3a003a, 0x0000dededede00de, + 0x0000111111110011, 0x0000323232320032, 0x00009c9c9c9c009c, + 0x0000535353530053, 0x0000f2f2f2f200f2, 0x0000fefefefe00fe, + 0x0000cfcfcfcf00cf, 0x0000c3c3c3c300c3, 0x00007a7a7a7a007a, + 0x0000242424240024, 0x0000e8e8e8e800e8, 0x0000606060600060, + 0x0000696969690069, 0x0000aaaaaaaa00aa, 0x0000a0a0a0a000a0, + 0x0000a1a1a1a100a1, 0x0000626262620062, 0x0000545454540054, + 0x00001e1e1e1e001e, 0x0000e0e0e0e000e0, 0x0000646464640064, + 0x0000101010100010, 0x0000000000000000, 0x0000a3a3a3a300a3, + 0x0000757575750075, 0x00008a8a8a8a008a, 0x0000e6e6e6e600e6, + 0x0000090909090009, 0x0000dddddddd00dd, 0x0000878787870087, + 0x0000838383830083, 0x0000cdcdcdcd00cd, 0x0000909090900090, + 0x0000737373730073, 0x0000f6f6f6f600f6, 0x00009d9d9d9d009d, + 0x0000bfbfbfbf00bf, 0x0000525252520052, 0x0000d8d8d8d800d8, + 0x0000c8c8c8c800c8, 0x0000c6c6c6c600c6, 0x0000818181810081, + 0x00006f6f6f6f006f, 0x0000131313130013, 0x0000636363630063, + 0x0000e9e9e9e900e9, 0x0000a7a7a7a700a7, 0x00009f9f9f9f009f, + 0x0000bcbcbcbc00bc, 0x0000292929290029, 0x0000f9f9f9f900f9, + 0x00002f2f2f2f002f, 0x0000b4b4b4b400b4, 0x0000787878780078, + 0x0000060606060006, 0x0000e7e7e7e700e7, 0x0000717171710071, + 0x0000d4d4d4d400d4, 0x0000abababab00ab, 0x0000888888880088, + 0x00008d8d8d8d008d, 0x0000727272720072, 0x0000b9b9b9b900b9, + 0x0000f8f8f8f800f8, 0x0000acacacac00ac, 0x0000363636360036, + 0x00002a2a2a2a002a, 0x00003c3c3c3c003c, 0x0000f1f1f1f100f1, + 0x0000404040400040, 0x0000d3d3d3d300d3, 0x0000bbbbbbbb00bb, + 0x0000434343430043, 0x0000151515150015, 0x0000adadadad00ad, + 0x0000777777770077, 0x0000808080800080, 0x0000828282820082, + 0x0000ecececec00ec, 0x0000272727270027, 0x0000e5e5e5e500e5, + 0x0000858585850085, 0x0000353535350035, 0x00000c0c0c0c000c, + 0x0000414141410041, 0x0000efefefef00ef, 0x0000939393930093, + 0x0000191919190019, 0x0000212121210021, 0x00000e0e0e0e000e, + 0x00004e4e4e4e004e, 0x0000656565650065, 0x0000bdbdbdbd00bd, + 0x0000b8b8b8b800b8, 0x00008f8f8f8f008f, 0x0000ebebebeb00eb, + 0x0000cececece00ce, 0x0000303030300030, 0x00005f5f5f5f005f, + 0x0000c5c5c5c500c5, 0x00001a1a1a1a001a, 0x0000e1e1e1e100e1, + 0x0000cacacaca00ca, 0x0000474747470047, 0x00003d3d3d3d003d, + 0x0000010101010001, 0x0000d6d6d6d600d6, 0x0000565656560056, + 0x00004d4d4d4d004d, 0x00000d0d0d0d000d, 0x0000666666660066, + 0x0000cccccccc00cc, 0x00002d2d2d2d002d, 0x0000121212120012, + 0x0000202020200020, 0x0000b1b1b1b100b1, 0x0000999999990099, + 0x00004c4c4c4c004c, 0x0000c2c2c2c200c2, 0x00007e7e7e7e007e, + 0x0000050505050005, 0x0000b7b7b7b700b7, 0x0000313131310031, + 0x0000171717170017, 0x0000d7d7d7d700d7, 0x0000585858580058, + 0x0000616161610061, 0x00001b1b1b1b001b, 0x00001c1c1c1c001c, + 0x00000f0f0f0f000f, 0x0000161616160016, 0x0000181818180018, + 0x0000222222220022, 0x0000444444440044, 0x0000b2b2b2b200b2, + 0x0000b5b5b5b500b5, 0x0000919191910091, 0x0000080808080008, + 0x0000a8a8a8a800a8, 0x0000fcfcfcfc00fc, 0x0000505050500050, + 0x0000d0d0d0d000d0, 0x00007d7d7d7d007d, 0x0000898989890089, + 0x0000979797970097, 0x00005b5b5b5b005b, 0x0000959595950095, + 0x0000ffffffff00ff, 0x0000d2d2d2d200d2, 0x0000c4c4c4c400c4, + 0x0000484848480048, 0x0000f7f7f7f700f7, 0x0000dbdbdbdb00db, + 0x0000030303030003, 0x0000dadadada00da, 0x00003f3f3f3f003f, + 0x0000949494940094, 0x00005c5c5c5c005c, 0x0000020202020002, + 0x00004a4a4a4a004a, 0x0000333333330033, 0x0000676767670067, + 0x0000f3f3f3f300f3, 0x00007f7f7f7f007f, 0x0000e2e2e2e200e2, + 0x00009b9b9b9b009b, 0x0000262626260026, 0x0000373737370037, + 0x00003b3b3b3b003b, 0x0000969696960096, 0x00004b4b4b4b004b, + 0x0000bebebebe00be, 0x00002e2e2e2e002e, 0x0000797979790079, + 0x00008c8c8c8c008c, 0x00006e6e6e6e006e, 0x00008e8e8e8e008e, + 0x0000f5f5f5f500f5, 0x0000b6b6b6b600b6, 0x0000fdfdfdfd00fd, + 0x0000595959590059, 0x0000989898980098, 0x00006a6a6a6a006a, + 0x0000464646460046, 0x0000babababa00ba, 0x0000252525250025, + 0x0000424242420042, 0x0000a2a2a2a200a2, 0x0000fafafafa00fa, + 0x0000070707070007, 0x0000555555550055, 0x0000eeeeeeee00ee, + 0x00000a0a0a0a000a, 0x0000494949490049, 0x0000686868680068, + 0x0000383838380038, 0x0000a4a4a4a400a4, 0x0000282828280028, + 0x00007b7b7b7b007b, 0x0000c9c9c9c900c9, 0x0000c1c1c1c100c1, + 0x0000e3e3e3e300e3, 0x0000f4f4f4f400f4, 0x0000c7c7c7c700c7, + 0x00009e9e9e9e009e, +}; + +const u64 camellia_sp02220222[256] = { + 0x00e0e0e000e0e0e0, 0x0005050500050505, 0x0058585800585858, + 0x00d9d9d900d9d9d9, 0x0067676700676767, 0x004e4e4e004e4e4e, + 0x0081818100818181, 0x00cbcbcb00cbcbcb, 0x00c9c9c900c9c9c9, + 0x000b0b0b000b0b0b, 0x00aeaeae00aeaeae, 0x006a6a6a006a6a6a, + 0x00d5d5d500d5d5d5, 0x0018181800181818, 0x005d5d5d005d5d5d, + 0x0082828200828282, 0x0046464600464646, 0x00dfdfdf00dfdfdf, + 0x00d6d6d600d6d6d6, 0x0027272700272727, 0x008a8a8a008a8a8a, + 0x0032323200323232, 0x004b4b4b004b4b4b, 0x0042424200424242, + 0x00dbdbdb00dbdbdb, 0x001c1c1c001c1c1c, 0x009e9e9e009e9e9e, + 0x009c9c9c009c9c9c, 0x003a3a3a003a3a3a, 0x00cacaca00cacaca, + 0x0025252500252525, 0x007b7b7b007b7b7b, 0x000d0d0d000d0d0d, + 0x0071717100717171, 0x005f5f5f005f5f5f, 0x001f1f1f001f1f1f, + 0x00f8f8f800f8f8f8, 0x00d7d7d700d7d7d7, 0x003e3e3e003e3e3e, + 0x009d9d9d009d9d9d, 0x007c7c7c007c7c7c, 0x0060606000606060, + 0x00b9b9b900b9b9b9, 0x00bebebe00bebebe, 0x00bcbcbc00bcbcbc, + 0x008b8b8b008b8b8b, 0x0016161600161616, 0x0034343400343434, + 0x004d4d4d004d4d4d, 0x00c3c3c300c3c3c3, 0x0072727200727272, + 0x0095959500959595, 0x00ababab00ababab, 0x008e8e8e008e8e8e, + 0x00bababa00bababa, 0x007a7a7a007a7a7a, 0x00b3b3b300b3b3b3, + 0x0002020200020202, 0x00b4b4b400b4b4b4, 0x00adadad00adadad, + 0x00a2a2a200a2a2a2, 0x00acacac00acacac, 0x00d8d8d800d8d8d8, + 0x009a9a9a009a9a9a, 0x0017171700171717, 0x001a1a1a001a1a1a, + 0x0035353500353535, 0x00cccccc00cccccc, 0x00f7f7f700f7f7f7, + 0x0099999900999999, 0x0061616100616161, 0x005a5a5a005a5a5a, + 0x00e8e8e800e8e8e8, 0x0024242400242424, 0x0056565600565656, + 0x0040404000404040, 0x00e1e1e100e1e1e1, 0x0063636300636363, + 0x0009090900090909, 0x0033333300333333, 0x00bfbfbf00bfbfbf, + 0x0098989800989898, 0x0097979700979797, 0x0085858500858585, + 0x0068686800686868, 0x00fcfcfc00fcfcfc, 0x00ececec00ececec, + 0x000a0a0a000a0a0a, 0x00dadada00dadada, 0x006f6f6f006f6f6f, + 0x0053535300535353, 0x0062626200626262, 0x00a3a3a300a3a3a3, + 0x002e2e2e002e2e2e, 0x0008080800080808, 0x00afafaf00afafaf, + 0x0028282800282828, 0x00b0b0b000b0b0b0, 0x0074747400747474, + 0x00c2c2c200c2c2c2, 0x00bdbdbd00bdbdbd, 0x0036363600363636, + 0x0022222200222222, 0x0038383800383838, 0x0064646400646464, + 0x001e1e1e001e1e1e, 0x0039393900393939, 0x002c2c2c002c2c2c, + 0x00a6a6a600a6a6a6, 0x0030303000303030, 0x00e5e5e500e5e5e5, + 0x0044444400444444, 0x00fdfdfd00fdfdfd, 0x0088888800888888, + 0x009f9f9f009f9f9f, 0x0065656500656565, 0x0087878700878787, + 0x006b6b6b006b6b6b, 0x00f4f4f400f4f4f4, 0x0023232300232323, + 0x0048484800484848, 0x0010101000101010, 0x00d1d1d100d1d1d1, + 0x0051515100515151, 0x00c0c0c000c0c0c0, 0x00f9f9f900f9f9f9, + 0x00d2d2d200d2d2d2, 0x00a0a0a000a0a0a0, 0x0055555500555555, + 0x00a1a1a100a1a1a1, 0x0041414100414141, 0x00fafafa00fafafa, + 0x0043434300434343, 0x0013131300131313, 0x00c4c4c400c4c4c4, + 0x002f2f2f002f2f2f, 0x00a8a8a800a8a8a8, 0x00b6b6b600b6b6b6, + 0x003c3c3c003c3c3c, 0x002b2b2b002b2b2b, 0x00c1c1c100c1c1c1, + 0x00ffffff00ffffff, 0x00c8c8c800c8c8c8, 0x00a5a5a500a5a5a5, + 0x0020202000202020, 0x0089898900898989, 0x0000000000000000, + 0x0090909000909090, 0x0047474700474747, 0x00efefef00efefef, + 0x00eaeaea00eaeaea, 0x00b7b7b700b7b7b7, 0x0015151500151515, + 0x0006060600060606, 0x00cdcdcd00cdcdcd, 0x00b5b5b500b5b5b5, + 0x0012121200121212, 0x007e7e7e007e7e7e, 0x00bbbbbb00bbbbbb, + 0x0029292900292929, 0x000f0f0f000f0f0f, 0x00b8b8b800b8b8b8, + 0x0007070700070707, 0x0004040400040404, 0x009b9b9b009b9b9b, + 0x0094949400949494, 0x0021212100212121, 0x0066666600666666, + 0x00e6e6e600e6e6e6, 0x00cecece00cecece, 0x00ededed00ededed, + 0x00e7e7e700e7e7e7, 0x003b3b3b003b3b3b, 0x00fefefe00fefefe, + 0x007f7f7f007f7f7f, 0x00c5c5c500c5c5c5, 0x00a4a4a400a4a4a4, + 0x0037373700373737, 0x00b1b1b100b1b1b1, 0x004c4c4c004c4c4c, + 0x0091919100919191, 0x006e6e6e006e6e6e, 0x008d8d8d008d8d8d, + 0x0076767600767676, 0x0003030300030303, 0x002d2d2d002d2d2d, + 0x00dedede00dedede, 0x0096969600969696, 0x0026262600262626, + 0x007d7d7d007d7d7d, 0x00c6c6c600c6c6c6, 0x005c5c5c005c5c5c, + 0x00d3d3d300d3d3d3, 0x00f2f2f200f2f2f2, 0x004f4f4f004f4f4f, + 0x0019191900191919, 0x003f3f3f003f3f3f, 0x00dcdcdc00dcdcdc, + 0x0079797900797979, 0x001d1d1d001d1d1d, 0x0052525200525252, + 0x00ebebeb00ebebeb, 0x00f3f3f300f3f3f3, 0x006d6d6d006d6d6d, + 0x005e5e5e005e5e5e, 0x00fbfbfb00fbfbfb, 0x0069696900696969, + 0x00b2b2b200b2b2b2, 0x00f0f0f000f0f0f0, 0x0031313100313131, + 0x000c0c0c000c0c0c, 0x00d4d4d400d4d4d4, 0x00cfcfcf00cfcfcf, + 0x008c8c8c008c8c8c, 0x00e2e2e200e2e2e2, 0x0075757500757575, + 0x00a9a9a900a9a9a9, 0x004a4a4a004a4a4a, 0x0057575700575757, + 0x0084848400848484, 0x0011111100111111, 0x0045454500454545, + 0x001b1b1b001b1b1b, 0x00f5f5f500f5f5f5, 0x00e4e4e400e4e4e4, + 0x000e0e0e000e0e0e, 0x0073737300737373, 0x00aaaaaa00aaaaaa, + 0x00f1f1f100f1f1f1, 0x00dddddd00dddddd, 0x0059595900595959, + 0x0014141400141414, 0x006c6c6c006c6c6c, 0x0092929200929292, + 0x0054545400545454, 0x00d0d0d000d0d0d0, 0x0078787800787878, + 0x0070707000707070, 0x00e3e3e300e3e3e3, 0x0049494900494949, + 0x0080808000808080, 0x0050505000505050, 0x00a7a7a700a7a7a7, + 0x00f6f6f600f6f6f6, 0x0077777700777777, 0x0093939300939393, + 0x0086868600868686, 0x0083838300838383, 0x002a2a2a002a2a2a, + 0x00c7c7c700c7c7c7, 0x005b5b5b005b5b5b, 0x00e9e9e900e9e9e9, + 0x00eeeeee00eeeeee, 0x008f8f8f008f8f8f, 0x0001010100010101, + 0x003d3d3d003d3d3d, +}; + +const u64 camellia_sp30333033[256] = { + 0x3800383838003838, 0x4100414141004141, 0x1600161616001616, + 0x7600767676007676, 0xd900d9d9d900d9d9, 0x9300939393009393, + 0x6000606060006060, 0xf200f2f2f200f2f2, 0x7200727272007272, + 0xc200c2c2c200c2c2, 0xab00ababab00abab, 0x9a009a9a9a009a9a, + 0x7500757575007575, 0x0600060606000606, 0x5700575757005757, + 0xa000a0a0a000a0a0, 0x9100919191009191, 0xf700f7f7f700f7f7, + 0xb500b5b5b500b5b5, 0xc900c9c9c900c9c9, 0xa200a2a2a200a2a2, + 0x8c008c8c8c008c8c, 0xd200d2d2d200d2d2, 0x9000909090009090, + 0xf600f6f6f600f6f6, 0x0700070707000707, 0xa700a7a7a700a7a7, + 0x2700272727002727, 0x8e008e8e8e008e8e, 0xb200b2b2b200b2b2, + 0x4900494949004949, 0xde00dedede00dede, 0x4300434343004343, + 0x5c005c5c5c005c5c, 0xd700d7d7d700d7d7, 0xc700c7c7c700c7c7, + 0x3e003e3e3e003e3e, 0xf500f5f5f500f5f5, 0x8f008f8f8f008f8f, + 0x6700676767006767, 0x1f001f1f1f001f1f, 0x1800181818001818, + 0x6e006e6e6e006e6e, 0xaf00afafaf00afaf, 0x2f002f2f2f002f2f, + 0xe200e2e2e200e2e2, 0x8500858585008585, 0x0d000d0d0d000d0d, + 0x5300535353005353, 0xf000f0f0f000f0f0, 0x9c009c9c9c009c9c, + 0x6500656565006565, 0xea00eaeaea00eaea, 0xa300a3a3a300a3a3, + 0xae00aeaeae00aeae, 0x9e009e9e9e009e9e, 0xec00ececec00ecec, + 0x8000808080008080, 0x2d002d2d2d002d2d, 0x6b006b6b6b006b6b, + 0xa800a8a8a800a8a8, 0x2b002b2b2b002b2b, 0x3600363636003636, + 0xa600a6a6a600a6a6, 0xc500c5c5c500c5c5, 0x8600868686008686, + 0x4d004d4d4d004d4d, 0x3300333333003333, 0xfd00fdfdfd00fdfd, + 0x6600666666006666, 0x5800585858005858, 0x9600969696009696, + 0x3a003a3a3a003a3a, 0x0900090909000909, 0x9500959595009595, + 0x1000101010001010, 0x7800787878007878, 0xd800d8d8d800d8d8, + 0x4200424242004242, 0xcc00cccccc00cccc, 0xef00efefef00efef, + 0x2600262626002626, 0xe500e5e5e500e5e5, 0x6100616161006161, + 0x1a001a1a1a001a1a, 0x3f003f3f3f003f3f, 0x3b003b3b3b003b3b, + 0x8200828282008282, 0xb600b6b6b600b6b6, 0xdb00dbdbdb00dbdb, + 0xd400d4d4d400d4d4, 0x9800989898009898, 0xe800e8e8e800e8e8, + 0x8b008b8b8b008b8b, 0x0200020202000202, 0xeb00ebebeb00ebeb, + 0x0a000a0a0a000a0a, 0x2c002c2c2c002c2c, 0x1d001d1d1d001d1d, + 0xb000b0b0b000b0b0, 0x6f006f6f6f006f6f, 0x8d008d8d8d008d8d, + 0x8800888888008888, 0x0e000e0e0e000e0e, 0x1900191919001919, + 0x8700878787008787, 0x4e004e4e4e004e4e, 0x0b000b0b0b000b0b, + 0xa900a9a9a900a9a9, 0x0c000c0c0c000c0c, 0x7900797979007979, + 0x1100111111001111, 0x7f007f7f7f007f7f, 0x2200222222002222, + 0xe700e7e7e700e7e7, 0x5900595959005959, 0xe100e1e1e100e1e1, + 0xda00dadada00dada, 0x3d003d3d3d003d3d, 0xc800c8c8c800c8c8, + 0x1200121212001212, 0x0400040404000404, 0x7400747474007474, + 0x5400545454005454, 0x3000303030003030, 0x7e007e7e7e007e7e, + 0xb400b4b4b400b4b4, 0x2800282828002828, 0x5500555555005555, + 0x6800686868006868, 0x5000505050005050, 0xbe00bebebe00bebe, + 0xd000d0d0d000d0d0, 0xc400c4c4c400c4c4, 0x3100313131003131, + 0xcb00cbcbcb00cbcb, 0x2a002a2a2a002a2a, 0xad00adadad00adad, + 0x0f000f0f0f000f0f, 0xca00cacaca00caca, 0x7000707070007070, + 0xff00ffffff00ffff, 0x3200323232003232, 0x6900696969006969, + 0x0800080808000808, 0x6200626262006262, 0x0000000000000000, + 0x2400242424002424, 0xd100d1d1d100d1d1, 0xfb00fbfbfb00fbfb, + 0xba00bababa00baba, 0xed00ededed00eded, 0x4500454545004545, + 0x8100818181008181, 0x7300737373007373, 0x6d006d6d6d006d6d, + 0x8400848484008484, 0x9f009f9f9f009f9f, 0xee00eeeeee00eeee, + 0x4a004a4a4a004a4a, 0xc300c3c3c300c3c3, 0x2e002e2e2e002e2e, + 0xc100c1c1c100c1c1, 0x0100010101000101, 0xe600e6e6e600e6e6, + 0x2500252525002525, 0x4800484848004848, 0x9900999999009999, + 0xb900b9b9b900b9b9, 0xb300b3b3b300b3b3, 0x7b007b7b7b007b7b, + 0xf900f9f9f900f9f9, 0xce00cecece00cece, 0xbf00bfbfbf00bfbf, + 0xdf00dfdfdf00dfdf, 0x7100717171007171, 0x2900292929002929, + 0xcd00cdcdcd00cdcd, 0x6c006c6c6c006c6c, 0x1300131313001313, + 0x6400646464006464, 0x9b009b9b9b009b9b, 0x6300636363006363, + 0x9d009d9d9d009d9d, 0xc000c0c0c000c0c0, 0x4b004b4b4b004b4b, + 0xb700b7b7b700b7b7, 0xa500a5a5a500a5a5, 0x8900898989008989, + 0x5f005f5f5f005f5f, 0xb100b1b1b100b1b1, 0x1700171717001717, + 0xf400f4f4f400f4f4, 0xbc00bcbcbc00bcbc, 0xd300d3d3d300d3d3, + 0x4600464646004646, 0xcf00cfcfcf00cfcf, 0x3700373737003737, + 0x5e005e5e5e005e5e, 0x4700474747004747, 0x9400949494009494, + 0xfa00fafafa00fafa, 0xfc00fcfcfc00fcfc, 0x5b005b5b5b005b5b, + 0x9700979797009797, 0xfe00fefefe00fefe, 0x5a005a5a5a005a5a, + 0xac00acacac00acac, 0x3c003c3c3c003c3c, 0x4c004c4c4c004c4c, + 0x0300030303000303, 0x3500353535003535, 0xf300f3f3f300f3f3, + 0x2300232323002323, 0xb800b8b8b800b8b8, 0x5d005d5d5d005d5d, + 0x6a006a6a6a006a6a, 0x9200929292009292, 0xd500d5d5d500d5d5, + 0x2100212121002121, 0x4400444444004444, 0x5100515151005151, + 0xc600c6c6c600c6c6, 0x7d007d7d7d007d7d, 0x3900393939003939, + 0x8300838383008383, 0xdc00dcdcdc00dcdc, 0xaa00aaaaaa00aaaa, + 0x7c007c7c7c007c7c, 0x7700777777007777, 0x5600565656005656, + 0x0500050505000505, 0x1b001b1b1b001b1b, 0xa400a4a4a400a4a4, + 0x1500151515001515, 0x3400343434003434, 0x1e001e1e1e001e1e, + 0x1c001c1c1c001c1c, 0xf800f8f8f800f8f8, 0x5200525252005252, + 0x2000202020002020, 0x1400141414001414, 0xe900e9e9e900e9e9, + 0xbd00bdbdbd00bdbd, 0xdd00dddddd00dddd, 0xe400e4e4e400e4e4, + 0xa100a1a1a100a1a1, 0xe000e0e0e000e0e0, 0x8a008a8a8a008a8a, + 0xf100f1f1f100f1f1, 0xd600d6d6d600d6d6, 0x7a007a7a7a007a7a, + 0xbb00bbbbbb00bbbb, 0xe300e3e3e300e3e3, 0x4000404040004040, + 0x4f004f4f4f004f4f, +}; + +const u64 camellia_sp44044404[256] = { + 0x7070007070700070, 0x2c2c002c2c2c002c, 0xb3b300b3b3b300b3, + 0xc0c000c0c0c000c0, 0xe4e400e4e4e400e4, 0x5757005757570057, + 0xeaea00eaeaea00ea, 0xaeae00aeaeae00ae, 0x2323002323230023, + 0x6b6b006b6b6b006b, 0x4545004545450045, 0xa5a500a5a5a500a5, + 0xeded00ededed00ed, 0x4f4f004f4f4f004f, 0x1d1d001d1d1d001d, + 0x9292009292920092, 0x8686008686860086, 0xafaf00afafaf00af, + 0x7c7c007c7c7c007c, 0x1f1f001f1f1f001f, 0x3e3e003e3e3e003e, + 0xdcdc00dcdcdc00dc, 0x5e5e005e5e5e005e, 0x0b0b000b0b0b000b, + 0xa6a600a6a6a600a6, 0x3939003939390039, 0xd5d500d5d5d500d5, + 0x5d5d005d5d5d005d, 0xd9d900d9d9d900d9, 0x5a5a005a5a5a005a, + 0x5151005151510051, 0x6c6c006c6c6c006c, 0x8b8b008b8b8b008b, + 0x9a9a009a9a9a009a, 0xfbfb00fbfbfb00fb, 0xb0b000b0b0b000b0, + 0x7474007474740074, 0x2b2b002b2b2b002b, 0xf0f000f0f0f000f0, + 0x8484008484840084, 0xdfdf00dfdfdf00df, 0xcbcb00cbcbcb00cb, + 0x3434003434340034, 0x7676007676760076, 0x6d6d006d6d6d006d, + 0xa9a900a9a9a900a9, 0xd1d100d1d1d100d1, 0x0404000404040004, + 0x1414001414140014, 0x3a3a003a3a3a003a, 0xdede00dedede00de, + 0x1111001111110011, 0x3232003232320032, 0x9c9c009c9c9c009c, + 0x5353005353530053, 0xf2f200f2f2f200f2, 0xfefe00fefefe00fe, + 0xcfcf00cfcfcf00cf, 0xc3c300c3c3c300c3, 0x7a7a007a7a7a007a, + 0x2424002424240024, 0xe8e800e8e8e800e8, 0x6060006060600060, + 0x6969006969690069, 0xaaaa00aaaaaa00aa, 0xa0a000a0a0a000a0, + 0xa1a100a1a1a100a1, 0x6262006262620062, 0x5454005454540054, + 0x1e1e001e1e1e001e, 0xe0e000e0e0e000e0, 0x6464006464640064, + 0x1010001010100010, 0x0000000000000000, 0xa3a300a3a3a300a3, + 0x7575007575750075, 0x8a8a008a8a8a008a, 0xe6e600e6e6e600e6, + 0x0909000909090009, 0xdddd00dddddd00dd, 0x8787008787870087, + 0x8383008383830083, 0xcdcd00cdcdcd00cd, 0x9090009090900090, + 0x7373007373730073, 0xf6f600f6f6f600f6, 0x9d9d009d9d9d009d, + 0xbfbf00bfbfbf00bf, 0x5252005252520052, 0xd8d800d8d8d800d8, + 0xc8c800c8c8c800c8, 0xc6c600c6c6c600c6, 0x8181008181810081, + 0x6f6f006f6f6f006f, 0x1313001313130013, 0x6363006363630063, + 0xe9e900e9e9e900e9, 0xa7a700a7a7a700a7, 0x9f9f009f9f9f009f, + 0xbcbc00bcbcbc00bc, 0x2929002929290029, 0xf9f900f9f9f900f9, + 0x2f2f002f2f2f002f, 0xb4b400b4b4b400b4, 0x7878007878780078, + 0x0606000606060006, 0xe7e700e7e7e700e7, 0x7171007171710071, + 0xd4d400d4d4d400d4, 0xabab00ababab00ab, 0x8888008888880088, + 0x8d8d008d8d8d008d, 0x7272007272720072, 0xb9b900b9b9b900b9, + 0xf8f800f8f8f800f8, 0xacac00acacac00ac, 0x3636003636360036, + 0x2a2a002a2a2a002a, 0x3c3c003c3c3c003c, 0xf1f100f1f1f100f1, + 0x4040004040400040, 0xd3d300d3d3d300d3, 0xbbbb00bbbbbb00bb, + 0x4343004343430043, 0x1515001515150015, 0xadad00adadad00ad, + 0x7777007777770077, 0x8080008080800080, 0x8282008282820082, + 0xecec00ececec00ec, 0x2727002727270027, 0xe5e500e5e5e500e5, + 0x8585008585850085, 0x3535003535350035, 0x0c0c000c0c0c000c, + 0x4141004141410041, 0xefef00efefef00ef, 0x9393009393930093, + 0x1919001919190019, 0x2121002121210021, 0x0e0e000e0e0e000e, + 0x4e4e004e4e4e004e, 0x6565006565650065, 0xbdbd00bdbdbd00bd, + 0xb8b800b8b8b800b8, 0x8f8f008f8f8f008f, 0xebeb00ebebeb00eb, + 0xcece00cecece00ce, 0x3030003030300030, 0x5f5f005f5f5f005f, + 0xc5c500c5c5c500c5, 0x1a1a001a1a1a001a, 0xe1e100e1e1e100e1, + 0xcaca00cacaca00ca, 0x4747004747470047, 0x3d3d003d3d3d003d, + 0x0101000101010001, 0xd6d600d6d6d600d6, 0x5656005656560056, + 0x4d4d004d4d4d004d, 0x0d0d000d0d0d000d, 0x6666006666660066, + 0xcccc00cccccc00cc, 0x2d2d002d2d2d002d, 0x1212001212120012, + 0x2020002020200020, 0xb1b100b1b1b100b1, 0x9999009999990099, + 0x4c4c004c4c4c004c, 0xc2c200c2c2c200c2, 0x7e7e007e7e7e007e, + 0x0505000505050005, 0xb7b700b7b7b700b7, 0x3131003131310031, + 0x1717001717170017, 0xd7d700d7d7d700d7, 0x5858005858580058, + 0x6161006161610061, 0x1b1b001b1b1b001b, 0x1c1c001c1c1c001c, + 0x0f0f000f0f0f000f, 0x1616001616160016, 0x1818001818180018, + 0x2222002222220022, 0x4444004444440044, 0xb2b200b2b2b200b2, + 0xb5b500b5b5b500b5, 0x9191009191910091, 0x0808000808080008, + 0xa8a800a8a8a800a8, 0xfcfc00fcfcfc00fc, 0x5050005050500050, + 0xd0d000d0d0d000d0, 0x7d7d007d7d7d007d, 0x8989008989890089, + 0x9797009797970097, 0x5b5b005b5b5b005b, 0x9595009595950095, + 0xffff00ffffff00ff, 0xd2d200d2d2d200d2, 0xc4c400c4c4c400c4, + 0x4848004848480048, 0xf7f700f7f7f700f7, 0xdbdb00dbdbdb00db, + 0x0303000303030003, 0xdada00dadada00da, 0x3f3f003f3f3f003f, + 0x9494009494940094, 0x5c5c005c5c5c005c, 0x0202000202020002, + 0x4a4a004a4a4a004a, 0x3333003333330033, 0x6767006767670067, + 0xf3f300f3f3f300f3, 0x7f7f007f7f7f007f, 0xe2e200e2e2e200e2, + 0x9b9b009b9b9b009b, 0x2626002626260026, 0x3737003737370037, + 0x3b3b003b3b3b003b, 0x9696009696960096, 0x4b4b004b4b4b004b, + 0xbebe00bebebe00be, 0x2e2e002e2e2e002e, 0x7979007979790079, + 0x8c8c008c8c8c008c, 0x6e6e006e6e6e006e, 0x8e8e008e8e8e008e, + 0xf5f500f5f5f500f5, 0xb6b600b6b6b600b6, 0xfdfd00fdfdfd00fd, + 0x5959005959590059, 0x9898009898980098, 0x6a6a006a6a6a006a, + 0x4646004646460046, 0xbaba00bababa00ba, 0x2525002525250025, + 0x4242004242420042, 0xa2a200a2a2a200a2, 0xfafa00fafafa00fa, + 0x0707000707070007, 0x5555005555550055, 0xeeee00eeeeee00ee, + 0x0a0a000a0a0a000a, 0x4949004949490049, 0x6868006868680068, + 0x3838003838380038, 0xa4a400a4a4a400a4, 0x2828002828280028, + 0x7b7b007b7b7b007b, 0xc9c900c9c9c900c9, 0xc1c100c1c1c100c1, + 0xe3e300e3e3e300e3, 0xf4f400f4f4f400f4, 0xc7c700c7c7c700c7, + 0x9e9e009e9e9e009e, +}; + +const u64 camellia_sp11101110[256] = { + 0x7070700070707000, 0x8282820082828200, 0x2c2c2c002c2c2c00, + 0xececec00ececec00, 0xb3b3b300b3b3b300, 0x2727270027272700, + 0xc0c0c000c0c0c000, 0xe5e5e500e5e5e500, 0xe4e4e400e4e4e400, + 0x8585850085858500, 0x5757570057575700, 0x3535350035353500, + 0xeaeaea00eaeaea00, 0x0c0c0c000c0c0c00, 0xaeaeae00aeaeae00, + 0x4141410041414100, 0x2323230023232300, 0xefefef00efefef00, + 0x6b6b6b006b6b6b00, 0x9393930093939300, 0x4545450045454500, + 0x1919190019191900, 0xa5a5a500a5a5a500, 0x2121210021212100, + 0xededed00ededed00, 0x0e0e0e000e0e0e00, 0x4f4f4f004f4f4f00, + 0x4e4e4e004e4e4e00, 0x1d1d1d001d1d1d00, 0x6565650065656500, + 0x9292920092929200, 0xbdbdbd00bdbdbd00, 0x8686860086868600, + 0xb8b8b800b8b8b800, 0xafafaf00afafaf00, 0x8f8f8f008f8f8f00, + 0x7c7c7c007c7c7c00, 0xebebeb00ebebeb00, 0x1f1f1f001f1f1f00, + 0xcecece00cecece00, 0x3e3e3e003e3e3e00, 0x3030300030303000, + 0xdcdcdc00dcdcdc00, 0x5f5f5f005f5f5f00, 0x5e5e5e005e5e5e00, + 0xc5c5c500c5c5c500, 0x0b0b0b000b0b0b00, 0x1a1a1a001a1a1a00, + 0xa6a6a600a6a6a600, 0xe1e1e100e1e1e100, 0x3939390039393900, + 0xcacaca00cacaca00, 0xd5d5d500d5d5d500, 0x4747470047474700, + 0x5d5d5d005d5d5d00, 0x3d3d3d003d3d3d00, 0xd9d9d900d9d9d900, + 0x0101010001010100, 0x5a5a5a005a5a5a00, 0xd6d6d600d6d6d600, + 0x5151510051515100, 0x5656560056565600, 0x6c6c6c006c6c6c00, + 0x4d4d4d004d4d4d00, 0x8b8b8b008b8b8b00, 0x0d0d0d000d0d0d00, + 0x9a9a9a009a9a9a00, 0x6666660066666600, 0xfbfbfb00fbfbfb00, + 0xcccccc00cccccc00, 0xb0b0b000b0b0b000, 0x2d2d2d002d2d2d00, + 0x7474740074747400, 0x1212120012121200, 0x2b2b2b002b2b2b00, + 0x2020200020202000, 0xf0f0f000f0f0f000, 0xb1b1b100b1b1b100, + 0x8484840084848400, 0x9999990099999900, 0xdfdfdf00dfdfdf00, + 0x4c4c4c004c4c4c00, 0xcbcbcb00cbcbcb00, 0xc2c2c200c2c2c200, + 0x3434340034343400, 0x7e7e7e007e7e7e00, 0x7676760076767600, + 0x0505050005050500, 0x6d6d6d006d6d6d00, 0xb7b7b700b7b7b700, + 0xa9a9a900a9a9a900, 0x3131310031313100, 0xd1d1d100d1d1d100, + 0x1717170017171700, 0x0404040004040400, 0xd7d7d700d7d7d700, + 0x1414140014141400, 0x5858580058585800, 0x3a3a3a003a3a3a00, + 0x6161610061616100, 0xdedede00dedede00, 0x1b1b1b001b1b1b00, + 0x1111110011111100, 0x1c1c1c001c1c1c00, 0x3232320032323200, + 0x0f0f0f000f0f0f00, 0x9c9c9c009c9c9c00, 0x1616160016161600, + 0x5353530053535300, 0x1818180018181800, 0xf2f2f200f2f2f200, + 0x2222220022222200, 0xfefefe00fefefe00, 0x4444440044444400, + 0xcfcfcf00cfcfcf00, 0xb2b2b200b2b2b200, 0xc3c3c300c3c3c300, + 0xb5b5b500b5b5b500, 0x7a7a7a007a7a7a00, 0x9191910091919100, + 0x2424240024242400, 0x0808080008080800, 0xe8e8e800e8e8e800, + 0xa8a8a800a8a8a800, 0x6060600060606000, 0xfcfcfc00fcfcfc00, + 0x6969690069696900, 0x5050500050505000, 0xaaaaaa00aaaaaa00, + 0xd0d0d000d0d0d000, 0xa0a0a000a0a0a000, 0x7d7d7d007d7d7d00, + 0xa1a1a100a1a1a100, 0x8989890089898900, 0x6262620062626200, + 0x9797970097979700, 0x5454540054545400, 0x5b5b5b005b5b5b00, + 0x1e1e1e001e1e1e00, 0x9595950095959500, 0xe0e0e000e0e0e000, + 0xffffff00ffffff00, 0x6464640064646400, 0xd2d2d200d2d2d200, + 0x1010100010101000, 0xc4c4c400c4c4c400, 0x0000000000000000, + 0x4848480048484800, 0xa3a3a300a3a3a300, 0xf7f7f700f7f7f700, + 0x7575750075757500, 0xdbdbdb00dbdbdb00, 0x8a8a8a008a8a8a00, + 0x0303030003030300, 0xe6e6e600e6e6e600, 0xdadada00dadada00, + 0x0909090009090900, 0x3f3f3f003f3f3f00, 0xdddddd00dddddd00, + 0x9494940094949400, 0x8787870087878700, 0x5c5c5c005c5c5c00, + 0x8383830083838300, 0x0202020002020200, 0xcdcdcd00cdcdcd00, + 0x4a4a4a004a4a4a00, 0x9090900090909000, 0x3333330033333300, + 0x7373730073737300, 0x6767670067676700, 0xf6f6f600f6f6f600, + 0xf3f3f300f3f3f300, 0x9d9d9d009d9d9d00, 0x7f7f7f007f7f7f00, + 0xbfbfbf00bfbfbf00, 0xe2e2e200e2e2e200, 0x5252520052525200, + 0x9b9b9b009b9b9b00, 0xd8d8d800d8d8d800, 0x2626260026262600, + 0xc8c8c800c8c8c800, 0x3737370037373700, 0xc6c6c600c6c6c600, + 0x3b3b3b003b3b3b00, 0x8181810081818100, 0x9696960096969600, + 0x6f6f6f006f6f6f00, 0x4b4b4b004b4b4b00, 0x1313130013131300, + 0xbebebe00bebebe00, 0x6363630063636300, 0x2e2e2e002e2e2e00, + 0xe9e9e900e9e9e900, 0x7979790079797900, 0xa7a7a700a7a7a700, + 0x8c8c8c008c8c8c00, 0x9f9f9f009f9f9f00, 0x6e6e6e006e6e6e00, + 0xbcbcbc00bcbcbc00, 0x8e8e8e008e8e8e00, 0x2929290029292900, + 0xf5f5f500f5f5f500, 0xf9f9f900f9f9f900, 0xb6b6b600b6b6b600, + 0x2f2f2f002f2f2f00, 0xfdfdfd00fdfdfd00, 0xb4b4b400b4b4b400, + 0x5959590059595900, 0x7878780078787800, 0x9898980098989800, + 0x0606060006060600, 0x6a6a6a006a6a6a00, 0xe7e7e700e7e7e700, + 0x4646460046464600, 0x7171710071717100, 0xbababa00bababa00, + 0xd4d4d400d4d4d400, 0x2525250025252500, 0xababab00ababab00, + 0x4242420042424200, 0x8888880088888800, 0xa2a2a200a2a2a200, + 0x8d8d8d008d8d8d00, 0xfafafa00fafafa00, 0x7272720072727200, + 0x0707070007070700, 0xb9b9b900b9b9b900, 0x5555550055555500, + 0xf8f8f800f8f8f800, 0xeeeeee00eeeeee00, 0xacacac00acacac00, + 0x0a0a0a000a0a0a00, 0x3636360036363600, 0x4949490049494900, + 0x2a2a2a002a2a2a00, 0x6868680068686800, 0x3c3c3c003c3c3c00, + 0x3838380038383800, 0xf1f1f100f1f1f100, 0xa4a4a400a4a4a400, + 0x4040400040404000, 0x2828280028282800, 0xd3d3d300d3d3d300, + 0x7b7b7b007b7b7b00, 0xbbbbbb00bbbbbb00, 0xc9c9c900c9c9c900, + 0x4343430043434300, 0xc1c1c100c1c1c100, 0x1515150015151500, + 0xe3e3e300e3e3e300, 0xadadad00adadad00, 0xf4f4f400f4f4f400, + 0x7777770077777700, 0xc7c7c700c7c7c700, 0x8080800080808000, + 0x9e9e9e009e9e9e00, +}; + +/* key constants */ +#define CAMELLIA_SIGMA1L (0xA09E667FL) +#define CAMELLIA_SIGMA1R (0x3BCC908BL) +#define CAMELLIA_SIGMA2L (0xB67AE858L) +#define CAMELLIA_SIGMA2R (0x4CAA73B2L) +#define CAMELLIA_SIGMA3L (0xC6EF372FL) +#define CAMELLIA_SIGMA3R (0xE94F82BEL) +#define CAMELLIA_SIGMA4L (0x54FF53A5L) +#define CAMELLIA_SIGMA4R (0xF1D36F1CL) +#define CAMELLIA_SIGMA5L (0x10E527FAL) +#define CAMELLIA_SIGMA5R (0xDE682D1DL) +#define CAMELLIA_SIGMA6L (0xB05688C2L) +#define CAMELLIA_SIGMA6R (0xB3E6C1FDL) + +/* macros */ +#define ROLDQ(l, r, bits) ({ \ + u64 t = l; \ + l = (l << bits) | (r >> (64 - bits)); \ + r = (r << bits) | (t >> (64 - bits)); \ +}) + +#define CAMELLIA_F(x, kl, kr, y) ({ \ + u64 ii = x ^ (((u64)kl << 32) | kr); \ + y = camellia_sp11101110[(uint8_t)ii]; \ + y ^= camellia_sp44044404[(uint8_t)(ii >> 8)]; \ + ii >>= 16; \ + y ^= camellia_sp30333033[(uint8_t)ii]; \ + y ^= camellia_sp02220222[(uint8_t)(ii >> 8)]; \ + ii >>= 16; \ + y ^= camellia_sp00444404[(uint8_t)ii]; \ + y ^= camellia_sp03303033[(uint8_t)(ii >> 8)]; \ + ii >>= 16; \ + y ^= camellia_sp22000222[(uint8_t)ii]; \ + y ^= camellia_sp10011110[(uint8_t)(ii >> 8)]; \ + y = ror64(y, 32); \ +}) + +#define SET_SUBKEY_LR(INDEX, sRL) (subkey[(INDEX)] = ror64((sRL), 32)) + +static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) +{ + u64 kw4, tt; + u32 dw, tl, tr; + + /* absorb kw2 to other subkeys */ + /* round 2 */ + subRL[3] ^= subRL[1]; + /* round 4 */ + subRL[5] ^= subRL[1]; + /* round 6 */ + subRL[7] ^= subRL[1]; + + subRL[1] ^= (subRL[1] & ~subRL[9]) << 32; + /* modified for FLinv(kl2) */ + dw = (subRL[1] & subRL[9]) >> 32, + subRL[1] ^= rol32(dw, 1); + + /* round 8 */ + subRL[11] ^= subRL[1]; + /* round 10 */ + subRL[13] ^= subRL[1]; + /* round 12 */ + subRL[15] ^= subRL[1]; + + subRL[1] ^= (subRL[1] & ~subRL[17]) << 32; + /* modified for FLinv(kl4) */ + dw = (subRL[1] & subRL[17]) >> 32, + subRL[1] ^= rol32(dw, 1); + + /* round 14 */ + subRL[19] ^= subRL[1]; + /* round 16 */ + subRL[21] ^= subRL[1]; + /* round 18 */ + subRL[23] ^= subRL[1]; + + if (max == 24) { + /* kw3 */ + subRL[24] ^= subRL[1]; + + /* absorb kw4 to other subkeys */ + kw4 = subRL[25]; + } else { + subRL[1] ^= (subRL[1] & ~subRL[25]) << 32; + /* modified for FLinv(kl6) */ + dw = (subRL[1] & subRL[25]) >> 32, + subRL[1] ^= rol32(dw, 1); + + /* round 20 */ + subRL[27] ^= subRL[1]; + /* round 22 */ + subRL[29] ^= subRL[1]; + /* round 24 */ + subRL[31] ^= subRL[1]; + /* kw3 */ + subRL[32] ^= subRL[1]; + + /* absorb kw4 to other subkeys */ + kw4 = subRL[33]; + /* round 23 */ + subRL[30] ^= kw4; + /* round 21 */ + subRL[28] ^= kw4; + /* round 19 */ + subRL[26] ^= kw4; + + kw4 ^= (kw4 & ~subRL[24]) << 32; + /* modified for FL(kl5) */ + dw = (kw4 & subRL[24]) >> 32, + kw4 ^= rol32(dw, 1); + } + + /* round 17 */ + subRL[22] ^= kw4; + /* round 15 */ + subRL[20] ^= kw4; + /* round 13 */ + subRL[18] ^= kw4; + + kw4 ^= (kw4 & ~subRL[16]) << 32; + /* modified for FL(kl3) */ + dw = (kw4 & subRL[16]) >> 32, + kw4 ^= rol32(dw, 1); + + /* round 11 */ + subRL[14] ^= kw4; + /* round 9 */ + subRL[12] ^= kw4; + /* round 7 */ + subRL[10] ^= kw4; + + kw4 ^= (kw4 & ~subRL[8]) << 32; + /* modified for FL(kl1) */ + dw = (kw4 & subRL[8]) >> 32, + kw4 ^= rol32(dw, 1); + + /* round 5 */ + subRL[6] ^= kw4; + /* round 3 */ + subRL[4] ^= kw4; + /* round 1 */ + subRL[2] ^= kw4; + /* kw1 */ + subRL[0] ^= kw4; + + /* key XOR is end of F-function */ + SET_SUBKEY_LR(0, subRL[0] ^ subRL[2]); /* kw1 */ + SET_SUBKEY_LR(2, subRL[3]); /* round 1 */ + SET_SUBKEY_LR(3, subRL[2] ^ subRL[4]); /* round 2 */ + SET_SUBKEY_LR(4, subRL[3] ^ subRL[5]); /* round 3 */ + SET_SUBKEY_LR(5, subRL[4] ^ subRL[6]); /* round 4 */ + SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]); /* round 5 */ + + tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]); + dw = tl & (subRL[8] >> 32), /* FL(kl1) */ + tr = subRL[10] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(7, subRL[6] ^ tt); /* round 6 */ + SET_SUBKEY_LR(8, subRL[8]); /* FL(kl1) */ + SET_SUBKEY_LR(9, subRL[9]); /* FLinv(kl2) */ + + tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]); + dw = tl & (subRL[9] >> 32), /* FLinv(kl2) */ + tr = subRL[7] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(10, subRL[11] ^ tt); /* round 7 */ + SET_SUBKEY_LR(11, subRL[10] ^ subRL[12]); /* round 8 */ + SET_SUBKEY_LR(12, subRL[11] ^ subRL[13]); /* round 9 */ + SET_SUBKEY_LR(13, subRL[12] ^ subRL[14]); /* round 10 */ + SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]); /* round 11 */ + + tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]); + dw = tl & (subRL[16] >> 32), /* FL(kl3) */ + tr = subRL[18] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(15, subRL[14] ^ tt); /* round 12 */ + SET_SUBKEY_LR(16, subRL[16]); /* FL(kl3) */ + SET_SUBKEY_LR(17, subRL[17]); /* FLinv(kl4) */ + + tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]); + dw = tl & (subRL[17] >> 32), /* FLinv(kl4) */ + tr = subRL[15] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(18, subRL[19] ^ tt); /* round 13 */ + SET_SUBKEY_LR(19, subRL[18] ^ subRL[20]); /* round 14 */ + SET_SUBKEY_LR(20, subRL[19] ^ subRL[21]); /* round 15 */ + SET_SUBKEY_LR(21, subRL[20] ^ subRL[22]); /* round 16 */ + SET_SUBKEY_LR(22, subRL[21] ^ subRL[23]); /* round 17 */ + + if (max == 24) { + SET_SUBKEY_LR(23, subRL[22]); /* round 18 */ + SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]); /* kw3 */ + } else { + tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]); + dw = tl & (subRL[24] >> 32), /* FL(kl5) */ + tr = subRL[26] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(23, subRL[22] ^ tt); /* round 18 */ + SET_SUBKEY_LR(24, subRL[24]); /* FL(kl5) */ + SET_SUBKEY_LR(25, subRL[25]); /* FLinv(kl6) */ + + tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]); + dw = tl & (subRL[25] >> 32), /* FLinv(kl6) */ + tr = subRL[23] ^ rol32(dw, 1); + tt = (tr | ((u64)tl << 32)); + + SET_SUBKEY_LR(26, subRL[27] ^ tt); /* round 19 */ + SET_SUBKEY_LR(27, subRL[26] ^ subRL[28]); /* round 20 */ + SET_SUBKEY_LR(28, subRL[27] ^ subRL[29]); /* round 21 */ + SET_SUBKEY_LR(29, subRL[28] ^ subRL[30]); /* round 22 */ + SET_SUBKEY_LR(30, subRL[29] ^ subRL[31]); /* round 23 */ + SET_SUBKEY_LR(31, subRL[30]); /* round 24 */ + SET_SUBKEY_LR(32, subRL[32] ^ subRL[31]); /* kw3 */ + } +} + +static void camellia_setup128(const unsigned char *key, u64 *subkey) +{ + u64 kl, kr, ww; + u64 subRL[26]; + + /** + * k == kl || kr (|| is concatenation) + */ + kl = get_unaligned_be64(key); + kr = get_unaligned_be64(key + 8); + + /* generate KL dependent subkeys */ + /* kw1 */ + subRL[0] = kl; + /* kw2 */ + subRL[1] = kr; + + /* rotation left shift 15bit */ + ROLDQ(kl, kr, 15); + + /* k3 */ + subRL[4] = kl; + /* k4 */ + subRL[5] = kr; + + /* rotation left shift 15+30bit */ + ROLDQ(kl, kr, 30); + + /* k7 */ + subRL[10] = kl; + /* k8 */ + subRL[11] = kr; + + /* rotation left shift 15+30+15bit */ + ROLDQ(kl, kr, 15); + + /* k10 */ + subRL[13] = kr; + /* rotation left shift 15+30+15+17 bit */ + ROLDQ(kl, kr, 17); + + /* kl3 */ + subRL[16] = kl; + /* kl4 */ + subRL[17] = kr; + + /* rotation left shift 15+30+15+17+17 bit */ + ROLDQ(kl, kr, 17); + + /* k13 */ + subRL[18] = kl; + /* k14 */ + subRL[19] = kr; + + /* rotation left shift 15+30+15+17+17+17 bit */ + ROLDQ(kl, kr, 17); + + /* k17 */ + subRL[22] = kl; + /* k18 */ + subRL[23] = kr; + + /* generate KA */ + kl = subRL[0]; + kr = subRL[1]; + CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww); + kr ^= ww; + CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl); + + /* current status == (kll, klr, w0, w1) */ + CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr); + kr ^= ww; + CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww); + kl ^= ww; + + /* generate KA dependent subkeys */ + /* k1, k2 */ + subRL[2] = kl; + subRL[3] = kr; + ROLDQ(kl, kr, 15); + /* k5,k6 */ + subRL[6] = kl; + subRL[7] = kr; + ROLDQ(kl, kr, 15); + /* kl1, kl2 */ + subRL[8] = kl; + subRL[9] = kr; + ROLDQ(kl, kr, 15); + /* k9 */ + subRL[12] = kl; + ROLDQ(kl, kr, 15); + /* k11, k12 */ + subRL[14] = kl; + subRL[15] = kr; + ROLDQ(kl, kr, 34); + /* k15, k16 */ + subRL[20] = kl; + subRL[21] = kr; + ROLDQ(kl, kr, 17); + /* kw3, kw4 */ + subRL[24] = kl; + subRL[25] = kr; + + camellia_setup_tail(subkey, subRL, 24); +} + +static void camellia_setup256(const unsigned char *key, u64 *subkey) +{ + u64 kl, kr; /* left half of key */ + u64 krl, krr; /* right half of key */ + u64 ww; /* temporary variables */ + u64 subRL[34]; + + /** + * key = (kl || kr || krl || krr) (|| is concatenation) + */ + kl = get_unaligned_be64(key); + kr = get_unaligned_be64(key + 8); + krl = get_unaligned_be64(key + 16); + krr = get_unaligned_be64(key + 24); + + /* generate KL dependent subkeys */ + /* kw1 */ + subRL[0] = kl; + /* kw2 */ + subRL[1] = kr; + ROLDQ(kl, kr, 45); + /* k9 */ + subRL[12] = kl; + /* k10 */ + subRL[13] = kr; + ROLDQ(kl, kr, 15); + /* kl3 */ + subRL[16] = kl; + /* kl4 */ + subRL[17] = kr; + ROLDQ(kl, kr, 17); + /* k17 */ + subRL[22] = kl; + /* k18 */ + subRL[23] = kr; + ROLDQ(kl, kr, 34); + /* k23 */ + subRL[30] = kl; + /* k24 */ + subRL[31] = kr; + + /* generate KR dependent subkeys */ + ROLDQ(krl, krr, 15); + /* k3 */ + subRL[4] = krl; + /* k4 */ + subRL[5] = krr; + ROLDQ(krl, krr, 15); + /* kl1 */ + subRL[8] = krl; + /* kl2 */ + subRL[9] = krr; + ROLDQ(krl, krr, 30); + /* k13 */ + subRL[18] = krl; + /* k14 */ + subRL[19] = krr; + ROLDQ(krl, krr, 34); + /* k19 */ + subRL[26] = krl; + /* k20 */ + subRL[27] = krr; + ROLDQ(krl, krr, 34); + + /* generate KA */ + kl = subRL[0] ^ krl; + kr = subRL[1] ^ krr; + + CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww); + kr ^= ww; + CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl); + kl ^= krl; + CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr); + kr ^= ww ^ krr; + CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww); + kl ^= ww; + + /* generate KB */ + krl ^= kl; + krr ^= kr; + CAMELLIA_F(krl, CAMELLIA_SIGMA5L, CAMELLIA_SIGMA5R, ww); + krr ^= ww; + CAMELLIA_F(krr, CAMELLIA_SIGMA6L, CAMELLIA_SIGMA6R, ww); + krl ^= ww; + + /* generate KA dependent subkeys */ + ROLDQ(kl, kr, 15); + /* k5 */ + subRL[6] = kl; + /* k6 */ + subRL[7] = kr; + ROLDQ(kl, kr, 30); + /* k11 */ + subRL[14] = kl; + /* k12 */ + subRL[15] = kr; + /* rotation left shift 32bit */ + ROLDQ(kl, kr, 32); + /* kl5 */ + subRL[24] = kl; + /* kl6 */ + subRL[25] = kr; + /* rotation left shift 17 from k11,k12 -> k21,k22 */ + ROLDQ(kl, kr, 17); + /* k21 */ + subRL[28] = kl; + /* k22 */ + subRL[29] = kr; + + /* generate KB dependent subkeys */ + /* k1 */ + subRL[2] = krl; + /* k2 */ + subRL[3] = krr; + ROLDQ(krl, krr, 30); + /* k7 */ + subRL[10] = krl; + /* k8 */ + subRL[11] = krr; + ROLDQ(krl, krr, 30); + /* k15 */ + subRL[20] = krl; + /* k16 */ + subRL[21] = krr; + ROLDQ(krl, krr, 51); + /* kw3 */ + subRL[32] = krl; + /* kw4 */ + subRL[33] = krr; + + camellia_setup_tail(subkey, subRL, 32); +} + +static void camellia_setup192(const unsigned char *key, u64 *subkey) +{ + unsigned char kk[32]; + u64 krl, krr; + + memcpy(kk, key, 24); + memcpy((unsigned char *)&krl, key+16, 8); + krr = ~krl; + memcpy(kk+24, (unsigned char *)&krr, 8); + camellia_setup256(kk, subkey); +} + +static int __camellia_setkey(struct camellia_ctx *cctx, + const unsigned char *key, + unsigned int key_len, u32 *flags) +{ + if (key_len != 16 && key_len != 24 && key_len != 32) { + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + cctx->key_length = key_len; + + switch (key_len) { + case 16: + camellia_setup128(key, cctx->key_table); + break; + case 24: + camellia_setup192(key, cctx->key_table); + break; + case 32: + camellia_setup256(key, cctx->key_table); + break; + } + + return 0; +} + +static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len, + &tfm->crt_flags); +} + +static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, + void (*fn)(struct camellia_ctx *, u8 *, const u8 *), + void (*fn_2way)(struct camellia_ctx *, u8 *, const u8 *)) +{ + struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = CAMELLIA_BLOCK_SIZE; + unsigned int nbytes; + int err; + + err = blkcipher_walk_virt(desc, walk); + + while ((nbytes = walk->nbytes)) { + u8 *wsrc = walk->src.virt.addr; + u8 *wdst = walk->dst.virt.addr; + + /* Process two block batch */ + if (nbytes >= bsize * 2) { + do { + fn_2way(ctx, wdst, wsrc); + + wsrc += bsize * 2; + wdst += bsize * 2; + nbytes -= bsize * 2; + } while (nbytes >= bsize * 2); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + do { + fn(ctx, wdst, wsrc); + + wsrc += bsize; + wdst += bsize; + nbytes -= bsize; + } while (nbytes >= bsize); + +done: + err = blkcipher_walk_done(desc, walk, nbytes); + } + + return err; +} + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_crypt(desc, &walk, camellia_enc_blk, camellia_enc_blk_2way); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_crypt(desc, &walk, camellia_dec_blk, camellia_dec_blk_2way); +} + +static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = CAMELLIA_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 *iv = (u128 *)walk->iv; + + do { + u128_xor(dst, src, iv); + camellia_enc_blk(ctx, (u8 *)dst, (u8 *)dst); + iv = dst; + + src += 1; + dst += 1; + nbytes -= bsize; + } while (nbytes >= bsize); + + u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); + return nbytes; +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + nbytes = __cbc_encrypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + return err; +} + +static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = CAMELLIA_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 ivs[2 - 1]; + u128 last_iv; + + /* Start of the last block. */ + src += nbytes / bsize - 1; + dst += nbytes / bsize - 1; + + last_iv = *src; + + /* Process two block batch */ + if (nbytes >= bsize * 2) { + do { + nbytes -= bsize * (2 - 1); + src -= 2 - 1; + dst -= 2 - 1; + + ivs[0] = src[0]; + + camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src); + + u128_xor(dst + 1, dst + 1, ivs + 0); + + nbytes -= bsize; + if (nbytes < bsize) + goto done; + + u128_xor(dst, dst, src - 1); + src -= 1; + dst -= 1; + } while (nbytes >= bsize * 2); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + for (;;) { + camellia_dec_blk(ctx, (u8 *)dst, (u8 *)src); + + nbytes -= bsize; + if (nbytes < bsize) + break; + + u128_xor(dst, dst, src - 1); + src -= 1; + dst -= 1; + } + +done: + u128_xor(dst, dst, (u128 *)walk->iv); + *(u128 *)walk->iv = last_iv; + + return nbytes; +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + nbytes = __cbc_decrypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + return err; +} + +static inline void u128_to_be128(be128 *dst, const u128 *src) +{ + dst->a = cpu_to_be64(src->a); + dst->b = cpu_to_be64(src->b); +} + +static inline void be128_to_u128(u128 *dst, const be128 *src) +{ + dst->a = be64_to_cpu(src->a); + dst->b = be64_to_cpu(src->b); +} + +static inline void u128_inc(u128 *i) +{ + i->b++; + if (!i->b) + i->a++; +} + +static void ctr_crypt_final(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + u8 keystream[CAMELLIA_BLOCK_SIZE]; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + u128 ctrblk; + + memcpy(keystream, src, nbytes); + camellia_enc_blk_xor(ctx, keystream, walk->iv); + memcpy(dst, keystream, nbytes); + + be128_to_u128(&ctrblk, (be128 *)walk->iv); + u128_inc(&ctrblk); + u128_to_be128((be128 *)walk->iv, &ctrblk); +} + +static unsigned int __ctr_crypt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int bsize = CAMELLIA_BLOCK_SIZE; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 ctrblk; + be128 ctrblocks[2]; + + be128_to_u128(&ctrblk, (be128 *)walk->iv); + + /* Process two block batch */ + if (nbytes >= bsize * 2) { + do { + if (dst != src) { + dst[0] = src[0]; + dst[1] = src[1]; + } + + /* create ctrblks for parallel encrypt */ + u128_to_be128(&ctrblocks[0], &ctrblk); + u128_inc(&ctrblk); + u128_to_be128(&ctrblocks[1], &ctrblk); + u128_inc(&ctrblk); + + camellia_enc_blk_xor_2way(ctx, (u8 *)dst, + (u8 *)ctrblocks); + + src += 2; + dst += 2; + nbytes -= bsize * 2; + } while (nbytes >= bsize * 2); + + if (nbytes < bsize) + goto done; + } + + /* Handle leftovers */ + do { + if (dst != src) + *dst = *src; + + u128_to_be128(&ctrblocks[0], &ctrblk); + u128_inc(&ctrblk); + + camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); + + src += 1; + dst += 1; + nbytes -= bsize; + } while (nbytes >= bsize); + +done: + u128_to_be128((be128 *)walk->iv, &ctrblk); + return nbytes; +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, CAMELLIA_BLOCK_SIZE); + + while ((nbytes = walk.nbytes) >= CAMELLIA_BLOCK_SIZE) { + nbytes = __ctr_crypt(desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + if (walk.nbytes) { + ctr_crypt_final(desc, &walk); + err = blkcipher_walk_done(desc, &walk, 0); + } + + return err; +} + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = CAMELLIA_BLOCK_SIZE; + struct camellia_ctx *ctx = priv; + int i; + + while (nbytes >= 2 * bsize) { + camellia_enc_blk_2way(ctx, srcdst, srcdst); + srcdst += bsize * 2; + nbytes -= bsize * 2; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + camellia_enc_blk(ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = CAMELLIA_BLOCK_SIZE; + struct camellia_ctx *ctx = priv; + int i; + + while (nbytes >= 2 * bsize) { + camellia_dec_blk_2way(ctx, srcdst, srcdst); + srcdst += bsize * 2; + nbytes -= bsize * 2; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + camellia_dec_blk(ctx, srcdst, srcdst); +} + +struct camellia_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct camellia_ctx camellia_ctx; +}; + +static int lrw_camellia_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct camellia_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + + err = __camellia_setkey(&ctx->camellia_ctx, key, + keylen - CAMELLIA_BLOCK_SIZE, + &tfm->crt_flags); + if (err) + return err; + + return lrw_init_table(&ctx->lrw_table, + key + keylen - CAMELLIA_BLOCK_SIZE); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[2 * 4]; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &ctx->camellia_ctx, + .crypt_fn = encrypt_callback, + }; + + return lrw_crypt(desc, dst, src, nbytes, &req); +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[2 * 4]; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &ctx->camellia_ctx, + .crypt_fn = decrypt_callback, + }; + + return lrw_crypt(desc, dst, src, nbytes, &req); +} + +static void lrw_exit_tfm(struct crypto_tfm *tfm) +{ + struct camellia_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + + lrw_free_table(&ctx->lrw_table); +} + +struct camellia_xts_ctx { + struct camellia_ctx tweak_ctx; + struct camellia_ctx crypt_ctx; +}; + +static int xts_camellia_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct camellia_xts_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *flags = &tfm->crt_flags; + int err; + + /* key consists of keys of equal size concatenated, therefore + * the length must be even + */ + if (keylen % 2) { + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + /* first half of xts-key is for crypt */ + err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); + if (err) + return err; + + /* second half of xts-key is for tweak */ + return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, + flags); +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[2 * 4]; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk), + .crypt_ctx = &ctx->crypt_ctx, + .crypt_fn = encrypt_callback, + }; + + return xts_crypt(desc, dst, src, nbytes, &req); +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[2 * 4]; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(camellia_enc_blk), + .crypt_ctx = &ctx->crypt_ctx, + .crypt_fn = decrypt_callback, + }; + + return xts_crypt(desc, dst, src, nbytes, &req); +} + +static struct crypto_alg camellia_algs[6] = { { + .cra_name = "camellia", + .cra_driver_name = "camellia-asm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[0].cra_list), + .cra_u = { + .cipher = { + .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, + .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE, + .cia_setkey = camellia_setkey, + .cia_encrypt = camellia_encrypt, + .cia_decrypt = camellia_decrypt + } + } +}, { + .cra_name = "ecb(camellia)", + .cra_driver_name = "ecb-camellia-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(camellia)", + .cra_driver_name = "cbc-camellia-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = camellia_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "ctr(camellia)", + .cra_driver_name = "ctr-camellia-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct camellia_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[3].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = camellia_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "lrw(camellia)", + .cra_driver_name = "lrw-camellia-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[4].cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE + + CAMELLIA_BLOCK_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = lrw_camellia_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "xts(camellia)", + .cra_driver_name = "xts-camellia-asm", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(camellia_algs[5].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, + .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = xts_camellia_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +} }; + +static bool is_blacklisted_cpu(void) +{ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return false; + + if (boot_cpu_data.x86 == 0x0f) { + /* + * On Pentium 4, camellia-asm is slower than original assembler + * implementation because excessive uses of 64bit rotate and + * left-shifts (which are really slow on P4) needed to store and + * handle 128bit block in two 64bit registers. + */ + return true; + } + + return false; +} + +static int force; +module_param(force, int, 0); +MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); + +int __init init(void) +{ + if (!force && is_blacklisted_cpu()) { + printk(KERN_INFO + "camellia-x86_64: performance on this CPU " + "would be suboptimal: disabling " + "camellia-x86_64.\n"); + return -ENODEV; + } + + return crypto_register_algs(camellia_algs, ARRAY_SIZE(camellia_algs)); +} + +void __exit fini(void) +{ + crypto_unregister_algs(camellia_algs, ARRAY_SIZE(camellia_algs)); +} + +module_init(init); +module_exit(fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Camellia Cipher Algorithm, asm optimized"); +MODULE_ALIAS("camellia"); +MODULE_ALIAS("camellia-asm"); diff --git a/arch/x86/crypto/crc32c-intel.c b/arch/x86/crypto/crc32c-intel.c index b9d0026..493f959 100644 --- a/arch/x86/crypto/crc32c-intel.c +++ b/arch/x86/crypto/crc32c-intel.c @@ -31,6 +31,7 @@ #include <crypto/internal/hash.h> #include <asm/cpufeature.h> +#include <asm/cpu_device_id.h> #define CHKSUM_BLOCK_SIZE 1 #define CHKSUM_DIGEST_SIZE 4 @@ -173,13 +174,17 @@ static struct shash_alg alg = { } }; +static const struct x86_cpu_id crc32c_cpu_id[] = { + X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); static int __init crc32c_intel_mod_init(void) { - if (cpu_has_xmm4_2) - return crypto_register_shash(&alg); - else + if (!x86_match_cpu(crc32c_cpu_id)) return -ENODEV; + return crypto_register_shash(&alg); } static void __exit crc32c_intel_mod_fini(void) diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index 976aa64..b4bf0a6 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -20,6 +20,7 @@ #include <crypto/gf128mul.h> #include <crypto/internal/hash.h> #include <asm/i387.h> +#include <asm/cpu_device_id.h> #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 @@ -294,15 +295,18 @@ static struct ahash_alg ghash_async_alg = { }, }; +static const struct x86_cpu_id pcmul_cpu_id[] = { + X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */ + {} +}; +MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); + static int __init ghash_pclmulqdqni_mod_init(void) { int err; - if (!cpu_has_pclmulqdq) { - printk(KERN_INFO "Intel PCLMULQDQ-NI instructions are not" - " detected.\n"); + if (!x86_match_cpu(pcmul_cpu_id)) return -ENODEV; - } err = crypto_register_shash(&ghash_alg); if (err) diff --git a/arch/x86/crypto/serpent-sse2-i586-asm_32.S b/arch/x86/crypto/serpent-sse2-i586-asm_32.S index 4e37677..c00053d 100644 --- a/arch/x86/crypto/serpent-sse2-i586-asm_32.S +++ b/arch/x86/crypto/serpent-sse2-i586-asm_32.S @@ -463,23 +463,20 @@ pand x0, x4; \ pxor x2, x4; -#define transpose_4x4(x0, x1, x2, x3, t1, t2, t3) \ - movdqa x2, t3; \ - movdqa x0, t1; \ - unpcklps x3, t3; \ +#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ movdqa x0, t2; \ - unpcklps x1, t1; \ - unpckhps x1, t2; \ - movdqa t3, x1; \ - unpckhps x3, x2; \ - movdqa t1, x0; \ - movhlps t1, x1; \ - movdqa t2, t1; \ - movlhps t3, x0; \ - movlhps x2, t1; \ - movhlps t2, x2; \ - movdqa x2, x3; \ - movdqa t1, x2; + punpckldq x1, x0; \ + punpckhdq x1, t2; \ + movdqa x2, t1; \ + punpckhdq x3, x2; \ + punpckldq x3, t1; \ + movdqa x0, x1; \ + punpcklqdq t1, x0; \ + punpckhqdq t1, x1; \ + movdqa t2, x3; \ + punpcklqdq x2, t2; \ + punpckhqdq x2, x3; \ + movdqa t2, x2; #define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \ movdqu (0*4*4)(in), x0; \ diff --git a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S index 7f24a15..3ee1ff0 100644 --- a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S @@ -585,23 +585,20 @@ get_key(i, 1, RK1); \ SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ -#define transpose_4x4(x0, x1, x2, x3, t1, t2, t3) \ - movdqa x2, t3; \ - movdqa x0, t1; \ - unpcklps x3, t3; \ +#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ movdqa x0, t2; \ - unpcklps x1, t1; \ - unpckhps x1, t2; \ - movdqa t3, x1; \ - unpckhps x3, x2; \ - movdqa t1, x0; \ - movhlps t1, x1; \ - movdqa t2, t1; \ - movlhps t3, x0; \ - movlhps x2, t1; \ - movhlps t2, x2; \ - movdqa x2, x3; \ - movdqa t1, x2; + punpckldq x1, x0; \ + punpckhdq x1, t2; \ + movdqa x2, t1; \ + punpckhdq x3, x2; \ + punpckldq x3, t1; \ + movdqa x0, x1; \ + punpcklqdq t1, x0; \ + punpckhqdq t1, x1; \ + movdqa t2, x3; \ + punpcklqdq x2, t2; \ + punpckhqdq x2, x3; \ + movdqa t2, x2; #define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \ movdqu (0*4*4)(in), x0; \ diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 7955a9b..4b21be8 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -145,28 +145,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ecb_crypt(desc, &walk, false); } -static struct crypto_alg blk_ecb_alg = { - .cra_name = "__ecb-serpent-sse2", - .cra_driver_name = "__driver-ecb-serpent-sse2", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .setkey = serpent_setkey, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}; - static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { @@ -295,28 +273,6 @@ static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_cbc_alg = { - .cra_name = "__cbc-serpent-sse2", - .cra_driver_name = "__driver-cbc-serpent-sse2", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .setkey = serpent_setkey, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -}; - static inline void u128_to_be128(be128 *dst, const u128 *src) { dst->a = cpu_to_be64(src->a); @@ -439,29 +395,6 @@ static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_ctr_alg = { - .cra_name = "__ctr-serpent-sse2", - .cra_driver_name = "__driver-ctr-serpent-sse2", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, - }, -}; - struct crypt_priv { struct serpent_ctx *ctx; bool fpu_enabled; @@ -580,32 +513,6 @@ static void lrw_exit_tfm(struct crypto_tfm *tfm) lrw_free_table(&ctx->lrw_table); } -static struct crypto_alg blk_lrw_alg = { - .cra_name = "__lrw-serpent-sse2", - .cra_driver_name = "__driver-lrw-serpent-sse2", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_lrw_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list), - .cra_exit = lrw_exit_tfm, - .cra_u = { - .blkcipher = { - .min_keysize = SERPENT_MIN_KEY_SIZE + - SERPENT_BLOCK_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE + - SERPENT_BLOCK_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = lrw_serpent_setkey, - .encrypt = lrw_encrypt, - .decrypt = lrw_decrypt, - }, - }, -}; - struct serpent_xts_ctx { struct serpent_ctx tweak_ctx; struct serpent_ctx crypt_ctx; @@ -689,29 +596,6 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ret; } -static struct crypto_alg blk_xts_alg = { - .cra_name = "__xts-serpent-sse2", - .cra_driver_name = "__driver-xts-serpent-sse2", - .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_xts_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_xts_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = SERPENT_MIN_KEY_SIZE * 2, - .max_keysize = SERPENT_MAX_KEY_SIZE * 2, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = xts_serpent_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, - }, - }, -}; - static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) { @@ -792,28 +676,133 @@ static void ablk_exit(struct crypto_tfm *tfm) cryptd_free_ablkcipher(ctx->cryptd_tfm); } -static void ablk_init_common(struct crypto_tfm *tfm, - struct cryptd_ablkcipher *cryptd_tfm) +static int ablk_init(struct crypto_tfm *tfm) { struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm); + struct cryptd_ablkcipher *cryptd_tfm; + char drv_name[CRYPTO_MAX_ALG_NAME]; + + snprintf(drv_name, sizeof(drv_name), "__driver-%s", + crypto_tfm_alg_driver_name(tfm)); + + cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); ctx->cryptd_tfm = cryptd_tfm; tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + crypto_ablkcipher_reqsize(&cryptd_tfm->base); -} - -static int ablk_ecb_init(struct crypto_tfm *tfm) -{ - struct cryptd_ablkcipher *cryptd_tfm; - cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-serpent-sse2", 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ablk_init_common(tfm, cryptd_tfm); return 0; } -static struct crypto_alg ablk_ecb_alg = { +static struct crypto_alg serpent_algs[10] = { { + .cra_name = "__ecb-serpent-sse2", + .cra_driver_name = "__driver-ecb-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-serpent-sse2", + .cra_driver_name = "__driver-cbc-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-serpent-sse2", + .cra_driver_name = "__driver-ctr-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = serpent_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-serpent-sse2", + .cra_driver_name = "__driver-lrw-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = lrw_serpent_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-serpent-sse2", + .cra_driver_name = "__driver-xts-serpent-sse2", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE * 2, + .max_keysize = SERPENT_MAX_KEY_SIZE * 2, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = xts_serpent_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { .cra_name = "ecb(serpent)", .cra_driver_name = "ecb-serpent-sse2", .cra_priority = 400, @@ -823,8 +812,8 @@ static struct crypto_alg ablk_ecb_alg = { .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), - .cra_init = ablk_ecb_init, + .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), + .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { @@ -835,20 +824,7 @@ static struct crypto_alg ablk_ecb_alg = { .decrypt = ablk_decrypt, }, }, -}; - -static int ablk_cbc_init(struct crypto_tfm *tfm) -{ - struct cryptd_ablkcipher *cryptd_tfm; - - cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-serpent-sse2", 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ablk_init_common(tfm, cryptd_tfm); - return 0; -} - -static struct crypto_alg ablk_cbc_alg = { +}, { .cra_name = "cbc(serpent)", .cra_driver_name = "cbc-serpent-sse2", .cra_priority = 400, @@ -858,8 +834,8 @@ static struct crypto_alg ablk_cbc_alg = { .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), - .cra_init = ablk_cbc_init, + .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), + .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { @@ -871,20 +847,7 @@ static struct crypto_alg ablk_cbc_alg = { .decrypt = ablk_decrypt, }, }, -}; - -static int ablk_ctr_init(struct crypto_tfm *tfm) -{ - struct cryptd_ablkcipher *cryptd_tfm; - - cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-serpent-sse2", 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ablk_init_common(tfm, cryptd_tfm); - return 0; -} - -static struct crypto_alg ablk_ctr_alg = { +}, { .cra_name = "ctr(serpent)", .cra_driver_name = "ctr-serpent-sse2", .cra_priority = 400, @@ -894,8 +857,8 @@ static struct crypto_alg ablk_ctr_alg = { .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), - .cra_init = ablk_ctr_init, + .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), + .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { @@ -908,20 +871,7 @@ static struct crypto_alg ablk_ctr_alg = { .geniv = "chainiv", }, }, -}; - -static int ablk_lrw_init(struct crypto_tfm *tfm) -{ - struct cryptd_ablkcipher *cryptd_tfm; - - cryptd_tfm = cryptd_alloc_ablkcipher("__driver-lrw-serpent-sse2", 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ablk_init_common(tfm, cryptd_tfm); - return 0; -} - -static struct crypto_alg ablk_lrw_alg = { +}, { .cra_name = "lrw(serpent)", .cra_driver_name = "lrw-serpent-sse2", .cra_priority = 400, @@ -931,8 +881,8 @@ static struct crypto_alg ablk_lrw_alg = { .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), - .cra_init = ablk_lrw_init, + .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), + .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { @@ -946,20 +896,7 @@ static struct crypto_alg ablk_lrw_alg = { .decrypt = ablk_decrypt, }, }, -}; - -static int ablk_xts_init(struct crypto_tfm *tfm) -{ - struct cryptd_ablkcipher *cryptd_tfm; - - cryptd_tfm = cryptd_alloc_ablkcipher("__driver-xts-serpent-sse2", 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - ablk_init_common(tfm, cryptd_tfm); - return 0; -} - -static struct crypto_alg ablk_xts_alg = { +}, { .cra_name = "xts(serpent)", .cra_driver_name = "xts-serpent-sse2", .cra_priority = 400, @@ -969,8 +906,8 @@ static struct crypto_alg ablk_xts_alg = { .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), - .cra_init = ablk_xts_init, + .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), + .cra_init = ablk_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { @@ -982,84 +919,21 @@ static struct crypto_alg ablk_xts_alg = { .decrypt = ablk_decrypt, }, }, -}; +} }; static int __init serpent_sse2_init(void) { - int err; - if (!cpu_has_xmm2) { printk(KERN_INFO "SSE2 instructions are not detected.\n"); return -ENODEV; } - err = crypto_register_alg(&blk_ecb_alg); - if (err) - goto blk_ecb_err; - err = crypto_register_alg(&blk_cbc_alg); - if (err) - goto blk_cbc_err; - err = crypto_register_alg(&blk_ctr_alg); - if (err) - goto blk_ctr_err; - err = crypto_register_alg(&ablk_ecb_alg); - if (err) - goto ablk_ecb_err; - err = crypto_register_alg(&ablk_cbc_alg); - if (err) - goto ablk_cbc_err; - err = crypto_register_alg(&ablk_ctr_alg); - if (err) - goto ablk_ctr_err; - err = crypto_register_alg(&blk_lrw_alg); - if (err) - goto blk_lrw_err; - err = crypto_register_alg(&ablk_lrw_alg); - if (err) - goto ablk_lrw_err; - err = crypto_register_alg(&blk_xts_alg); - if (err) - goto blk_xts_err; - err = crypto_register_alg(&ablk_xts_alg); - if (err) - goto ablk_xts_err; - return err; - - crypto_unregister_alg(&ablk_xts_alg); -ablk_xts_err: - crypto_unregister_alg(&blk_xts_alg); -blk_xts_err: - crypto_unregister_alg(&ablk_lrw_alg); -ablk_lrw_err: - crypto_unregister_alg(&blk_lrw_alg); -blk_lrw_err: - crypto_unregister_alg(&ablk_ctr_alg); -ablk_ctr_err: - crypto_unregister_alg(&ablk_cbc_alg); -ablk_cbc_err: - crypto_unregister_alg(&ablk_ecb_alg); -ablk_ecb_err: - crypto_unregister_alg(&blk_ctr_alg); -blk_ctr_err: - crypto_unregister_alg(&blk_cbc_alg); -blk_cbc_err: - crypto_unregister_alg(&blk_ecb_alg); -blk_ecb_err: - return err; + return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); } static void __exit serpent_sse2_exit(void) { - crypto_unregister_alg(&ablk_xts_alg); - crypto_unregister_alg(&blk_xts_alg); - crypto_unregister_alg(&ablk_lrw_alg); - crypto_unregister_alg(&blk_lrw_alg); - crypto_unregister_alg(&ablk_ctr_alg); - crypto_unregister_alg(&ablk_cbc_alg); - crypto_unregister_alg(&ablk_ecb_alg); - crypto_unregister_alg(&blk_ctr_alg); - crypto_unregister_alg(&blk_cbc_alg); - crypto_unregister_alg(&blk_ecb_alg); + crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); } module_init(serpent_sse2_init); diff --git a/arch/x86/crypto/twofish_glue.c b/arch/x86/crypto/twofish_glue.c index dc6b3fb..359ae08 100644 --- a/arch/x86/crypto/twofish_glue.c +++ b/arch/x86/crypto/twofish_glue.c @@ -68,7 +68,7 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 3, + .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(alg.cra_list), .cra_u = { diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 7fee8c1..408fc0c 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -25,6 +25,7 @@ * */ +#include <asm/processor.h> #include <linux/crypto.h> #include <linux/init.h> #include <linux/module.h> @@ -122,28 +123,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way); } -static struct crypto_alg blk_ecb_alg = { - .cra_name = "ecb(twofish)", - .cra_driver_name = "ecb-twofish-3way", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = TF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .setkey = twofish_setkey, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}; - static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { @@ -267,29 +246,6 @@ static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_cbc_alg = { - .cra_name = "cbc(twofish)", - .cra_driver_name = "cbc-twofish-3way", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = TF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .setkey = twofish_setkey, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -}; - static inline void u128_to_be128(be128 *dst, const u128 *src) { dst->a = cpu_to_be64(src->a); @@ -411,29 +367,6 @@ static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, return err; } -static struct crypto_alg blk_ctr_alg = { - .cra_name = "ctr(twofish)", - .cra_driver_name = "ctr-twofish-3way", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .setkey = twofish_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, - }, -}; - static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) { const unsigned int bsize = TF_BLOCK_SIZE; @@ -524,30 +457,6 @@ static void lrw_exit_tfm(struct crypto_tfm *tfm) lrw_free_table(&ctx->lrw_table); } -static struct crypto_alg blk_lrw_alg = { - .cra_name = "lrw(twofish)", - .cra_driver_name = "lrw-twofish-3way", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = TF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct twofish_lrw_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list), - .cra_exit = lrw_exit_tfm, - .cra_u = { - .blkcipher = { - .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, - .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, - .ivsize = TF_BLOCK_SIZE, - .setkey = lrw_twofish_setkey, - .encrypt = lrw_encrypt, - .decrypt = lrw_decrypt, - }, - }, -}; - struct twofish_xts_ctx { struct twofish_ctx tweak_ctx; struct twofish_ctx crypt_ctx; @@ -614,7 +523,91 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return xts_crypt(desc, dst, src, nbytes, &req); } -static struct crypto_alg blk_xts_alg = { +static struct crypto_alg tf_algs[5] = { { + .cra_name = "ecb(twofish)", + .cra_driver_name = "ecb-twofish-3way", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = twofish_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(twofish)", + .cra_driver_name = "cbc-twofish-3way", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = twofish_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "ctr(twofish)", + .cra_driver_name = "ctr-twofish-3way", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = twofish_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "lrw(twofish)", + .cra_driver_name = "lrw-twofish-3way", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = lrw_twofish_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { .cra_name = "xts(twofish)", .cra_driver_name = "xts-twofish-3way", .cra_priority = 300, @@ -624,7 +617,7 @@ static struct crypto_alg blk_xts_alg = { .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(blk_xts_alg.cra_list), + .cra_list = LIST_HEAD_INIT(tf_algs[4].cra_list), .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE * 2, @@ -635,50 +628,62 @@ static struct crypto_alg blk_xts_alg = { .decrypt = xts_decrypt, }, }, -}; +} }; + +static bool is_blacklisted_cpu(void) +{ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return false; + + if (boot_cpu_data.x86 == 0x06 && + (boot_cpu_data.x86_model == 0x1c || + boot_cpu_data.x86_model == 0x26 || + boot_cpu_data.x86_model == 0x36)) { + /* + * On Atom, twofish-3way is slower than original assembler + * implementation. Twofish-3way trades off some performance in + * storing blocks in 64bit registers to allow three blocks to + * be processed parallel. Parallel operation then allows gaining + * more performance than was trade off, on out-of-order CPUs. + * However Atom does not benefit from this parallellism and + * should be blacklisted. + */ + return true; + } + + if (boot_cpu_data.x86 == 0x0f) { + /* + * On Pentium 4, twofish-3way is slower than original assembler + * implementation because excessive uses of 64bit rotate and + * left-shifts (which are really slow on P4) needed to store and + * handle 128bit block in two 64bit registers. + */ + return true; + } + + return false; +} + +static int force; +module_param(force, int, 0); +MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); int __init init(void) { - int err; + if (!force && is_blacklisted_cpu()) { + printk(KERN_INFO + "twofish-x86_64-3way: performance on this CPU " + "would be suboptimal: disabling " + "twofish-x86_64-3way.\n"); + return -ENODEV; + } - err = crypto_register_alg(&blk_ecb_alg); - if (err) - goto ecb_err; - err = crypto_register_alg(&blk_cbc_alg); - if (err) - goto cbc_err; - err = crypto_register_alg(&blk_ctr_alg); - if (err) - goto ctr_err; - err = crypto_register_alg(&blk_lrw_alg); - if (err) - goto blk_lrw_err; - err = crypto_register_alg(&blk_xts_alg); - if (err) - goto blk_xts_err; - - return 0; - - crypto_unregister_alg(&blk_xts_alg); -blk_xts_err: - crypto_unregister_alg(&blk_lrw_alg); -blk_lrw_err: - crypto_unregister_alg(&blk_ctr_alg); -ctr_err: - crypto_unregister_alg(&blk_cbc_alg); -cbc_err: - crypto_unregister_alg(&blk_ecb_alg); -ecb_err: - return err; + return crypto_register_algs(tf_algs, ARRAY_SIZE(tf_algs)); } void __exit fini(void) { - crypto_unregister_alg(&blk_xts_alg); - crypto_unregister_alg(&blk_lrw_alg); - crypto_unregister_alg(&blk_ctr_alg); - crypto_unregister_alg(&blk_cbc_alg); - crypto_unregister_alg(&blk_ecb_alg); + crypto_unregister_algs(tf_algs, ARRAY_SIZE(tf_algs)); } module_init(init); diff --git a/arch/x86/include/asm/cpu_device_id.h b/arch/x86/include/asm/cpu_device_id.h new file mode 100644 index 0000000..ff501e5 --- /dev/null +++ b/arch/x86/include/asm/cpu_device_id.h @@ -0,0 +1,13 @@ +#ifndef _CPU_DEVICE_ID +#define _CPU_DEVICE_ID 1 + +/* + * Declare drivers belonging to specific x86 CPUs + * Similar in spirit to pci_device_id and related PCI functions + */ + +#include <linux/mod_devicetable.h> + +extern const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match); + +#endif diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 8d67d42..dcb839e 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -177,6 +177,7 @@ #define X86_FEATURE_PLN (7*32+ 5) /* Intel Power Limit Notification */ #define X86_FEATURE_PTS (7*32+ 6) /* Intel Package Thermal Status */ #define X86_FEATURE_DTS (7*32+ 7) /* Digital Thermal Sensor */ +#define X86_FEATURE_HW_PSTATE (7*32+ 8) /* AMD HW-PState */ /* Virtualization flags: Linux defined, word 8 */ #define X86_FEATURE_TPR_SHADOW (8*32+ 0) /* Intel TPR Shadow */ diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h index da0b3ca..382f75d 100644 --- a/arch/x86/include/asm/hardirq.h +++ b/arch/x86/include/asm/hardirq.h @@ -7,7 +7,6 @@ typedef struct { unsigned int __softirq_pending; unsigned int __nmi_count; /* arch dependent */ - unsigned int irq0_irqs; #ifdef CONFIG_X86_LOCAL_APIC unsigned int apic_timer_irqs; /* arch dependent */ unsigned int irq_spurious_count; diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h index 3bd0402..302a323 100644 --- a/arch/x86/include/asm/highmem.h +++ b/arch/x86/include/asm/highmem.h @@ -61,7 +61,7 @@ void *kmap(struct page *page); void kunmap(struct page *page); void *kmap_atomic_prot(struct page *page, pgprot_t prot); -void *__kmap_atomic(struct page *page); +void *kmap_atomic(struct page *page); void __kunmap_atomic(void *kvaddr); void *kmap_atomic_pfn(unsigned long pfn); void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h index 205b063..74a2e31 100644 --- a/arch/x86/include/asm/inat.h +++ b/arch/x86/include/asm/inat.h @@ -97,11 +97,12 @@ /* Attribute search APIs */ extern insn_attr_t inat_get_opcode_attribute(insn_byte_t opcode); +extern int inat_get_last_prefix_id(insn_byte_t last_pfx); extern insn_attr_t inat_get_escape_attribute(insn_byte_t opcode, - insn_byte_t last_pfx, + int lpfx_id, insn_attr_t esc_attr); extern insn_attr_t inat_get_group_attribute(insn_byte_t modrm, - insn_byte_t last_pfx, + int lpfx_id, insn_attr_t esc_attr); extern insn_attr_t inat_get_avx_attribute(insn_byte_t opcode, insn_byte_t vex_m, diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index 74df3f1..48eb30a 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -96,12 +96,6 @@ struct insn { #define X86_VEX_P(vex) ((vex) & 0x03) /* VEX3 Byte2, VEX2 Byte1 */ #define X86_VEX_M_MAX 0x1f /* VEX3.M Maximum value */ -/* The last prefix is needed for two-byte and three-byte opcodes */ -static inline insn_byte_t insn_last_prefix(struct insn *insn) -{ - return insn->prefixes.bytes[3]; -} - extern void insn_init(struct insn *insn, const void *kaddr, int x86_64); extern void insn_get_prefixes(struct insn *insn); extern void insn_get_opcode(struct insn *insn); @@ -160,6 +154,18 @@ static inline insn_byte_t insn_vex_p_bits(struct insn *insn) return X86_VEX_P(insn->vex_prefix.bytes[2]); } +/* Get the last prefix id from last prefix or VEX prefix */ +static inline int insn_last_prefix_id(struct insn *insn) +{ + if (insn_is_avx(insn)) + return insn_vex_p_bits(insn); /* VEX_p is a SIMD prefix id */ + + if (insn->prefixes.bytes[3]) + return inat_get_last_prefix_id(insn->prefixes.bytes[3]); + + return 0; +} + /* Offset of each field from kaddr */ static inline int insn_offset_rex_prefix(struct insn *insn) { diff --git a/arch/x86/include/asm/irq_controller.h b/arch/x86/include/asm/irq_controller.h deleted file mode 100644 index 423bbbd..0000000 --- a/arch/x86/include/asm/irq_controller.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __IRQ_CONTROLLER__ -#define __IRQ_CONTROLLER__ - -struct irq_domain { - int (*xlate)(struct irq_domain *h, const u32 *intspec, u32 intsize, - u32 *out_hwirq, u32 *out_type); - void *priv; - struct device_node *controller; - struct list_head l; -}; - -#endif diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index a32b18c..3a16c14 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -9,12 +9,12 @@ #define JUMP_LABEL_NOP_SIZE 5 -#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" +#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" -static __always_inline bool arch_static_branch(struct jump_label_key *key) +static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("1:" - JUMP_LABEL_INITIAL_NOP + STATIC_KEY_INITIAL_NOP ".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN "\n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t" diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index a6962d9..ccb8059 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -56,6 +56,13 @@ #define MSR_OFFCORE_RSP_0 0x000001a6 #define MSR_OFFCORE_RSP_1 0x000001a7 +#define MSR_LBR_SELECT 0x000001c8 +#define MSR_LBR_TOS 0x000001c9 +#define MSR_LBR_NHM_FROM 0x00000680 +#define MSR_LBR_NHM_TO 0x000006c0 +#define MSR_LBR_CORE_FROM 0x00000040 +#define MSR_LBR_CORE_TO 0x00000060 + #define MSR_IA32_PEBS_ENABLE 0x000003f1 #define MSR_IA32_DS_AREA 0x00000600 #define MSR_IA32_PERF_CAPABILITIES 0x00000345 diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index a7d2db9..c0180fd 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -230,9 +230,9 @@ static inline unsigned long long paravirt_sched_clock(void) return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); } -struct jump_label_key; -extern struct jump_label_key paravirt_steal_enabled; -extern struct jump_label_key paravirt_steal_rq_enabled; +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; static inline u64 paravirt_steal_clock(int cpu) { diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 461ce43..e8fb2c7 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -188,8 +188,6 @@ extern u32 get_ibs_caps(void); #ifdef CONFIG_PERF_EVENTS extern void perf_events_lapic_init(void); -#define PERF_EVENT_INDEX_OFFSET 0 - /* * Abuse bit 3 of the cpu eflags register to indicate proper PEBS IP fixups. * This flag is otherwise unused and ABI specified to be 0, so nobody should diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h index 644dd885..60bef66 100644 --- a/arch/x86/include/asm/prom.h +++ b/arch/x86/include/asm/prom.h @@ -21,7 +21,6 @@ #include <asm/irq.h> #include <linux/atomic.h> #include <asm/setup.h> -#include <asm/irq_controller.h> #ifdef CONFIG_OF extern int of_ioapic; @@ -43,15 +42,6 @@ extern char cmd_line[COMMAND_LINE_SIZE]; #define pci_address_to_pio pci_address_to_pio unsigned long pci_address_to_pio(phys_addr_t addr); -/** - * irq_dispose_mapping - Unmap an interrupt - * @virq: linux virq number of the interrupt to unmap - * - * FIXME: We really should implement proper virq handling like power, - * but that's going to be major surgery. - */ -static inline void irq_dispose_mapping(unsigned int virq) { } - #define HAVE_ARCH_DEVTREE_FIXUPS #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h index 431793e..34baa0e 100644 --- a/arch/x86/include/asm/timer.h +++ b/arch/x86/include/asm/timer.h @@ -57,14 +57,10 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset); static inline unsigned long long __cycles_2_ns(unsigned long long cyc) { - unsigned long long quot; - unsigned long long rem; int cpu = smp_processor_id(); unsigned long long ns = per_cpu(cyc2ns_offset, cpu); - quot = (cyc >> CYC2NS_SCALE_FACTOR); - rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1); - ns += quot * per_cpu(cyc2ns, cpu) + - ((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR); + ns += mult_frac(cyc, per_cpu(cyc2ns, cpu), + (1UL << CYC2NS_SCALE_FACTOR)); return ns; } diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 5369059..532d2e0 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -69,6 +69,7 @@ obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_OPTPROBES) += kprobes-opt.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o obj-$(CONFIG_KGDB) += kgdb.o diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index f76623c..5d56931 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -1234,8 +1234,7 @@ static int suspend(int vetoable) struct apm_user *as; dpm_suspend_start(PMSG_SUSPEND); - - dpm_suspend_noirq(PMSG_SUSPEND); + dpm_suspend_end(PMSG_SUSPEND); local_irq_disable(); syscore_suspend(); @@ -1259,9 +1258,9 @@ static int suspend(int vetoable) syscore_resume(); local_irq_enable(); - dpm_resume_noirq(PMSG_RESUME); - + dpm_resume_start(PMSG_RESUME); dpm_resume_end(PMSG_RESUME); + queue_event(APM_NORMAL_RESUME, NULL); spin_lock(&user_list_lock); for (as = user_list; as != NULL; as = as->next) { @@ -1277,7 +1276,7 @@ static void standby(void) { int err; - dpm_suspend_noirq(PMSG_SUSPEND); + dpm_suspend_end(PMSG_SUSPEND); local_irq_disable(); syscore_suspend(); @@ -1291,7 +1290,7 @@ static void standby(void) syscore_resume(); local_irq_enable(); - dpm_resume_noirq(PMSG_RESUME); + dpm_resume_start(PMSG_RESUME); } static apm_event_t get_event(void) diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 25f24dc..6ab6aa2 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -16,6 +16,7 @@ obj-y := intel_cacheinfo.o scattered.o topology.o obj-y += proc.o capflags.o powerflags.o common.o obj-y += vmware.o hypervisor.o sched.o mshyperv.o obj-y += rdrand.o +obj-y += match.o obj-$(CONFIG_X86_32) += bugs.o obj-$(CONFIG_X86_64) += bugs_64.o diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index f4773f4..0a44b90 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -5,6 +5,7 @@ #include <linux/mm.h> #include <linux/io.h> +#include <linux/sched.h> #include <asm/processor.h> #include <asm/apic.h> #include <asm/cpu.h> @@ -456,6 +457,8 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c) if (c->x86_power & (1 << 8)) { set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); + if (!check_tsc_unstable()) + sched_clock_stable = 1; } #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c new file mode 100644 index 0000000..5502b28 --- /dev/null +++ b/arch/x86/kernel/cpu/match.c @@ -0,0 +1,91 @@ +#include <asm/cpu_device_id.h> +#include <asm/processor.h> +#include <linux/cpu.h> +#include <linux/module.h> +#include <linux/slab.h> + +/** + * x86_match_cpu - match current CPU again an array of x86_cpu_ids + * @match: Pointer to array of x86_cpu_ids. Last entry terminated with + * {}. + * + * Return the entry if the current CPU matches the entries in the + * passed x86_cpu_id match table. Otherwise NULL. The match table + * contains vendor (X86_VENDOR_*), family, model and feature bits or + * respective wildcard entries. + * + * A typical table entry would be to match a specific CPU + * { X86_VENDOR_INTEL, 6, 0x12 } + * or to match a specific CPU feature + * { X86_FEATURE_MATCH(X86_FEATURE_FOOBAR) } + * + * Fields can be wildcarded with %X86_VENDOR_ANY, %X86_FAMILY_ANY, + * %X86_MODEL_ANY, %X86_FEATURE_ANY or 0 (except for vendor) + * + * Arrays used to match for this should also be declared using + * MODULE_DEVICE_TABLE(x86_cpu, ...) + * + * This always matches against the boot cpu, assuming models and features are + * consistent over all CPUs. + */ +const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match) +{ + const struct x86_cpu_id *m; + struct cpuinfo_x86 *c = &boot_cpu_data; + + for (m = match; m->vendor | m->family | m->model | m->feature; m++) { + if (m->vendor != X86_VENDOR_ANY && c->x86_vendor != m->vendor) + continue; + if (m->family != X86_FAMILY_ANY && c->x86 != m->family) + continue; + if (m->model != X86_MODEL_ANY && c->x86_model != m->model) + continue; + if (m->feature != X86_FEATURE_ANY && !cpu_has(c, m->feature)) + continue; + return m; + } + return NULL; +} +EXPORT_SYMBOL(x86_match_cpu); + +ssize_t arch_print_cpu_modalias(struct device *dev, + struct device_attribute *attr, + char *bufptr) +{ + int size = PAGE_SIZE; + int i, n; + char *buf = bufptr; + + n = snprintf(buf, size, "x86cpu:vendor:%04X:family:%04X:" + "model:%04X:feature:", + boot_cpu_data.x86_vendor, + boot_cpu_data.x86, + boot_cpu_data.x86_model); + size -= n; + buf += n; + size -= 1; + for (i = 0; i < NCAPINTS*32; i++) { + if (boot_cpu_has(i)) { + n = snprintf(buf, size, ",%04X", i); + if (n >= size) { + WARN(1, "x86 features overflow page\n"); + break; + } + size -= n; + buf += n; + } + } + *buf++ = '\n'; + return buf - bufptr; +} + +int arch_cpu_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (buf) { + arch_print_cpu_modalias(NULL, NULL, buf); + add_uevent_var(env, "MODALIAS=%s", buf); + kfree(buf); + } + return 0; +} diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 5adce10..0a18d16 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -24,6 +24,7 @@ #include <linux/slab.h> #include <linux/cpu.h> #include <linux/bitops.h> +#include <linux/device.h> #include <asm/apic.h> #include <asm/stacktrace.h> @@ -31,6 +32,7 @@ #include <asm/compat.h> #include <asm/smp.h> #include <asm/alternative.h> +#include <asm/timer.h> #include "perf_event.h" @@ -351,6 +353,36 @@ int x86_setup_perfctr(struct perf_event *event) return 0; } +/* + * check that branch_sample_type is compatible with + * settings needed for precise_ip > 1 which implies + * using the LBR to capture ALL taken branches at the + * priv levels of the measurement + */ +static inline int precise_br_compat(struct perf_event *event) +{ + u64 m = event->attr.branch_sample_type; + u64 b = 0; + + /* must capture all branches */ + if (!(m & PERF_SAMPLE_BRANCH_ANY)) + return 0; + + m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; + + if (!event->attr.exclude_user) + b |= PERF_SAMPLE_BRANCH_USER; + + if (!event->attr.exclude_kernel) + b |= PERF_SAMPLE_BRANCH_KERNEL; + + /* + * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 + */ + + return m == b; +} + int x86_pmu_hw_config(struct perf_event *event) { if (event->attr.precise_ip) { @@ -367,6 +399,36 @@ int x86_pmu_hw_config(struct perf_event *event) if (event->attr.precise_ip > precise) return -EOPNOTSUPP; + /* + * check that PEBS LBR correction does not conflict with + * whatever the user is asking with attr->branch_sample_type + */ + if (event->attr.precise_ip > 1) { + u64 *br_type = &event->attr.branch_sample_type; + + if (has_branch_stack(event)) { + if (!precise_br_compat(event)) + return -EOPNOTSUPP; + + /* branch_sample_type is compatible */ + + } else { + /* + * user did not specify branch_sample_type + * + * For PEBS fixups, we capture all + * the branches at the priv level of the + * event. + */ + *br_type = PERF_SAMPLE_BRANCH_ANY; + + if (!event->attr.exclude_user) + *br_type |= PERF_SAMPLE_BRANCH_USER; + + if (!event->attr.exclude_kernel) + *br_type |= PERF_SAMPLE_BRANCH_KERNEL; + } + } } /* @@ -424,6 +486,10 @@ static int __x86_pmu_event_init(struct perf_event *event) /* mark unused */ event->hw.extra_reg.idx = EXTRA_REG_NONE; + /* mark not used */ + event->hw.extra_reg.idx = EXTRA_REG_NONE; + event->hw.branch_reg.idx = EXTRA_REG_NONE; + return x86_pmu.hw_config(event); } @@ -1210,6 +1276,8 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) break; case CPU_STARTING: + if (x86_pmu.attr_rdpmc) + set_in_cr4(X86_CR4_PCE); if (x86_pmu.cpu_starting) x86_pmu.cpu_starting(cpu); break; @@ -1319,6 +1387,8 @@ static int __init init_hw_perf_events(void) } } + x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ + pr_info("... version: %d\n", x86_pmu.version); pr_info("... bit width: %d\n", x86_pmu.cntval_bits); pr_info("... generic registers: %d\n", x86_pmu.num_counters); @@ -1542,23 +1612,106 @@ static int x86_pmu_event_init(struct perf_event *event) return err; } +static int x86_pmu_event_idx(struct perf_event *event) +{ + int idx = event->hw.idx; + + if (x86_pmu.num_counters_fixed && idx >= X86_PMC_IDX_FIXED) { + idx -= X86_PMC_IDX_FIXED; + idx |= 1 << 30; + } + + return idx + 1; +} + +static ssize_t get_attr_rdpmc(struct device *cdev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); +} + +static void change_rdpmc(void *info) +{ + bool enable = !!(unsigned long)info; + + if (enable) + set_in_cr4(X86_CR4_PCE); + else + clear_in_cr4(X86_CR4_PCE); +} + +static ssize_t set_attr_rdpmc(struct device *cdev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned long val = simple_strtoul(buf, NULL, 0); + + if (!!val != !!x86_pmu.attr_rdpmc) { + x86_pmu.attr_rdpmc = !!val; + smp_call_function(change_rdpmc, (void *)val, 1); + } + + return count; +} + +static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); + +static struct attribute *x86_pmu_attrs[] = { + &dev_attr_rdpmc.attr, + NULL, +}; + +static struct attribute_group x86_pmu_attr_group = { + .attrs = x86_pmu_attrs, +}; + +static const struct attribute_group *x86_pmu_attr_groups[] = { + &x86_pmu_attr_group, + NULL, +}; + +static void x86_pmu_flush_branch_stack(void) +{ + if (x86_pmu.flush_branch_stack) + x86_pmu.flush_branch_stack(); +} + static struct pmu pmu = { - .pmu_enable = x86_pmu_enable, - .pmu_disable = x86_pmu_disable, + .pmu_enable = x86_pmu_enable, + .pmu_disable = x86_pmu_disable, + + .attr_groups = x86_pmu_attr_groups, .event_init = x86_pmu_event_init, - .add = x86_pmu_add, - .del = x86_pmu_del, - .start = x86_pmu_start, - .stop = x86_pmu_stop, - .read = x86_pmu_read, + .add = x86_pmu_add, + .del = x86_pmu_del, + .start = x86_pmu_start, + .stop = x86_pmu_stop, + .read = x86_pmu_read, .start_txn = x86_pmu_start_txn, .cancel_txn = x86_pmu_cancel_txn, .commit_txn = x86_pmu_commit_txn, + + .event_idx = x86_pmu_event_idx, + .flush_branch_stack = x86_pmu_flush_branch_stack, }; +void perf_update_user_clock(struct perf_event_mmap_page *userpg, u64 now) +{ + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) + return; + + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) + return; + + userpg->time_mult = this_cpu_read(cyc2ns); + userpg->time_shift = CYC2NS_SCALE_FACTOR; + userpg->time_offset = this_cpu_read(cyc2ns_offset) - now; +} + /* * callchain support */ diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h index c30c807..8484e77 100644 --- a/arch/x86/kernel/cpu/perf_event.h +++ b/arch/x86/kernel/cpu/perf_event.h @@ -33,6 +33,7 @@ enum extra_reg_type { EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */ EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */ + EXTRA_REG_LBR = 2, /* lbr_select */ EXTRA_REG_MAX /* number of entries needed */ }; @@ -130,6 +131,8 @@ struct cpu_hw_events { void *lbr_context; struct perf_branch_stack lbr_stack; struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES]; + struct er_account *lbr_sel; + u64 br_sel; /* * Intel host/guest exclude bits @@ -268,6 +271,29 @@ struct x86_pmu_quirk { void (*func)(void); }; +union x86_pmu_config { + struct { + u64 event:8, + umask:8, + usr:1, + os:1, + edge:1, + pc:1, + interrupt:1, + __reserved1:1, + en:1, + inv:1, + cmask:8, + event2:4, + __reserved2:4, + go:1, + ho:1; + } bits; + u64 value; +}; + +#define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value + /* * struct x86_pmu - generic x86 pmu */ @@ -309,10 +335,19 @@ struct x86_pmu { struct x86_pmu_quirk *quirks; int perfctr_second_write; + /* + * sysfs attrs + */ + int attr_rdpmc; + + /* + * CPU Hotplug hooks + */ int (*cpu_prepare)(int cpu); void (*cpu_starting)(int cpu); void (*cpu_dying)(int cpu); void (*cpu_dead)(int cpu); + void (*flush_branch_stack)(void); /* * Intel Arch Perfmon v2+ @@ -334,6 +369,8 @@ struct x86_pmu { */ unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */ int lbr_nr; /* hardware stack size */ + u64 lbr_sel_mask; /* LBR_SELECT valid bits */ + const int *lbr_sel_map; /* lbr_select mappings */ /* * Extra registers for events @@ -447,6 +484,15 @@ extern struct event_constraint emptyconstraint; extern struct event_constraint unconstrained; +static inline bool kernel_ip(unsigned long ip) +{ +#ifdef CONFIG_X86_32 + return ip > PAGE_OFFSET; +#else + return (long)ip < 0; +#endif +} + #ifdef CONFIG_CPU_SUP_AMD int amd_pmu_init(void); @@ -527,6 +573,10 @@ void intel_pmu_lbr_init_nhm(void); void intel_pmu_lbr_init_atom(void); +void intel_pmu_lbr_init_snb(void); + +int intel_pmu_setup_lbr_filter(struct perf_event *event); + int p4_pmu_init(void); int p6_pmu_init(void); diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c index 67250a5..dd002fa 100644 --- a/arch/x86/kernel/cpu/perf_event_amd.c +++ b/arch/x86/kernel/cpu/perf_event_amd.c @@ -139,6 +139,9 @@ static int amd_pmu_hw_config(struct perf_event *event) if (ret) return ret; + if (has_branch_stack(event)) + return -EOPNOTSUPP; + if (event->attr.exclude_host && event->attr.exclude_guest) /* * When HO == GO == 1 the hardware treats that as GO == HO == 0 diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 61d4f79..6a84e7f 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -728,6 +728,19 @@ static __initconst const u64 atom_hw_cache_event_ids }, }; +static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event) +{ + /* user explicitly requested branch sampling */ + if (has_branch_stack(event)) + return true; + + /* implicit branch sampling to correct PEBS skid */ + if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1) + return true; + + return false; +} + static void intel_pmu_disable_all(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); @@ -882,6 +895,13 @@ static void intel_pmu_disable_event(struct perf_event *event) cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx); cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx); + /* + * must disable before any actual event + * because any event may be combined with LBR + */ + if (intel_pmu_needs_lbr_smpl(event)) + intel_pmu_lbr_disable(event); + if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { intel_pmu_disable_fixed(hwc); return; @@ -936,6 +956,12 @@ static void intel_pmu_enable_event(struct perf_event *event) intel_pmu_enable_bts(hwc->config); return; } + /* + * must enabled before any actual event + * because any event may be combined with LBR + */ + if (intel_pmu_needs_lbr_smpl(event)) + intel_pmu_lbr_enable(event); if (event->attr.exclude_host) cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx); @@ -1058,6 +1084,9 @@ again: data.period = event->hw.last_period; + if (has_branch_stack(event)) + data.br_stack = &cpuc->lbr_stack; + if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); } @@ -1124,17 +1153,17 @@ static bool intel_try_alt_er(struct perf_event *event, int orig_idx) */ static struct event_constraint * __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, - struct perf_event *event) + struct perf_event *event, + struct hw_perf_event_extra *reg) { struct event_constraint *c = &emptyconstraint; - struct hw_perf_event_extra *reg = &event->hw.extra_reg; struct er_account *era; unsigned long flags; int orig_idx = reg->idx; /* already allocated shared msr */ if (reg->alloc) - return &unconstrained; + return NULL; /* call x86_get_event_constraint() */ again: era = &cpuc->shared_regs->regs[reg->idx]; @@ -1157,14 +1186,10 @@ again: reg->alloc = 1; /* - * All events using extra_reg are unconstrained. - * Avoids calling x86_get_event_constraints() - * - * Must revisit if extra_reg controlling events - * ever have constraints. Worst case we go through - * the regular event constraint table. + * need to call x86_get_event_constraint() + * to check if associated event has constraints */ - c = &unconstrained; + c = NULL; } else if (intel_try_alt_er(event, orig_idx)) { raw_spin_unlock_irqrestore(&era->lock, flags); goto again; @@ -1201,11 +1226,23 @@ static struct event_constraint * intel_shared_regs_constraints(struct cpu_hw_events *cpuc, struct perf_event *event) { - struct event_constraint *c = NULL; - - if (event->hw.extra_reg.idx != EXTRA_REG_NONE) - c = __intel_shared_reg_get_constraints(cpuc, event); - + struct event_constraint *c = NULL, *d; + struct hw_perf_event_extra *xreg, *breg; + + xreg = &event->hw.extra_reg; + if (xreg->idx != EXTRA_REG_NONE) { + c = __intel_shared_reg_get_constraints(cpuc, event, xreg); + if (c == &emptyconstraint) + return c; + } + breg = &event->hw.branch_reg; + if (breg->idx != EXTRA_REG_NONE) { + d = __intel_shared_reg_get_constraints(cpuc, event, breg); + if (d == &emptyconstraint) { + __intel_shared_reg_put_constraints(cpuc, xreg); + c = d; + } + } return c; } @@ -1253,6 +1290,10 @@ intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc, reg = &event->hw.extra_reg; if (reg->idx != EXTRA_REG_NONE) __intel_shared_reg_put_constraints(cpuc, reg); + + reg = &event->hw.branch_reg; + if (reg->idx != EXTRA_REG_NONE) + __intel_shared_reg_put_constraints(cpuc, reg); } static void intel_put_event_constraints(struct cpu_hw_events *cpuc, @@ -1288,12 +1329,19 @@ static int intel_pmu_hw_config(struct perf_event *event) * * Thereby we gain a PEBS capable cycle counter. */ - u64 alt_config = 0x108000c0; /* INST_RETIRED.TOTAL_CYCLES */ + u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16); + alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK); event->hw.config = alt_config; } + if (intel_pmu_needs_lbr_smpl(event)) { + ret = intel_pmu_setup_lbr_filter(event); + if (ret) + return ret; + } + if (event->attr.type != PERF_TYPE_RAW) return 0; @@ -1432,7 +1480,7 @@ static int intel_pmu_cpu_prepare(int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); - if (!x86_pmu.extra_regs) + if (!(x86_pmu.extra_regs || x86_pmu.lbr_sel_map)) return NOTIFY_OK; cpuc->shared_regs = allocate_shared_regs(cpu); @@ -1454,22 +1502,28 @@ static void intel_pmu_cpu_starting(int cpu) */ intel_pmu_lbr_reset(); - if (!cpuc->shared_regs || (x86_pmu.er_flags & ERF_NO_HT_SHARING)) + cpuc->lbr_sel = NULL; + + if (!cpuc->shared_regs) return; - for_each_cpu(i, topology_thread_cpumask(cpu)) { - struct intel_shared_regs *pc; + if (!(x86_pmu.er_flags & ERF_NO_HT_SHARING)) { + for_each_cpu(i, topology_thread_cpumask(cpu)) { + struct intel_shared_regs *pc; - pc = per_cpu(cpu_hw_events, i).shared_regs; - if (pc && pc->core_id == core_id) { - cpuc->kfree_on_online = cpuc->shared_regs; - cpuc->shared_regs = pc; - break; + pc = per_cpu(cpu_hw_events, i).shared_regs; + if (pc && pc->core_id == core_id) { + cpuc->kfree_on_online = cpuc->shared_regs; + cpuc->shared_regs = pc; + break; + } } + cpuc->shared_regs->core_id = core_id; + cpuc->shared_regs->refcnt++; } - cpuc->shared_regs->core_id = core_id; - cpuc->shared_regs->refcnt++; + if (x86_pmu.lbr_sel_map) + cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR]; } static void intel_pmu_cpu_dying(int cpu) @@ -1487,6 +1541,18 @@ static void intel_pmu_cpu_dying(int cpu) fini_debug_store_on_cpu(cpu); } +static void intel_pmu_flush_branch_stack(void) +{ + /* + * Intel LBR does not tag entries with the + * PID of the current task, then we need to + * flush it on ctxsw + * For now, we simply reset it + */ + if (x86_pmu.lbr_nr) + intel_pmu_lbr_reset(); +} + static __initconst const struct x86_pmu intel_pmu = { .name = "Intel", .handle_irq = intel_pmu_handle_irq, @@ -1514,6 +1580,7 @@ static __initconst const struct x86_pmu intel_pmu = { .cpu_starting = intel_pmu_cpu_starting, .cpu_dying = intel_pmu_cpu_dying, .guest_get_msrs = intel_guest_get_msrs, + .flush_branch_stack = intel_pmu_flush_branch_stack, }; static __init void intel_clovertown_quirk(void) @@ -1690,9 +1757,11 @@ __init int intel_pmu_init(void) x86_pmu.extra_regs = intel_nehalem_extra_regs; /* UOPS_ISSUED.STALLED_CYCLES */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = + X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = + X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); x86_add_quirk(intel_nehalem_quirk); @@ -1727,9 +1796,11 @@ __init int intel_pmu_init(void) x86_pmu.er_flags |= ERF_HAS_RSP_1; /* UOPS_ISSUED.STALLED_CYCLES */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = + X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = + X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); pr_cont("Westmere events, "); break; @@ -1740,7 +1811,7 @@ __init int intel_pmu_init(void) memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, sizeof(hw_cache_event_ids)); - intel_pmu_lbr_init_nhm(); + intel_pmu_lbr_init_snb(); x86_pmu.event_constraints = intel_snb_event_constraints; x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints; @@ -1750,9 +1821,11 @@ __init int intel_pmu_init(void) x86_pmu.er_flags |= ERF_NO_HT_SHARING; /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = + X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1; + intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = + X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1); pr_cont("SandyBridge events, "); break; diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c index d6bd49f..7f64df1 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c @@ -3,6 +3,7 @@ #include <linux/slab.h> #include <asm/perf_event.h> +#include <asm/insn.h> #include "perf_event.h" @@ -439,9 +440,6 @@ void intel_pmu_pebs_enable(struct perf_event *event) hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT; cpuc->pebs_enabled |= 1ULL << hwc->idx; - - if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1) - intel_pmu_lbr_enable(event); } void intel_pmu_pebs_disable(struct perf_event *event) @@ -454,9 +452,6 @@ void intel_pmu_pebs_disable(struct perf_event *event) wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); hwc->config |= ARCH_PERFMON_EVENTSEL_INT; - - if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1) - intel_pmu_lbr_disable(event); } void intel_pmu_pebs_enable_all(void) @@ -475,17 +470,6 @@ void intel_pmu_pebs_disable_all(void) wrmsrl(MSR_IA32_PEBS_ENABLE, 0); } -#include <asm/insn.h> - -static inline bool kernel_ip(unsigned long ip) -{ -#ifdef CONFIG_X86_32 - return ip > PAGE_OFFSET; -#else - return (long)ip < 0; -#endif -} - static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); @@ -572,6 +556,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, * both formats and we don't use the other fields in this * routine. */ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); struct pebs_record_core *pebs = __pebs; struct perf_sample_data data; struct pt_regs regs; @@ -602,6 +587,9 @@ static void __intel_pmu_pebs_event(struct perf_event *event, else regs.flags &= ~PERF_EFLAGS_EXACT; + if (has_branch_stack(event)) + data.br_stack = &cpuc->lbr_stack; + if (perf_event_overflow(event, &data, ®s)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c index 47a7e63..520b426 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c +++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c @@ -3,6 +3,7 @@ #include <asm/perf_event.h> #include <asm/msr.h> +#include <asm/insn.h> #include "perf_event.h" @@ -14,6 +15,100 @@ enum { }; /* + * Intel LBR_SELECT bits + * Intel Vol3a, April 2011, Section 16.7 Table 16-10 + * + * Hardware branch filter (not available on all CPUs) + */ +#define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ +#define LBR_USER_BIT 1 /* do not capture at ring > 0 */ +#define LBR_JCC_BIT 2 /* do not capture conditional branches */ +#define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ +#define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ +#define LBR_RETURN_BIT 5 /* do not capture near returns */ +#define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ +#define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ +#define LBR_FAR_BIT 8 /* do not capture far branches */ + +#define LBR_KERNEL (1 << LBR_KERNEL_BIT) +#define LBR_USER (1 << LBR_USER_BIT) +#define LBR_JCC (1 << LBR_JCC_BIT) +#define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) +#define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) +#define LBR_RETURN (1 << LBR_RETURN_BIT) +#define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) +#define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) +#define LBR_FAR (1 << LBR_FAR_BIT) + +#define LBR_PLM (LBR_KERNEL | LBR_USER) + +#define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */ +#define LBR_NOT_SUPP -1 /* LBR filter not supported */ +#define LBR_IGN 0 /* ignored */ + +#define LBR_ANY \ + (LBR_JCC |\ + LBR_REL_CALL |\ + LBR_IND_CALL |\ + LBR_RETURN |\ + LBR_REL_JMP |\ + LBR_IND_JMP |\ + LBR_FAR) + +#define LBR_FROM_FLAG_MISPRED (1ULL << 63) + +#define for_each_branch_sample_type(x) \ + for ((x) = PERF_SAMPLE_BRANCH_USER; \ + (x) < PERF_SAMPLE_BRANCH_MAX; (x) <<= 1) + +/* + * x86control flow change classification + * x86control flow changes include branches, interrupts, traps, faults + */ +enum { + X86_BR_NONE = 0, /* unknown */ + + X86_BR_USER = 1 << 0, /* branch target is user */ + X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ + + X86_BR_CALL = 1 << 2, /* call */ + X86_BR_RET = 1 << 3, /* return */ + X86_BR_SYSCALL = 1 << 4, /* syscall */ + X86_BR_SYSRET = 1 << 5, /* syscall return */ + X86_BR_INT = 1 << 6, /* sw interrupt */ + X86_BR_IRET = 1 << 7, /* return from interrupt */ + X86_BR_JCC = 1 << 8, /* conditional */ + X86_BR_JMP = 1 << 9, /* jump */ + X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ + X86_BR_IND_CALL = 1 << 11,/* indirect calls */ +}; + +#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) + +#define X86_BR_ANY \ + (X86_BR_CALL |\ + X86_BR_RET |\ + X86_BR_SYSCALL |\ + X86_BR_SYSRET |\ + X86_BR_INT |\ + X86_BR_IRET |\ + X86_BR_JCC |\ + X86_BR_JMP |\ + X86_BR_IRQ |\ + X86_BR_IND_CALL) + +#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) + +#define X86_BR_ANY_CALL \ + (X86_BR_CALL |\ + X86_BR_IND_CALL |\ + X86_BR_SYSCALL |\ + X86_BR_IRQ |\ + X86_BR_INT) + +static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); + +/* * We only support LBR implementations that have FREEZE_LBRS_ON_PMI * otherwise it becomes near impossible to get a reliable stack. */ @@ -21,6 +116,10 @@ enum { static void __intel_pmu_lbr_enable(void) { u64 debugctl; + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + + if (cpuc->lbr_sel) + wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config); rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); @@ -76,11 +175,11 @@ void intel_pmu_lbr_enable(struct perf_event *event) * Reset the LBR stack if we changed task context to * avoid data leaks. */ - if (event->ctx->task && cpuc->lbr_context != event->ctx) { intel_pmu_lbr_reset(); cpuc->lbr_context = event->ctx; } + cpuc->br_sel = event->hw.branch_reg.reg; cpuc->lbr_users++; } @@ -95,8 +194,11 @@ void intel_pmu_lbr_disable(struct perf_event *event) cpuc->lbr_users--; WARN_ON_ONCE(cpuc->lbr_users < 0); - if (cpuc->enabled && !cpuc->lbr_users) + if (cpuc->enabled && !cpuc->lbr_users) { __intel_pmu_lbr_disable(); + /* avoid stale pointer */ + cpuc->lbr_context = NULL; + } } void intel_pmu_lbr_enable_all(void) @@ -115,6 +217,9 @@ void intel_pmu_lbr_disable_all(void) __intel_pmu_lbr_disable(); } +/* + * TOS = most recently recorded branch + */ static inline u64 intel_pmu_lbr_tos(void) { u64 tos; @@ -142,15 +247,15 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); - cpuc->lbr_entries[i].from = msr_lastbranch.from; - cpuc->lbr_entries[i].to = msr_lastbranch.to; - cpuc->lbr_entries[i].flags = 0; + cpuc->lbr_entries[i].from = msr_lastbranch.from; + cpuc->lbr_entries[i].to = msr_lastbranch.to; + cpuc->lbr_entries[i].mispred = 0; + cpuc->lbr_entries[i].predicted = 0; + cpuc->lbr_entries[i].reserved = 0; } cpuc->lbr_stack.nr = i; } -#define LBR_FROM_FLAG_MISPRED (1ULL << 63) - /* * Due to lack of segmentation in Linux the effective address (offset) * is the same as the linear address, allowing us to merge the LIP and EIP @@ -165,19 +270,22 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) for (i = 0; i < x86_pmu.lbr_nr; i++) { unsigned long lbr_idx = (tos - i) & mask; - u64 from, to, flags = 0; + u64 from, to, mis = 0, pred = 0; rdmsrl(x86_pmu.lbr_from + lbr_idx, from); rdmsrl(x86_pmu.lbr_to + lbr_idx, to); if (lbr_format == LBR_FORMAT_EIP_FLAGS) { - flags = !!(from & LBR_FROM_FLAG_MISPRED); + mis = !!(from & LBR_FROM_FLAG_MISPRED); + pred = !mis; from = (u64)((((s64)from) << 1) >> 1); } - cpuc->lbr_entries[i].from = from; - cpuc->lbr_entries[i].to = to; - cpuc->lbr_entries[i].flags = flags; + cpuc->lbr_entries[i].from = from; + cpuc->lbr_entries[i].to = to; + cpuc->lbr_entries[i].mispred = mis; + cpuc->lbr_entries[i].predicted = pred; + cpuc->lbr_entries[i].reserved = 0; } cpuc->lbr_stack.nr = i; } @@ -193,28 +301,404 @@ void intel_pmu_lbr_read(void) intel_pmu_lbr_read_32(cpuc); else intel_pmu_lbr_read_64(cpuc); + + intel_pmu_lbr_filter(cpuc); +} + +/* + * SW filter is used: + * - in case there is no HW filter + * - in case the HW filter has errata or limitations + */ +static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event) +{ + u64 br_type = event->attr.branch_sample_type; + int mask = 0; + + if (br_type & PERF_SAMPLE_BRANCH_USER) + mask |= X86_BR_USER; + + if (br_type & PERF_SAMPLE_BRANCH_KERNEL) + mask |= X86_BR_KERNEL; + + /* we ignore BRANCH_HV here */ + + if (br_type & PERF_SAMPLE_BRANCH_ANY) + mask |= X86_BR_ANY; + + if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) + mask |= X86_BR_ANY_CALL; + + if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) + mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; + + if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) + mask |= X86_BR_IND_CALL; + /* + * stash actual user request into reg, it may + * be used by fixup code for some CPU + */ + event->hw.branch_reg.reg = mask; +} + +/* + * setup the HW LBR filter + * Used only when available, may not be enough to disambiguate + * all branches, may need the help of the SW filter + */ +static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) +{ + struct hw_perf_event_extra *reg; + u64 br_type = event->attr.branch_sample_type; + u64 mask = 0, m; + u64 v; + + for_each_branch_sample_type(m) { + if (!(br_type & m)) + continue; + + v = x86_pmu.lbr_sel_map[m]; + if (v == LBR_NOT_SUPP) + return -EOPNOTSUPP; + + if (v != LBR_IGN) + mask |= v; + } + reg = &event->hw.branch_reg; + reg->idx = EXTRA_REG_LBR; + + /* LBR_SELECT operates in suppress mode so invert mask */ + reg->config = ~mask & x86_pmu.lbr_sel_mask; + + return 0; +} + +int intel_pmu_setup_lbr_filter(struct perf_event *event) +{ + int ret = 0; + + /* + * no LBR on this PMU + */ + if (!x86_pmu.lbr_nr) + return -EOPNOTSUPP; + + /* + * setup SW LBR filter + */ + intel_pmu_setup_sw_lbr_filter(event); + + /* + * setup HW LBR filter, if any + */ + if (x86_pmu.lbr_sel_map) + ret = intel_pmu_setup_hw_lbr_filter(event); + + return ret; } +/* + * return the type of control flow change at address "from" + * intruction is not necessarily a branch (in case of interrupt). + * + * The branch type returned also includes the priv level of the + * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). + * + * If a branch type is unknown OR the instruction cannot be + * decoded (e.g., text page not present), then X86_BR_NONE is + * returned. + */ +static int branch_type(unsigned long from, unsigned long to) +{ + struct insn insn; + void *addr; + int bytes, size = MAX_INSN_SIZE; + int ret = X86_BR_NONE; + int ext, to_plm, from_plm; + u8 buf[MAX_INSN_SIZE]; + int is64 = 0; + + to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + + /* + * maybe zero if lbr did not fill up after a reset by the time + * we get a PMU interrupt + */ + if (from == 0 || to == 0) + return X86_BR_NONE; + + if (from_plm == X86_BR_USER) { + /* + * can happen if measuring at the user level only + * and we interrupt in a kernel thread, e.g., idle. + */ + if (!current->mm) + return X86_BR_NONE; + + /* may fail if text not present */ + bytes = copy_from_user_nmi(buf, (void __user *)from, size); + if (bytes != size) + return X86_BR_NONE; + + addr = buf; + } else + addr = (void *)from; + + /* + * decoder needs to know the ABI especially + * on 64-bit systems running 32-bit apps + */ +#ifdef CONFIG_X86_64 + is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32); +#endif + insn_init(&insn, addr, is64); + insn_get_opcode(&insn); + + switch (insn.opcode.bytes[0]) { + case 0xf: + switch (insn.opcode.bytes[1]) { + case 0x05: /* syscall */ + case 0x34: /* sysenter */ + ret = X86_BR_SYSCALL; + break; + case 0x07: /* sysret */ + case 0x35: /* sysexit */ + ret = X86_BR_SYSRET; + break; + case 0x80 ... 0x8f: /* conditional */ + ret = X86_BR_JCC; + break; + default: + ret = X86_BR_NONE; + } + break; + case 0x70 ... 0x7f: /* conditional */ + ret = X86_BR_JCC; + break; + case 0xc2: /* near ret */ + case 0xc3: /* near ret */ + case 0xca: /* far ret */ + case 0xcb: /* far ret */ + ret = X86_BR_RET; + break; + case 0xcf: /* iret */ + ret = X86_BR_IRET; + break; + case 0xcc ... 0xce: /* int */ + ret = X86_BR_INT; + break; + case 0xe8: /* call near rel */ + case 0x9a: /* call far absolute */ + ret = X86_BR_CALL; + break; + case 0xe0 ... 0xe3: /* loop jmp */ + ret = X86_BR_JCC; + break; + case 0xe9 ... 0xeb: /* jmp */ + ret = X86_BR_JMP; + break; + case 0xff: /* call near absolute, call far absolute ind */ + insn_get_modrm(&insn); + ext = (insn.modrm.bytes[0] >> 3) & 0x7; + switch (ext) { + case 2: /* near ind call */ + case 3: /* far ind call */ + ret = X86_BR_IND_CALL; + break; + case 4: + case 5: + ret = X86_BR_JMP; + break; + } + break; + default: + ret = X86_BR_NONE; + } + /* + * interrupts, traps, faults (and thus ring transition) may + * occur on any instructions. Thus, to classify them correctly, + * we need to first look at the from and to priv levels. If they + * are different and to is in the kernel, then it indicates + * a ring transition. If the from instruction is not a ring + * transition instr (syscall, systenter, int), then it means + * it was a irq, trap or fault. + * + * we have no way of detecting kernel to kernel faults. + */ + if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL + && ret != X86_BR_SYSCALL && ret != X86_BR_INT) + ret = X86_BR_IRQ; + + /* + * branch priv level determined by target as + * is done by HW when LBR_SELECT is implemented + */ + if (ret != X86_BR_NONE) + ret |= to_plm; + + return ret; +} + +/* + * implement actual branch filter based on user demand. + * Hardware may not exactly satisfy that request, thus + * we need to inspect opcodes. Mismatched branches are + * discarded. Therefore, the number of branches returned + * in PERF_SAMPLE_BRANCH_STACK sample may vary. + */ +static void +intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) +{ + u64 from, to; + int br_sel = cpuc->br_sel; + int i, j, type; + bool compress = false; + + /* if sampling all branches, then nothing to filter */ + if ((br_sel & X86_BR_ALL) == X86_BR_ALL) + return; + + for (i = 0; i < cpuc->lbr_stack.nr; i++) { + + from = cpuc->lbr_entries[i].from; + to = cpuc->lbr_entries[i].to; + + type = branch_type(from, to); + + /* if type does not correspond, then discard */ + if (type == X86_BR_NONE || (br_sel & type) != type) { + cpuc->lbr_entries[i].from = 0; + compress = true; + } + } + + if (!compress) + return; + + /* remove all entries with from=0 */ + for (i = 0; i < cpuc->lbr_stack.nr; ) { + if (!cpuc->lbr_entries[i].from) { + j = i; + while (++j < cpuc->lbr_stack.nr) + cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; + cpuc->lbr_stack.nr--; + if (!cpuc->lbr_entries[i].from) + continue; + } + i++; + } +} + +/* + * Map interface branch filters onto LBR filters + */ +static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = { + [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY, + [PERF_SAMPLE_BRANCH_USER] = LBR_USER, + [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL, + [PERF_SAMPLE_BRANCH_HV] = LBR_IGN, + [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_REL_JMP + | LBR_IND_JMP | LBR_FAR, + /* + * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches + */ + [PERF_SAMPLE_BRANCH_ANY_CALL] = + LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, + /* + * NHM/WSM erratum: must include IND_JMP to capture IND_CALL + */ + [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP, +}; + +static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = { + [PERF_SAMPLE_BRANCH_ANY] = LBR_ANY, + [PERF_SAMPLE_BRANCH_USER] = LBR_USER, + [PERF_SAMPLE_BRANCH_KERNEL] = LBR_KERNEL, + [PERF_SAMPLE_BRANCH_HV] = LBR_IGN, + [PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_FAR, + [PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL | LBR_IND_CALL + | LBR_FAR, + [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL, +}; + +/* core */ void intel_pmu_lbr_init_core(void) { x86_pmu.lbr_nr = 4; - x86_pmu.lbr_tos = 0x01c9; - x86_pmu.lbr_from = 0x40; - x86_pmu.lbr_to = 0x60; + x86_pmu.lbr_tos = MSR_LBR_TOS; + x86_pmu.lbr_from = MSR_LBR_CORE_FROM; + x86_pmu.lbr_to = MSR_LBR_CORE_TO; + + /* + * SW branch filter usage: + * - compensate for lack of HW filter + */ + pr_cont("4-deep LBR, "); } +/* nehalem/westmere */ void intel_pmu_lbr_init_nhm(void) { x86_pmu.lbr_nr = 16; - x86_pmu.lbr_tos = 0x01c9; - x86_pmu.lbr_from = 0x680; - x86_pmu.lbr_to = 0x6c0; + x86_pmu.lbr_tos = MSR_LBR_TOS; + x86_pmu.lbr_from = MSR_LBR_NHM_FROM; + x86_pmu.lbr_to = MSR_LBR_NHM_TO; + + x86_pmu.lbr_sel_mask = LBR_SEL_MASK; + x86_pmu.lbr_sel_map = nhm_lbr_sel_map; + + /* + * SW branch filter usage: + * - workaround LBR_SEL errata (see above) + * - support syscall, sysret capture. + * That requires LBR_FAR but that means far + * jmp need to be filtered out + */ + pr_cont("16-deep LBR, "); +} + +/* sandy bridge */ +void intel_pmu_lbr_init_snb(void) +{ + x86_pmu.lbr_nr = 16; + x86_pmu.lbr_tos = MSR_LBR_TOS; + x86_pmu.lbr_from = MSR_LBR_NHM_FROM; + x86_pmu.lbr_to = MSR_LBR_NHM_TO; + + x86_pmu.lbr_sel_mask = LBR_SEL_MASK; + x86_pmu.lbr_sel_map = snb_lbr_sel_map; + + /* + * SW branch filter usage: + * - support syscall, sysret capture. + * That requires LBR_FAR but that means far + * jmp need to be filtered out + */ + pr_cont("16-deep LBR, "); } +/* atom */ void intel_pmu_lbr_init_atom(void) { + /* + * only models starting at stepping 10 seems + * to have an operational LBR which can freeze + * on PMU interrupt + */ + if (boot_cpu_data.x86_mask < 10) { + pr_cont("LBR disabled due to erratum"); + return; + } + x86_pmu.lbr_nr = 8; - x86_pmu.lbr_tos = 0x01c9; - x86_pmu.lbr_from = 0x40; - x86_pmu.lbr_to = 0x60; + x86_pmu.lbr_tos = MSR_LBR_TOS; + x86_pmu.lbr_from = MSR_LBR_CORE_FROM; + x86_pmu.lbr_to = MSR_LBR_CORE_TO; + + /* + * SW branch filter usage: + * - compensate for lack of HW filter + */ + pr_cont("8-deep LBR, "); } diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index c7f64e6..addf9e8 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -40,6 +40,7 @@ void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c) { X86_FEATURE_EPB, CR_ECX, 3, 0x00000006, 0 }, { X86_FEATURE_XSAVEOPT, CR_EAX, 0, 0x0000000d, 1 }, { X86_FEATURE_CPB, CR_EDX, 9, 0x80000007, 0 }, + { X86_FEATURE_HW_PSTATE, CR_EDX, 7, 0x80000007, 0 }, { X86_FEATURE_NPT, CR_EDX, 0, 0x8000000a, 0 }, { X86_FEATURE_LBRV, CR_EDX, 1, 0x8000000a, 0 }, { X86_FEATURE_SVML, CR_EDX, 2, 0x8000000a, 0 }, diff --git a/arch/x86/kernel/crash_dump_32.c b/arch/x86/kernel/crash_dump_32.c index 642f75a..11891ca 100644 --- a/arch/x86/kernel/crash_dump_32.c +++ b/arch/x86/kernel/crash_dump_32.c @@ -62,16 +62,16 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf, if (!userbuf) { memcpy(buf, (vaddr + offset), csize); - kunmap_atomic(vaddr, KM_PTE0); + kunmap_atomic(vaddr); } else { if (!kdump_buf_page) { printk(KERN_WARNING "Kdump: Kdump buffer page not" " allocated\n"); - kunmap_atomic(vaddr, KM_PTE0); + kunmap_atomic(vaddr); return -EFAULT; } copy_page(kdump_buf_page, vaddr); - kunmap_atomic(vaddr, KM_PTE0); + kunmap_atomic(vaddr); if (copy_to_user(buf, (kdump_buf_page + offset), csize)) return -EFAULT; } diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 5282179..3ae2ced 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -4,6 +4,7 @@ #include <linux/bootmem.h> #include <linux/export.h> #include <linux/io.h> +#include <linux/irqdomain.h> #include <linux/interrupt.h> #include <linux/list.h> #include <linux/of.h> @@ -17,64 +18,14 @@ #include <linux/initrd.h> #include <asm/hpet.h> -#include <asm/irq_controller.h> #include <asm/apic.h> #include <asm/pci_x86.h> __initdata u64 initial_dtb; char __initdata cmd_line[COMMAND_LINE_SIZE]; -static LIST_HEAD(irq_domains); -static DEFINE_RAW_SPINLOCK(big_irq_lock); int __initdata of_ioapic; -#ifdef CONFIG_X86_IO_APIC -static void add_interrupt_host(struct irq_domain *ih) -{ - unsigned long flags; - - raw_spin_lock_irqsave(&big_irq_lock, flags); - list_add(&ih->l, &irq_domains); - raw_spin_unlock_irqrestore(&big_irq_lock, flags); -} -#endif - -static struct irq_domain *get_ih_from_node(struct device_node *controller) -{ - struct irq_domain *ih, *found = NULL; - unsigned long flags; - - raw_spin_lock_irqsave(&big_irq_lock, flags); - list_for_each_entry(ih, &irq_domains, l) { - if (ih->controller == controller) { - found = ih; - break; - } - } - raw_spin_unlock_irqrestore(&big_irq_lock, flags); - return found; -} - -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) -{ - struct irq_domain *ih; - u32 virq, type; - int ret; - - ih = get_ih_from_node(controller); - if (!ih) - return 0; - ret = ih->xlate(ih, intspec, intsize, &virq, &type); - if (ret) - return 0; - if (type == IRQ_TYPE_NONE) - return virq; - irq_set_irq_type(virq, type); - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); - unsigned long pci_address_to_pio(phys_addr_t address) { /* @@ -354,36 +305,43 @@ static struct of_ioapic_type of_ioapic_type[] = }, }; -static int ioapic_xlate(struct irq_domain *id, const u32 *intspec, u32 intsize, - u32 *out_hwirq, u32 *out_type) +static int ioapic_xlate(struct irq_domain *domain, + struct device_node *controller, + const u32 *intspec, u32 intsize, + irq_hw_number_t *out_hwirq, u32 *out_type) { - struct mp_ioapic_gsi *gsi_cfg; struct io_apic_irq_attr attr; struct of_ioapic_type *it; - u32 line, idx, type; + u32 line, idx; + int rc; - if (intsize < 2) + if (WARN_ON(intsize < 2)) return -EINVAL; - line = *intspec; - idx = (u32) id->priv; - gsi_cfg = mp_ioapic_gsi_routing(idx); - *out_hwirq = line + gsi_cfg->gsi_base; - - intspec++; - type = *intspec; + line = intspec[0]; - if (type >= ARRAY_SIZE(of_ioapic_type)) + if (intspec[1] >= ARRAY_SIZE(of_ioapic_type)) return -EINVAL; - it = of_ioapic_type + type; - *out_type = it->out_type; + it = &of_ioapic_type[intspec[1]]; + idx = (u32) domain->host_data; set_io_apic_irq_attr(&attr, idx, line, it->trigger, it->polarity); - return io_apic_setup_irq_pin_once(*out_hwirq, cpu_to_node(0), &attr); + rc = io_apic_setup_irq_pin_once(irq_find_mapping(domain, line), + cpu_to_node(0), &attr); + if (rc) + return rc; + + *out_hwirq = line; + *out_type = it->out_type; + return 0; } +const struct irq_domain_ops ioapic_irq_domain_ops = { + .xlate = ioapic_xlate, +}; + static void __init ioapic_add_ofnode(struct device_node *np) { struct resource r; @@ -399,13 +357,14 @@ static void __init ioapic_add_ofnode(struct device_node *np) for (i = 0; i < nr_ioapics; i++) { if (r.start == mpc_ioapic_addr(i)) { struct irq_domain *id; + struct mp_ioapic_gsi *gsi_cfg; + + gsi_cfg = mp_ioapic_gsi_routing(i); - id = kzalloc(sizeof(*id), GFP_KERNEL); + id = irq_domain_add_legacy(np, 32, gsi_cfg->gsi_base, 0, + &ioapic_irq_domain_ops, + (void*)i); BUG_ON(!id); - id->controller = np; - id->xlate = ioapic_xlate; - id->priv = (void *)i; - add_interrupt_host(id); return; } } diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 40fc861..58b7f27 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -100,13 +100,8 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) irqctx->tinfo.task = curctx->tinfo.task; irqctx->tinfo.previous_esp = current_stack_pointer; - /* - * Copy the softirq bits in preempt_count so that the - * softirq checks work in the hardirq context. - */ - irqctx->tinfo.preempt_count = - (irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) | - (curctx->tinfo.preempt_count & SOFTIRQ_MASK); + /* Copy the preempt_count so that the [soft]irq checks work. */ + irqctx->tinfo.preempt_count = curctx->tinfo.preempt_count; if (unlikely(overflow)) call_on_stack(print_stack_overflow, isp); @@ -196,7 +191,7 @@ bool handle_irq(unsigned irq, struct pt_regs *regs) if (unlikely(!desc)) return false; - if (!execute_on_irq_stack(overflow, desc, irq)) { + if (user_mode_vm(regs) || !execute_on_irq_stack(overflow, desc, irq)) { if (unlikely(overflow)) print_stack_overflow(); desc->handle_irq(irq, desc); diff --git a/arch/x86/kernel/kprobes-common.h b/arch/x86/kernel/kprobes-common.h new file mode 100644 index 0000000..3230b68 --- /dev/null +++ b/arch/x86/kernel/kprobes-common.h @@ -0,0 +1,102 @@ +#ifndef __X86_KERNEL_KPROBES_COMMON_H +#define __X86_KERNEL_KPROBES_COMMON_H + +/* Kprobes and Optprobes common header */ + +#ifdef CONFIG_X86_64 +#define SAVE_REGS_STRING \ + /* Skip cs, ip, orig_ax. */ \ + " subq $24, %rsp\n" \ + " pushq %rdi\n" \ + " pushq %rsi\n" \ + " pushq %rdx\n" \ + " pushq %rcx\n" \ + " pushq %rax\n" \ + " pushq %r8\n" \ + " pushq %r9\n" \ + " pushq %r10\n" \ + " pushq %r11\n" \ + " pushq %rbx\n" \ + " pushq %rbp\n" \ + " pushq %r12\n" \ + " pushq %r13\n" \ + " pushq %r14\n" \ + " pushq %r15\n" +#define RESTORE_REGS_STRING \ + " popq %r15\n" \ + " popq %r14\n" \ + " popq %r13\n" \ + " popq %r12\n" \ + " popq %rbp\n" \ + " popq %rbx\n" \ + " popq %r11\n" \ + " popq %r10\n" \ + " popq %r9\n" \ + " popq %r8\n" \ + " popq %rax\n" \ + " popq %rcx\n" \ + " popq %rdx\n" \ + " popq %rsi\n" \ + " popq %rdi\n" \ + /* Skip orig_ax, ip, cs */ \ + " addq $24, %rsp\n" +#else +#define SAVE_REGS_STRING \ + /* Skip cs, ip, orig_ax and gs. */ \ + " subl $16, %esp\n" \ + " pushl %fs\n" \ + " pushl %es\n" \ + " pushl %ds\n" \ + " pushl %eax\n" \ + " pushl %ebp\n" \ + " pushl %edi\n" \ + " pushl %esi\n" \ + " pushl %edx\n" \ + " pushl %ecx\n" \ + " pushl %ebx\n" +#define RESTORE_REGS_STRING \ + " popl %ebx\n" \ + " popl %ecx\n" \ + " popl %edx\n" \ + " popl %esi\n" \ + " popl %edi\n" \ + " popl %ebp\n" \ + " popl %eax\n" \ + /* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\ + " addl $24, %esp\n" +#endif + +/* Ensure if the instruction can be boostable */ +extern int can_boost(kprobe_opcode_t *instruction); +/* Recover instruction if given address is probed */ +extern unsigned long recover_probed_instruction(kprobe_opcode_t *buf, + unsigned long addr); +/* + * Copy an instruction and adjust the displacement if the instruction + * uses the %rip-relative addressing mode. + */ +extern int __copy_instruction(u8 *dest, u8 *src); + +/* Generate a relative-jump/call instruction */ +extern void synthesize_reljump(void *from, void *to); +extern void synthesize_relcall(void *from, void *to); + +#ifdef CONFIG_OPTPROBES +extern int arch_init_optprobes(void); +extern int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter); +extern unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr); +#else /* !CONFIG_OPTPROBES */ +static inline int arch_init_optprobes(void) +{ + return 0; +} +static inline int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) +{ + return 0; +} +static inline unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) +{ + return addr; +} +#endif +#endif diff --git a/arch/x86/kernel/kprobes-opt.c b/arch/x86/kernel/kprobes-opt.c new file mode 100644 index 0000000..c5e410e --- /dev/null +++ b/arch/x86/kernel/kprobes-opt.c @@ -0,0 +1,512 @@ +/* + * Kernel Probes Jump Optimization (Optprobes) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2002, 2004 + * Copyright (C) Hitachi Ltd., 2012 + */ +#include <linux/kprobes.h> +#include <linux/ptrace.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/hardirq.h> +#include <linux/preempt.h> +#include <linux/module.h> +#include <linux/kdebug.h> +#include <linux/kallsyms.h> +#include <linux/ftrace.h> + +#include <asm/cacheflush.h> +#include <asm/desc.h> +#include <asm/pgtable.h> +#include <asm/uaccess.h> +#include <asm/alternative.h> +#include <asm/insn.h> +#include <asm/debugreg.h> + +#include "kprobes-common.h" + +unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) +{ + struct optimized_kprobe *op; + struct kprobe *kp; + long offs; + int i; + + for (i = 0; i < RELATIVEJUMP_SIZE; i++) { + kp = get_kprobe((void *)addr - i); + /* This function only handles jump-optimized kprobe */ + if (kp && kprobe_optimized(kp)) { + op = container_of(kp, struct optimized_kprobe, kp); + /* If op->list is not empty, op is under optimizing */ + if (list_empty(&op->list)) + goto found; + } + } + + return addr; +found: + /* + * If the kprobe can be optimized, original bytes which can be + * overwritten by jump destination address. In this case, original + * bytes must be recovered from op->optinsn.copied_insn buffer. + */ + memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); + if (addr == (unsigned long)kp->addr) { + buf[0] = kp->opcode; + memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); + } else { + offs = addr - (unsigned long)kp->addr - 1; + memcpy(buf, op->optinsn.copied_insn + offs, RELATIVE_ADDR_SIZE - offs); + } + + return (unsigned long)buf; +} + +/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ +static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) +{ +#ifdef CONFIG_X86_64 + *addr++ = 0x48; + *addr++ = 0xbf; +#else + *addr++ = 0xb8; +#endif + *(unsigned long *)addr = val; +} + +static void __used __kprobes kprobes_optinsn_template_holder(void) +{ + asm volatile ( + ".global optprobe_template_entry\n" + "optprobe_template_entry:\n" +#ifdef CONFIG_X86_64 + /* We don't bother saving the ss register */ + " pushq %rsp\n" + " pushfq\n" + SAVE_REGS_STRING + " movq %rsp, %rsi\n" + ".global optprobe_template_val\n" + "optprobe_template_val:\n" + ASM_NOP5 + ASM_NOP5 + ".global optprobe_template_call\n" + "optprobe_template_call:\n" + ASM_NOP5 + /* Move flags to rsp */ + " movq 144(%rsp), %rdx\n" + " movq %rdx, 152(%rsp)\n" + RESTORE_REGS_STRING + /* Skip flags entry */ + " addq $8, %rsp\n" + " popfq\n" +#else /* CONFIG_X86_32 */ + " pushf\n" + SAVE_REGS_STRING + " movl %esp, %edx\n" + ".global optprobe_template_val\n" + "optprobe_template_val:\n" + ASM_NOP5 + ".global optprobe_template_call\n" + "optprobe_template_call:\n" + ASM_NOP5 + RESTORE_REGS_STRING + " addl $4, %esp\n" /* skip cs */ + " popf\n" +#endif + ".global optprobe_template_end\n" + "optprobe_template_end:\n"); +} + +#define TMPL_MOVE_IDX \ + ((long)&optprobe_template_val - (long)&optprobe_template_entry) +#define TMPL_CALL_IDX \ + ((long)&optprobe_template_call - (long)&optprobe_template_entry) +#define TMPL_END_IDX \ + ((long)&optprobe_template_end - (long)&optprobe_template_entry) + +#define INT3_SIZE sizeof(kprobe_opcode_t) + +/* Optimized kprobe call back function: called from optinsn */ +static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + unsigned long flags; + + /* This is possible if op is under delayed unoptimizing */ + if (kprobe_disabled(&op->kp)) + return; + + local_irq_save(flags); + if (kprobe_running()) { + kprobes_inc_nmissed_count(&op->kp); + } else { + /* Save skipped registers */ +#ifdef CONFIG_X86_64 + regs->cs = __KERNEL_CS; +#else + regs->cs = __KERNEL_CS | get_kernel_rpl(); + regs->gs = 0; +#endif + regs->ip = (unsigned long)op->kp.addr + INT3_SIZE; + regs->orig_ax = ~0UL; + + __this_cpu_write(current_kprobe, &op->kp); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + opt_pre_handler(&op->kp, regs); + __this_cpu_write(current_kprobe, NULL); + } + local_irq_restore(flags); +} + +static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src) +{ + int len = 0, ret; + + while (len < RELATIVEJUMP_SIZE) { + ret = __copy_instruction(dest + len, src + len); + if (!ret || !can_boost(dest + len)) + return -EINVAL; + len += ret; + } + /* Check whether the address range is reserved */ + if (ftrace_text_reserved(src, src + len - 1) || + alternatives_text_reserved(src, src + len - 1) || + jump_label_text_reserved(src, src + len - 1)) + return -EBUSY; + + return len; +} + +/* Check whether insn is indirect jump */ +static int __kprobes insn_is_indirect_jump(struct insn *insn) +{ + return ((insn->opcode.bytes[0] == 0xff && + (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ + insn->opcode.bytes[0] == 0xea); /* Segment based jump */ +} + +/* Check whether insn jumps into specified address range */ +static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) +{ + unsigned long target = 0; + + switch (insn->opcode.bytes[0]) { + case 0xe0: /* loopne */ + case 0xe1: /* loope */ + case 0xe2: /* loop */ + case 0xe3: /* jcxz */ + case 0xe9: /* near relative jump */ + case 0xeb: /* short relative jump */ + break; + case 0x0f: + if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */ + break; + return 0; + default: + if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */ + break; + return 0; + } + target = (unsigned long)insn->next_byte + insn->immediate.value; + + return (start <= target && target <= start + len); +} + +/* Decode whole function to ensure any instructions don't jump into target */ +static int __kprobes can_optimize(unsigned long paddr) +{ + unsigned long addr, size = 0, offset = 0; + struct insn insn; + kprobe_opcode_t buf[MAX_INSN_SIZE]; + + /* Lookup symbol including addr */ + if (!kallsyms_lookup_size_offset(paddr, &size, &offset)) + return 0; + + /* + * Do not optimize in the entry code due to the unstable + * stack handling. + */ + if ((paddr >= (unsigned long)__entry_text_start) && + (paddr < (unsigned long)__entry_text_end)) + return 0; + + /* Check there is enough space for a relative jump. */ + if (size - offset < RELATIVEJUMP_SIZE) + return 0; + + /* Decode instructions */ + addr = paddr - offset; + while (addr < paddr - offset + size) { /* Decode until function end */ + if (search_exception_tables(addr)) + /* + * Since some fixup code will jumps into this function, + * we can't optimize kprobe in this function. + */ + return 0; + kernel_insn_init(&insn, (void *)recover_probed_instruction(buf, addr)); + insn_get_length(&insn); + /* Another subsystem puts a breakpoint */ + if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) + return 0; + /* Recover address */ + insn.kaddr = (void *)addr; + insn.next_byte = (void *)(addr + insn.length); + /* Check any instructions don't jump into target */ + if (insn_is_indirect_jump(&insn) || + insn_jump_into_range(&insn, paddr + INT3_SIZE, + RELATIVE_ADDR_SIZE)) + return 0; + addr += insn.length; + } + + return 1; +} + +/* Check optimized_kprobe can actually be optimized. */ +int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op) +{ + int i; + struct kprobe *p; + + for (i = 1; i < op->optinsn.size; i++) { + p = get_kprobe(op->kp.addr + i); + if (p && !kprobe_disabled(p)) + return -EEXIST; + } + + return 0; +} + +/* Check the addr is within the optimized instructions. */ +int __kprobes +arch_within_optimized_kprobe(struct optimized_kprobe *op, unsigned long addr) +{ + return ((unsigned long)op->kp.addr <= addr && + (unsigned long)op->kp.addr + op->optinsn.size > addr); +} + +/* Free optimized instruction slot */ +static __kprobes +void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) +{ + if (op->optinsn.insn) { + free_optinsn_slot(op->optinsn.insn, dirty); + op->optinsn.insn = NULL; + op->optinsn.size = 0; + } +} + +void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op) +{ + __arch_remove_optimized_kprobe(op, 1); +} + +/* + * Copy replacing target instructions + * Target instructions MUST be relocatable (checked inside) + * This is called when new aggr(opt)probe is allocated or reused. + */ +int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) +{ + u8 *buf; + int ret; + long rel; + + if (!can_optimize((unsigned long)op->kp.addr)) + return -EILSEQ; + + op->optinsn.insn = get_optinsn_slot(); + if (!op->optinsn.insn) + return -ENOMEM; + + /* + * Verify if the address gap is in 2GB range, because this uses + * a relative jump. + */ + rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE; + if (abs(rel) > 0x7fffffff) + return -ERANGE; + + buf = (u8 *)op->optinsn.insn; + + /* Copy instructions into the out-of-line buffer */ + ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr); + if (ret < 0) { + __arch_remove_optimized_kprobe(op, 0); + return ret; + } + op->optinsn.size = ret; + + /* Copy arch-dep-instance from template */ + memcpy(buf, &optprobe_template_entry, TMPL_END_IDX); + + /* Set probe information */ + synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); + + /* Set probe function call */ + synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback); + + /* Set returning jmp instruction at the tail of out-of-line buffer */ + synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size, + (u8 *)op->kp.addr + op->optinsn.size); + + flush_icache_range((unsigned long) buf, + (unsigned long) buf + TMPL_END_IDX + + op->optinsn.size + RELATIVEJUMP_SIZE); + return 0; +} + +#define MAX_OPTIMIZE_PROBES 256 +static struct text_poke_param *jump_poke_params; +static struct jump_poke_buffer { + u8 buf[RELATIVEJUMP_SIZE]; +} *jump_poke_bufs; + +static void __kprobes setup_optimize_kprobe(struct text_poke_param *tprm, + u8 *insn_buf, + struct optimized_kprobe *op) +{ + s32 rel = (s32)((long)op->optinsn.insn - + ((long)op->kp.addr + RELATIVEJUMP_SIZE)); + + /* Backup instructions which will be replaced by jump address */ + memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, + RELATIVE_ADDR_SIZE); + + insn_buf[0] = RELATIVEJUMP_OPCODE; + *(s32 *)(&insn_buf[1]) = rel; + + tprm->addr = op->kp.addr; + tprm->opcode = insn_buf; + tprm->len = RELATIVEJUMP_SIZE; +} + +/* + * Replace breakpoints (int3) with relative jumps. + * Caller must call with locking kprobe_mutex and text_mutex. + */ +void __kprobes arch_optimize_kprobes(struct list_head *oplist) +{ + struct optimized_kprobe *op, *tmp; + int c = 0; + + list_for_each_entry_safe(op, tmp, oplist, list) { + WARN_ON(kprobe_disabled(&op->kp)); + /* Setup param */ + setup_optimize_kprobe(&jump_poke_params[c], + jump_poke_bufs[c].buf, op); + list_del_init(&op->list); + if (++c >= MAX_OPTIMIZE_PROBES) + break; + } + + /* + * text_poke_smp doesn't support NMI/MCE code modifying. + * However, since kprobes itself also doesn't support NMI/MCE + * code probing, it's not a problem. + */ + text_poke_smp_batch(jump_poke_params, c); +} + +static void __kprobes setup_unoptimize_kprobe(struct text_poke_param *tprm, + u8 *insn_buf, + struct optimized_kprobe *op) +{ + /* Set int3 to first byte for kprobes */ + insn_buf[0] = BREAKPOINT_INSTRUCTION; + memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); + + tprm->addr = op->kp.addr; + tprm->opcode = insn_buf; + tprm->len = RELATIVEJUMP_SIZE; +} + +/* + * Recover original instructions and breakpoints from relative jumps. + * Caller must call with locking kprobe_mutex. + */ +extern void arch_unoptimize_kprobes(struct list_head *oplist, + struct list_head *done_list) +{ + struct optimized_kprobe *op, *tmp; + int c = 0; + + list_for_each_entry_safe(op, tmp, oplist, list) { + /* Setup param */ + setup_unoptimize_kprobe(&jump_poke_params[c], + jump_poke_bufs[c].buf, op); + list_move(&op->list, done_list); + if (++c >= MAX_OPTIMIZE_PROBES) + break; + } + + /* + * text_poke_smp doesn't support NMI/MCE code modifying. + * However, since kprobes itself also doesn't support NMI/MCE + * code probing, it's not a problem. + */ + text_poke_smp_batch(jump_poke_params, c); +} + +/* Replace a relative jump with a breakpoint (int3). */ +void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) +{ + u8 buf[RELATIVEJUMP_SIZE]; + + /* Set int3 to first byte for kprobes */ + buf[0] = BREAKPOINT_INSTRUCTION; + memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); + text_poke_smp(op->kp.addr, buf, RELATIVEJUMP_SIZE); +} + +int __kprobes +setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) +{ + struct optimized_kprobe *op; + + if (p->flags & KPROBE_FLAG_OPTIMIZED) { + /* This kprobe is really able to run optimized path. */ + op = container_of(p, struct optimized_kprobe, kp); + /* Detour through copied instructions */ + regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; + if (!reenter) + reset_current_kprobe(); + preempt_enable_no_resched(); + return 1; + } + return 0; +} + +int __kprobes arch_init_optprobes(void) +{ + /* Allocate code buffer and parameter array */ + jump_poke_bufs = kmalloc(sizeof(struct jump_poke_buffer) * + MAX_OPTIMIZE_PROBES, GFP_KERNEL); + if (!jump_poke_bufs) + return -ENOMEM; + + jump_poke_params = kmalloc(sizeof(struct text_poke_param) * + MAX_OPTIMIZE_PROBES, GFP_KERNEL); + if (!jump_poke_params) { + kfree(jump_poke_bufs); + jump_poke_bufs = NULL; + return -ENOMEM; + } + + return 0; +} diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c index 7da647d..e213fc8 100644 --- a/arch/x86/kernel/kprobes.c +++ b/arch/x86/kernel/kprobes.c @@ -30,16 +30,15 @@ * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi * <prasanna@in.ibm.com> added function-return probes. * 2005-May Rusty Lynch <rusty.lynch@intel.com> - * Added function return probes functionality + * Added function return probes functionality * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added - * kprobe-booster and kretprobe-booster for i386. + * kprobe-booster and kretprobe-booster for i386. * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster - * and kretprobe-booster for x86-64 + * and kretprobe-booster for x86-64 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven - * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> - * unified x86 kprobes code. + * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> + * unified x86 kprobes code. */ - #include <linux/kprobes.h> #include <linux/ptrace.h> #include <linux/string.h> @@ -59,6 +58,8 @@ #include <asm/insn.h> #include <asm/debugreg.h> +#include "kprobes-common.h" + void jprobe_return_end(void); DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; @@ -108,6 +109,7 @@ struct kretprobe_blackpoint kretprobe_blacklist[] = { doesn't switch kernel stack.*/ {NULL, NULL} /* Terminator */ }; + const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op) @@ -123,11 +125,17 @@ static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op) } /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ -static void __kprobes synthesize_reljump(void *from, void *to) +void __kprobes synthesize_reljump(void *from, void *to) { __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE); } +/* Insert a call instruction at address 'from', which calls address 'to'.*/ +void __kprobes synthesize_relcall(void *from, void *to) +{ + __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE); +} + /* * Skip the prefixes of the instruction. */ @@ -151,7 +159,7 @@ static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn) * Returns non-zero if opcode is boostable. * RIP relative instructions are adjusted at copying time in 64 bits mode */ -static int __kprobes can_boost(kprobe_opcode_t *opcodes) +int __kprobes can_boost(kprobe_opcode_t *opcodes) { kprobe_opcode_t opcode; kprobe_opcode_t *orig_opcodes = opcodes; @@ -207,13 +215,15 @@ retry: } } -/* Recover the probed instruction at addr for further analysis. */ -static int recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) +static unsigned long +__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) { struct kprobe *kp; + kp = get_kprobe((void *)addr); + /* There is no probe, return original address */ if (!kp) - return -EINVAL; + return addr; /* * Basically, kp->ainsn.insn has an original instruction. @@ -230,14 +240,29 @@ static int recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) */ memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); buf[0] = kp->opcode; - return 0; + return (unsigned long)buf; +} + +/* + * Recover the probed instruction at addr for further analysis. + * Caller must lock kprobes by kprobe_mutex, or disable preemption + * for preventing to release referencing kprobes. + */ +unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) +{ + unsigned long __addr; + + __addr = __recover_optprobed_insn(buf, addr); + if (__addr != addr) + return __addr; + + return __recover_probed_insn(buf, addr); } /* Check if paddr is at an instruction boundary */ static int __kprobes can_probe(unsigned long paddr) { - int ret; - unsigned long addr, offset = 0; + unsigned long addr, __addr, offset = 0; struct insn insn; kprobe_opcode_t buf[MAX_INSN_SIZE]; @@ -247,26 +272,24 @@ static int __kprobes can_probe(unsigned long paddr) /* Decode instructions */ addr = paddr - offset; while (addr < paddr) { - kernel_insn_init(&insn, (void *)addr); - insn_get_opcode(&insn); - /* * Check if the instruction has been modified by another * kprobe, in which case we replace the breakpoint by the * original instruction in our buffer. + * Also, jump optimization will change the breakpoint to + * relative-jump. Since the relative-jump itself is + * normally used, we just go through if there is no kprobe. */ - if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) { - ret = recover_probed_instruction(buf, addr); - if (ret) - /* - * Another debugging subsystem might insert - * this breakpoint. In that case, we can't - * recover it. - */ - return 0; - kernel_insn_init(&insn, buf); - } + __addr = recover_probed_instruction(buf, addr); + kernel_insn_init(&insn, (void *)__addr); insn_get_length(&insn); + + /* + * Another debugging subsystem might insert this breakpoint. + * In that case, we can't recover it. + */ + if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) + return 0; addr += insn.length; } @@ -299,24 +322,16 @@ static int __kprobes is_IF_modifier(kprobe_opcode_t *insn) * If not, return null. * Only applicable to 64-bit x86. */ -static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover) +int __kprobes __copy_instruction(u8 *dest, u8 *src) { struct insn insn; - int ret; kprobe_opcode_t buf[MAX_INSN_SIZE]; - kernel_insn_init(&insn, src); - if (recover) { - insn_get_opcode(&insn); - if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) { - ret = recover_probed_instruction(buf, - (unsigned long)src); - if (ret) - return 0; - kernel_insn_init(&insn, buf); - } - } + kernel_insn_init(&insn, (void *)recover_probed_instruction(buf, (unsigned long)src)); insn_get_length(&insn); + /* Another subsystem puts a breakpoint, failed to recover */ + if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) + return 0; memcpy(dest, insn.kaddr, insn.length); #ifdef CONFIG_X86_64 @@ -337,8 +352,7 @@ static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover) * extension of the original signed 32-bit displacement would * have given. */ - newdisp = (u8 *) src + (s64) insn.displacement.value - - (u8 *) dest; + newdisp = (u8 *) src + (s64) insn.displacement.value - (u8 *) dest; BUG_ON((s64) (s32) newdisp != newdisp); /* Sanity check. */ disp = (u8 *) dest + insn_offset_displacement(&insn); *(s32 *) disp = (s32) newdisp; @@ -349,18 +363,20 @@ static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover) static void __kprobes arch_copy_kprobe(struct kprobe *p) { + /* Copy an instruction with recovering if other optprobe modifies it.*/ + __copy_instruction(p->ainsn.insn, p->addr); + /* - * Copy an instruction without recovering int3, because it will be - * put by another subsystem. + * __copy_instruction can modify the displacement of the instruction, + * but it doesn't affect boostable check. */ - __copy_instruction(p->ainsn.insn, p->addr, 0); - - if (can_boost(p->addr)) + if (can_boost(p->ainsn.insn)) p->ainsn.boostable = 0; else p->ainsn.boostable = -1; - p->opcode = *p->addr; + /* Also, displacement change doesn't affect the first byte */ + p->opcode = p->ainsn.insn[0]; } int __kprobes arch_prepare_kprobe(struct kprobe *p) @@ -442,8 +458,8 @@ static void __kprobes restore_btf(void) } } -void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, - struct pt_regs *regs) +void __kprobes +arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { unsigned long *sara = stack_addr(regs); @@ -453,16 +469,8 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, *sara = (unsigned long) &kretprobe_trampoline; } -#ifdef CONFIG_OPTPROBES -static int __kprobes setup_detour_execution(struct kprobe *p, - struct pt_regs *regs, - int reenter); -#else -#define setup_detour_execution(p, regs, reenter) (0) -#endif - -static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb, int reenter) +static void __kprobes +setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter) { if (setup_detour_execution(p, regs, reenter)) return; @@ -504,8 +512,8 @@ static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs, * within the handler. We save the original kprobes variables and just single * step on the instruction of the new probe without calling any user handlers. */ -static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb) +static int __kprobes +reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { switch (kcb->kprobe_status) { case KPROBE_HIT_SSDONE: @@ -600,69 +608,6 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) return 0; } -#ifdef CONFIG_X86_64 -#define SAVE_REGS_STRING \ - /* Skip cs, ip, orig_ax. */ \ - " subq $24, %rsp\n" \ - " pushq %rdi\n" \ - " pushq %rsi\n" \ - " pushq %rdx\n" \ - " pushq %rcx\n" \ - " pushq %rax\n" \ - " pushq %r8\n" \ - " pushq %r9\n" \ - " pushq %r10\n" \ - " pushq %r11\n" \ - " pushq %rbx\n" \ - " pushq %rbp\n" \ - " pushq %r12\n" \ - " pushq %r13\n" \ - " pushq %r14\n" \ - " pushq %r15\n" -#define RESTORE_REGS_STRING \ - " popq %r15\n" \ - " popq %r14\n" \ - " popq %r13\n" \ - " popq %r12\n" \ - " popq %rbp\n" \ - " popq %rbx\n" \ - " popq %r11\n" \ - " popq %r10\n" \ - " popq %r9\n" \ - " popq %r8\n" \ - " popq %rax\n" \ - " popq %rcx\n" \ - " popq %rdx\n" \ - " popq %rsi\n" \ - " popq %rdi\n" \ - /* Skip orig_ax, ip, cs */ \ - " addq $24, %rsp\n" -#else -#define SAVE_REGS_STRING \ - /* Skip cs, ip, orig_ax and gs. */ \ - " subl $16, %esp\n" \ - " pushl %fs\n" \ - " pushl %es\n" \ - " pushl %ds\n" \ - " pushl %eax\n" \ - " pushl %ebp\n" \ - " pushl %edi\n" \ - " pushl %esi\n" \ - " pushl %edx\n" \ - " pushl %ecx\n" \ - " pushl %ebx\n" -#define RESTORE_REGS_STRING \ - " popl %ebx\n" \ - " popl %ecx\n" \ - " popl %edx\n" \ - " popl %esi\n" \ - " popl %edi\n" \ - " popl %ebp\n" \ - " popl %eax\n" \ - /* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\ - " addl $24, %esp\n" -#endif - /* * When a retprobed function returns, this code saves registers and * calls trampoline_handler() runs, which calls the kretprobe's handler. @@ -816,8 +761,8 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs) * jump instruction after the copied instruction, that jumps to the next * instruction after the probepoint. */ -static void __kprobes resume_execution(struct kprobe *p, - struct pt_regs *regs, struct kprobe_ctlblk *kcb) +static void __kprobes +resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { unsigned long *tos = stack_addr(regs); unsigned long copy_ip = (unsigned long)p->ainsn.insn; @@ -996,8 +941,8 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) /* * Wrapper routine for handling exceptions. */ -int __kprobes kprobe_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data) +int __kprobes +kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) { struct die_args *args = data; int ret = NOTIFY_DONE; @@ -1107,466 +1052,9 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) return 0; } - -#ifdef CONFIG_OPTPROBES - -/* Insert a call instruction at address 'from', which calls address 'to'.*/ -static void __kprobes synthesize_relcall(void *from, void *to) -{ - __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE); -} - -/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ -static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr, - unsigned long val) -{ -#ifdef CONFIG_X86_64 - *addr++ = 0x48; - *addr++ = 0xbf; -#else - *addr++ = 0xb8; -#endif - *(unsigned long *)addr = val; -} - -static void __used __kprobes kprobes_optinsn_template_holder(void) -{ - asm volatile ( - ".global optprobe_template_entry\n" - "optprobe_template_entry: \n" -#ifdef CONFIG_X86_64 - /* We don't bother saving the ss register */ - " pushq %rsp\n" - " pushfq\n" - SAVE_REGS_STRING - " movq %rsp, %rsi\n" - ".global optprobe_template_val\n" - "optprobe_template_val: \n" - ASM_NOP5 - ASM_NOP5 - ".global optprobe_template_call\n" - "optprobe_template_call: \n" - ASM_NOP5 - /* Move flags to rsp */ - " movq 144(%rsp), %rdx\n" - " movq %rdx, 152(%rsp)\n" - RESTORE_REGS_STRING - /* Skip flags entry */ - " addq $8, %rsp\n" - " popfq\n" -#else /* CONFIG_X86_32 */ - " pushf\n" - SAVE_REGS_STRING - " movl %esp, %edx\n" - ".global optprobe_template_val\n" - "optprobe_template_val: \n" - ASM_NOP5 - ".global optprobe_template_call\n" - "optprobe_template_call: \n" - ASM_NOP5 - RESTORE_REGS_STRING - " addl $4, %esp\n" /* skip cs */ - " popf\n" -#endif - ".global optprobe_template_end\n" - "optprobe_template_end: \n"); -} - -#define TMPL_MOVE_IDX \ - ((long)&optprobe_template_val - (long)&optprobe_template_entry) -#define TMPL_CALL_IDX \ - ((long)&optprobe_template_call - (long)&optprobe_template_entry) -#define TMPL_END_IDX \ - ((long)&optprobe_template_end - (long)&optprobe_template_entry) - -#define INT3_SIZE sizeof(kprobe_opcode_t) - -/* Optimized kprobe call back function: called from optinsn */ -static void __kprobes optimized_callback(struct optimized_kprobe *op, - struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long flags; - - /* This is possible if op is under delayed unoptimizing */ - if (kprobe_disabled(&op->kp)) - return; - - local_irq_save(flags); - if (kprobe_running()) { - kprobes_inc_nmissed_count(&op->kp); - } else { - /* Save skipped registers */ -#ifdef CONFIG_X86_64 - regs->cs = __KERNEL_CS; -#else - regs->cs = __KERNEL_CS | get_kernel_rpl(); - regs->gs = 0; -#endif - regs->ip = (unsigned long)op->kp.addr + INT3_SIZE; - regs->orig_ax = ~0UL; - - __this_cpu_write(current_kprobe, &op->kp); - kcb->kprobe_status = KPROBE_HIT_ACTIVE; - opt_pre_handler(&op->kp, regs); - __this_cpu_write(current_kprobe, NULL); - } - local_irq_restore(flags); -} - -static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src) -{ - int len = 0, ret; - - while (len < RELATIVEJUMP_SIZE) { - ret = __copy_instruction(dest + len, src + len, 1); - if (!ret || !can_boost(dest + len)) - return -EINVAL; - len += ret; - } - /* Check whether the address range is reserved */ - if (ftrace_text_reserved(src, src + len - 1) || - alternatives_text_reserved(src, src + len - 1) || - jump_label_text_reserved(src, src + len - 1)) - return -EBUSY; - - return len; -} - -/* Check whether insn is indirect jump */ -static int __kprobes insn_is_indirect_jump(struct insn *insn) -{ - return ((insn->opcode.bytes[0] == 0xff && - (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ - insn->opcode.bytes[0] == 0xea); /* Segment based jump */ -} - -/* Check whether insn jumps into specified address range */ -static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) -{ - unsigned long target = 0; - - switch (insn->opcode.bytes[0]) { - case 0xe0: /* loopne */ - case 0xe1: /* loope */ - case 0xe2: /* loop */ - case 0xe3: /* jcxz */ - case 0xe9: /* near relative jump */ - case 0xeb: /* short relative jump */ - break; - case 0x0f: - if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */ - break; - return 0; - default: - if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */ - break; - return 0; - } - target = (unsigned long)insn->next_byte + insn->immediate.value; - - return (start <= target && target <= start + len); -} - -/* Decode whole function to ensure any instructions don't jump into target */ -static int __kprobes can_optimize(unsigned long paddr) -{ - int ret; - unsigned long addr, size = 0, offset = 0; - struct insn insn; - kprobe_opcode_t buf[MAX_INSN_SIZE]; - - /* Lookup symbol including addr */ - if (!kallsyms_lookup_size_offset(paddr, &size, &offset)) - return 0; - - /* - * Do not optimize in the entry code due to the unstable - * stack handling. - */ - if ((paddr >= (unsigned long )__entry_text_start) && - (paddr < (unsigned long )__entry_text_end)) - return 0; - - /* Check there is enough space for a relative jump. */ - if (size - offset < RELATIVEJUMP_SIZE) - return 0; - - /* Decode instructions */ - addr = paddr - offset; - while (addr < paddr - offset + size) { /* Decode until function end */ - if (search_exception_tables(addr)) - /* - * Since some fixup code will jumps into this function, - * we can't optimize kprobe in this function. - */ - return 0; - kernel_insn_init(&insn, (void *)addr); - insn_get_opcode(&insn); - if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) { - ret = recover_probed_instruction(buf, addr); - if (ret) - return 0; - kernel_insn_init(&insn, buf); - } - insn_get_length(&insn); - /* Recover address */ - insn.kaddr = (void *)addr; - insn.next_byte = (void *)(addr + insn.length); - /* Check any instructions don't jump into target */ - if (insn_is_indirect_jump(&insn) || - insn_jump_into_range(&insn, paddr + INT3_SIZE, - RELATIVE_ADDR_SIZE)) - return 0; - addr += insn.length; - } - - return 1; -} - -/* Check optimized_kprobe can actually be optimized. */ -int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op) -{ - int i; - struct kprobe *p; - - for (i = 1; i < op->optinsn.size; i++) { - p = get_kprobe(op->kp.addr + i); - if (p && !kprobe_disabled(p)) - return -EEXIST; - } - - return 0; -} - -/* Check the addr is within the optimized instructions. */ -int __kprobes arch_within_optimized_kprobe(struct optimized_kprobe *op, - unsigned long addr) -{ - return ((unsigned long)op->kp.addr <= addr && - (unsigned long)op->kp.addr + op->optinsn.size > addr); -} - -/* Free optimized instruction slot */ -static __kprobes -void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) -{ - if (op->optinsn.insn) { - free_optinsn_slot(op->optinsn.insn, dirty); - op->optinsn.insn = NULL; - op->optinsn.size = 0; - } -} - -void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op) -{ - __arch_remove_optimized_kprobe(op, 1); -} - -/* - * Copy replacing target instructions - * Target instructions MUST be relocatable (checked inside) - */ -int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) -{ - u8 *buf; - int ret; - long rel; - - if (!can_optimize((unsigned long)op->kp.addr)) - return -EILSEQ; - - op->optinsn.insn = get_optinsn_slot(); - if (!op->optinsn.insn) - return -ENOMEM; - - /* - * Verify if the address gap is in 2GB range, because this uses - * a relative jump. - */ - rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE; - if (abs(rel) > 0x7fffffff) - return -ERANGE; - - buf = (u8 *)op->optinsn.insn; - - /* Copy instructions into the out-of-line buffer */ - ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr); - if (ret < 0) { - __arch_remove_optimized_kprobe(op, 0); - return ret; - } - op->optinsn.size = ret; - - /* Copy arch-dep-instance from template */ - memcpy(buf, &optprobe_template_entry, TMPL_END_IDX); - - /* Set probe information */ - synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); - - /* Set probe function call */ - synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback); - - /* Set returning jmp instruction at the tail of out-of-line buffer */ - synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size, - (u8 *)op->kp.addr + op->optinsn.size); - - flush_icache_range((unsigned long) buf, - (unsigned long) buf + TMPL_END_IDX + - op->optinsn.size + RELATIVEJUMP_SIZE); - return 0; -} - -#define MAX_OPTIMIZE_PROBES 256 -static struct text_poke_param *jump_poke_params; -static struct jump_poke_buffer { - u8 buf[RELATIVEJUMP_SIZE]; -} *jump_poke_bufs; - -static void __kprobes setup_optimize_kprobe(struct text_poke_param *tprm, - u8 *insn_buf, - struct optimized_kprobe *op) -{ - s32 rel = (s32)((long)op->optinsn.insn - - ((long)op->kp.addr + RELATIVEJUMP_SIZE)); - - /* Backup instructions which will be replaced by jump address */ - memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, - RELATIVE_ADDR_SIZE); - - insn_buf[0] = RELATIVEJUMP_OPCODE; - *(s32 *)(&insn_buf[1]) = rel; - - tprm->addr = op->kp.addr; - tprm->opcode = insn_buf; - tprm->len = RELATIVEJUMP_SIZE; -} - -/* - * Replace breakpoints (int3) with relative jumps. - * Caller must call with locking kprobe_mutex and text_mutex. - */ -void __kprobes arch_optimize_kprobes(struct list_head *oplist) -{ - struct optimized_kprobe *op, *tmp; - int c = 0; - - list_for_each_entry_safe(op, tmp, oplist, list) { - WARN_ON(kprobe_disabled(&op->kp)); - /* Setup param */ - setup_optimize_kprobe(&jump_poke_params[c], - jump_poke_bufs[c].buf, op); - list_del_init(&op->list); - if (++c >= MAX_OPTIMIZE_PROBES) - break; - } - - /* - * text_poke_smp doesn't support NMI/MCE code modifying. - * However, since kprobes itself also doesn't support NMI/MCE - * code probing, it's not a problem. - */ - text_poke_smp_batch(jump_poke_params, c); -} - -static void __kprobes setup_unoptimize_kprobe(struct text_poke_param *tprm, - u8 *insn_buf, - struct optimized_kprobe *op) -{ - /* Set int3 to first byte for kprobes */ - insn_buf[0] = BREAKPOINT_INSTRUCTION; - memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); - - tprm->addr = op->kp.addr; - tprm->opcode = insn_buf; - tprm->len = RELATIVEJUMP_SIZE; -} - -/* - * Recover original instructions and breakpoints from relative jumps. - * Caller must call with locking kprobe_mutex. - */ -extern void arch_unoptimize_kprobes(struct list_head *oplist, - struct list_head *done_list) -{ - struct optimized_kprobe *op, *tmp; - int c = 0; - - list_for_each_entry_safe(op, tmp, oplist, list) { - /* Setup param */ - setup_unoptimize_kprobe(&jump_poke_params[c], - jump_poke_bufs[c].buf, op); - list_move(&op->list, done_list); - if (++c >= MAX_OPTIMIZE_PROBES) - break; - } - - /* - * text_poke_smp doesn't support NMI/MCE code modifying. - * However, since kprobes itself also doesn't support NMI/MCE - * code probing, it's not a problem. - */ - text_poke_smp_batch(jump_poke_params, c); -} - -/* Replace a relative jump with a breakpoint (int3). */ -void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) -{ - u8 buf[RELATIVEJUMP_SIZE]; - - /* Set int3 to first byte for kprobes */ - buf[0] = BREAKPOINT_INSTRUCTION; - memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); - text_poke_smp(op->kp.addr, buf, RELATIVEJUMP_SIZE); -} - -static int __kprobes setup_detour_execution(struct kprobe *p, - struct pt_regs *regs, - int reenter) -{ - struct optimized_kprobe *op; - - if (p->flags & KPROBE_FLAG_OPTIMIZED) { - /* This kprobe is really able to run optimized path. */ - op = container_of(p, struct optimized_kprobe, kp); - /* Detour through copied instructions */ - regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; - if (!reenter) - reset_current_kprobe(); - preempt_enable_no_resched(); - return 1; - } - return 0; -} - -static int __kprobes init_poke_params(void) -{ - /* Allocate code buffer and parameter array */ - jump_poke_bufs = kmalloc(sizeof(struct jump_poke_buffer) * - MAX_OPTIMIZE_PROBES, GFP_KERNEL); - if (!jump_poke_bufs) - return -ENOMEM; - - jump_poke_params = kmalloc(sizeof(struct text_poke_param) * - MAX_OPTIMIZE_PROBES, GFP_KERNEL); - if (!jump_poke_params) { - kfree(jump_poke_bufs); - jump_poke_bufs = NULL; - return -ENOMEM; - } - - return 0; -} -#else /* !CONFIG_OPTPROBES */ -static int __kprobes init_poke_params(void) -{ - return 0; -} -#endif - int __init arch_init_kprobes(void) { - return init_poke_params(); + return arch_init_optprobes(); } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index f0c6fd6..694d801 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -438,9 +438,9 @@ void __init kvm_guest_init(void) static __init int activate_jump_labels(void) { if (has_steal_clock) { - jump_label_inc(¶virt_steal_enabled); + static_key_slow_inc(¶virt_steal_enabled); if (steal_acc) - jump_label_inc(¶virt_steal_rq_enabled); + static_key_slow_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c index fda91c3..87a0f86 100644 --- a/arch/x86/kernel/microcode_core.c +++ b/arch/x86/kernel/microcode_core.c @@ -86,6 +86,7 @@ #include <asm/microcode.h> #include <asm/processor.h> +#include <asm/cpu_device_id.h> MODULE_DESCRIPTION("Microcode Update Driver"); MODULE_AUTHOR("Tigran Aivazian <tigran@aivazian.fsnet.co.uk>"); @@ -504,6 +505,20 @@ static struct notifier_block __refdata mc_cpu_notifier = { .notifier_call = mc_cpu_callback, }; +#ifdef MODULE +/* Autoload on Intel and AMD systems */ +static const struct x86_cpu_id microcode_id[] = { +#ifdef CONFIG_MICROCODE_INTEL + { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, }, +#endif +#ifdef CONFIG_MICROCODE_AMD + { X86_VENDOR_AMD, X86_FAMILY_ANY, X86_MODEL_ANY, }, +#endif + {} +}; +MODULE_DEVICE_TABLE(x86cpu, microcode_id); +#endif + static int __init microcode_init(void) { struct cpuinfo_x86 *c = &cpu_data(0); diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index d90272e..ada2f99 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -202,8 +202,8 @@ static void native_flush_tlb_single(unsigned long addr) __native_flush_tlb_single(addr); } -struct jump_label_key paravirt_steal_enabled; -struct jump_label_key paravirt_steal_rq_enabled; +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; static u64 native_steal_clock(int cpu) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 15763af..44eefde 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -377,8 +377,8 @@ static inline int hlt_use_halt(void) void default_idle(void) { if (hlt_use_halt()) { - trace_power_start(POWER_CSTATE, 1, smp_processor_id()); - trace_cpu_idle(1, smp_processor_id()); + trace_power_start_rcuidle(POWER_CSTATE, 1, smp_processor_id()); + trace_cpu_idle_rcuidle(1, smp_processor_id()); current_thread_info()->status &= ~TS_POLLING; /* * TS_POLLING-cleared state must be visible before we @@ -391,8 +391,8 @@ void default_idle(void) else local_irq_enable(); current_thread_info()->status |= TS_POLLING; - trace_power_end(smp_processor_id()); - trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); + trace_power_end_rcuidle(smp_processor_id()); + trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); } else { local_irq_enable(); /* loop is done by the caller */ @@ -450,8 +450,8 @@ EXPORT_SYMBOL_GPL(cpu_idle_wait); static void mwait_idle(void) { if (!need_resched()) { - trace_power_start(POWER_CSTATE, 1, smp_processor_id()); - trace_cpu_idle(1, smp_processor_id()); + trace_power_start_rcuidle(POWER_CSTATE, 1, smp_processor_id()); + trace_cpu_idle_rcuidle(1, smp_processor_id()); if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) clflush((void *)¤t_thread_info()->flags); @@ -461,8 +461,8 @@ static void mwait_idle(void) __sti_mwait(0, 0); else local_irq_enable(); - trace_power_end(smp_processor_id()); - trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); + trace_power_end_rcuidle(smp_processor_id()); + trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); } else local_irq_enable(); } @@ -474,13 +474,13 @@ static void mwait_idle(void) */ static void poll_idle(void) { - trace_power_start(POWER_CSTATE, 0, smp_processor_id()); - trace_cpu_idle(0, smp_processor_id()); + trace_power_start_rcuidle(POWER_CSTATE, 0, smp_processor_id()); + trace_cpu_idle_rcuidle(0, smp_processor_id()); local_irq_enable(); while (!need_resched()) cpu_relax(); - trace_power_end(smp_processor_id()); - trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); + trace_power_end_rcuidle(smp_processor_id()); + trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); } /* diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index c08d1ff..49888fe 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -119,9 +119,7 @@ void cpu_idle(void) } rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index cfa5c90..e34257c 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -156,9 +156,7 @@ void cpu_idle(void) } tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 66d250c..58f7816 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -291,19 +291,6 @@ notrace static void __cpuinit start_secondary(void *unused) per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; x86_platform.nmi_init(); - /* - * Wait until the cpu which brought this one up marked it - * online before enabling interrupts. If we don't do that then - * we can end up waking up the softirq thread before this cpu - * reached the active state, which makes the scheduler unhappy - * and schedule the softirq thread on the wrong cpu. This is - * only observable with forced threaded interrupts, but in - * theory it could also happen w/o them. It's just way harder - * to achieve. - */ - while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask)) - cpu_relax(); - /* enable local interrupts */ local_irq_enable(); diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index dd5fbf4..c6eba2b 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -57,9 +57,6 @@ EXPORT_SYMBOL(profile_pc); */ static irqreturn_t timer_interrupt(int irq, void *dev_id) { - /* Keep nmi watchdog up to date */ - inc_irq_stat(irq0_irqs); - global_clock_event->event_handler(global_clock_event); /* MCA bus quirk: Acknowledge irq0 by setting bit 7 in port 0x61 */ diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index a62c201..183c592 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -620,7 +620,8 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) if (cpu_khz) { *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz; - *offset = ns_now - (tsc_now * *scale >> CYC2NS_SCALE_FACTOR); + *offset = ns_now - mult_frac(tsc_now, *scale, + (1UL << CYC2NS_SCALE_FACTOR)); } sched_clock_idle_wakeup_event(0); diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index 9eba29b..fc25e60 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -42,7 +42,7 @@ static __cpuinitdata int nr_warps; /* * TSC-warp measurement loop running on both CPUs: */ -static __cpuinit void check_tsc_warp(void) +static __cpuinit void check_tsc_warp(unsigned int timeout) { cycles_t start, now, prev, end; int i; @@ -51,9 +51,9 @@ static __cpuinit void check_tsc_warp(void) start = get_cycles(); rdtsc_barrier(); /* - * The measurement runs for 20 msecs: + * The measurement runs for 'timeout' msecs: */ - end = start + tsc_khz * 20ULL; + end = start + (cycles_t) tsc_khz * timeout; now = start; for (i = 0; ; i++) { @@ -99,6 +99,25 @@ static __cpuinit void check_tsc_warp(void) } /* + * If the target CPU coming online doesn't have any of its core-siblings + * online, a timeout of 20msec will be used for the TSC-warp measurement + * loop. Otherwise a smaller timeout of 2msec will be used, as we have some + * information about this socket already (and this information grows as we + * have more and more logical-siblings in that socket). + * + * Ideally we should be able to skip the TSC sync check on the other + * core-siblings, if the first logical CPU in a socket passed the sync test. + * But as the TSC is per-logical CPU and can potentially be modified wrongly + * by the bios, TSC sync test for smaller duration should be able + * to catch such errors. Also this will catch the condition where all the + * cores in the socket doesn't get reset at the same time. + */ +static inline unsigned int loop_timeout(int cpu) +{ + return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20; +} + +/* * Source CPU calls into this - it waits for the freshly booted * target CPU to arrive and then starts the measurement: */ @@ -135,7 +154,7 @@ void __cpuinit check_tsc_sync_source(int cpu) */ atomic_inc(&start_count); - check_tsc_warp(); + check_tsc_warp(loop_timeout(cpu)); while (atomic_read(&stop_count) != cpus-1) cpu_relax(); @@ -183,7 +202,7 @@ void __cpuinit check_tsc_sync_target(void) while (atomic_read(&start_count) != cpus) cpu_relax(); - check_tsc_warp(); + check_tsc_warp(loop_timeout(smp_processor_id())); /* * Ok, we are done: diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index cfdc6e0..31bfc69 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1283,9 +1283,9 @@ void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) if (!irqchip_in_kernel(vcpu->kvm) || !vcpu->arch.apic->vapic_addr) return; - vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0); + vapic = kmap_atomic(vcpu->arch.apic->vapic_page); data = *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)); - kunmap_atomic(vapic, KM_USER0); + kunmap_atomic(vapic); apic_set_tpr(vcpu->arch.apic, data & 0xff); } @@ -1310,9 +1310,9 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) max_isr = 0; data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); - vapic = kmap_atomic(vcpu->arch.apic->vapic_page, KM_USER0); + vapic = kmap_atomic(vcpu->arch.apic->vapic_page); *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)) = data; - kunmap_atomic(vapic, KM_USER0); + kunmap_atomic(vapic); } void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index fe15dcc..ea7b4fd 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c @@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu) } static bool mmu_audit; -static struct jump_label_key mmu_audit_key; +static struct static_key mmu_audit_key; static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { @@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { - if (static_branch((&mmu_audit_key))) + if (static_key_false((&mmu_audit_key))) __kvm_mmu_audit(vcpu, point); } @@ -259,7 +259,7 @@ static void mmu_audit_enable(void) if (mmu_audit) return; - jump_label_inc(&mmu_audit_key); + static_key_slow_inc(&mmu_audit_key); mmu_audit = true; } @@ -268,7 +268,7 @@ static void mmu_audit_disable(void) if (!mmu_audit) return; - jump_label_dec(&mmu_audit_key); + static_key_slow_dec(&mmu_audit_key); mmu_audit = false; } diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 1561028..df5a703 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -92,9 +92,9 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, if (unlikely(npages != 1)) return -EFAULT; - table = kmap_atomic(page, KM_USER0); + table = kmap_atomic(page); ret = CMPXCHG(&table[index], orig_pte, new_pte); - kunmap_atomic(table, KM_USER0); + kunmap_atomic(table); kvm_release_page_dirty(page); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 9cbfc06..bb4fd263 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1162,12 +1162,12 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) */ vcpu->hv_clock.version += 2; - shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0); + shared_kaddr = kmap_atomic(vcpu->time_page); memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock, sizeof(vcpu->hv_clock)); - kunmap_atomic(shared_kaddr, KM_USER0); + kunmap_atomic(shared_kaddr); mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); return 0; @@ -3848,7 +3848,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, goto emul_write; } - kaddr = kmap_atomic(page, KM_USER0); + kaddr = kmap_atomic(page); kaddr += offset_in_page(gpa); switch (bytes) { case 1: @@ -3866,7 +3866,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, default: BUG(); } - kunmap_atomic(kaddr, KM_USER0); + kunmap_atomic(kaddr); kvm_release_page_dirty(page); if (!exchanged) diff --git a/arch/x86/lib/inat.c b/arch/x86/lib/inat.c index 88ad5fb..c1f01a8 100644 --- a/arch/x86/lib/inat.c +++ b/arch/x86/lib/inat.c @@ -29,46 +29,46 @@ insn_attr_t inat_get_opcode_attribute(insn_byte_t opcode) return inat_primary_table[opcode]; } -insn_attr_t inat_get_escape_attribute(insn_byte_t opcode, insn_byte_t last_pfx, +int inat_get_last_prefix_id(insn_byte_t last_pfx) +{ + insn_attr_t lpfx_attr; + + lpfx_attr = inat_get_opcode_attribute(last_pfx); + return inat_last_prefix_id(lpfx_attr); +} + +insn_attr_t inat_get_escape_attribute(insn_byte_t opcode, int lpfx_id, insn_attr_t esc_attr) { const insn_attr_t *table; - insn_attr_t lpfx_attr; - int n, m = 0; + int n; n = inat_escape_id(esc_attr); - if (last_pfx) { - lpfx_attr = inat_get_opcode_attribute(last_pfx); - m = inat_last_prefix_id(lpfx_attr); - } + table = inat_escape_tables[n][0]; if (!table) return 0; - if (inat_has_variant(table[opcode]) && m) { - table = inat_escape_tables[n][m]; + if (inat_has_variant(table[opcode]) && lpfx_id) { + table = inat_escape_tables[n][lpfx_id]; if (!table) return 0; } return table[opcode]; } -insn_attr_t inat_get_group_attribute(insn_byte_t modrm, insn_byte_t last_pfx, +insn_attr_t inat_get_group_attribute(insn_byte_t modrm, int lpfx_id, insn_attr_t grp_attr) { const insn_attr_t *table; - insn_attr_t lpfx_attr; - int n, m = 0; + int n; n = inat_group_id(grp_attr); - if (last_pfx) { - lpfx_attr = inat_get_opcode_attribute(last_pfx); - m = inat_last_prefix_id(lpfx_attr); - } + table = inat_group_tables[n][0]; if (!table) return inat_group_common_attribute(grp_attr); - if (inat_has_variant(table[X86_MODRM_REG(modrm)]) && m) { - table = inat_group_tables[n][m]; + if (inat_has_variant(table[X86_MODRM_REG(modrm)]) && lpfx_id) { + table = inat_group_tables[n][lpfx_id]; if (!table) return inat_group_common_attribute(grp_attr); } diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 5a1f9f3..25feb1a 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -185,7 +185,8 @@ err_out: void insn_get_opcode(struct insn *insn) { struct insn_field *opcode = &insn->opcode; - insn_byte_t op, pfx; + insn_byte_t op; + int pfx_id; if (opcode->got) return; if (!insn->prefixes.got) @@ -212,8 +213,8 @@ void insn_get_opcode(struct insn *insn) /* Get escaped opcode */ op = get_next(insn_byte_t, insn); opcode->bytes[opcode->nbytes++] = op; - pfx = insn_last_prefix(insn); - insn->attr = inat_get_escape_attribute(op, pfx, insn->attr); + pfx_id = insn_last_prefix_id(insn); + insn->attr = inat_get_escape_attribute(op, pfx_id, insn->attr); } if (inat_must_vex(insn->attr)) insn->attr = 0; /* This instruction is bad */ @@ -235,7 +236,7 @@ err_out: void insn_get_modrm(struct insn *insn) { struct insn_field *modrm = &insn->modrm; - insn_byte_t pfx, mod; + insn_byte_t pfx_id, mod; if (modrm->got) return; if (!insn->opcode.got) @@ -246,8 +247,8 @@ void insn_get_modrm(struct insn *insn) modrm->value = mod; modrm->nbytes = 1; if (inat_is_group(insn->attr)) { - pfx = insn_last_prefix(insn); - insn->attr = inat_get_group_attribute(mod, pfx, + pfx_id = insn_last_prefix_id(insn); + insn->attr = inat_get_group_attribute(mod, pfx_id, insn->attr); if (insn_is_avx(insn) && !inat_accept_vex(insn->attr)) insn->attr = 0; /* This is bad */ diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index e218d5d..d9b094c 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -760,9 +760,9 @@ survive: break; } - maddr = kmap_atomic(pg, KM_USER0); + maddr = kmap_atomic(pg); memcpy(maddr + offset, from, len); - kunmap_atomic(maddr, KM_USER0); + kunmap_atomic(maddr); set_page_dirty_lock(pg); put_page(pg); up_read(¤t->mm->mmap_sem); diff --git a/arch/x86/mm/highmem_32.c b/arch/x86/mm/highmem_32.c index f4f29b1..6f31ee5 100644 --- a/arch/x86/mm/highmem_32.c +++ b/arch/x86/mm/highmem_32.c @@ -51,11 +51,11 @@ void *kmap_atomic_prot(struct page *page, pgprot_t prot) } EXPORT_SYMBOL(kmap_atomic_prot); -void *__kmap_atomic(struct page *page) +void *kmap_atomic(struct page *page) { return kmap_atomic_prot(page, kmap_prot); } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); /* * This is the same as kmap_atomic() but can map memory that doesn't diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 7c1b765..5671752 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -475,8 +475,10 @@ void bpf_jit_compile(struct sk_filter *fp) case BPF_S_LD_W_ABS: func = sk_load_word; common_load: seen |= SEEN_DATAREF; - if ((int)K < 0) + if ((int)K < 0) { + /* Abort the JIT because __load_pointer() is needed. */ goto out; + } t_offset = func - (image + addrs[i]); EMIT1_off32(0xbe, K); /* mov imm32,%esi */ EMIT1_off32(0xe8, t_offset); /* call */ @@ -489,14 +491,8 @@ common_load: seen |= SEEN_DATAREF; goto common_load; case BPF_S_LDX_B_MSH: if ((int)K < 0) { - if (pc_ret0 > 0) { - /* addrs[pc_ret0 - 1] is the start address */ - EMIT_JMP(addrs[pc_ret0 - 1] - addrs[i]); - break; - } - CLEAR_A(); - EMIT_JMP(cleanup_addr - addrs[i]); - break; + /* Abort the JIT because __load_pointer() is needed. */ + goto out; } seen |= SEEN_DATAREF | SEEN_XREG; t_offset = sk_load_byte_msh - (image + addrs[i]); diff --git a/arch/x86/platform/scx200/scx200_32.c b/arch/x86/platform/scx200/scx200_32.c index 7e004ac..7a9ad30 100644 --- a/arch/x86/platform/scx200/scx200_32.c +++ b/arch/x86/platform/scx200/scx200_32.c @@ -17,8 +17,6 @@ /* Verify that the configuration block really is there */ #define scx200_cb_probe(base) (inw((base) + SCx200_CBA) == (base)) -#define NAME "scx200" - MODULE_AUTHOR("Christer Weinigel <wingel@nano-system.com>"); MODULE_DESCRIPTION("NatSemi SCx200 Driver"); MODULE_LICENSE("GPL"); @@ -29,10 +27,10 @@ unsigned long scx200_gpio_shadow[2]; unsigned scx200_cb_base = 0; static struct pci_device_id scx200_tbl[] = { - { PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SCx200_BRIDGE) }, - { PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SC1100_BRIDGE) }, - { PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SCx200_XBUS) }, - { PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SC1100_XBUS) }, + { PCI_VDEVICE(NS, PCI_DEVICE_ID_NS_SCx200_BRIDGE) }, + { PCI_VDEVICE(NS, PCI_DEVICE_ID_NS_SC1100_BRIDGE) }, + { PCI_VDEVICE(NS, PCI_DEVICE_ID_NS_SCx200_XBUS) }, + { PCI_VDEVICE(NS, PCI_DEVICE_ID_NS_SC1100_XBUS) }, { }, }; MODULE_DEVICE_TABLE(pci,scx200_tbl); @@ -63,10 +61,11 @@ static int __devinit scx200_probe(struct pci_dev *pdev, const struct pci_device_ if (pdev->device == PCI_DEVICE_ID_NS_SCx200_BRIDGE || pdev->device == PCI_DEVICE_ID_NS_SC1100_BRIDGE) { base = pci_resource_start(pdev, 0); - printk(KERN_INFO NAME ": GPIO base 0x%x\n", base); + pr_info("GPIO base 0x%x\n", base); - if (!request_region(base, SCx200_GPIO_SIZE, "NatSemi SCx200 GPIO")) { - printk(KERN_ERR NAME ": can't allocate I/O for GPIOs\n"); + if (!request_region(base, SCx200_GPIO_SIZE, + "NatSemi SCx200 GPIO")) { + pr_err("can't allocate I/O for GPIOs\n"); return -EBUSY; } @@ -82,11 +81,11 @@ static int __devinit scx200_probe(struct pci_dev *pdev, const struct pci_device_ if (scx200_cb_probe(base)) { scx200_cb_base = base; } else { - printk(KERN_WARNING NAME ": Configuration Block not found\n"); + pr_warn("Configuration Block not found\n"); return -ENODEV; } } - printk(KERN_INFO NAME ": Configuration Block base 0x%x\n", scx200_cb_base); + pr_info("Configuration Block base 0x%x\n", scx200_cb_base); } return 0; @@ -111,8 +110,7 @@ u32 scx200_gpio_configure(unsigned index, u32 mask, u32 bits) static int __init scx200_init(void) { - printk(KERN_INFO NAME ": NatSemi SCx200 Driver\n"); - + pr_info("NatSemi SCx200 Driver\n"); return pci_register_driver(&scx200_pci_driver); } diff --git a/arch/xtensa/include/asm/socket.h b/arch/xtensa/include/asm/socket.h index bb06968..e36c681 100644 --- a/arch/xtensa/include/asm/socket.h +++ b/arch/xtensa/include/asm/socket.h @@ -75,5 +75,9 @@ #define SO_WIFI_STATUS 41 #define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 #endif /* _XTENSA_SOCKET_H */ diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index 47041e7..2c90047 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -113,9 +113,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) platform_idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c index 2c723e8..f9726f6 100644 --- a/arch/xtensa/platforms/iss/console.c +++ b/arch/xtensa/platforms/iss/console.c @@ -19,7 +19,6 @@ #include <linux/param.h> #include <linux/seq_file.h> #include <linux/serial.h> -#include <linux/serialP.h> #include <asm/uaccess.h> #include <asm/irq.h> @@ -37,6 +36,7 @@ #define SERIAL_TIMER_VALUE (20 * HZ) static struct tty_driver *serial_driver; +static struct tty_port serial_port; static struct timer_list serial_timer; static DEFINE_SPINLOCK(timer_lock); @@ -68,17 +68,10 @@ static void rs_poll(unsigned long); static int rs_open(struct tty_struct *tty, struct file * filp) { - int line = tty->index; - - if ((line < 0) || (line >= SERIAL_MAX_NUM_LINES)) - return -ENODEV; - + tty->port = &serial_port; spin_lock(&timer_lock); - if (tty->count == 1) { - init_timer(&serial_timer); - serial_timer.data = (unsigned long) tty; - serial_timer.function = rs_poll; + setup_timer(&serial_timer, rs_poll, (unsigned long)tty); mod_timer(&serial_timer, jiffies + SERIAL_TIMER_VALUE); } spin_unlock(&timer_lock); @@ -99,10 +92,10 @@ static int rs_open(struct tty_struct *tty, struct file * filp) */ static void rs_close(struct tty_struct *tty, struct file * filp) { - spin_lock(&timer_lock); + spin_lock_bh(&timer_lock); if (tty->count == 1) del_timer_sync(&serial_timer); - spin_unlock(&timer_lock); + spin_unlock_bh(&timer_lock); } @@ -210,13 +203,14 @@ static const struct tty_operations serial_ops = { int __init rs_init(void) { - serial_driver = alloc_tty_driver(1); + tty_port_init(&serial_port); + + serial_driver = alloc_tty_driver(SERIAL_MAX_NUM_LINES); printk ("%s %s\n", serial_name, serial_version); /* Initialize the tty_driver structure */ - serial_driver->owner = THIS_MODULE; serial_driver->driver_name = "iss_serial"; serial_driver->name = "ttyS"; serial_driver->major = TTY_MAJOR; |