diff options
Diffstat (limited to 'arch')
385 files changed, 9660 insertions, 2660 deletions
diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 0f1d849..c1f3e7c 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -506,7 +506,7 @@ set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds,RTC_SECONDS); CMOS_WRITE(real_minutes,RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c index 42ff90b..665ebf7 100644 --- a/arch/arm/common/it8152.c +++ b/arch/arm/common/it8152.c @@ -236,7 +236,7 @@ static struct resource it8152_mem = { /* * The following functions are needed for DMA bouncing. - * ITE8152 chip can addrees up to 64MByte, so all the devices + * ITE8152 chip can address up to 64MByte, so all the devices * connected to ITE8152 (PCI and USB) should have limited DMA window */ diff --git a/arch/arm/common/vic.c b/arch/arm/common/vic.c index ba65f6e..cb660bc 100644 --- a/arch/arm/common/vic.c +++ b/arch/arm/common/vic.c @@ -70,7 +70,7 @@ static inline struct vic_device *to_vic(struct sys_device *sys) * vic_init2 - common initialisation code * @base: Base of the VIC. * - * Common initialisation code for registeration + * Common initialisation code for registration * and resume. */ static void vic_init2(void __iomem *base) diff --git a/arch/arm/mach-at91/board-ecbat91.c b/arch/arm/mach-at91/board-ecbat91.c index 7b58c94..de2fd04 100644 --- a/arch/arm/mach-at91/board-ecbat91.c +++ b/arch/arm/mach-at91/board-ecbat91.c @@ -128,17 +128,17 @@ static struct spi_board_info __initdata ecb_at91spi_devices[] = { .platform_data = &my_flash0_platform, #endif }, - { /* User accessable spi - cs1 (250KHz) */ + { /* User accessible spi - cs1 (250KHz) */ .modalias = "spi-cs1", .chip_select = 1, .max_speed_hz = 250 * 1000, }, - { /* User accessable spi - cs2 (1MHz) */ + { /* User accessible spi - cs2 (1MHz) */ .modalias = "spi-cs2", .chip_select = 2, .max_speed_hz = 1 * 1000 * 1000, }, - { /* User accessable spi - cs3 (10MHz) */ + { /* User accessible spi - cs3 (10MHz) */ .modalias = "spi-cs3", .chip_select = 3, .max_speed_hz = 10 * 1000 * 1000, diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index dafbacc..ea53f4d 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -301,7 +301,7 @@ static void at91_pm_end(void) } -static struct platform_suspend_ops at91_pm_ops ={ +static const struct platform_suspend_ops at91_pm_ops = { .valid = at91_pm_valid_state, .begin = at91_pm_begin, .enter = at91_pm_enter, diff --git a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c index b3a61d8..96273ff 100644 --- a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c +++ b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c @@ -757,7 +757,7 @@ static int chipcHw_divide(int num, int denom) t = t << 1; } - /* Intialize the result */ + /* Initialize the result */ r = 0; do { diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c index 7b9bac2..6b9be2e 100644 --- a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c +++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c @@ -893,7 +893,7 @@ int dmacHw_setDataDescriptor(dmacHw_CONFIG_t *pConfig, /* [ IN ] Configuration */ /****************************************************************************/ uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ - dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controler attribute of type dmacHw_CONTROLLER_ATTRIB_e */ + dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controller attribute of type dmacHw_CONTROLLER_ATTRIB_e */ ) { dmacHw_CBLK_t *pCblk = dmacHw_HANDLE_TO_CBLK(handle); diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c index ff7b436..77f84b4 100644 --- a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c +++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c @@ -316,7 +316,7 @@ static void DisplayDescRing(void *pDescriptor, /* [ IN ] Descriptor buffer */ /** * @brief Check if DMA channel is the flow controller * -* @return 1 : If DMA is a flow controler +* @return 1 : If DMA is a flow controller * 0 : Peripheral is the flow controller * * @note diff --git a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c index 5c1c9a0..16225e4 100644 --- a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c +++ b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c @@ -558,7 +558,7 @@ static int tmrHw_divide(int num, int denom) t = t << 1; } - /* Intialize the result */ + /* Initialize the result */ r = 0; do { diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c index 77eb35c..8d1baf3 100644 --- a/arch/arm/mach-bcmring/dma.c +++ b/arch/arm/mach-bcmring/dma.c @@ -671,7 +671,7 @@ static int ConfigChannel(DMA_Handle_t handle) /****************************************************************************/ /** -* Intializes all of the data structures associated with the DMA. +* Initializes all of the data structures associated with the DMA. * @return * >= 0 - Initialization was successfull. * diff --git a/arch/arm/mach-bcmring/include/csp/dmacHw.h b/arch/arm/mach-bcmring/include/csp/dmacHw.h index 5d51013..6c8da2b 100644 --- a/arch/arm/mach-bcmring/include/csp/dmacHw.h +++ b/arch/arm/mach-bcmring/include/csp/dmacHw.h @@ -590,7 +590,7 @@ void dmacHw_printDebugInfo(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ /****************************************************************************/ uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ - dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controler attribute of type dmacHw_CONTROLLER_ATTRIB_e */ + dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controller attribute of type dmacHw_CONTROLLER_ATTRIB_e */ ); #endif /* _DMACHW_H */ diff --git a/arch/arm/mach-bcmring/include/csp/tmrHw.h b/arch/arm/mach-bcmring/include/csp/tmrHw.h index f1236d0..2cbb530 100644 --- a/arch/arm/mach-bcmring/include/csp/tmrHw.h +++ b/arch/arm/mach-bcmring/include/csp/tmrHw.h @@ -76,7 +76,7 @@ tmrHw_RATE_t tmrHw_setPeriodicTimerRate(tmrHw_ID_t timerId, /* [ IN ] Timer Id * certain time interval * * This function initializes a periodic timer to generate timer interrupt -* after every time interval in milisecond +* after every time interval in millisecond * * @return On success: Effective interval set in mili-second * On failure: 0 @@ -93,7 +93,7 @@ tmrHw_INTERVAL_t tmrHw_setPeriodicTimerInterval(tmrHw_ID_t timerId, /* [ IN ] T * after certain time interval * * This function initializes a periodic timer to generate a single ticks after -* certain time interval in milisecond +* certain time interval in millisecond * * @return On success: Effective interval set in mili-second * On failure: 0 diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h index cbf334d..d67e2f8 100644 --- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h +++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h @@ -28,7 +28,7 @@ /* Data type for DMA Link List Item */ typedef struct { - uint32_t sar; /* Source Adress Register. + uint32_t sar; /* Source Address Register. Address must be aligned to CTLx.SRC_TR_WIDTH. */ uint32_t dar; /* Destination Address Register. Address must be aligned to CTLx.DST_TR_WIDTH. */ diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h index 891cea8..f1ecf96 100644 --- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h +++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h @@ -35,7 +35,7 @@ typedef struct { /* Data type representing DMA channel registers */ typedef struct { - dmacHw_REG64_t ChannelSar; /* Source Adress Register. 64 bits (upper 32 bits are reserved) + dmacHw_REG64_t ChannelSar; /* Source Address Register. 64 bits (upper 32 bits are reserved) Address must be aligned to CTLx.SRC_TR_WIDTH. */ dmacHw_REG64_t ChannelDar; /* Destination Address Register.64 bits (upper 32 bits are reserved) diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c index fab953b..1bd73a0 100644 --- a/arch/arm/mach-davinci/pm.c +++ b/arch/arm/mach-davinci/pm.c @@ -110,7 +110,7 @@ static int davinci_pm_enter(suspend_state_t state) return ret; } -static struct platform_suspend_ops davinci_pm_ops = { +static const struct platform_suspend_ops davinci_pm_ops = { .enter = davinci_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-dove/common.c b/arch/arm/mach-dove/common.c index f7a1258..fe627ab 100644 --- a/arch/arm/mach-dove/common.c +++ b/arch/arm/mach-dove/common.c @@ -770,7 +770,7 @@ static struct resource dove_sdio0_resources[] = { }; static struct platform_device dove_sdio0 = { - .name = "sdhci-mv", + .name = "sdhci-dove", .id = 0, .dev = { .dma_mask = &sdio_dmamask, @@ -798,7 +798,7 @@ static struct resource dove_sdio1_resources[] = { }; static struct platform_device dove_sdio1 = { - .name = "sdhci-mv", + .name = "sdhci-dove", .id = 1, .dev = { .dma_mask = &sdio_dmamask, diff --git a/arch/arm/mach-gemini/include/mach/hardware.h b/arch/arm/mach-gemini/include/mach/hardware.h index 213a4fc..8c950e1 100644 --- a/arch/arm/mach-gemini/include/mach/hardware.h +++ b/arch/arm/mach-gemini/include/mach/hardware.h @@ -33,7 +33,7 @@ #define GEMINI_LPC_HOST_BASE 0x47000000 #define GEMINI_LPC_IO_BASE 0x47800000 #define GEMINI_INTERRUPT_BASE 0x48000000 -/* TODO: Different interrupt controlers when SMP +/* TODO: Different interrupt controllers when SMP * #define GEMINI_INTERRUPT0_BASE 0x48000000 * #define GEMINI_INTERRUPT1_BASE 0x49000000 */ diff --git a/arch/arm/mach-imx/mach-pcm038.c b/arch/arm/mach-imx/mach-pcm038.c index f667a26..5056148 100644 --- a/arch/arm/mach-imx/mach-pcm038.c +++ b/arch/arm/mach-imx/mach-pcm038.c @@ -254,10 +254,10 @@ static struct regulator_init_data cam_data = { static struct mc13783_regulator_init_data pcm038_regulators[] = { { - .id = MC13783_REGU_VCAM, + .id = MC13783_REG_VCAM, .init_data = &cam_data, }, { - .id = MC13783_REGU_VMMC1, + .id = MC13783_REG_VMMC1, .init_data = &sdhc1_data, }, }; diff --git a/arch/arm/mach-imx/pm-imx27.c b/arch/arm/mach-imx/pm-imx27.c index 6bf81ce..acf1769 100644 --- a/arch/arm/mach-imx/pm-imx27.c +++ b/arch/arm/mach-imx/pm-imx27.c @@ -32,7 +32,7 @@ static int mx27_suspend_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops mx27_suspend_ops = { +static const struct platform_suspend_ops mx27_suspend_ops = { .enter = mx27_suspend_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-kirkwood/openrd-setup.c b/arch/arm/mach-kirkwood/openrd-setup.c index c9d77fa..cfcca41 100644 --- a/arch/arm/mach-kirkwood/openrd-setup.c +++ b/arch/arm/mach-kirkwood/openrd-setup.c @@ -171,7 +171,7 @@ static void __init openrd_init(void) kirkwood_i2c_init(); - if (machine_is_openrd_client()) { + if (machine_is_openrd_client() || machine_is_openrd_ultimate()) { i2c_register_board_info(0, i2c_board_info, ARRAY_SIZE(i2c_board_info)); kirkwood_audio_init(); diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c index a6e2aed..e76d41b 100644 --- a/arch/arm/mach-lpc32xx/pm.c +++ b/arch/arm/mach-lpc32xx/pm.c @@ -123,7 +123,7 @@ static int lpc32xx_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops lpc32xx_pm_ops = { +static const struct platform_suspend_ops lpc32xx_pm_ops = { .valid = suspend_valid_only_mem, .enter = lpc32xx_pm_enter, }; diff --git a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S index 4dc99aa..1246715 100644 --- a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S +++ b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S @@ -26,7 +26,7 @@ * The interrupt numbering scheme is defined in the * interrupt controller spec. To wit: * - * Migrated the code from ARM MP port to be more consistant + * Migrated the code from ARM MP port to be more consistent * with interrupt processing , the following still holds true * however, all interrupts are treated the same regardless of * if they are local IPI or PPI diff --git a/arch/arm/mach-msm/io.c b/arch/arm/mach-msm/io.c index 800f327..1260007 100644 --- a/arch/arm/mach-msm/io.c +++ b/arch/arm/mach-msm/io.c @@ -154,7 +154,7 @@ __msm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) { if (mtype == MT_DEVICE) { /* The peripherals in the 88000000 - D0000000 range - * are only accessable by type MT_DEVICE_NONSHARED. + * are only accessible by type MT_DEVICE_NONSHARED. * Adjust mtype as necessary to make this "just work." */ if ((phys_addr >= 0x88000000) && (phys_addr < 0xD0000000)) diff --git a/arch/arm/mach-mx3/mach-mx31_3ds.c b/arch/arm/mach-mx3/mach-mx31_3ds.c index 4e516b4..899a969 100644 --- a/arch/arm/mach-mx3/mach-mx31_3ds.c +++ b/arch/arm/mach-mx3/mach-mx31_3ds.c @@ -140,10 +140,10 @@ static struct regulator_init_data gpo_init = { static struct mc13783_regulator_init_data mx31_3ds_regulators[] = { { - .id = MC13783_REGU_PWGT1SPI, /* Power Gate for ARM core. */ + .id = MC13783_REG_PWGT1SPI, /* Power Gate for ARM core. */ .init_data = &pwgtx_init, }, { - .id = MC13783_REGU_PWGT2SPI, /* Power Gate for L2 Cache. */ + .id = MC13783_REG_PWGT2SPI, /* Power Gate for L2 Cache. */ .init_data = &pwgtx_init, }, { diff --git a/arch/arm/mach-mx3/mach-mx31moboard.c b/arch/arm/mach-mx3/mach-mx31moboard.c index 203d21a..1aa8d65 100644 --- a/arch/arm/mach-mx3/mach-mx31moboard.c +++ b/arch/arm/mach-mx3/mach-mx31moboard.c @@ -216,11 +216,11 @@ static struct regulator_init_data cam_vreg_data = { static struct mc13783_regulator_init_data moboard_regulators[] = { { - .id = MC13783_REGU_VMMC1, + .id = MC13783_REG_VMMC1, .init_data = &sdhc_vreg_data, }, { - .id = MC13783_REGU_VCAM, + .id = MC13783_REG_VCAM, .init_data = &cam_vreg_data, }, }; diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c index 0cca23a..98ba978 100644 --- a/arch/arm/mach-omap1/pm.c +++ b/arch/arm/mach-omap1/pm.c @@ -647,7 +647,7 @@ static struct irqaction omap_wakeup_irq = { -static struct platform_suspend_ops omap_pm_ops ={ +static const struct platform_suspend_ops omap_pm_ops = { .prepare = omap_pm_prepare, .enter = omap_pm_enter, .finish = omap_pm_finish, diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile index cd7332f..1c0c2b0 100644 --- a/arch/arm/mach-omap2/Makefile +++ b/arch/arm/mach-omap2/Makefile @@ -187,16 +187,19 @@ obj-$(CONFIG_MACH_NOKIA_RX51) += board-rx51.o \ hsmmc.o obj-$(CONFIG_MACH_OMAP_ZOOM2) += board-zoom.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o \ board-zoom-debugboard.o obj-$(CONFIG_MACH_OMAP_ZOOM3) += board-zoom.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o \ board-zoom-debugboard.o obj-$(CONFIG_MACH_OMAP_3630SDP) += board-3630sdp.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o obj-$(CONFIG_MACH_CM_T35) += board-cm-t35.o \ diff --git a/arch/arm/mach-omap2/board-3430sdp.c b/arch/arm/mach-omap2/board-3430sdp.c index 3b39ef1..d4e41ef 100644 --- a/arch/arm/mach-omap2/board-3430sdp.c +++ b/arch/arm/mach-omap2/board-3430sdp.c @@ -38,6 +38,7 @@ #include <plat/dma.h> #include <plat/gpmc.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/gpmc-smc91x.h> @@ -270,13 +271,18 @@ static struct omap_dss_device sdp3430_lcd_device = { .platform_disable = sdp3430_panel_disable_lcd, }; +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = sdp3430_panel_enable_dvi, + .platform_disable = sdp3430_panel_disable_dvi, +}; + static struct omap_dss_device sdp3430_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = sdp3430_panel_enable_dvi, - .platform_disable = sdp3430_panel_disable_dvi, }; static struct omap_dss_device sdp3430_tv_device = { diff --git a/arch/arm/mach-omap2/board-3630sdp.c b/arch/arm/mach-omap2/board-3630sdp.c index 5d41dbe..6264564 100644 --- a/arch/arm/mach-omap2/board-3630sdp.c +++ b/arch/arm/mach-omap2/board-3630sdp.c @@ -207,6 +207,7 @@ static void __init omap_sdp_init(void) { omap3_mux_init(board_mux, OMAP_PACKAGE_CBP); zoom_peripherals_init(); + zoom_display_init(); board_smc91x_init(); board_flash_init(sdp_flash_partitions, chip_sel_sdp); enable_board_wakeup_source(); diff --git a/arch/arm/mach-omap2/board-am3517evm.c b/arch/arm/mach-omap2/board-am3517evm.c index bc15626..10d60b7 100644 --- a/arch/arm/mach-omap2/board-am3517evm.c +++ b/arch/arm/mach-omap2/board-am3517evm.c @@ -35,6 +35,7 @@ #include <plat/common.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include "mux.h" #include "control.h" @@ -303,13 +304,18 @@ static void am3517_evm_panel_disable_lcd(struct omap_dss_device *dssdev) lcd_enabled = 0; } +static struct panel_generic_dpi_data lcd_panel = { + .name = "sharp_lq", + .platform_enable = am3517_evm_panel_enable_lcd, + .platform_disable = am3517_evm_panel_disable_lcd, +}; + static struct omap_dss_device am3517_evm_lcd_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "lcd", - .driver_name = "sharp_lq_panel", + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 16, - .platform_enable = am3517_evm_panel_enable_lcd, - .platform_disable = am3517_evm_panel_disable_lcd, }; static int am3517_evm_panel_enable_tv(struct omap_dss_device *dssdev) @@ -346,13 +352,18 @@ static void am3517_evm_panel_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = am3517_evm_panel_enable_dvi, + .platform_disable = am3517_evm_panel_disable_dvi, +}; + static struct omap_dss_device am3517_evm_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = am3517_evm_panel_enable_dvi, - .platform_disable = am3517_evm_panel_disable_dvi, }; static struct omap_dss_device *am3517_evm_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c index 486a3de..dac1416 100644 --- a/arch/arm/mach-omap2/board-cm-t35.c +++ b/arch/arm/mach-omap2/board-cm-t35.c @@ -46,6 +46,7 @@ #include <plat/gpmc.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <mach/hardware.h> @@ -351,22 +352,32 @@ static void cm_t35_panel_disable_tv(struct omap_dss_device *dssdev) { } +static struct panel_generic_dpi_data lcd_panel = { + .name = "toppoly_tdo35s", + .platform_enable = cm_t35_panel_enable_lcd, + .platform_disable = cm_t35_panel_disable_lcd, +}; + static struct omap_dss_device cm_t35_lcd_device = { .name = "lcd", - .driver_name = "toppoly_tdo35s_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 18, - .platform_enable = cm_t35_panel_enable_lcd, - .platform_disable = cm_t35_panel_disable_lcd, +}; + +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = cm_t35_panel_enable_dvi, + .platform_disable = cm_t35_panel_disable_dvi, }; static struct omap_dss_device cm_t35_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = cm_t35_panel_enable_dvi, - .platform_disable = cm_t35_panel_disable_dvi, }; static struct omap_dss_device cm_t35_tv_device = { diff --git a/arch/arm/mach-omap2/board-devkit8000.c b/arch/arm/mach-omap2/board-devkit8000.c index 451e7ff..00bb1fc 100644 --- a/arch/arm/mach-omap2/board-devkit8000.c +++ b/arch/arm/mach-omap2/board-devkit8000.c @@ -46,6 +46,7 @@ #include <plat/nand.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <linux/input/matrix_keypad.h> @@ -149,23 +150,32 @@ static struct regulator_consumer_supply devkit8000_vmmc1_supply = static struct regulator_consumer_supply devkit8000_vio_supply = REGULATOR_SUPPLY("vcc", "spi2.0"); +static struct panel_generic_dpi_data lcd_panel = { + .name = "generic", + .platform_enable = devkit8000_panel_enable_lcd, + .platform_disable = devkit8000_panel_disable_lcd, +}; + static struct omap_dss_device devkit8000_lcd_device = { .name = "lcd", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 24, - .reset_gpio = -EINVAL, /* will be replaced */ - .platform_enable = devkit8000_panel_enable_lcd, - .platform_disable = devkit8000_panel_disable_lcd, }; + +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = devkit8000_panel_enable_dvi, + .platform_disable = devkit8000_panel_disable_dvi, +}; + static struct omap_dss_device devkit8000_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .reset_gpio = -EINVAL, /* will be replaced */ - .platform_enable = devkit8000_panel_enable_dvi, - .platform_disable = devkit8000_panel_disable_dvi, }; static struct omap_dss_device devkit8000_tv_device = { diff --git a/arch/arm/mach-omap2/board-igep0020.c b/arch/arm/mach-omap2/board-igep0020.c index 0afa301..ebaa230 100644 --- a/arch/arm/mach-omap2/board-igep0020.c +++ b/arch/arm/mach-omap2/board-igep0020.c @@ -31,6 +31,7 @@ #include <plat/gpmc.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/onenand.h> #include "mux.h" @@ -459,13 +460,18 @@ static void igep2_disable_dvi(struct omap_dss_device *dssdev) gpio_direction_output(IGEP2_GPIO_DVI_PUP, 0); } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = igep2_enable_dvi, + .platform_disable = igep2_disable_dvi, +}; + static struct omap_dss_device igep2_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = igep2_enable_dvi, - .platform_disable = igep2_disable_dvi, }; static struct omap_dss_device *igep2_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-omap3beagle.c b/arch/arm/mach-omap2/board-omap3beagle.c index 6c12760..a4fe8e1 100644 --- a/arch/arm/mach-omap2/board-omap3beagle.c +++ b/arch/arm/mach-omap2/board-omap3beagle.c @@ -41,6 +41,7 @@ #include <plat/board.h> #include <plat/common.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/gpmc.h> #include <plat/nand.h> #include <plat/usb.h> @@ -194,14 +195,19 @@ static void beagle_disable_dvi(struct omap_dss_device *dssdev) gpio_set_value(dssdev->reset_gpio, 0); } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = beagle_enable_dvi, + .platform_disable = beagle_disable_dvi, +}; + static struct omap_dss_device beagle_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, .reset_gpio = 170, - .platform_enable = beagle_enable_dvi, - .platform_disable = beagle_disable_dvi, }; static struct omap_dss_device beagle_tv_device = { diff --git a/arch/arm/mach-omap2/board-omap3evm.c b/arch/arm/mach-omap2/board-omap3evm.c index 3de8d9b..323c380 100644 --- a/arch/arm/mach-omap2/board-omap3evm.c +++ b/arch/arm/mach-omap2/board-omap3evm.c @@ -43,6 +43,7 @@ #include <plat/common.h> #include <plat/mcspi.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include "mux.h" #include "sdram-micron-mt46h32m32lf-6.h" @@ -301,13 +302,18 @@ static void omap3_evm_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = omap3_evm_enable_dvi, + .platform_disable = omap3_evm_disable_dvi, +}; + static struct omap_dss_device omap3_evm_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = omap3_evm_enable_dvi, - .platform_disable = omap3_evm_disable_dvi, }; static struct omap_dss_device *omap3_evm_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-omap3stalker.c b/arch/arm/mach-omap2/board-omap3stalker.c index 9df9d93..2a2dad4 100644 --- a/arch/arm/mach-omap2/board-omap3stalker.c +++ b/arch/arm/mach-omap2/board-omap3stalker.c @@ -40,6 +40,7 @@ #include <plat/nand.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <linux/input/matrix_keypad.h> @@ -160,13 +161,18 @@ static void omap3_stalker_disable_lcd(struct omap_dss_device *dssdev) lcd_enabled = 0; } +static struct panel_generic_dpi_data lcd_panel = { + .name = "generic", + .platform_enable = omap3_stalker_enable_lcd, + .platform_disable = omap3_stalker_disable_lcd, +}; + static struct omap_dss_device omap3_stalker_lcd_device = { .name = "lcd", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 24, .type = OMAP_DISPLAY_TYPE_DPI, - .platform_enable = omap3_stalker_enable_lcd, - .platform_disable = omap3_stalker_disable_lcd, }; static int omap3_stalker_enable_tv(struct omap_dss_device *dssdev) @@ -208,13 +214,18 @@ static void omap3_stalker_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = omap3_stalker_enable_dvi, + .platform_disable = omap3_stalker_disable_dvi, +}; + static struct omap_dss_device omap3_stalker_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = omap3_stalker_enable_dvi, - .platform_disable = omap3_stalker_disable_dvi, }; static struct omap_dss_device *omap3_stalker_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-zoom-display.c b/arch/arm/mach-omap2/board-zoom-display.c new file mode 100644 index 0000000..6bcd436 --- /dev/null +++ b/arch/arm/mach-omap2/board-zoom-display.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2010 Texas Instruments Inc. + * + * Modified from mach-omap2/board-zoom-peripherals.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/gpio.h> +#include <linux/i2c/twl.h> +#include <linux/spi/spi.h> +#include <plat/mcspi.h> +#include <plat/display.h> + +#define LCD_PANEL_RESET_GPIO_PROD 96 +#define LCD_PANEL_RESET_GPIO_PILOT 55 +#define LCD_PANEL_QVGA_GPIO 56 + +static void zoom_lcd_panel_init(void) +{ + int ret; + unsigned char lcd_panel_reset_gpio; + + lcd_panel_reset_gpio = (omap_rev() > OMAP3430_REV_ES3_0) ? + LCD_PANEL_RESET_GPIO_PROD : + LCD_PANEL_RESET_GPIO_PILOT; + + ret = gpio_request(lcd_panel_reset_gpio, "lcd reset"); + if (ret) { + pr_err("Failed to get LCD reset GPIO (gpio%d).\n", + lcd_panel_reset_gpio); + return; + } + gpio_direction_output(lcd_panel_reset_gpio, 1); + + ret = gpio_request(LCD_PANEL_QVGA_GPIO, "lcd qvga"); + if (ret) { + pr_err("Failed to get LCD_PANEL_QVGA_GPIO (gpio%d).\n", + LCD_PANEL_QVGA_GPIO); + goto err0; + } + gpio_direction_output(LCD_PANEL_QVGA_GPIO, 1); + + return; +err0: + gpio_free(lcd_panel_reset_gpio); +} + +static int zoom_panel_enable_lcd(struct omap_dss_device *dssdev) +{ + return 0; +} + +static void zoom_panel_disable_lcd(struct omap_dss_device *dssdev) +{ +} + +/* + * PWMA/B register offsets (TWL4030_MODULE_PWMA) + */ +#define TWL_INTBR_PMBR1 0xD +#define TWL_INTBR_GPBR1 0xC +#define TWL_LED_PWMON 0x0 +#define TWL_LED_PWMOFF 0x1 + +static int zoom_set_bl_intensity(struct omap_dss_device *dssdev, int level) +{ + unsigned char c; + u8 mux_pwm, enb_pwm; + + if (level > 100) + return -1; + + twl_i2c_read_u8(TWL4030_MODULE_INTBR, &mux_pwm, TWL_INTBR_PMBR1); + twl_i2c_read_u8(TWL4030_MODULE_INTBR, &enb_pwm, TWL_INTBR_GPBR1); + + if (level == 0) { + /* disable pwm1 output and clock */ + enb_pwm = enb_pwm & 0xF5; + /* change pwm1 pin to gpio pin */ + mux_pwm = mux_pwm & 0xCF; + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + enb_pwm, TWL_INTBR_GPBR1); + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + mux_pwm, TWL_INTBR_PMBR1); + return 0; + } + + if (!((enb_pwm & 0xA) && (mux_pwm & 0x30))) { + /* change gpio pin to pwm1 pin */ + mux_pwm = mux_pwm | 0x30; + /* enable pwm1 output and clock*/ + enb_pwm = enb_pwm | 0x0A; + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + mux_pwm, TWL_INTBR_PMBR1); + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + enb_pwm, TWL_INTBR_GPBR1); + } + + c = ((50 * (100 - level)) / 100) + 1; + twl_i2c_write_u8(TWL4030_MODULE_PWM1, 0x7F, TWL_LED_PWMOFF); + twl_i2c_write_u8(TWL4030_MODULE_PWM1, c, TWL_LED_PWMON); + + return 0; +} + +static struct omap_dss_device zoom_lcd_device = { + .name = "lcd", + .driver_name = "NEC_8048_panel", + .type = OMAP_DISPLAY_TYPE_DPI, + .phy.dpi.data_lines = 24, + .platform_enable = zoom_panel_enable_lcd, + .platform_disable = zoom_panel_disable_lcd, + .max_backlight_level = 100, + .set_backlight = zoom_set_bl_intensity, +}; + +static struct omap_dss_device *zoom_dss_devices[] = { + &zoom_lcd_device, +}; + +static struct omap_dss_board_info zoom_dss_data = { + .num_devices = ARRAY_SIZE(zoom_dss_devices), + .devices = zoom_dss_devices, + .default_device = &zoom_lcd_device, +}; + +static struct platform_device zoom_dss_device = { + .name = "omapdss", + .id = -1, + .dev = { + .platform_data = &zoom_dss_data, + }, +}; + +static struct omap2_mcspi_device_config dss_lcd_mcspi_config = { + .turbo_mode = 1, + .single_channel = 1, /* 0: slave, 1: master */ +}; + +static struct spi_board_info nec_8048_spi_board_info[] __initdata = { + [0] = { + .modalias = "nec_8048_spi", + .bus_num = 1, + .chip_select = 2, + .max_speed_hz = 375000, + .controller_data = &dss_lcd_mcspi_config, + }, +}; + +static struct platform_device *zoom_display_devices[] __initdata = { + &zoom_dss_device, +}; + +void __init zoom_display_init(void) +{ + platform_add_devices(zoom_display_devices, + ARRAY_SIZE(zoom_display_devices)); + spi_register_board_info(nec_8048_spi_board_info, + ARRAY_SIZE(nec_8048_spi_board_info)); + zoom_lcd_panel_init(); +} + diff --git a/arch/arm/mach-omap2/board-zoom-peripherals.c b/arch/arm/mach-omap2/board-zoom-peripherals.c index 3fbd0ed..14d95af 100644 --- a/arch/arm/mach-omap2/board-zoom-peripherals.c +++ b/arch/arm/mach-omap2/board-zoom-peripherals.c @@ -35,6 +35,8 @@ #define OMAP_ZOOM_WLAN_PMENA_GPIO (101) #define OMAP_ZOOM_WLAN_IRQ_GPIO (162) +#define LCD_PANEL_ENABLE_GPIO (7 + OMAP_MAX_GPIO_LINES) + /* Zoom2 has Qwerty keyboard*/ static uint32_t board_keymap[] = { KEY(0, 0, KEY_E), @@ -224,9 +226,43 @@ static struct omap2_hsmmc_info mmc[] = { {} /* Terminator */ }; +static struct regulator_consumer_supply zoom_vpll2_supply = + REGULATOR_SUPPLY("vdds_dsi", "omapdss"); + +static struct regulator_consumer_supply zoom_vdda_dac_supply = + REGULATOR_SUPPLY("vdda_dac", "omapdss"); + +static struct regulator_init_data zoom_vpll2 = { + .constraints = { + .min_uV = 1800000, + .max_uV = 1800000, + .valid_modes_mask = REGULATOR_MODE_NORMAL + | REGULATOR_MODE_STANDBY, + .valid_ops_mask = REGULATOR_CHANGE_MODE + | REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = 1, + .consumer_supplies = &zoom_vpll2_supply, +}; + +static struct regulator_init_data zoom_vdac = { + .constraints = { + .min_uV = 1800000, + .max_uV = 1800000, + .valid_modes_mask = REGULATOR_MODE_NORMAL + | REGULATOR_MODE_STANDBY, + .valid_ops_mask = REGULATOR_CHANGE_MODE + | REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = 1, + .consumer_supplies = &zoom_vdda_dac_supply, +}; + static int zoom_twl_gpio_setup(struct device *dev, unsigned gpio, unsigned ngpio) { + int ret; + /* gpio + 0 is "mmc0_cd" (input/IRQ) */ mmc[0].gpio_cd = gpio + 0; omap2_hsmmc_init(mmc); @@ -238,7 +274,15 @@ static int zoom_twl_gpio_setup(struct device *dev, zoom_vsim_supply.dev = mmc[0].dev; zoom_vmmc2_supply.dev = mmc[1].dev; - return 0; + ret = gpio_request(LCD_PANEL_ENABLE_GPIO, "lcd enable"); + if (ret) { + pr_err("Failed to get LCD_PANEL_ENABLE_GPIO (gpio%d).\n", + LCD_PANEL_ENABLE_GPIO); + return ret; + } + gpio_direction_output(LCD_PANEL_ENABLE_GPIO, 0); + + return ret; } /* EXTMUTE callback function */ @@ -301,7 +345,8 @@ static struct twl4030_platform_data zoom_twldata = { .vmmc1 = &zoom_vmmc1, .vmmc2 = &zoom_vmmc2, .vsim = &zoom_vsim, - + .vpll2 = &zoom_vpll2, + .vdac = &zoom_vdac, }; static struct i2c_board_info __initdata zoom_i2c_boardinfo[] = { diff --git a/arch/arm/mach-omap2/board-zoom.c b/arch/arm/mach-omap2/board-zoom.c index e041c53..e26754c 100644 --- a/arch/arm/mach-omap2/board-zoom.c +++ b/arch/arm/mach-omap2/board-zoom.c @@ -130,6 +130,7 @@ static void __init omap_zoom_init(void) ARRAY_SIZE(zoom_nand_partitions), ZOOM_NAND_CS); zoom_debugboard_init(); zoom_peripherals_init(); + zoom_display_init(); } MACHINE_START(OMAP_ZOOM2, "OMAP Zoom2 board") diff --git a/arch/arm/mach-omap2/cpuidle34xx.c b/arch/arm/mach-omap2/cpuidle34xx.c index f3e043f..11b89e9 100644 --- a/arch/arm/mach-omap2/cpuidle34xx.c +++ b/arch/arm/mach-omap2/cpuidle34xx.c @@ -252,7 +252,7 @@ static int omap3_enter_idle_bm(struct cpuidle_device *dev, * FIXME: we currently manage device-specific idle states * for PER and CORE in combination with CPU-specific * idle states. This is wrong, and device-specific - * idle managment needs to be separated out into + * idle management needs to be separated out into * its own code. */ diff --git a/arch/arm/mach-omap2/include/mach/board-zoom.h b/arch/arm/mach-omap2/include/mach/board-zoom.h index f93ca39..d20bd9c 100644 --- a/arch/arm/mach-omap2/include/mach/board-zoom.h +++ b/arch/arm/mach-omap2/include/mach/board-zoom.h @@ -1,9 +1,12 @@ /* * Defines for zoom boards */ +#include <plat/display.h> + #define ZOOM_NAND_CS 0 extern int __init zoom_debugboard_init(void); extern void __init zoom_peripherals_init(void); +extern void __init zoom_display_init(void); #define ZOOM2_HEADSET_EXTMUTE_GPIO 153 diff --git a/arch/arm/mach-omap2/pm24xx.c b/arch/arm/mach-omap2/pm24xx.c index dac2d1d..9e5dc8e 100644 --- a/arch/arm/mach-omap2/pm24xx.c +++ b/arch/arm/mach-omap2/pm24xx.c @@ -350,7 +350,7 @@ static void omap2_pm_end(void) enable_hlt(); } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap2_pm_begin, .enter = omap2_pm_enter, .end = omap2_pm_end, diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c index 5b323f2..8cbbead 100644 --- a/arch/arm/mach-omap2/pm34xx.c +++ b/arch/arm/mach-omap2/pm34xx.c @@ -605,7 +605,7 @@ static void omap3_pm_end(void) return; } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap3_pm_begin, .end = omap3_pm_end, .enter = omap3_pm_enter, diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c index e9f4862c..76cfff2 100644 --- a/arch/arm/mach-omap2/pm44xx.c +++ b/arch/arm/mach-omap2/pm44xx.c @@ -65,7 +65,7 @@ static void omap4_pm_end(void) return; } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap4_pm_begin, .end = omap4_pm_end, .enter = omap4_pm_enter, diff --git a/arch/arm/mach-omap2/serial.c b/arch/arm/mach-omap2/serial.c index c645788..302da74 100644 --- a/arch/arm/mach-omap2/serial.c +++ b/arch/arm/mach-omap2/serial.c @@ -852,7 +852,7 @@ void __init omap_serial_init_port(struct omap_board_data *bdata) } /** - * omap_serial_init() - intialize all supported serial ports + * omap_serial_init() - initialize all supported serial ports * * Initializes all available UARTs as serial ports. Platforms * can call this function when they want to have default behaviour diff --git a/arch/arm/mach-pnx4008/pm.c b/arch/arm/mach-pnx4008/pm.c index ee3c29c..f3e60a0 100644 --- a/arch/arm/mach-pnx4008/pm.c +++ b/arch/arm/mach-pnx4008/pm.c @@ -119,7 +119,7 @@ static int pnx4008_pm_valid(suspend_state_t state) (state == PM_SUSPEND_MEM); } -static struct platform_suspend_ops pnx4008_pm_ops = { +static const struct platform_suspend_ops pnx4008_pm_ops = { .enter = pnx4008_pm_enter, .valid = pnx4008_pm_valid, }; diff --git a/arch/arm/mach-pxa/mxm8x10.c b/arch/arm/mach-pxa/mxm8x10.c index 462167a..cdf7f41 100644 --- a/arch/arm/mach-pxa/mxm8x10.c +++ b/arch/arm/mach-pxa/mxm8x10.c @@ -337,7 +337,7 @@ void __init mxm_8x10_mmc_init(void) } #endif -/* USB Open Host Controler Interface */ +/* USB Open Host Controller Interface */ static struct pxaohci_platform_data mxm_8x10_ohci_platform_data = { .port_mode = PMM_NPS_MODE, .flags = ENABLE_PORT_ALL diff --git a/arch/arm/mach-pxa/pm.c b/arch/arm/mach-pxa/pm.c index 166c15f..978e1b2 100644 --- a/arch/arm/mach-pxa/pm.c +++ b/arch/arm/mach-pxa/pm.c @@ -96,7 +96,7 @@ void pxa_pm_finish(void) pxa_cpu_pm_fns->finish(); } -static struct platform_suspend_ops pxa_pm_ops = { +static const struct platform_suspend_ops pxa_pm_ops = { .valid = pxa_pm_valid, .enter = pxa_pm_enter, .prepare = pxa_pm_prepare, diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c index e68d46d..785880f 100644 --- a/arch/arm/mach-pxa/sharpsl_pm.c +++ b/arch/arm/mach-pxa/sharpsl_pm.c @@ -869,7 +869,7 @@ static void sharpsl_apm_get_power_status(struct apm_power_info *info) } #ifdef CONFIG_PM -static struct platform_suspend_ops sharpsl_pm_ops = { +static const struct platform_suspend_ops sharpsl_pm_ops = { .prepare = pxa_pm_prepare, .finish = pxa_pm_finish, .enter = corgi_pxa_pm_enter, diff --git a/arch/arm/mach-s3c2410/mach-h1940.c b/arch/arm/mach-s3c2410/mach-h1940.c index d7ada8c..1a81fe1 100644 --- a/arch/arm/mach-s3c2410/mach-h1940.c +++ b/arch/arm/mach-s3c2410/mach-h1940.c @@ -387,7 +387,7 @@ static struct platform_device *h1940_devices[] __initdata = { &s3c_device_wdt, &s3c_device_i2c0, &s3c_device_iis, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c_device_usbgadget, &h1940_device_leds, &h1940_device_bluetooth, diff --git a/arch/arm/mach-s3c2440/mach-rx1950.c b/arch/arm/mach-s3c2440/mach-rx1950.c index e0622bb..eab6ae5 100644 --- a/arch/arm/mach-s3c2440/mach-rx1950.c +++ b/arch/arm/mach-s3c2440/mach-rx1950.c @@ -692,7 +692,7 @@ static struct platform_device *rx1950_devices[] __initdata = { &s3c_device_wdt, &s3c_device_i2c0, &s3c_device_iis, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c_device_usbgadget, &s3c_device_rtc, &s3c_device_nand, diff --git a/arch/arm/mach-s3c64xx/clock.c b/arch/arm/mach-s3c64xx/clock.c index 7e03f0a..1c98d2ff 100644 --- a/arch/arm/mach-s3c64xx/clock.c +++ b/arch/arm/mach-s3c64xx/clock.c @@ -695,7 +695,7 @@ static struct clksrc_clk clksrcs[] = { }, { .clk = { .name = "audio-bus", - .id = -1, /* There's only one IISv4 port */ + .id = 2, .ctrlbit = S3C6410_CLKCON_SCLK_AUDIO2, .enable = s3c64xx_sclk_ctrl, }, diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c index 76426a3..cad6702 100644 --- a/arch/arm/mach-s3c64xx/dev-audio.c +++ b/arch/arm/mach-s3c64xx/dev-audio.c @@ -22,7 +22,12 @@ #include <plat/audio.h> #include <plat/gpio-cfg.h> -static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) +static const char *rclksrc[] = { + [0] = "iis", + [1] = "audio-bus", +}; + +static int s3c64xx_i2s_cfg_gpio(struct platform_device *pdev) { unsigned int base; @@ -33,6 +38,12 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) case 1: base = S3C64XX_GPE(0); break; + case 2: + s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5)); + return 0; default: printk(KERN_DEBUG "Invalid I2S Controller number: %d\n", pdev->id); @@ -44,16 +55,6 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) return 0; } -static int s3c64xx_i2sv4_cfg_gpio(struct platform_device *pdev) -{ - s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5)); - - return 0; -} - static struct resource s3c64xx_iis0_resource[] = { [0] = { .start = S3C64XX_PA_IIS0, @@ -72,17 +73,22 @@ static struct resource s3c64xx_iis0_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2s0_pdata = { - .cfg_gpio = s3c64xx_i2sv3_cfg_gpio, +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .type = { + .i2s = { + .src_clk = rclksrc, + }, + }, }; struct platform_device s3c64xx_device_iis0 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 0, .num_resources = ARRAY_SIZE(s3c64xx_iis0_resource), .resource = s3c64xx_iis0_resource, .dev = { - .platform_data = &s3c_i2s0_pdata, + .platform_data = &i2sv3_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis0); @@ -105,17 +111,13 @@ static struct resource s3c64xx_iis1_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2s1_pdata = { - .cfg_gpio = s3c64xx_i2sv3_cfg_gpio, -}; - struct platform_device s3c64xx_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s3c64xx_iis1_resource), .resource = s3c64xx_iis1_resource, .dev = { - .platform_data = &s3c_i2s1_pdata, + .platform_data = &i2sv3_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis1); @@ -138,17 +140,23 @@ static struct resource s3c64xx_iisv4_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2sv4_pdata = { - .cfg_gpio = s3c64xx_i2sv4_cfg_gpio, +static struct s3c_audio_pdata i2sv4_pdata = { + .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN, + .src_clk = rclksrc, + }, + }, }; struct platform_device s3c64xx_device_iisv4 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 2, .num_resources = ARRAY_SIZE(s3c64xx_iisv4_resource), .resource = s3c64xx_iisv4_resource, .dev = { - .platform_data = &s3c_i2sv4_pdata, + .platform_data = &i2sv4_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iisv4); @@ -288,7 +296,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata; static u64 s3c64xx_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s3c64xx_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s3c64xx_ac97_resource), .resource = s3c64xx_ac97_resource, @@ -307,16 +315,3 @@ void __init s3c64xx_ac97_setup_gpio(int num) else s3c_ac97_pdata.cfg_gpio = s3c64xx_ac97_cfg_gpe; } - -static u64 s3c_device_audio_dmamask = 0xffffffffUL; - -struct platform_device s3c_device_pcm = { - .name = "s3c24xx-pcm-audio", - .id = -1, - .dev = { - .dma_mask = &s3c_device_audio_dmamask, - .coherent_dma_mask = 0xffffffffUL - } -}; -EXPORT_SYMBOL(s3c_device_pcm); - diff --git a/arch/arm/mach-s3c64xx/dma.c b/arch/arm/mach-s3c64xx/dma.c index e7d03ab..372ea68 100644 --- a/arch/arm/mach-s3c64xx/dma.c +++ b/arch/arm/mach-s3c64xx/dma.c @@ -740,7 +740,7 @@ static int __init s3c64xx_dma_init(void) /* Set all DMA configuration to be DMA, not SDMA */ writel(0xffffff, S3C_SYSREG(0x110)); - /* Register standard DMA controlers */ + /* Register standard DMA controllers */ s3c64xx_dma_init1(0, DMACH_UART0, IRQ_DMA0, 0x75000000); s3c64xx_dma_init1(8, DMACH_PCM1_TX, IRQ_DMA1, 0x75100000); diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c index 77488fa..e85192a 100644 --- a/arch/arm/mach-s3c64xx/mach-smdk6410.c +++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c @@ -283,7 +283,7 @@ static struct platform_device *smdk6410_devices[] __initdata = { &s3c_device_fb, &s3c_device_ohci, &s3c_device_usb_hsotg, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c64xx_device_iisv4, &samsung_device_keypad, diff --git a/arch/arm/mach-s5p6442/dev-audio.c b/arch/arm/mach-s5p6442/dev-audio.c index 3462197..8719dc4 100644 --- a/arch/arm/mach-s5p6442/dev-audio.c +++ b/arch/arm/mach-s5p6442/dev-audio.c @@ -29,7 +29,7 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev) base = S5P6442_GPC1(0); break; - case -1: + case 0: base = S5P6442_GPC0(0); break; @@ -42,8 +42,19 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static const char *rclksrc_v35[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + +static struct s3c_audio_pdata i2sv35_pdata = { .cfg_gpio = s5p6442_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_SEC_DAI | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc_v35, + }, + }, }; static struct resource s5p6442_iis0_resource[] = { @@ -62,15 +73,34 @@ static struct resource s5p6442_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5p6442_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p6442_iis0_resource), .resource = s5p6442_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv35_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "sclk_audio", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5p6442_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -93,12 +123,12 @@ static struct resource s5p6442_iis1_resource[] = { }; struct platform_device s5p6442_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5p6442_iis1_resource), .resource = s5p6442_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; diff --git a/arch/arm/mach-s5p64x0/clock-s5p6440.c b/arch/arm/mach-s5p64x0/clock-s5p6440.c index e4883dc..409c5fc 100644 --- a/arch/arm/mach-s5p64x0/clock-s5p6440.c +++ b/arch/arm/mach-s5p64x0/clock-s5p6440.c @@ -261,7 +261,7 @@ static struct clk init_clocks_disable[] = { .enable = s5p64x0_pclk_ctrl, .ctrlbit = (1 << 25), }, { - .name = "i2s_v40", + .name = "iis", .id = 0, .parent = &clk_pclk_low.clk, .enable = s5p64x0_pclk_ctrl, diff --git a/arch/arm/mach-s5p64x0/clock-s5p6450.c b/arch/arm/mach-s5p64x0/clock-s5p6450.c index 7dbf3c9..7fc6abd 100644 --- a/arch/arm/mach-s5p64x0/clock-s5p6450.c +++ b/arch/arm/mach-s5p64x0/clock-s5p6450.c @@ -256,7 +256,7 @@ static struct clk init_clocks_disable[] = { .ctrlbit = (1 << 22), }, { .name = "iis", - .id = -1, + .id = 0, .parent = &clk_pclk_low.clk, .enable = s5p64x0_pclk_ctrl, .ctrlbit = (1 << 26), diff --git a/arch/arm/mach-s5p64x0/dev-audio.c b/arch/arm/mach-s5p64x0/dev-audio.c index 396bacc..14f89e73 100644 --- a/arch/arm/mach-s5p64x0/dev-audio.c +++ b/arch/arm/mach-s5p64x0/dev-audio.c @@ -19,15 +19,19 @@ #include <mach/dma.h> #include <mach/irqs.h> -static int s5p6440_cfg_i2s(struct platform_device *pdev) +static const char *rclksrc[] = { + [0] = "iis", + [1] = "sclk_audio2", +}; + +static int s5p64x0_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { - case -1: + case 0: s3c_gpio_cfgpin_range(S5P6440_GPR(4), 5, S3C_GPIO_SFN(5)); s3c_gpio_cfgpin_range(S5P6440_GPR(13), 2, S3C_GPIO_SFN(5)); break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -36,31 +40,14 @@ static int s5p6440_cfg_i2s(struct platform_device *pdev) return 0; } -static int s5p6450_cfg_i2s(struct platform_device *pdev) -{ - /* configure GPIO for i2s port */ - switch (pdev->id) { - case -1: - s3c_gpio_cfgpin(S5P6450_GPB(4), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S5P6450_GPR(4), 5, S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S5P6450_GPR(13), 2, S3C_GPIO_SFN(5)); - - break; - - default: - printk(KERN_ERR "Invalid Device %d\n", pdev->id); - return -EINVAL; - } - - return 0; -} - -static struct s3c_audio_pdata s5p6440_i2s_pdata = { - .cfg_gpio = s5p6440_cfg_i2s, -}; - -static struct s3c_audio_pdata s5p6450_i2s_pdata = { - .cfg_gpio = s5p6450_cfg_i2s, +static struct s3c_audio_pdata s5p64x0_i2s_pdata = { + .cfg_gpio = s5p64x0_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN, + .src_clk = rclksrc, + }, + }, }; static struct resource s5p64x0_iis0_resource[] = { @@ -82,22 +69,22 @@ static struct resource s5p64x0_iis0_resource[] = { }; struct platform_device s5p6440_device_iis = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p64x0_iis0_resource), .resource = s5p64x0_iis0_resource, .dev = { - .platform_data = &s5p6440_i2s_pdata, + .platform_data = &s5p64x0_i2s_pdata, }, }; struct platform_device s5p6450_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p64x0_iis0_resource), .resource = s5p64x0_iis0_resource, .dev = { - .platform_data = &s5p6450_i2s_pdata, + .platform_data = &s5p64x0_i2s_pdata, }, }; diff --git a/arch/arm/mach-s5pc100/dev-audio.c b/arch/arm/mach-s5pc100/dev-audio.c index 564e195..ab2d271 100644 --- a/arch/arm/mach-s5pc100/dev-audio.c +++ b/arch/arm/mach-s5pc100/dev-audio.c @@ -23,17 +23,14 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { + case 0: /* Dedicated pins */ + break; case 1: s3c_gpio_cfgpin_range(S5PC100_GPC(0), 5, S3C_GPIO_SFN(2)); break; - case 2: s3c_gpio_cfgpin_range(S5PC100_GPG3(0), 5, S3C_GPIO_SFN(4)); break; - - case -1: /* Dedicated pins */ - break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -42,8 +39,20 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static const char *rclksrc_v5[] = { + [0] = "iis", + [1] = "i2sclkd2", +}; + +static struct s3c_audio_pdata i2sv5_pdata = { .cfg_gpio = s5pc100_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc_v5, + }, + }, }; static struct resource s5pc100_iis0_resource[] = { @@ -62,15 +71,34 @@ static struct resource s5pc100_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5pc100_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5pc100_iis0_resource), .resource = s5pc100_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "sclk_audio", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pc100_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -93,12 +121,12 @@ static struct resource s5pc100_iis1_resource[] = { }; struct platform_device s5pc100_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5pc100_iis1_resource), .resource = s5pc100_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -121,12 +149,12 @@ static struct resource s5pc100_iis2_resource[] = { }; struct platform_device s5pc100_device_iis2 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 2, .num_resources = ARRAY_SIZE(s5pc100_iis2_resource), .resource = s5pc100_iis2_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -253,7 +281,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = { static u64 s5pc100_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s5pc100_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s5pc100_ac97_resource), .resource = s5pc100_ac97_resource, diff --git a/arch/arm/mach-s5pc100/mach-smdkc100.c b/arch/arm/mach-s5pc100/mach-smdkc100.c index 18b405d..dd192a2 100644 --- a/arch/arm/mach-s5pc100/mach-smdkc100.c +++ b/arch/arm/mach-s5pc100/mach-smdkc100.c @@ -96,6 +96,7 @@ static struct s3c2410_uartcfg smdkc100_uartcfgs[] __initdata = { /* I2C0 */ static struct i2c_board_info i2c_devs0[] __initdata = { + {I2C_BOARD_INFO("wm8580", 0x1b),}, }; /* I2C1 */ @@ -190,6 +191,7 @@ static struct platform_device *smdkc100_devices[] __initdata = { &s3c_device_ts, &s3c_device_wdt, &smdkc100_lcd_powerdev, + &samsung_asoc_dma, &s5pc100_device_iis0, &samsung_device_keypad, &s5pc100_device_ac97, diff --git a/arch/arm/mach-s5pv210/clock.c b/arch/arm/mach-s5pv210/clock.c index 019c3a6..b774ff1 100644 --- a/arch/arm/mach-s5pv210/clock.c +++ b/arch/arm/mach-s5pv210/clock.c @@ -467,20 +467,20 @@ static struct clk init_clocks_disable[] = { .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1<<21), }, { - .name = "i2s_v50", + .name = "iis", .id = 0, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1<<4), }, { - .name = "i2s_v32", - .id = 0, + .name = "iis", + .id = 1, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1 << 5), }, { - .name = "i2s_v32", - .id = 1, + .name = "iis", + .id = 2, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1 << 6), diff --git a/arch/arm/mach-s5pv210/dev-audio.c b/arch/arm/mach-s5pv210/dev-audio.c index 1303fcb..8d58f19 100644 --- a/arch/arm/mach-s5pv210/dev-audio.c +++ b/arch/arm/mach-s5pv210/dev-audio.c @@ -19,22 +19,24 @@ #include <mach/dma.h> #include <mach/irqs.h> +static const char *rclksrc[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + static int s5pv210_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2)); + break; case 1: s3c_gpio_cfgpin_range(S5PV210_GPC0(0), 5, S3C_GPIO_SFN(2)); break; - case 2: s3c_gpio_cfgpin_range(S5PV210_GPC1(0), 5, S3C_GPIO_SFN(4)); break; - - case -1: - s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2)); - break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -43,8 +45,15 @@ static int s5pv210_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static struct s3c_audio_pdata i2sv5_pdata = { .cfg_gpio = s5pv210_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc, + }, + }, }; static struct resource s5pv210_iis0_resource[] = { @@ -63,15 +72,34 @@ static struct resource s5pv210_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5pv210_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5pv210_iis0_resource), .resource = s5pv210_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "audio-bus", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pv210_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -94,12 +122,12 @@ static struct resource s5pv210_iis1_resource[] = { }; struct platform_device s5pv210_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5pv210_iis1_resource), .resource = s5pv210_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -122,12 +150,12 @@ static struct resource s5pv210_iis2_resource[] = { }; struct platform_device s5pv210_device_iis2 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 2, .num_resources = ARRAY_SIZE(s5pv210_iis2_resource), .resource = s5pv210_iis2_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -283,7 +311,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = { static u64 s5pv210_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s5pv210_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s5pv210_ac97_resource), .resource = s5pv210_ac97_resource, diff --git a/arch/arm/mach-s5pv310/Kconfig b/arch/arm/mach-s5pv310/Kconfig index 1150b36..d64efe0 100644 --- a/arch/arm/mach-s5pv310/Kconfig +++ b/arch/arm/mach-s5pv310/Kconfig @@ -11,6 +11,7 @@ if ARCH_S5PV310 config CPU_S5PV310 bool + select S3C_PL330_DMA help Enable S5PV310 CPU support diff --git a/arch/arm/mach-s5pv310/Makefile b/arch/arm/mach-s5pv310/Makefile index 84afc64..61e3cb6 100644 --- a/arch/arm/mach-s5pv310/Makefile +++ b/arch/arm/mach-s5pv310/Makefile @@ -13,7 +13,7 @@ obj- := # Core support for S5PV310 system obj-$(CONFIG_CPU_S5PV310) += cpu.o init.o clock.o irq-combiner.o -obj-$(CONFIG_CPU_S5PV310) += setup-i2c0.o time.o gpiolib.o irq-eint.o +obj-$(CONFIG_CPU_S5PV310) += setup-i2c0.o time.o gpiolib.o irq-eint.o dma.o obj-$(CONFIG_SMP) += platsmp.o headsmp.o obj-$(CONFIG_LOCAL_TIMERS) += localtimer.o @@ -27,6 +27,7 @@ obj-$(CONFIG_MACH_UNIVERSAL_C210) += mach-universal_c210.o # device support +obj-y += dev-audio.o obj-$(CONFIG_S5PV310_SETUP_I2C1) += setup-i2c1.o obj-$(CONFIG_S5PV310_SETUP_I2C2) += setup-i2c2.o obj-$(CONFIG_S5PV310_SETUP_I2C3) += setup-i2c3.o diff --git a/arch/arm/mach-s5pv310/dev-audio.c b/arch/arm/mach-s5pv310/dev-audio.c new file mode 100644 index 0000000..a196424 --- /dev/null +++ b/arch/arm/mach-s5pv310/dev-audio.c @@ -0,0 +1,364 @@ +/* linux/arch/arm/mach-s5pv310/dev-audio.c + * + * Copyright (c) 2010 Samsung Electronics Co. Ltd + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <linux/gpio.h> + +#include <plat/gpio-cfg.h> +#include <plat/audio.h> + +#include <mach/map.h> +#include <mach/dma.h> +#include <mach/irqs.h> + +static const char *rclksrc[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + +static int s5pv310_cfg_i2s(struct platform_device *pdev) +{ + /* configure GPIO for i2s port */ + switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 7, S3C_GPIO_SFN(2)); + break; + case 1: + s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(2)); + break; + case 2: + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(4)); + break; + default: + printk(KERN_ERR "Invalid Device %d\n", pdev->id); + return -EINVAL; + } + + return 0; +} + +static struct s3c_audio_pdata i2sv5_pdata = { + .cfg_gpio = s5pv310_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc, + }, + }, +}; + +static struct resource s5pv310_i2s0_resource[] = { + [0] = { + .start = S5PV310_PA_I2S0, + .end = S5PV310_PA_I2S0 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S0_TX, + .end = DMACH_I2S0_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S0_RX, + .end = DMACH_I2S0_RX, + .flags = IORESOURCE_DMA, + }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s0 = { + .name = "samsung-i2s", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_i2s0_resource), + .resource = s5pv310_i2s0_resource, + .dev = { + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "sclk_i2s", + [1] = "no_such_clock", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pv310_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_NO_MUXPSR, + .src_clk = rclksrc_v3, + }, + }, +}; + +static struct resource s5pv310_i2s1_resource[] = { + [0] = { + .start = S5PV310_PA_I2S1, + .end = S5PV310_PA_I2S1 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S1_TX, + .end = DMACH_I2S1_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S1_RX, + .end = DMACH_I2S1_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s1 = { + .name = "samsung-i2s", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_i2s1_resource), + .resource = s5pv310_i2s1_resource, + .dev = { + .platform_data = &i2sv3_pdata, + }, +}; + +static struct resource s5pv310_i2s2_resource[] = { + [0] = { + .start = S5PV310_PA_I2S2, + .end = S5PV310_PA_I2S2 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S2_TX, + .end = DMACH_I2S2_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S2_RX, + .end = DMACH_I2S2_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s2 = { + .name = "samsung-i2s", + .id = 2, + .num_resources = ARRAY_SIZE(s5pv310_i2s2_resource), + .resource = s5pv310_i2s2_resource, + .dev = { + .platform_data = &i2sv3_pdata, + }, +}; + +/* PCM Controller platform_devices */ + +static int s5pv310_pcm_cfg_gpio(struct platform_device *pdev) +{ + switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 5, S3C_GPIO_SFN(3)); + break; + case 1: + s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(3)); + break; + case 2: + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(3)); + break; + default: + printk(KERN_DEBUG "Invalid PCM Controller number!"); + return -EINVAL; + } + + return 0; +} + +static struct s3c_audio_pdata s3c_pcm_pdata = { + .cfg_gpio = s5pv310_pcm_cfg_gpio, +}; + +static struct resource s5pv310_pcm0_resource[] = { + [0] = { + .start = S5PV310_PA_PCM0, + .end = S5PV310_PA_PCM0 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM0_TX, + .end = DMACH_PCM0_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM0_RX, + .end = DMACH_PCM0_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm0 = { + .name = "samsung-pcm", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_pcm0_resource), + .resource = s5pv310_pcm0_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +static struct resource s5pv310_pcm1_resource[] = { + [0] = { + .start = S5PV310_PA_PCM1, + .end = S5PV310_PA_PCM1 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM1_TX, + .end = DMACH_PCM1_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM1_RX, + .end = DMACH_PCM1_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm1 = { + .name = "samsung-pcm", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_pcm1_resource), + .resource = s5pv310_pcm1_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +static struct resource s5pv310_pcm2_resource[] = { + [0] = { + .start = S5PV310_PA_PCM2, + .end = S5PV310_PA_PCM2 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM2_TX, + .end = DMACH_PCM2_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM2_RX, + .end = DMACH_PCM2_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm2 = { + .name = "samsung-pcm", + .id = 2, + .num_resources = ARRAY_SIZE(s5pv310_pcm2_resource), + .resource = s5pv310_pcm2_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +/* AC97 Controller platform devices */ + +static int s5pv310_ac97_cfg_gpio(struct platform_device *pdev) +{ + return s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(4)); +} + +static struct resource s5pv310_ac97_resource[] = { + [0] = { + .start = S5PV310_PA_AC97, + .end = S5PV310_PA_AC97 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_AC97_PCMOUT, + .end = DMACH_AC97_PCMOUT, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_AC97_PCMIN, + .end = DMACH_AC97_PCMIN, + .flags = IORESOURCE_DMA, + }, + [3] = { + .start = DMACH_AC97_MICIN, + .end = DMACH_AC97_MICIN, + .flags = IORESOURCE_DMA, + }, + [4] = { + .start = IRQ_AC97, + .end = IRQ_AC97, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_audio_pdata s3c_ac97_pdata = { + .cfg_gpio = s5pv310_ac97_cfg_gpio, +}; + +static u64 s5pv310_ac97_dmamask = DMA_BIT_MASK(32); + +struct platform_device s5pv310_device_ac97 = { + .name = "samsung-ac97", + .id = -1, + .num_resources = ARRAY_SIZE(s5pv310_ac97_resource), + .resource = s5pv310_ac97_resource, + .dev = { + .platform_data = &s3c_ac97_pdata, + .dma_mask = &s5pv310_ac97_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + }, +}; + +/* S/PDIF Controller platform_device */ + +static int s5pv310_spdif_cfg_gpio(struct platform_device *pdev) +{ + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 2, S3C_GPIO_SFN(3)); + + return 0; +} + +static struct resource s5pv310_spdif_resource[] = { + [0] = { + .start = S5PV310_PA_SPDIF, + .end = S5PV310_PA_SPDIF + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_SPDIF, + .end = DMACH_SPDIF, + .flags = IORESOURCE_DMA, + }, +}; + +static struct s3c_audio_pdata samsung_spdif_pdata = { + .cfg_gpio = s5pv310_spdif_cfg_gpio, +}; + +static u64 s5pv310_spdif_dmamask = DMA_BIT_MASK(32); + +struct platform_device s5pv310_device_spdif = { + .name = "samsung-spdif", + .id = -1, + .num_resources = ARRAY_SIZE(s5pv310_spdif_resource), + .resource = s5pv310_spdif_resource, + .dev = { + .platform_data = &samsung_spdif_pdata, + .dma_mask = &s5pv310_spdif_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + }, +}; diff --git a/arch/arm/mach-s5pv310/dma.c b/arch/arm/mach-s5pv310/dma.c new file mode 100644 index 0000000..20066c7 --- /dev/null +++ b/arch/arm/mach-s5pv310/dma.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2010 Samsung Electronics Co. Ltd. + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> + +#include <plat/devs.h> +#include <plat/irqs.h> + +#include <mach/map.h> +#include <mach/irqs.h> + +#include <plat/s3c-pl330-pdata.h> + +static u64 dma_dmamask = DMA_BIT_MASK(32); + +static struct resource s5pv310_pdma0_resource[] = { + [0] = { + .start = S5PV310_PA_PDMA0, + .end = S5PV310_PA_PDMA0 + SZ_4K, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = IRQ_PDMA0, + .end = IRQ_PDMA0, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_pl330_platdata s5pv310_pdma0_pdata = { + .peri = { + [0] = DMACH_PCM0_RX, + [1] = DMACH_PCM0_TX, + [2] = DMACH_PCM2_RX, + [3] = DMACH_PCM2_TX, + [4] = DMACH_MSM_REQ0, + [5] = DMACH_MSM_REQ2, + [6] = DMACH_SPI0_RX, + [7] = DMACH_SPI0_TX, + [8] = DMACH_SPI2_RX, + [9] = DMACH_SPI2_TX, + [10] = DMACH_I2S0S_TX, + [11] = DMACH_I2S0_RX, + [12] = DMACH_I2S0_TX, + [13] = DMACH_I2S2_RX, + [14] = DMACH_I2S2_TX, + [15] = DMACH_UART0_RX, + [16] = DMACH_UART0_TX, + [17] = DMACH_UART2_RX, + [18] = DMACH_UART2_TX, + [19] = DMACH_UART4_RX, + [20] = DMACH_UART4_TX, + [21] = DMACH_SLIMBUS0_RX, + [22] = DMACH_SLIMBUS0_TX, + [23] = DMACH_SLIMBUS2_RX, + [24] = DMACH_SLIMBUS2_TX, + [25] = DMACH_SLIMBUS4_RX, + [26] = DMACH_SLIMBUS4_TX, + [27] = DMACH_AC97_MICIN, + [28] = DMACH_AC97_PCMIN, + [29] = DMACH_AC97_PCMOUT, + [30] = DMACH_MAX, + [31] = DMACH_MAX, + }, +}; + +static struct platform_device s5pv310_device_pdma0 = { + .name = "s3c-pl330", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_pdma0_resource), + .resource = s5pv310_pdma0_resource, + .dev = { + .dma_mask = &dma_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &s5pv310_pdma0_pdata, + }, +}; + +static struct resource s5pv310_pdma1_resource[] = { + [0] = { + .start = S5PV310_PA_PDMA1, + .end = S5PV310_PA_PDMA1 + SZ_4K, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = IRQ_PDMA1, + .end = IRQ_PDMA1, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_pl330_platdata s5pv310_pdma1_pdata = { + .peri = { + [0] = DMACH_PCM0_RX, + [1] = DMACH_PCM0_TX, + [2] = DMACH_PCM1_RX, + [3] = DMACH_PCM1_TX, + [4] = DMACH_MSM_REQ1, + [5] = DMACH_MSM_REQ3, + [6] = DMACH_SPI1_RX, + [7] = DMACH_SPI1_TX, + [8] = DMACH_I2S0S_TX, + [9] = DMACH_I2S0_RX, + [10] = DMACH_I2S0_TX, + [11] = DMACH_I2S1_RX, + [12] = DMACH_I2S1_TX, + [13] = DMACH_UART0_RX, + [14] = DMACH_UART0_TX, + [15] = DMACH_UART1_RX, + [16] = DMACH_UART1_TX, + [17] = DMACH_UART3_RX, + [18] = DMACH_UART3_TX, + [19] = DMACH_SLIMBUS1_RX, + [20] = DMACH_SLIMBUS1_TX, + [21] = DMACH_SLIMBUS3_RX, + [22] = DMACH_SLIMBUS3_TX, + [23] = DMACH_SLIMBUS5_RX, + [24] = DMACH_SLIMBUS5_TX, + [25] = DMACH_SLIMBUS0AUX_RX, + [26] = DMACH_SLIMBUS0AUX_TX, + [27] = DMACH_SPDIF, + [28] = DMACH_MAX, + [29] = DMACH_MAX, + [30] = DMACH_MAX, + [31] = DMACH_MAX, + }, +}; + +static struct platform_device s5pv310_device_pdma1 = { + .name = "s3c-pl330", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_pdma1_resource), + .resource = s5pv310_pdma1_resource, + .dev = { + .dma_mask = &dma_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &s5pv310_pdma1_pdata, + }, +}; + +static struct platform_device *s5pv310_dmacs[] __initdata = { + &s5pv310_device_pdma0, + &s5pv310_device_pdma1, +}; + +static int __init s5pv310_dma_init(void) +{ + platform_add_devices(s5pv310_dmacs, ARRAY_SIZE(s5pv310_dmacs)); + + return 0; +} +arch_initcall(s5pv310_dma_init); diff --git a/arch/arm/mach-s5pv310/include/mach/dma.h b/arch/arm/mach-s5pv310/include/mach/dma.h new file mode 100644 index 0000000..81209eb --- /dev/null +++ b/arch/arm/mach-s5pv310/include/mach/dma.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2010 Samsung Electronics Co. Ltd. + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifndef __MACH_DMA_H +#define __MACH_DMA_H + +/* This platform uses the common S3C DMA API driver for PL330 */ +#include <plat/s3c-dma-pl330.h> + +#endif /* __MACH_DMA_H */ diff --git a/arch/arm/mach-s5pv310/include/mach/irqs.h b/arch/arm/mach-s5pv310/include/mach/irqs.h index 99e7dad..3c05c58 100644 --- a/arch/arm/mach-s5pv310/include/mach/irqs.h +++ b/arch/arm/mach-s5pv310/include/mach/irqs.h @@ -54,6 +54,9 @@ #define COMBINER_GROUP(x) ((x) * MAX_IRQ_IN_COMBINER + IRQ_SPI(64)) #define COMBINER_IRQ(x, y) (COMBINER_GROUP(x) + y) +#define IRQ_PDMA0 COMBINER_IRQ(21, 0) +#define IRQ_PDMA1 COMBINER_IRQ(21, 1) + #define IRQ_TIMER0_VIC COMBINER_IRQ(22, 0) #define IRQ_TIMER1_VIC COMBINER_IRQ(22, 1) #define IRQ_TIMER2_VIC COMBINER_IRQ(22, 2) diff --git a/arch/arm/mach-s5pv310/include/mach/map.h b/arch/arm/mach-s5pv310/include/mach/map.h index 7acf4e7..5399446 100644 --- a/arch/arm/mach-s5pv310/include/mach/map.h +++ b/arch/arm/mach-s5pv310/include/mach/map.h @@ -52,6 +52,11 @@ #define S5PV310_PA_GIC_DIST (0x10501000) #define S5PV310_PA_L2CC (0x10502000) +/* DMA */ +#define S5PV310_PA_MDMA 0x10810000 +#define S5PV310_PA_PDMA0 0x12680000 +#define S5PV310_PA_PDMA1 0x12690000 + #define S5PV310_PA_GPIO1 (0x11400000) #define S5PV310_PA_GPIO2 (0x11000000) #define S5PV310_PA_GPIO3 (0x03860000) @@ -60,6 +65,22 @@ #define S5PV310_PA_SROMC (0x12570000) +/* S/PDIF */ +#define S5PV310_PA_SPDIF 0xE1100000 + +/* I2S */ +#define S5PV310_PA_I2S0 0x03830000 +#define S5PV310_PA_I2S1 0xE3100000 +#define S5PV310_PA_I2S2 0xE2A00000 + +/* PCM */ +#define S5PV310_PA_PCM0 0x03840000 +#define S5PV310_PA_PCM1 0x13980000 +#define S5PV310_PA_PCM2 0x13990000 + +/* AC97 */ +#define S5PV310_PA_AC97 0x139A0000 + #define S5PV310_PA_UART (0x13800000) #define S5P_PA_UART(x) (S5PV310_PA_UART + ((x) * S3C_UART_OFFSET)) diff --git a/arch/arm/mach-sa1100/pm.c b/arch/arm/mach-sa1100/pm.c index c83fdc8..ab9fc44 100644 --- a/arch/arm/mach-sa1100/pm.c +++ b/arch/arm/mach-sa1100/pm.c @@ -120,7 +120,7 @@ unsigned long sleep_phys_sp(void *sp) return virt_to_phys(sp); } -static struct platform_suspend_ops sa11x0_pm_ops = { +static const struct platform_suspend_ops sa11x0_pm_ops = { .enter = sa11x0_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c index cd79d7c..3cf0951 100644 --- a/arch/arm/mach-shmobile/board-ap4evb.c +++ b/arch/arm/mach-shmobile/board-ap4evb.c @@ -247,10 +247,7 @@ static struct platform_device smc911x_device = { */ static int slot_cn7_get_cd(struct platform_device *pdev) { - if (gpio_is_valid(GPIO_PORT41)) - return !gpio_get_value(GPIO_PORT41); - else - return -ENXIO; + return !gpio_get_value(GPIO_PORT41); } /* SH_MMCIF */ @@ -308,6 +305,7 @@ static struct platform_device sh_mmcif_device = { static struct sh_mobile_sdhi_info sdhi0_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct resource sdhi0_resources[] = { @@ -339,7 +337,7 @@ static struct sh_mobile_sdhi_info sdhi1_info = { .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, .tmio_ocr_mask = MMC_VDD_165_195, .tmio_flags = TMIO_MMC_WRPROTECT_DISABLE, - .tmio_caps = MMC_CAP_NEEDS_POLL, + .tmio_caps = MMC_CAP_NEEDS_POLL | MMC_CAP_SDIO_IRQ, .get_cd = slot_cn7_get_cd, }; @@ -711,6 +709,10 @@ static struct platform_device fsi_device = { }, }; +static struct platform_device fsi_ak4643_device = { + .name = "sh_fsi2_a_ak4643", +}; + static struct sh_mobile_lcdc_info sh_mobile_lcdc1_info = { .clock_source = LCDC_CLK_EXTERNAL, .ch[0] = { @@ -933,6 +935,7 @@ static struct platform_device *ap4evb_devices[] __initdata = { &sdhi1_device, &usb1_host_device, &fsi_device, + &fsi_ak4643_device, &sh_mmcif_device, &lcdc1_device, &lcdc_device, diff --git a/arch/arm/mach-shmobile/board-g4evm.c b/arch/arm/mach-shmobile/board-g4evm.c index c13f012..dee3e92 100644 --- a/arch/arm/mach-shmobile/board-g4evm.c +++ b/arch/arm/mach-shmobile/board-g4evm.c @@ -30,6 +30,7 @@ #include <linux/io.h> #include <linux/input.h> #include <linux/input/sh_keysc.h> +#include <linux/mmc/host.h> #include <linux/mfd/sh_mobile_sdhi.h> #include <linux/gpio.h> #include <mach/sh7377.h> @@ -196,6 +197,10 @@ static struct platform_device keysc_device = { }; /* SDHI */ +static struct sh_mobile_sdhi_info sdhi0_info = { + .tmio_caps = MMC_CAP_SDIO_IRQ, +}; + static struct resource sdhi0_resources[] = { [0] = { .name = "SDHI0", @@ -214,6 +219,13 @@ static struct platform_device sdhi0_device = { .num_resources = ARRAY_SIZE(sdhi0_resources), .resource = sdhi0_resources, .id = 0, + .dev = { + .platform_data = &sdhi0_info, + }, +}; + +static struct sh_mobile_sdhi_info sdhi1_info = { + .tmio_caps = MMC_CAP_NONREMOVABLE | MMC_CAP_SDIO_IRQ, }; static struct resource sdhi1_resources[] = { @@ -234,6 +246,9 @@ static struct platform_device sdhi1_device = { .num_resources = ARRAY_SIZE(sdhi1_resources), .resource = sdhi1_resources, .id = 1, + .dev = { + .platform_data = &sdhi1_info, + }, }; static struct platform_device *g4evm_devices[] __initdata = { diff --git a/arch/arm/mach-shmobile/board-mackerel.c b/arch/arm/mach-shmobile/board-mackerel.c index 5bcf5c1..7b15d21 100644 --- a/arch/arm/mach-shmobile/board-mackerel.c +++ b/arch/arm/mach-shmobile/board-mackerel.c @@ -657,17 +657,14 @@ static struct platform_device fsi_ak4643_device = { */ static int slot_cn7_get_cd(struct platform_device *pdev) { - if (gpio_is_valid(GPIO_PORT41)) - return !gpio_get_value(GPIO_PORT41); - else - return -ENXIO; + return !gpio_get_value(GPIO_PORT41); } /* SDHI0 */ static struct sh_mobile_sdhi_info sdhi0_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, - .tmio_caps = MMC_CAP_SD_HIGHSPEED, + .tmio_caps = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ, }; static struct resource sdhi0_resources[] = { @@ -700,7 +697,7 @@ static struct sh_mobile_sdhi_info sdhi1_info = { .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, .tmio_ocr_mask = MMC_VDD_165_195, .tmio_flags = TMIO_MMC_WRPROTECT_DISABLE, - .tmio_caps = MMC_CAP_SD_HIGHSPEED | + .tmio_caps = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | MMC_CAP_NEEDS_POLL, .get_cd = slot_cn7_get_cd, }; @@ -729,13 +726,23 @@ static struct platform_device sdhi1_device = { }; #endif +/* + * The card detect pin of the top SD/MMC slot (CN23) is active low and is + * connected to GPIO SCIFB_SCK of SH7372 (GPIO_PORT162). + */ +static int slot_cn23_get_cd(struct platform_device *pdev) +{ + return !gpio_get_value(GPIO_PORT162); +} + /* SDHI2 */ static struct sh_mobile_sdhi_info sdhi2_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI2_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI2_RX, .tmio_flags = TMIO_MMC_WRPROTECT_DISABLE, - .tmio_caps = MMC_CAP_SD_HIGHSPEED | + .tmio_caps = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | MMC_CAP_NEEDS_POLL, + .get_cd = slot_cn23_get_cd, }; static struct resource sdhi2_resources[] = { @@ -953,6 +960,7 @@ static struct tca6416_keys_platform_data mackerel_tca6416_keys_info = { }; /* I2C */ +#define IRQ7 evt2irq(0x02e0) #define IRQ9 evt2irq(0x0320) static struct i2c_board_info i2c0_devices[] = { @@ -965,6 +973,11 @@ static struct i2c_board_info i2c0_devices[] = { .platform_data = &mackerel_tca6416_keys_info, .irq = IRQ9, }, + /* Touchscreen */ + { + I2C_BOARD_INFO("st1232-ts", 0x55), + .irq = IRQ7, + }, }; #define IRQ21 evt2irq(0x32a0) @@ -1092,6 +1105,10 @@ static void __init mackerel_init(void) gpio_request(GPIO_FN_IRQ9_42, NULL); set_irq_type(IRQ9, IRQ_TYPE_LEVEL_HIGH); + /* enable Touchscreen */ + gpio_request(GPIO_FN_IRQ7_40, NULL); + set_irq_type(IRQ7, IRQ_TYPE_LEVEL_LOW); + /* enable Accelerometer */ gpio_request(GPIO_FN_IRQ21, NULL); set_irq_type(IRQ21, IRQ_TYPE_LEVEL_HIGH); @@ -1127,6 +1144,10 @@ static void __init mackerel_init(void) gpio_request(GPIO_FN_SDHID2_1, NULL); gpio_request(GPIO_FN_SDHID2_0, NULL); + /* card detect pin for microSD slot (CN23) */ + gpio_request(GPIO_PORT162, NULL); + gpio_direction_input(GPIO_PORT162); + /* MMCIF */ gpio_request(GPIO_FN_MMCD0_0, NULL); gpio_request(GPIO_FN_MMCD0_1, NULL); diff --git a/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt b/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt index e3ebfa7..efd3687 100644 --- a/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt +++ b/arch/arm/mach-shmobile/include/mach/head-ap4evb.txt @@ -85,3 +85,10 @@ ED 0xE6150004, 0x80331050 WAIT 1, 0xFE40009C ED 0xE6150354, 0x00000002 + +LIST "SCIF0 - Serial port for earlyprintk" +EB 0xE6053098, 0x11 +EB 0xE6053098, 0xe1 +EW 0xE6C40000, 0x0000 +EB 0xE6C40004, 0x19 +EW 0xE6C40008, 0x3000 diff --git a/arch/arm/mach-shmobile/include/mach/head-mackerel.txt b/arch/arm/mach-shmobile/include/mach/head-mackerel.txt index e3ebfa7..efd3687 100644 --- a/arch/arm/mach-shmobile/include/mach/head-mackerel.txt +++ b/arch/arm/mach-shmobile/include/mach/head-mackerel.txt @@ -85,3 +85,10 @@ ED 0xE6150004, 0x80331050 WAIT 1, 0xFE40009C ED 0xE6150354, 0x00000002 + +LIST "SCIF0 - Serial port for earlyprintk" +EB 0xE6053098, 0x11 +EB 0xE6053098, 0xe1 +EW 0xE6C40000, 0x0000 +EB 0xE6C40004, 0x19 +EW 0xE6C40008, 0x3000 diff --git a/arch/arm/mach-shmobile/intc-sh7367.c b/arch/arm/mach-shmobile/intc-sh7367.c index 1a20c48..2fe9704 100644 --- a/arch/arm/mach-shmobile/intc-sh7367.c +++ b/arch/arm/mach-shmobile/intc-sh7367.c @@ -189,10 +189,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = { { SCIFB, SCIFA5, SCIFA4, MSIOF1, 0, 0, MSIOF2, 0 } }, { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } }, { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, TTI20, USBDMAC_USHDMI, SPU, SIU } }, { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */ { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10, @@ -207,7 +207,7 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = { { 0, 0, TPU0, TPU1, TPU2, TPU3, TPU4, 0 } }, { 0xe69400b4, 0xe69400f4, 8, /* IMR13A / IMCR13A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, MISTY, CMT3, RWDT1, RWDT0 } }, }; diff --git a/arch/arm/mach-shmobile/intc-sh7372.c b/arch/arm/mach-shmobile/intc-sh7372.c index 30b2f40..f78a1ea 100644 --- a/arch/arm/mach-shmobile/intc-sh7372.c +++ b/arch/arm/mach-shmobile/intc-sh7372.c @@ -230,10 +230,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = { { SCIFB, SCIFA5, SCIFA4, MSIOF1, 0, 0, MSIOF2, 0 } }, { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } }, { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */ - { 0, DISABLED, ENABLED, ENABLED, + { 0, ENABLED, ENABLED, ENABLED, TTI20, USBHSDMAC0_USHDMI, 0, 0 } }, { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */ { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10, diff --git a/arch/arm/mach-shmobile/intc-sh7377.c b/arch/arm/mach-shmobile/intc-sh7377.c index 2cdeb8c..dd56838 100644 --- a/arch/arm/mach-shmobile/intc-sh7377.c +++ b/arch/arm/mach-shmobile/intc-sh7377.c @@ -234,10 +234,10 @@ static struct intc_mask_reg intca_mask_registers[] __initdata = { { SCIFB, SCIFA5, SCIFA4, MSIOF1, 0, 0, MSIOF2, 0 } }, { 0xe694009c, 0xe69400dc, 8, /* IMR7A / IMCR7A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } }, { 0xe69400a0, 0xe69400e0, 8, /* IMR8A / IMCR8A */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, TTI20, USBDMAC_USHDMI, 0, MSUG } }, { 0xe69400a4, 0xe69400e4, 8, /* IMR9A / IMCR9A */ { CMT1_CMT13, CMT1_CMT12, CMT1_CMT11, CMT1_CMT10, diff --git a/arch/arm/mach-shmobile/setup-sh7367.c b/arch/arm/mach-shmobile/setup-sh7367.c index 003008c..ce28141 100644 --- a/arch/arm/mach-shmobile/setup-sh7367.c +++ b/arch/arm/mach-shmobile/setup-sh7367.c @@ -35,6 +35,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00) }, @@ -52,6 +54,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20) }, @@ -69,6 +73,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40) }, @@ -86,6 +92,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60) }, @@ -103,6 +111,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20) }, @@ -120,6 +130,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40) }, @@ -137,6 +149,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60) }, diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c index 2e3e11e..ff0494f 100644 --- a/arch/arm/mach-shmobile/setup-sh7372.c +++ b/arch/arm/mach-shmobile/setup-sh7372.c @@ -38,6 +38,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c00), evt2irq(0x0c00), evt2irq(0x0c00), evt2irq(0x0c00) }, @@ -55,6 +57,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c20), evt2irq(0x0c20), evt2irq(0x0c20), evt2irq(0x0c20) }, @@ -72,6 +76,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c40), evt2irq(0x0c40), evt2irq(0x0c40), evt2irq(0x0c40) }, @@ -89,6 +95,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c60), evt2irq(0x0c60), evt2irq(0x0c60), evt2irq(0x0c60) }, @@ -106,6 +114,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0d20), evt2irq(0x0d20), evt2irq(0x0d20), evt2irq(0x0d20) }, @@ -123,6 +133,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0d40), evt2irq(0x0d40), evt2irq(0x0d40), evt2irq(0x0d40) }, @@ -140,6 +152,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFB, .irqs = { evt2irq(0x0d60), evt2irq(0x0d60), evt2irq(0x0d60), evt2irq(0x0d60) }, diff --git a/arch/arm/mach-shmobile/setup-sh7377.c b/arch/arm/mach-shmobile/setup-sh7377.c index 575dbd6..8099b0b 100644 --- a/arch/arm/mach-shmobile/setup-sh7377.c +++ b/arch/arm/mach-shmobile/setup-sh7377.c @@ -36,6 +36,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00) }, @@ -53,6 +55,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20) }, @@ -70,6 +74,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40) }, @@ -87,6 +93,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60) }, @@ -104,6 +112,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20) }, @@ -121,6 +131,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40) }, @@ -138,6 +150,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6cc0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80) }, @@ -155,6 +169,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60) }, diff --git a/arch/arm/mach-shmobile/setup-sh73a0.c b/arch/arm/mach-shmobile/setup-sh73a0.c index f1eff8b..685c40a 100644 --- a/arch/arm/mach-shmobile/setup-sh73a0.c +++ b/arch/arm/mach-shmobile/setup-sh73a0.c @@ -36,6 +36,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(72), gic_spi(72), gic_spi(72), gic_spi(72) }, @@ -52,6 +54,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(73), gic_spi(73), gic_spi(73), gic_spi(73) }, @@ -68,6 +72,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(74), gic_spi(74), gic_spi(74), gic_spi(74) }, @@ -84,6 +90,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(75), gic_spi(75), gic_spi(75), gic_spi(75) }, @@ -100,6 +108,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(78), gic_spi(78), gic_spi(78), gic_spi(78) }, @@ -116,6 +126,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(79), gic_spi(79), gic_spi(79), gic_spi(79) }, @@ -132,6 +144,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6cc0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(156), gic_spi(156), gic_spi(156), gic_spi(156) }, @@ -148,6 +162,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xe6cd0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(143), gic_spi(143), gic_spi(143), gic_spi(143) }, @@ -164,6 +180,8 @@ static struct platform_device scif7_device = { static struct plat_sci_port scif8_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFB, .irqs = { gic_spi(80), gic_spi(80), gic_spi(80), gic_spi(80) }, diff --git a/arch/arm/mach-spear3xx/spear300.c b/arch/arm/mach-spear3xx/spear300.c index 3560f8c..5aa2d54 100644 --- a/arch/arm/mach-spear3xx/spear300.c +++ b/arch/arm/mach-spear3xx/spear300.c @@ -371,7 +371,7 @@ struct pmx_driver pmx_driver = { }; /* Add spear300 specific devices here */ -/* arm gpio1 device registeration */ +/* arm gpio1 device registration */ static struct pl061_platform_data gpio1_plat_data = { .gpio_base = 8, .irq_base = SPEAR_GPIO1_INT_BASE, @@ -451,7 +451,7 @@ void __init spear300_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ shirq_ras1.regs.base = ioremap(SPEAR300_TELECOM_BASE, SPEAR300_TELECOM_REG_SIZE); if (shirq_ras1.regs.base) { diff --git a/arch/arm/mach-spear3xx/spear310.c b/arch/arm/mach-spear3xx/spear310.c index 96a1ab8..53b41b5 100644 --- a/arch/arm/mach-spear3xx/spear310.c +++ b/arch/arm/mach-spear3xx/spear310.c @@ -266,7 +266,7 @@ void __init spear310_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ base = ioremap(SPEAR310_SOC_CONFIG_BASE, SPEAR310_SOC_CONFIG_SIZE); if (base) { /* shirq 1 */ diff --git a/arch/arm/mach-spear3xx/spear320.c b/arch/arm/mach-spear3xx/spear320.c index 6a12195..88b4652 100644 --- a/arch/arm/mach-spear3xx/spear320.c +++ b/arch/arm/mach-spear3xx/spear320.c @@ -519,7 +519,7 @@ void __init spear320_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ base = ioremap(SPEAR320_SOC_CONFIG_BASE, SPEAR320_SOC_CONFIG_SIZE); if (base) { /* shirq 1 */ diff --git a/arch/arm/mach-spear3xx/spear3xx.c b/arch/arm/mach-spear3xx/spear3xx.c index e87313a..52f553c 100644 --- a/arch/arm/mach-spear3xx/spear3xx.c +++ b/arch/arm/mach-spear3xx/spear3xx.c @@ -22,7 +22,7 @@ #include <mach/spear.h> /* Add spear3xx machines common devices here */ -/* gpio device registeration */ +/* gpio device registration */ static struct pl061_platform_data gpio_plat_data = { .gpio_base = 0, .irq_base = SPEAR_GPIO_INT_BASE, @@ -41,7 +41,7 @@ struct amba_device gpio_device = { .irq = {IRQ_BASIC_GPIO, NO_IRQ}, }; -/* uart device registeration */ +/* uart device registration */ struct amba_device uart_device = { .dev = { .init_name = "uart", @@ -543,6 +543,6 @@ void spear_pmx_init(struct pmx_driver *pmx_driver, uint base, uint size) pmx_fail: if (ret) - printk(KERN_ERR "padmux: registeration failed. err no: %d\n", + printk(KERN_ERR "padmux: registration failed. err no: %d\n", ret); } diff --git a/arch/arm/mach-spear6xx/spear6xx.c b/arch/arm/mach-spear6xx/spear6xx.c index baf6bcc..f2fe14e 100644 --- a/arch/arm/mach-spear6xx/spear6xx.c +++ b/arch/arm/mach-spear6xx/spear6xx.c @@ -23,7 +23,7 @@ #include <mach/spear.h> /* Add spear6xx machines common devices here */ -/* uart device registeration */ +/* uart device registration */ struct amba_device uart_device[] = { { .dev = { @@ -50,7 +50,7 @@ struct amba_device uart_device[] = { } }; -/* gpio device registeration */ +/* gpio device registration */ static struct pl061_platform_data gpio_plat_data[] = { { .gpio_base = 0, diff --git a/arch/arm/mach-tegra/include/mach/sdhci.h b/arch/arm/mach-tegra/include/mach/sdhci.h new file mode 100644 index 0000000..3ad086e --- /dev/null +++ b/arch/arm/mach-tegra/include/mach/sdhci.h @@ -0,0 +1,29 @@ +/* + * include/asm-arm/arch-tegra/include/mach/sdhci.h + * + * Copyright (C) 2009 Palm, Inc. + * Author: Yvonne Yip <y@palm.com> + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef __ASM_ARM_ARCH_TEGRA_SDHCI_H +#define __ASM_ARM_ARCH_TEGRA_SDHCI_H + +#include <linux/mmc/host.h> + +struct tegra_sdhci_platform_data { + int cd_gpio; + int wp_gpio; + int power_gpio; + int is_8bit; +}; + +#endif diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig index 801b21e..32a7b0f 100644 --- a/arch/arm/mach-u300/Kconfig +++ b/arch/arm/mach-u300/Kconfig @@ -64,7 +64,7 @@ config MACH_U300_DUAL_RAM bool "Dual RAM" help Select this if you want support for Dual RAM phones. - This is two RAM memorys on different EMIFs. + This is two RAM memories on different EMIFs. endchoice config U300_DEBUG diff --git a/arch/arm/mach-u300/include/mach/coh901318.h b/arch/arm/mach-u300/include/mach/coh901318.h index 193da2d..6193aaa 100644 --- a/arch/arm/mach-u300/include/mach/coh901318.h +++ b/arch/arm/mach-u300/include/mach/coh901318.h @@ -24,7 +24,7 @@ * @src_addr: transfer source address * @dst_addr: transfer destination address * @link_addr: physical address to next lli - * @virt_link_addr: virtual addres of next lli (only used by pool_free) + * @virt_link_addr: virtual address of next lli (only used by pool_free) * @phy_this: physical address of current lli (only used by pool_free) */ struct coh901318_lli { @@ -90,7 +90,7 @@ struct powersave { * struct coh901318_platform - platform arch structure * @chans_slave: specifying dma slave channels * @chans_memcpy: specifying dma memcpy channels - * @access_memory_state: requesting DMA memeory access (on / off) + * @access_memory_state: requesting DMA memory access (on / off) * @chan_conf: dma channel configurations * @max_channels: max number of dma chanenls */ diff --git a/arch/arm/mach-ux500/board-mop500-regulators.c b/arch/arm/mach-ux500/board-mop500-regulators.c index 1187f1f..533967c 100644 --- a/arch/arm/mach-ux500/board-mop500-regulators.c +++ b/arch/arm/mach-ux500/board-mop500-regulators.c @@ -3,99 +3,94 @@ * * License Terms: GNU General Public License v2 * - * Author: Sundar Iyer <sundar.iyer@stericsson.com> + * Authors: Sundar Iyer <sundar.iyer@stericsson.com> + * Bengt Jonsson <bengt.g.jonsson@stericsson.com> * * MOP500 board specific initialization for regulators */ #include <linux/kernel.h> #include <linux/regulator/machine.h> +#include <linux/regulator/ab8500.h> -/* supplies to the display/camera */ -static struct regulator_init_data ab8500_vaux1_regulator = { - .constraints = { - .name = "V-DISPLAY", - .min_uV = 2500000, - .max_uV = 2900000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, +/* AB8500 regulators */ +struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS] = { + /* supplies to the display/camera */ + [AB8500_LDO_AUX1] = { + .constraints = { + .name = "V-DISPLAY", + .min_uV = 2500000, + .max_uV = 2900000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supplies to the on-board eMMC */ -static struct regulator_init_data ab8500_vaux2_regulator = { - .constraints = { - .name = "V-eMMC1", - .min_uV = 1100000, - .max_uV = 3300000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, + /* supplies to the on-board eMMC */ + [AB8500_LDO_AUX2] = { + .constraints = { + .name = "V-eMMC1", + .min_uV = 1100000, + .max_uV = 3300000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for VAUX3, supplies to SDcard slots */ -static struct regulator_init_data ab8500_vaux3_regulator = { - .constraints = { - .name = "V-MMC-SD", - .min_uV = 1100000, - .max_uV = 3300000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, + /* supply for VAUX3, supplies to SDcard slots */ + [AB8500_LDO_AUX3] = { + .constraints = { + .name = "V-MMC-SD", + .min_uV = 1100000, + .max_uV = 3300000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for tvout, gpadc, TVOUT LDO */ -static struct regulator_init_data ab8500_vtvout_init = { - .constraints = { - .name = "V-TVOUT", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for tvout, gpadc, TVOUT LDO */ + [AB8500_LDO_TVOUT] = { + .constraints = { + .name = "V-TVOUT", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for ab8500-vaudio, VAUDIO LDO */ -static struct regulator_init_data ab8500_vaudio_init = { - .constraints = { - .name = "V-AUD", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for ab8500-vaudio, VAUDIO LDO */ + [AB8500_LDO_AUDIO] = { + .constraints = { + .name = "V-AUD", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-anamic1 VAMic1-LDO */ -static struct regulator_init_data ab8500_vamic1_init = { - .constraints = { - .name = "V-AMIC1", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-anamic1 VAMic1-LDO */ + [AB8500_LDO_ANAMIC1] = { + .constraints = { + .name = "V-AMIC1", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */ -static struct regulator_init_data ab8500_vamic2_init = { - .constraints = { - .name = "V-AMIC2", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */ + [AB8500_LDO_ANAMIC2] = { + .constraints = { + .name = "V-AMIC2", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-dmic, VDMIC LDO */ -static struct regulator_init_data ab8500_vdmic_init = { - .constraints = { - .name = "V-DMIC", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-dmic, VDMIC LDO */ + [AB8500_LDO_DMIC] = { + .constraints = { + .name = "V-DMIC", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-intcore12, VINTCORE12 LDO */ -static struct regulator_init_data ab8500_vintcore_init = { - .constraints = { - .name = "V-INTCORE", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-intcore12, VINTCORE12 LDO */ + [AB8500_LDO_INTCORE] = { + .constraints = { + .name = "V-INTCORE", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for U8500 CSI/DSI, VANA LDO */ -static struct regulator_init_data ab8500_vana_init = { - .constraints = { - .name = "V-CSI/DSI", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for U8500 CSI/DSI, VANA LDO */ + [AB8500_LDO_ANA] = { + .constraints = { + .name = "V-CSI/DSI", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, }; - diff --git a/arch/arm/mach-ux500/board-mop500-regulators.h b/arch/arm/mach-ux500/board-mop500-regulators.h new file mode 100644 index 0000000..2675fae --- /dev/null +++ b/arch/arm/mach-ux500/board-mop500-regulators.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) ST-Ericsson SA 2010 + * + * License Terms: GNU General Public License v2 + * + * Author: Bengt Jonsson <bengt.g.jonsson@stericsson.com> for ST-Ericsson + * + * MOP500 board specific initialization for regulators + */ + +#ifndef __BOARD_MOP500_REGULATORS_H +#define __BOARD_MOP500_REGULATORS_H + +#include <linux/regulator/machine.h> +#include <linux/regulator/ab8500.h> + +extern struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS]; + +#endif diff --git a/arch/arm/mach-ux500/board-mop500.c b/arch/arm/mach-ux500/board-mop500.c index a1c9ea1..a393f57 100644 --- a/arch/arm/mach-ux500/board-mop500.c +++ b/arch/arm/mach-ux500/board-mop500.c @@ -35,6 +35,7 @@ #include "devices-db8500.h" #include "pins-db8500.h" #include "board-mop500.h" +#include "board-mop500-regulators.h" static pin_cfg_t mop500_pins[] = { /* SSP0 */ @@ -80,6 +81,8 @@ static pin_cfg_t mop500_pins[] = { static struct ab8500_platform_data ab8500_platdata = { .irq_base = MOP500_AB8500_IRQ_BASE, + .regulator = ab8500_regulators, + .num_regulator = ARRAY_SIZE(ab8500_regulators), }; static struct resource ab8500_resources[] = { diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index c29f283..2b269c9 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -18,7 +18,6 @@ #include <asm/smp_plat.h> #include <asm/system.h> #include <asm/tlbflush.h> -#include <asm/smp_plat.h> #include "mm.h" diff --git a/arch/arm/plat-mxc/include/mach/irqs.h b/arch/arm/plat-mxc/include/mach/irqs.h index 58a49cc..ba65c92 100644 --- a/arch/arm/plat-mxc/include/mach/irqs.h +++ b/arch/arm/plat-mxc/include/mach/irqs.h @@ -70,7 +70,7 @@ extern int imx_irq_set_priority(unsigned char irq, unsigned char prio); /* all normal IRQs can be FIQs */ #define FIQ_START 0 -/* switch betwean IRQ and FIQ */ +/* switch between IRQ and FIQ */ extern int mxc_set_irq_fiq(unsigned int irq, unsigned int type); #endif /* __ASM_ARCH_MXC_IRQS_H__ */ diff --git a/arch/arm/plat-omap/include/plat/display.h b/arch/arm/plat-omap/include/plat/display.h index c915a66..537f4e4 100644 --- a/arch/arm/plat-omap/include/plat/display.h +++ b/arch/arm/plat-omap/include/plat/display.h @@ -42,6 +42,10 @@ #define DISPC_IRQ_SYNC_LOST (1 << 14) #define DISPC_IRQ_SYNC_LOST_DIGIT (1 << 15) #define DISPC_IRQ_WAKEUP (1 << 16) +#define DISPC_IRQ_SYNC_LOST2 (1 << 17) +#define DISPC_IRQ_VSYNC2 (1 << 18) +#define DISPC_IRQ_ACBIAS_COUNT_STAT2 (1 << 21) +#define DISPC_IRQ_FRAMEDONE2 (1 << 22) struct omap_dss_device; struct omap_overlay_manager; @@ -64,6 +68,7 @@ enum omap_plane { enum omap_channel { OMAP_DSS_CHANNEL_LCD = 0, OMAP_DSS_CHANNEL_DIGIT = 1, + OMAP_DSS_CHANNEL_LCD2 = 2, }; enum omap_color_mode { @@ -142,6 +147,7 @@ enum omap_dss_display_state { enum omap_dss_overlay_managers { OMAP_DSS_OVL_MGR_LCD, OMAP_DSS_OVL_MGR_TV, + OMAP_DSS_OVL_MGR_LCD2, }; enum omap_dss_rotation_type { @@ -268,6 +274,7 @@ struct omap_overlay_info { u16 out_width; /* if 0, out_width == width */ u16 out_height; /* if 0, out_height == height */ u8 global_alpha; + u8 pre_mult_alpha; }; struct omap_overlay { @@ -351,6 +358,8 @@ struct omap_dss_device { enum omap_display_type type; + enum omap_channel channel; + union { struct { u8 data_lines; diff --git a/arch/arm/plat-omap/include/plat/omap_hwmod.h b/arch/arm/plat-omap/include/plat/omap_hwmod.h index 6864a99..1eee85a 100644 --- a/arch/arm/plat-omap/include/plat/omap_hwmod.h +++ b/arch/arm/plat-omap/include/plat/omap_hwmod.h @@ -351,7 +351,7 @@ struct omap_hwmod_omap2_prcm { /** * struct omap_hwmod_omap4_prcm - OMAP4-specific PRCM data * @clkctrl_reg: PRCM address of the clock control register - * @rstctrl_reg: adress of the XXX_RSTCTRL register located in the PRM + * @rstctrl_reg: address of the XXX_RSTCTRL register located in the PRM * @submodule_wkdep_bit: bit shift of the WKDEP range */ struct omap_hwmod_omap4_prcm { diff --git a/arch/arm/plat-omap/include/plat/panel-generic-dpi.h b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h new file mode 100644 index 0000000..7906197 --- /dev/null +++ b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h @@ -0,0 +1,37 @@ +/* + * Header for generic DPI panel driver + * + * Copyright (C) 2010 Canonical Ltd. + * Author: Bryan Wu <bryan.wu@canonical.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H +#define __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H + +#include "display.h" + +/** + * struct panel_generic_dpi_data - panel driver configuration data + * @name: panel name + * @platform_enable: platform specific panel enable function + * @platform_disable: platform specific panel disable function + */ +struct panel_generic_dpi_data { + const char *name; + int (*platform_enable)(struct omap_dss_device *dssdev); + void (*platform_disable)(struct omap_dss_device *dssdev); +}; + +#endif /* __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H */ diff --git a/arch/arm/plat-s3c24xx/devs.c b/arch/arm/plat-s3c24xx/devs.c index 2f91057..8a42bc4 100644 --- a/arch/arm/plat-s3c24xx/devs.c +++ b/arch/arm/plat-s3c24xx/devs.c @@ -259,21 +259,6 @@ struct platform_device s3c_device_iis = { EXPORT_SYMBOL(s3c_device_iis); -/* ASoC PCM DMA */ - -static u64 s3c_device_audio_dmamask = 0xffffffffUL; - -struct platform_device s3c_device_pcm = { - .name = "s3c24xx-pcm-audio", - .id = -1, - .dev = { - .dma_mask = &s3c_device_audio_dmamask, - .coherent_dma_mask = 0xffffffffUL - } -}; - -EXPORT_SYMBOL(s3c_device_pcm); - /* RTC */ static struct resource s3c_rtc_resource[] = { @@ -496,8 +481,10 @@ static struct resource s3c_ac97_resource[] = { }, }; +static u64 s3c_device_audio_dmamask = 0xffffffffUL; + struct platform_device s3c_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s3c_ac97_resource), .resource = s3c_ac97_resource, diff --git a/arch/arm/plat-samsung/Makefile b/arch/arm/plat-samsung/Makefile index afcce47..19d8a16 100644 --- a/arch/arm/plat-samsung/Makefile +++ b/arch/arm/plat-samsung/Makefile @@ -17,6 +17,7 @@ obj-y += clock.o obj-y += pwm-clock.o obj-y += gpio.o obj-y += gpio-config.o +obj-y += dev-asocdma.o obj-$(CONFIG_SAMSUNG_GPIOLIB_4BIT) += gpiolib.o obj-$(CONFIG_SAMSUNG_CLKSRC) += clock-clksrc.o diff --git a/arch/arm/plat-samsung/dev-asocdma.c b/arch/arm/plat-samsung/dev-asocdma.c new file mode 100644 index 0000000..a068c4f --- /dev/null +++ b/arch/arm/plat-samsung/dev-asocdma.c @@ -0,0 +1,25 @@ +/* linux/arch/arm/plat-samsung/dev-asocdma.c + * + * Copyright (c) 2010 Samsung Electronics Co. Ltd + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <plat/devs.h> + +static u64 audio_dmamask = DMA_BIT_MASK(32); + +struct platform_device samsung_asoc_dma = { + .name = "samsung-audio", + .id = -1, + .dev = { + .dma_mask = &audio_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + } +}; +EXPORT_SYMBOL(samsung_asoc_dma); diff --git a/arch/arm/plat-samsung/include/plat/audio.h b/arch/arm/plat-samsung/include/plat/audio.h index 7712ff6..a0826ed 100644 --- a/arch/arm/plat-samsung/include/plat/audio.h +++ b/arch/arm/plat-samsung/include/plat/audio.h @@ -25,10 +25,34 @@ extern void s3c64xx_ac97_setup_gpio(int); #define S5PC100_SPDIF_GPG3 1 extern void s5pc100_spdif_setup_gpio(int); +struct samsung_i2s { +/* If the Primary DAI has 5.1 Channels */ +#define QUIRK_PRI_6CHAN (1 << 0) +/* If the I2S block has a Stereo Overlay Channel */ +#define QUIRK_SEC_DAI (1 << 1) +/* + * If the I2S block has no internal prescalar or MUX (I2SMOD[10] bit) + * The Machine driver must provide suitably set clock to the I2S block. + */ +#define QUIRK_NO_MUXPSR (1 << 2) +#define QUIRK_NEED_RSTCLR (1 << 3) + /* Quirks of the I2S controller */ + u32 quirks; + + /* + * Array of clock names that can be used to generate I2S signals. + * Also corresponds to clocks of I2SMOD[10] + */ + const char **src_clk; +}; + /** * struct s3c_audio_pdata - common platform data for audio device drivers * @cfg_gpio: Callback function to setup mux'ed pins in I2S/PCM/AC97 mode */ struct s3c_audio_pdata { int (*cfg_gpio)(struct platform_device *); + union { + struct samsung_i2s i2s; + } type; }; diff --git a/arch/arm/plat-samsung/include/plat/devs.h b/arch/arm/plat-samsung/include/plat/devs.h index 2d82a6c..e9e3b6e 100644 --- a/arch/arm/plat-samsung/include/plat/devs.h +++ b/arch/arm/plat-samsung/include/plat/devs.h @@ -32,7 +32,7 @@ extern struct platform_device s3c64xx_device_iisv4; extern struct platform_device s3c64xx_device_spi0; extern struct platform_device s3c64xx_device_spi1; -extern struct platform_device s3c_device_pcm; +extern struct platform_device samsung_asoc_dma; extern struct platform_device s3c64xx_device_pcm0; extern struct platform_device s3c64xx_device_pcm1; @@ -96,6 +96,15 @@ extern struct platform_device s5pv210_device_iis1; extern struct platform_device s5pv210_device_iis2; extern struct platform_device s5pv210_device_spdif; +extern struct platform_device s5pv310_device_ac97; +extern struct platform_device s5pv310_device_pcm0; +extern struct platform_device s5pv310_device_pcm1; +extern struct platform_device s5pv310_device_pcm2; +extern struct platform_device s5pv310_device_i2s0; +extern struct platform_device s5pv310_device_i2s1; +extern struct platform_device s5pv310_device_i2s2; +extern struct platform_device s5pv310_device_spdif; + extern struct platform_device s5p6442_device_pcm0; extern struct platform_device s5p6442_device_pcm1; extern struct platform_device s5p6442_device_iis0; diff --git a/arch/arm/plat-samsung/pm.c b/arch/arm/plat-samsung/pm.c index 27cfca5..5bf3f2f 100644 --- a/arch/arm/plat-samsung/pm.c +++ b/arch/arm/plat-samsung/pm.c @@ -355,7 +355,7 @@ static void s3c_pm_finish(void) s3c_pm_check_cleanup(); } -static struct platform_suspend_ops s3c_pm_ops = { +static const struct platform_suspend_ops s3c_pm_ops = { .enter = s3c_pm_enter, .prepare = s3c_pm_prepare, .finish = s3c_pm_finish, diff --git a/arch/avr32/mach-at32ap/pm.c b/arch/avr32/mach-at32ap/pm.c index f021edf..32d680e 100644 --- a/arch/avr32/mach-at32ap/pm.c +++ b/arch/avr32/mach-at32ap/pm.c @@ -176,7 +176,7 @@ out: return 0; } -static struct platform_suspend_ops avr32_pm_ops = { +static const struct platform_suspend_ops avr32_pm_ops = { .valid = avr32_pm_valid_state, .enter = avr32_pm_enter, }; diff --git a/arch/blackfin/mach-bf537/include/mach/defBF534.h b/arch/blackfin/mach-bf537/include/mach/defBF534.h index 32529a0..725bb35 100644 --- a/arch/blackfin/mach-bf537/include/mach/defBF534.h +++ b/arch/blackfin/mach-bf537/include/mach/defBF534.h @@ -1418,7 +1418,7 @@ #define SADD_LEN 0x0002 /* Slave Address Length */ #define STDVAL 0x0004 /* Slave Transmit Data Valid */ #define NAK 0x0008 /* NAK/ACK* Generated At Conclusion Of Transfer */ -#define GEN 0x0010 /* General Call Adrress Matching Enabled */ +#define GEN 0x0010 /* General Call Address Matching Enabled */ /* TWI_SLAVE_STAT Masks */ #define SDIR 0x0001 /* Slave Transfer Direction (Transmit/Receive*) */ diff --git a/arch/blackfin/mach-common/pm.c b/arch/blackfin/mach-common/pm.c index 42fa87e..3c648a0 100644 --- a/arch/blackfin/mach-common/pm.c +++ b/arch/blackfin/mach-common/pm.c @@ -223,7 +223,7 @@ static int bfin_pm_enter(suspend_state_t state) return 0; } -struct platform_suspend_ops bfin_pm_ops = { +static const struct platform_suspend_ops bfin_pm_ops = { .enter = bfin_pm_enter, .valid = bfin_pm_valid, }; diff --git a/arch/cris/arch-v32/lib/nand_init.S b/arch/cris/arch-v32/lib/nand_init.S index e705f5c..d671fed 100644 --- a/arch/cris/arch-v32/lib/nand_init.S +++ b/arch/cris/arch-v32/lib/nand_init.S @@ -139,7 +139,7 @@ copy_nand_to_ram: lsrq 8, $r4 move.b $r4, [$r1] ; Row address lsrq 8, $r4 - move.b $r4, [$r1] ; Row adddress + move.b $r4, [$r1] ; Row address moveq 20, $r4 2: bne 2b subq 1, $r4 diff --git a/arch/cris/include/asm/etraxgpio.h b/arch/cris/include/asm/etraxgpio.h index d474818..461c089 100644 --- a/arch/cris/include/asm/etraxgpio.h +++ b/arch/cris/include/asm/etraxgpio.h @@ -1,5 +1,5 @@ /* - * The following devices are accessable using this driver using + * The following devices are accessible using this driver using * GPIO_MAJOR (120) and a couple of minor numbers. * * For ETRAX 100LX (CONFIG_ETRAX_ARCH_V10): diff --git a/arch/cris/kernel/time.c b/arch/cris/kernel/time.c index b509643..4e73092 100644 --- a/arch/cris/kernel/time.c +++ b/arch/cris/kernel/time.c @@ -86,7 +86,7 @@ int set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds,RTC_SECONDS); CMOS_WRITE(real_minutes,RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/h8300/Kconfig.debug b/arch/h8300/Kconfig.debug index ee671c3..e8d1b23 100644 --- a/arch/h8300/Kconfig.debug +++ b/arch/h8300/Kconfig.debug @@ -48,7 +48,7 @@ config DEFAULT_CMDLINE builtin kernel commandline enabled. config KERNEL_COMMAND - string "Buildin commmand string" + string "Buildin command string" depends on DEFAULT_CMDLINE help builtin kernel commandline strings. diff --git a/arch/ia64/include/asm/kvm_host.h b/arch/ia64/include/asm/kvm_host.h index 2f229e5..2689ee5 100644 --- a/arch/ia64/include/asm/kvm_host.h +++ b/arch/ia64/include/asm/kvm_host.h @@ -590,6 +590,10 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu); int kvm_pal_emul(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); void kvm_sal_emul(struct kvm_vcpu *vcpu); +#define __KVM_HAVE_ARCH_VM_ALLOC 1 +struct kvm *kvm_arch_alloc_vm(void); +void kvm_arch_free_vm(struct kvm *kvm); + #endif /* __ASSEMBLY__*/ #endif diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index d92d5b5..ac76da0 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -617,11 +617,14 @@ pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, return get_unmapped_area(file, addr, len, pgoff, flags); } +/* forward declaration */ +static static const struct dentry_operations pfmfs_dentry_operations; static struct dentry * pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data) { - return mount_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC); + return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations, + PFMFS_MAGIC); } static struct file_system_type pfm_fs_type = { @@ -2232,7 +2235,6 @@ pfm_alloc_file(pfm_context_t *ctx) } path.mnt = mntget(pfmfs_mnt); - d_set_d_op(path.dentry, &pfmfs_dentry_operations); d_add(path.dentry, inode); file = alloc_file(&path, FMODE_READ, &pfm_file_ops); diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c index f56a631..70d224d 100644 --- a/arch/ia64/kvm/kvm-ia64.c +++ b/arch/ia64/kvm/kvm-ia64.c @@ -749,7 +749,7 @@ out: return r; } -static struct kvm *kvm_alloc_kvm(void) +struct kvm *kvm_arch_alloc_vm(void) { struct kvm *kvm; @@ -760,7 +760,7 @@ static struct kvm *kvm_alloc_kvm(void) vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE)); if (!vm_base) - return ERR_PTR(-ENOMEM); + return NULL; memset((void *)vm_base, 0, KVM_VM_DATA_SIZE); kvm = (struct kvm *)(vm_base + @@ -806,10 +806,12 @@ static void kvm_build_io_pmt(struct kvm *kvm) #define GUEST_PHYSICAL_RR4 0x2739 #define VMM_INIT_RR 0x1660 -static void kvm_init_vm(struct kvm *kvm) +int kvm_arch_init_vm(struct kvm *kvm) { BUG_ON(!kvm); + kvm->arch.is_sn2 = ia64_platform_is("sn2"); + kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0; kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4; kvm->arch.vmm_init_rr = VMM_INIT_RR; @@ -823,21 +825,8 @@ static void kvm_init_vm(struct kvm *kvm) /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); -} - -struct kvm *kvm_arch_create_vm(void) -{ - struct kvm *kvm = kvm_alloc_kvm(); - - if (IS_ERR(kvm)) - return ERR_PTR(-ENOMEM); - - kvm->arch.is_sn2 = ia64_platform_is("sn2"); - - kvm_init_vm(kvm); - - return kvm; + return 0; } static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, @@ -962,7 +951,9 @@ long kvm_arch_vm_ioctl(struct file *filp, goto out; r = kvm_setup_default_irq_routing(kvm); if (r) { + mutex_lock(&kvm->slots_lock); kvm_ioapic_destroy(kvm); + mutex_unlock(&kvm->slots_lock); goto out; } break; @@ -1357,7 +1348,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return -EINVAL; } -static void free_kvm(struct kvm *kvm) +void kvm_arch_free_vm(struct kvm *kvm) { unsigned long vm_base = kvm->arch.vm_base; @@ -1399,9 +1390,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm) #endif kfree(kvm->arch.vioapic); kvm_release_vm_pages(kvm); - kvm_free_physmem(kvm); - cleanup_srcu_struct(&kvm->srcu); - free_kvm(kvm); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) diff --git a/arch/ia64/kvm/mmio.c b/arch/ia64/kvm/mmio.c index fb8f9f5..f1e17d3 100644 --- a/arch/ia64/kvm/mmio.c +++ b/arch/ia64/kvm/mmio.c @@ -130,7 +130,7 @@ static void mmio_access(struct kvm_vcpu *vcpu, u64 src_pa, u64 *dest, local_irq_save(psr); - /*Intercept the acces for PIB range*/ + /*Intercept the access for PIB range*/ if (iot == GPFN_PIB) { if (!dir) lsapic_write(vcpu, src_pa, s, *dest); diff --git a/arch/m68k/ifpsp060/src/fpsp.S b/arch/m68k/ifpsp060/src/fpsp.S index 73613b5..26e85e2 100644 --- a/arch/m68k/ifpsp060/src/fpsp.S +++ b/arch/m68k/ifpsp060/src/fpsp.S @@ -3881,7 +3881,7 @@ _fpsp_fline: # FP Unimplemented Instruction stack frame and jump to that entry # point. # -# but, if the FPU is disabled, then we need to jump to the FPU diabled +# but, if the FPU is disabled, then we need to jump to the FPU disabled # entry point. movc %pcr,%d0 btst &0x1,%d0 diff --git a/arch/m68k/include/asm/m548xgpt.h b/arch/m68k/include/asm/m548xgpt.h index c8ef158..33b2eef 100644 --- a/arch/m68k/include/asm/m548xgpt.h +++ b/arch/m68k/include/asm/m548xgpt.h @@ -59,11 +59,13 @@ #define MCF_GPT_GMS_GPIO_INPUT (0x00000000) #define MCF_GPT_GMS_GPIO_OUTLO (0x00000020) #define MCF_GPT_GMS_GPIO_OUTHI (0x00000030) +#define MCF_GPT_GMS_GPIO_MASK (0x00000030) #define MCF_GPT_GMS_TMS_DISABLE (0x00000000) #define MCF_GPT_GMS_TMS_INCAPT (0x00000001) #define MCF_GPT_GMS_TMS_OUTCAPT (0x00000002) #define MCF_GPT_GMS_TMS_PWM (0x00000003) #define MCF_GPT_GMS_TMS_GPIO (0x00000004) +#define MCF_GPT_GMS_TMS_MASK (0x00000007) /* Bit definitions and macros for MCF_GPT_GCIR */ #define MCF_GPT_GCIR_CNT(x) (((x)&0x0000FFFF)<<0) diff --git a/arch/m68k/mac/psc.c b/arch/m68k/mac/psc.c index ba6ccab..a4c3eb6 100644 --- a/arch/m68k/mac/psc.c +++ b/arch/m68k/mac/psc.c @@ -88,7 +88,7 @@ void __init psc_init(void) /* * The PSC is always at the same spot, but using psc - * keeps things consisant with the psc_xxxx functions. + * keeps things consistent with the psc_xxxx functions. */ psc = (void *) PSC_BASE; diff --git a/arch/microblaze/Kconfig.debug b/arch/microblaze/Kconfig.debug index e66e25c..012e377 100644 --- a/arch/microblaze/Kconfig.debug +++ b/arch/microblaze/Kconfig.debug @@ -23,8 +23,4 @@ config HEART_BEAT This option turns on/off heart beat kernel functionality. First GPIO node is taken. -config DEBUG_BOOTMEM - depends on DEBUG_KERNEL - bool "Debug BOOTMEM initialization" - endmenu diff --git a/arch/microblaze/Makefile b/arch/microblaze/Makefile index 15f1f1d..6f432e6 100644 --- a/arch/microblaze/Makefile +++ b/arch/microblaze/Makefile @@ -17,7 +17,7 @@ export CPU_VER CPU_MAJOR CPU_MINOR CPU_REV # The various CONFIG_XILINX cpu features options are integers 0/1/2... # rather than bools y/n -# Work out HW multipler support. This is icky. +# Work out HW multipler support. This is tricky. # 1. Spartan2 has no HW multiplers. # 2. MicroBlaze v3.x always uses them, except in Spartan 2 # 3. All other FPGa/CPU ver combos, we can trust the CONFIG_ settings diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 8b422b1..ab8fbe7 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -66,5 +66,4 @@ CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_INFO=y # CONFIG_RCU_CPU_STALL_DETECTOR is not set CONFIG_EARLY_PRINTK=y -CONFIG_DEBUG_BOOTMEM=y # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/microblaze/include/asm/pvr.h b/arch/microblaze/include/asm/pvr.h index 37db96a..a10bec6 100644 --- a/arch/microblaze/include/asm/pvr.h +++ b/arch/microblaze/include/asm/pvr.h @@ -1,9 +1,9 @@ /* * Support for the MicroBlaze PVR (Processor Version Register) * - * Copyright (C) 2009 Michal Simek <monstr@monstr.eu> + * Copyright (C) 2009 - 2011 Michal Simek <monstr@monstr.eu> * Copyright (C) 2007 John Williams <john.williams@petalogix.com> - * Copyright (C) 2007 - 2009 PetaLogix + * Copyright (C) 2007 - 2011 PetaLogix * * This file is subject to the terms and conditions of the GNU General * Public License. See the file COPYING in the main directory of this @@ -46,11 +46,11 @@ struct pvr_s { #define PVR2_I_LMB_MASK 0x10000000 #define PVR2_INTERRUPT_IS_EDGE_MASK 0x08000000 #define PVR2_EDGE_IS_POSITIVE_MASK 0x04000000 -#define PVR2_D_PLB_MASK 0x02000000 /* new */ -#define PVR2_I_PLB_MASK 0x01000000 /* new */ -#define PVR2_INTERCONNECT 0x00800000 /* new */ -#define PVR2_USE_EXTEND_FSL 0x00080000 /* new */ -#define PVR2_USE_FSL_EXC 0x00040000 /* new */ +#define PVR2_D_PLB_MASK 0x02000000 /* new */ +#define PVR2_I_PLB_MASK 0x01000000 /* new */ +#define PVR2_INTERCONNECT 0x00800000 /* new */ +#define PVR2_USE_EXTEND_FSL 0x00080000 /* new */ +#define PVR2_USE_FSL_EXC 0x00040000 /* new */ #define PVR2_USE_MSR_INSTR 0x00020000 #define PVR2_USE_PCMP_INSTR 0x00010000 #define PVR2_AREA_OPTIMISED 0x00008000 @@ -59,7 +59,7 @@ struct pvr_s { #define PVR2_USE_HW_MUL_MASK 0x00001000 #define PVR2_USE_FPU_MASK 0x00000800 #define PVR2_USE_MUL64_MASK 0x00000400 -#define PVR2_USE_FPU2_MASK 0x00000200 /* new */ +#define PVR2_USE_FPU2_MASK 0x00000200 /* new */ #define PVR2_USE_IPLBEXC 0x00000100 #define PVR2_USE_DPLBEXC 0x00000080 #define PVR2_OPCODE_0x0_ILL_MASK 0x00000040 @@ -122,96 +122,103 @@ struct pvr_s { /* PVR access macros */ -#define PVR_IS_FULL(pvr) (pvr.pvr[0] & PVR0_PVR_FULL_MASK) -#define PVR_USE_BARREL(pvr) (pvr.pvr[0] & PVR0_USE_BARREL_MASK) -#define PVR_USE_DIV(pvr) (pvr.pvr[0] & PVR0_USE_DIV_MASK) -#define PVR_USE_HW_MUL(pvr) (pvr.pvr[0] & PVR0_USE_HW_MUL_MASK) -#define PVR_USE_FPU(pvr) (pvr.pvr[0] & PVR0_USE_FPU_MASK) -#define PVR_USE_FPU2(pvr) (pvr.pvr[2] & PVR2_USE_FPU2_MASK) -#define PVR_USE_ICACHE(pvr) (pvr.pvr[0] & PVR0_USE_ICACHE_MASK) -#define PVR_USE_DCACHE(pvr) (pvr.pvr[0] & PVR0_USE_DCACHE_MASK) -#define PVR_VERSION(pvr) ((pvr.pvr[0] & PVR0_VERSION_MASK) >> 8) -#define PVR_USER1(pvr) (pvr.pvr[0] & PVR0_USER1_MASK) -#define PVR_USER2(pvr) (pvr.pvr[1] & PVR1_USER2_MASK) - -#define PVR_D_OPB(pvr) (pvr.pvr[2] & PVR2_D_OPB_MASK) -#define PVR_D_LMB(pvr) (pvr.pvr[2] & PVR2_D_LMB_MASK) -#define PVR_I_OPB(pvr) (pvr.pvr[2] & PVR2_I_OPB_MASK) -#define PVR_I_LMB(pvr) (pvr.pvr[2] & PVR2_I_LMB_MASK) -#define PVR_INTERRUPT_IS_EDGE(pvr) \ - (pvr.pvr[2] & PVR2_INTERRUPT_IS_EDGE_MASK) -#define PVR_EDGE_IS_POSITIVE(pvr) \ - (pvr.pvr[2] & PVR2_EDGE_IS_POSITIVE_MASK) -#define PVR_USE_MSR_INSTR(pvr) (pvr.pvr[2] & PVR2_USE_MSR_INSTR) -#define PVR_USE_PCMP_INSTR(pvr) (pvr.pvr[2] & PVR2_USE_PCMP_INSTR) -#define PVR_AREA_OPTIMISED(pvr) (pvr.pvr[2] & PVR2_AREA_OPTIMISED) -#define PVR_USE_MUL64(pvr) (pvr.pvr[2] & PVR2_USE_MUL64_MASK) -#define PVR_OPCODE_0x0_ILLEGAL(pvr) \ - (pvr.pvr[2] & PVR2_OPCODE_0x0_ILL_MASK) -#define PVR_UNALIGNED_EXCEPTION(pvr) \ - (pvr.pvr[2] & PVR2_UNALIGNED_EXC_MASK) -#define PVR_ILL_OPCODE_EXCEPTION(pvr) \ - (pvr.pvr[2] & PVR2_ILL_OPCODE_EXC_MASK) -#define PVR_IOPB_BUS_EXCEPTION(pvr) \ - (pvr.pvr[2] & PVR2_IOPB_BUS_EXC_MASK) -#define PVR_DOPB_BUS_EXCEPTION(pvr) \ - (pvr.pvr[2] & PVR2_DOPB_BUS_EXC_MASK) -#define PVR_DIV_ZERO_EXCEPTION(pvr) \ - (pvr.pvr[2] & PVR2_DIV_ZERO_EXC_MASK) -#define PVR_FPU_EXCEPTION(pvr) (pvr.pvr[2] & PVR2_FPU_EXC_MASK) -#define PVR_FSL_EXCEPTION(pvr) (pvr.pvr[2] & PVR2_USE_EXTEND_FSL) - -#define PVR_DEBUG_ENABLED(pvr) (pvr.pvr[3] & PVR3_DEBUG_ENABLED_MASK) -#define PVR_NUMBER_OF_PC_BRK(pvr) \ - ((pvr.pvr[3] & PVR3_NUMBER_OF_PC_BRK_MASK) >> 25) -#define PVR_NUMBER_OF_RD_ADDR_BRK(pvr) \ - ((pvr.pvr[3] & PVR3_NUMBER_OF_RD_ADDR_BRK_MASK) >> 19) -#define PVR_NUMBER_OF_WR_ADDR_BRK(pvr) \ - ((pvr.pvr[3] & PVR3_NUMBER_OF_WR_ADDR_BRK_MASK) >> 13) -#define PVR_FSL_LINKS(pvr) ((pvr.pvr[3] & PVR3_FSL_LINKS_MASK) >> 7) - -#define PVR_ICACHE_ADDR_TAG_BITS(pvr) \ - ((pvr.pvr[4] & PVR4_ICACHE_ADDR_TAG_BITS_MASK) >> 26) -#define PVR_ICACHE_USE_FSL(pvr) (pvr.pvr[4] & PVR4_ICACHE_USE_FSL_MASK) -#define PVR_ICACHE_ALLOW_WR(pvr) (pvr.pvr[4] & PVR4_ICACHE_ALLOW_WR_MASK) -#define PVR_ICACHE_LINE_LEN(pvr) \ - (1 << ((pvr.pvr[4] & PVR4_ICACHE_LINE_LEN_MASK) >> 21)) -#define PVR_ICACHE_BYTE_SIZE(pvr) \ - (1 << ((pvr.pvr[4] & PVR4_ICACHE_BYTE_SIZE_MASK) >> 16)) - -#define PVR_DCACHE_ADDR_TAG_BITS(pvr) \ - ((pvr.pvr[5] & PVR5_DCACHE_ADDR_TAG_BITS_MASK) >> 26) -#define PVR_DCACHE_USE_FSL(pvr) (pvr.pvr[5] & PVR5_DCACHE_USE_FSL_MASK) -#define PVR_DCACHE_ALLOW_WR(pvr) (pvr.pvr[5] & PVR5_DCACHE_ALLOW_WR_MASK) +#define PVR_IS_FULL(_pvr) (_pvr.pvr[0] & PVR0_PVR_FULL_MASK) +#define PVR_USE_BARREL(_pvr) (_pvr.pvr[0] & PVR0_USE_BARREL_MASK) +#define PVR_USE_DIV(_pvr) (_pvr.pvr[0] & PVR0_USE_DIV_MASK) +#define PVR_USE_HW_MUL(_pvr) (_pvr.pvr[0] & PVR0_USE_HW_MUL_MASK) +#define PVR_USE_FPU(_pvr) (_pvr.pvr[0] & PVR0_USE_FPU_MASK) +#define PVR_USE_FPU2(_pvr) (_pvr.pvr[2] & PVR2_USE_FPU2_MASK) +#define PVR_USE_ICACHE(_pvr) (_pvr.pvr[0] & PVR0_USE_ICACHE_MASK) +#define PVR_USE_DCACHE(_pvr) (_pvr.pvr[0] & PVR0_USE_DCACHE_MASK) +#define PVR_VERSION(_pvr) ((_pvr.pvr[0] & PVR0_VERSION_MASK) >> 8) +#define PVR_USER1(_pvr) (_pvr.pvr[0] & PVR0_USER1_MASK) +#define PVR_USER2(_pvr) (_pvr.pvr[1] & PVR1_USER2_MASK) + +#define PVR_D_OPB(_pvr) (_pvr.pvr[2] & PVR2_D_OPB_MASK) +#define PVR_D_LMB(_pvr) (_pvr.pvr[2] & PVR2_D_LMB_MASK) +#define PVR_I_OPB(_pvr) (_pvr.pvr[2] & PVR2_I_OPB_MASK) +#define PVR_I_LMB(_pvr) (_pvr.pvr[2] & PVR2_I_LMB_MASK) +#define PVR_INTERRUPT_IS_EDGE(_pvr) \ + (_pvr.pvr[2] & PVR2_INTERRUPT_IS_EDGE_MASK) +#define PVR_EDGE_IS_POSITIVE(_pvr) \ + (_pvr.pvr[2] & PVR2_EDGE_IS_POSITIVE_MASK) +#define PVR_USE_MSR_INSTR(_pvr) (_pvr.pvr[2] & PVR2_USE_MSR_INSTR) +#define PVR_USE_PCMP_INSTR(_pvr) (_pvr.pvr[2] & PVR2_USE_PCMP_INSTR) +#define PVR_AREA_OPTIMISED(_pvr) (_pvr.pvr[2] & PVR2_AREA_OPTIMISED) +#define PVR_USE_MUL64(_pvr) (_pvr.pvr[2] & PVR2_USE_MUL64_MASK) +#define PVR_OPCODE_0x0_ILLEGAL(_pvr) \ + (_pvr.pvr[2] & PVR2_OPCODE_0x0_ILL_MASK) +#define PVR_UNALIGNED_EXCEPTION(_pvr) \ + (_pvr.pvr[2] & PVR2_UNALIGNED_EXC_MASK) +#define PVR_ILL_OPCODE_EXCEPTION(_pvr) \ + (_pvr.pvr[2] & PVR2_ILL_OPCODE_EXC_MASK) +#define PVR_IOPB_BUS_EXCEPTION(_pvr) \ + (_pvr.pvr[2] & PVR2_IOPB_BUS_EXC_MASK) +#define PVR_DOPB_BUS_EXCEPTION(_pvr) \ + (_pvr.pvr[2] & PVR2_DOPB_BUS_EXC_MASK) +#define PVR_DIV_ZERO_EXCEPTION(_pvr) \ + (_pvr.pvr[2] & PVR2_DIV_ZERO_EXC_MASK) +#define PVR_FPU_EXCEPTION(_pvr) (_pvr.pvr[2] & PVR2_FPU_EXC_MASK) +#define PVR_FSL_EXCEPTION(_pvr) (_pvr.pvr[2] & PVR2_USE_EXTEND_FSL) + +#define PVR_DEBUG_ENABLED(_pvr) (_pvr.pvr[3] & PVR3_DEBUG_ENABLED_MASK) +#define PVR_NUMBER_OF_PC_BRK(_pvr) \ + ((_pvr.pvr[3] & PVR3_NUMBER_OF_PC_BRK_MASK) >> 25) +#define PVR_NUMBER_OF_RD_ADDR_BRK(_pvr) \ + ((_pvr.pvr[3] & PVR3_NUMBER_OF_RD_ADDR_BRK_MASK) >> 19) +#define PVR_NUMBER_OF_WR_ADDR_BRK(_pvr) \ + ((_pvr.pvr[3] & PVR3_NUMBER_OF_WR_ADDR_BRK_MASK) >> 13) +#define PVR_FSL_LINKS(_pvr) ((_pvr.pvr[3] & PVR3_FSL_LINKS_MASK) >> 7) + +#define PVR_ICACHE_ADDR_TAG_BITS(_pvr) \ + ((_pvr.pvr[4] & PVR4_ICACHE_ADDR_TAG_BITS_MASK) >> 26) +#define PVR_ICACHE_USE_FSL(_pvr) \ + (_pvr.pvr[4] & PVR4_ICACHE_USE_FSL_MASK) +#define PVR_ICACHE_ALLOW_WR(_pvr) \ + (_pvr.pvr[4] & PVR4_ICACHE_ALLOW_WR_MASK) +#define PVR_ICACHE_LINE_LEN(_pvr) \ + (1 << ((_pvr.pvr[4] & PVR4_ICACHE_LINE_LEN_MASK) >> 21)) +#define PVR_ICACHE_BYTE_SIZE(_pvr) \ + (1 << ((_pvr.pvr[4] & PVR4_ICACHE_BYTE_SIZE_MASK) >> 16)) + +#define PVR_DCACHE_ADDR_TAG_BITS(_pvr) \ + ((_pvr.pvr[5] & PVR5_DCACHE_ADDR_TAG_BITS_MASK) >> 26) +#define PVR_DCACHE_USE_FSL(_pvr) (_pvr.pvr[5] & PVR5_DCACHE_USE_FSL_MASK) +#define PVR_DCACHE_ALLOW_WR(_pvr) \ + (_pvr.pvr[5] & PVR5_DCACHE_ALLOW_WR_MASK) /* FIXME two shifts on one line needs any comment */ -#define PVR_DCACHE_LINE_LEN(pvr) \ - (1 << ((pvr.pvr[5] & PVR5_DCACHE_LINE_LEN_MASK) >> 21)) -#define PVR_DCACHE_BYTE_SIZE(pvr) \ - (1 << ((pvr.pvr[5] & PVR5_DCACHE_BYTE_SIZE_MASK) >> 16)) +#define PVR_DCACHE_LINE_LEN(_pvr) \ + (1 << ((_pvr.pvr[5] & PVR5_DCACHE_LINE_LEN_MASK) >> 21)) +#define PVR_DCACHE_BYTE_SIZE(_pvr) \ + (1 << ((_pvr.pvr[5] & PVR5_DCACHE_BYTE_SIZE_MASK) >> 16)) -#define PVR_DCACHE_USE_WRITEBACK(pvr) \ - ((pvr.pvr[5] & PVR5_DCACHE_USE_WRITEBACK) >> 14) +#define PVR_DCACHE_USE_WRITEBACK(_pvr) \ + ((_pvr.pvr[5] & PVR5_DCACHE_USE_WRITEBACK) >> 14) -#define PVR_ICACHE_BASEADDR(pvr) (pvr.pvr[6] & PVR6_ICACHE_BASEADDR_MASK) -#define PVR_ICACHE_HIGHADDR(pvr) (pvr.pvr[7] & PVR7_ICACHE_HIGHADDR_MASK) +#define PVR_ICACHE_BASEADDR(_pvr) \ + (_pvr.pvr[6] & PVR6_ICACHE_BASEADDR_MASK) +#define PVR_ICACHE_HIGHADDR(_pvr) \ + (_pvr.pvr[7] & PVR7_ICACHE_HIGHADDR_MASK) +#define PVR_DCACHE_BASEADDR(_pvr) \ + (_pvr.pvr[8] & PVR8_DCACHE_BASEADDR_MASK) +#define PVR_DCACHE_HIGHADDR(_pvr) \ + (_pvr.pvr[9] & PVR9_DCACHE_HIGHADDR_MASK) -#define PVR_DCACHE_BASEADDR(pvr) (pvr.pvr[8] & PVR8_DCACHE_BASEADDR_MASK) -#define PVR_DCACHE_HIGHADDR(pvr) (pvr.pvr[9] & PVR9_DCACHE_HIGHADDR_MASK) +#define PVR_TARGET_FAMILY(_pvr) \ + ((_pvr.pvr[10] & PVR10_TARGET_FAMILY_MASK) >> 24) -#define PVR_TARGET_FAMILY(pvr) ((pvr.pvr[10] & PVR10_TARGET_FAMILY_MASK) >> 24) - -#define PVR_MSR_RESET_VALUE(pvr) \ - (pvr.pvr[11] & PVR11_MSR_RESET_VALUE_MASK) +#define PVR_MSR_RESET_VALUE(_pvr) \ + (_pvr.pvr[11] & PVR11_MSR_RESET_VALUE_MASK) /* mmu */ -#define PVR_USE_MMU(pvr) ((pvr.pvr[11] & PVR11_USE_MMU) >> 30) -#define PVR_MMU_ITLB_SIZE(pvr) (pvr.pvr[11] & PVR11_MMU_ITLB_SIZE) -#define PVR_MMU_DTLB_SIZE(pvr) (pvr.pvr[11] & PVR11_MMU_DTLB_SIZE) -#define PVR_MMU_TLB_ACCESS(pvr) (pvr.pvr[11] & PVR11_MMU_TLB_ACCESS) -#define PVR_MMU_ZONES(pvr) (pvr.pvr[11] & PVR11_MMU_ZONES) +#define PVR_USE_MMU(_pvr) ((_pvr.pvr[11] & PVR11_USE_MMU) >> 30) +#define PVR_MMU_ITLB_SIZE(_pvr) (_pvr.pvr[11] & PVR11_MMU_ITLB_SIZE) +#define PVR_MMU_DTLB_SIZE(_pvr) (_pvr.pvr[11] & PVR11_MMU_DTLB_SIZE) +#define PVR_MMU_TLB_ACCESS(_pvr) (_pvr.pvr[11] & PVR11_MMU_TLB_ACCESS) +#define PVR_MMU_ZONES(_pvr) (_pvr.pvr[11] & PVR11_MMU_ZONES) /* endian */ -#define PVR_ENDIAN(pvr) (pvr.pvr[0] & PVR0_ENDI) +#define PVR_ENDIAN(_pvr) (_pvr.pvr[0] & PVR0_ENDI) int cpu_has_pvr(void); void get_pvr(struct pvr_s *pvr); diff --git a/arch/microblaze/kernel/cpu/cpuinfo.c b/arch/microblaze/kernel/cpu/cpuinfo.c index 87c79fa..2c309fc 100644 --- a/arch/microblaze/kernel/cpu/cpuinfo.c +++ b/arch/microblaze/kernel/cpu/cpuinfo.c @@ -32,6 +32,7 @@ const struct cpu_ver_key cpu_ver_lookup[] = { {"7.30.a", 0x10}, {"7.30.b", 0x11}, {"8.00.a", 0x12}, + {"8.00.b", 0x13}, {NULL, 0}, }; diff --git a/arch/microblaze/kernel/entry.S b/arch/microblaze/kernel/entry.S index 819238b..41c30cd 100644 --- a/arch/microblaze/kernel/entry.S +++ b/arch/microblaze/kernel/entry.S @@ -287,25 +287,44 @@ * are masked. This is nice, means we don't have to CLI before state save */ C_ENTRY(_user_exception): - addi r14, r14, 4 /* return address is 4 byte after call */ swi r1, r0, TOPHYS(PER_CPU(ENTRY_SP)) /* save stack */ + addi r14, r14, 4 /* return address is 4 byte after call */ + + mfs r1, rmsr + nop + andi r1, r1, MSR_UMS + bnei r1, 1f + +/* Kernel-mode state save - kernel execve */ + lwi r1, r0, TOPHYS(PER_CPU(ENTRY_SP)); /* Reload kernel stack-ptr*/ + tophys(r1,r1); + + addik r1, r1, -STATE_SAVE_SIZE; /* Make room on the stack. */ + SAVE_REGS + swi r1, r1, PTO + PT_MODE; /* pt_regs -> kernel mode */ + brid 2f; + nop; /* Fill delay slot */ + +/* User-mode state save. */ +1: lwi r1, r0, TOPHYS(PER_CPU(CURRENT_SAVE)); /* get saved current */ tophys(r1,r1); lwi r1, r1, TS_THREAD_INFO; /* get stack from task_struct */ - /* MS these three instructions can be added to one */ - /* addik r1, r1, THREAD_SIZE; */ - /* tophys(r1,r1); */ - /* addik r1, r1, -STATE_SAVE_SIZE; */ - addik r1, r1, THREAD_SIZE + CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - STATE_SAVE_SIZE; +/* calculate kernel stack pointer from task struct 8k */ + addik r1, r1, THREAD_SIZE; + tophys(r1,r1); + + addik r1, r1, -STATE_SAVE_SIZE; /* Make room on the stack. */ SAVE_REGS swi r0, r1, PTO + PT_R3 swi r0, r1, PTO + PT_R4 + swi r0, r1, PTO + PT_MODE; /* Was in user-mode. */ lwi r11, r0, TOPHYS(PER_CPU(ENTRY_SP)); swi r11, r1, PTO+PT_R1; /* Store user SP. */ clear_ums; - lwi CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE)); +2: lwi CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE)); /* Save away the syscall number. */ swi r12, r1, PTO+PT_R0; tovirt(r1,r1) @@ -375,6 +394,9 @@ C_ENTRY(ret_from_trap): swi r3, r1, PTO + PT_R3 swi r4, r1, PTO + PT_R4 + lwi r11, r1, PTO + PT_MODE; +/* See if returning to kernel mode, if so, skip resched &c. */ + bnei r11, 2f; /* We're returning to user mode, so check for various conditions that * trigger rescheduling. */ /* FIXME: Restructure all these flag checks. */ @@ -417,6 +439,16 @@ C_ENTRY(ret_from_trap): RESTORE_REGS; addik r1, r1, STATE_SAVE_SIZE /* Clean up stack space. */ lwi r1, r1, PT_R1 - PT_SIZE;/* Restore user stack pointer. */ + bri 6f; + +/* Return to kernel state. */ +2: set_bip; /* Ints masked for state restore */ + VM_OFF; + tophys(r1,r1); + RESTORE_REGS; + addik r1, r1, STATE_SAVE_SIZE /* Clean up stack space. */ + tovirt(r1,r1); +6: TRAP_return: /* Make global symbol for debugging */ rtbd r14, 0; /* Instructions to return from an IRQ */ nop; diff --git a/arch/microblaze/kernel/exceptions.c b/arch/microblaze/kernel/exceptions.c index 478f294..a7fa6ae7 100644 --- a/arch/microblaze/kernel/exceptions.c +++ b/arch/microblaze/kernel/exceptions.c @@ -25,6 +25,7 @@ #include <linux/errno.h> #include <linux/ptrace.h> #include <asm/current.h> +#include <asm/cacheflush.h> #define MICROBLAZE_ILL_OPCODE_EXCEPTION 0x02 #define MICROBLAZE_IBUS_EXCEPTION 0x03 @@ -52,6 +53,8 @@ void die(const char *str, struct pt_regs *fp, long err) void sw_exception(struct pt_regs *regs) { _exception(SIGTRAP, regs, TRAP_BRKPT, regs->r16); + flush_dcache_range(regs->r16, regs->r16 + 0x4); + flush_icache_range(regs->r16, regs->r16 + 0x4); } void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) diff --git a/arch/microblaze/kernel/hw_exception_handler.S b/arch/microblaze/kernel/hw_exception_handler.S index 7811954..25f6e07 100644 --- a/arch/microblaze/kernel/hw_exception_handler.S +++ b/arch/microblaze/kernel/hw_exception_handler.S @@ -945,11 +945,20 @@ store3: sbi r3, r4, 2; store4: sbi r3, r4, 3; /* Delay slot */ ex_shw_vm: /* Store the lower half-word, byte-by-byte into destination address */ +#ifdef __MICROBLAZEEL__ + lbui r3, r5, 0; +store5: sbi r3, r4, 0; + lbui r3, r5, 1; + brid ret_from_exc; +store6: sbi r3, r4, 1; /* Delay slot */ +#else lbui r3, r5, 2; store5: sbi r3, r4, 0; lbui r3, r5, 3; brid ret_from_exc; store6: sbi r3, r4, 1; /* Delay slot */ +#endif + ex_sw_end_vm: /* Exception handling of store word, ends. */ /* We have to prevent cases that get/put_user macros get unaligned pointer diff --git a/arch/microblaze/kernel/prom.c b/arch/microblaze/kernel/prom.c index a105301..c881393 100644 --- a/arch/microblaze/kernel/prom.c +++ b/arch/microblaze/kernel/prom.c @@ -61,14 +61,12 @@ static int __init early_init_dt_scan_serial(unsigned long node, char *p; int *addr; - pr_debug("search \"chosen\", depth: %d, uname: %s\n", depth, uname); + pr_debug("search \"serial\", depth: %d, uname: %s\n", depth, uname); /* find all serial nodes */ if (strncmp(uname, "serial", 6) != 0) return 0; - early_init_dt_check_for_initrd(node); - /* find compatible node with uartlite */ p = of_get_flat_dt_prop(node, "compatible", &l); if ((strncmp(p, "xlnx,xps-uartlite", 17) != 0) && diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S index 96a88c3..3451bde 100644 --- a/arch/microblaze/kernel/vmlinux.lds.S +++ b/arch/microblaze/kernel/vmlinux.lds.S @@ -123,20 +123,10 @@ SECTIONS { __init_end_before_initramfs = .; - .init.ramfs ALIGN(PAGE_SIZE) : AT(ADDR(.init.ramfs) - LOAD_OFFSET) { - __initramfs_start = .; - *(.init.ramfs) - __initramfs_end = .; - . = ALIGN(4); - LONG(0); -/* - * FIXME this can break initramfs for MMU. - * Pad init.ramfs up to page boundary, - * so that __init_end == __bss_start. This will make image.elf - * consistent with the image.bin - */ - /* . = ALIGN(PAGE_SIZE); */ + .init.ramfs : AT(ADDR(.init.ramfs) - LOAD_OFFSET) { + INIT_RAM_FS } + __init_end = .; .bss ALIGN (PAGE_SIZE) : AT(ADDR(.bss) - LOAD_OFFSET) { diff --git a/arch/microblaze/lib/memmove.c b/arch/microblaze/lib/memmove.c index 123e361..810fd68 100644 --- a/arch/microblaze/lib/memmove.c +++ b/arch/microblaze/lib/memmove.c @@ -182,7 +182,7 @@ void *memmove(void *v_dst, const void *v_src, __kernel_size_t c) for (; c >= 4; c -= 4) { value = *--i_src; *--i_dst = buf_hold | ((value & 0xFF000000)>> 24); - buf_hold = (value & 0xFFFFFF) << 8;; + buf_hold = (value & 0xFFFFFF) << 8; } #endif /* Realign the source */ diff --git a/arch/microblaze/lib/muldi3.S b/arch/microblaze/lib/muldi3.S deleted file mode 100644 index ceeaa8c..0000000 --- a/arch/microblaze/lib/muldi3.S +++ /dev/null @@ -1,121 +0,0 @@ -#include <linux/linkage.h> - -/* - * Multiply operation for 64 bit integers, for devices with hard multiply - * Input : Operand1[H] in Reg r5 - * Operand1[L] in Reg r6 - * Operand2[H] in Reg r7 - * Operand2[L] in Reg r8 - * Output: Result[H] in Reg r3 - * Result[L] in Reg r4 - * - * Explaination: - * - * Both the input numbers are divided into 16 bit number as follows - * op1 = A B C D - * op2 = E F G H - * result = D * H - * + (C * H + D * G) << 16 - * + (B * H + C * G + D * F) << 32 - * + (A * H + B * G + C * F + D * E) << 48 - * - * Only 64 bits of the output are considered - */ - - .text - .globl __muldi3 - .type __muldi3, @function - .ent __muldi3 - -__muldi3: - addi r1, r1, -40 - -/* Save the input operands on the caller's stack */ - swi r5, r1, 44 - swi r6, r1, 48 - swi r7, r1, 52 - swi r8, r1, 56 - -/* Store all the callee saved registers */ - sw r20, r1, r0 - swi r21, r1, 4 - swi r22, r1, 8 - swi r23, r1, 12 - swi r24, r1, 16 - swi r25, r1, 20 - swi r26, r1, 24 - swi r27, r1, 28 - -/* Load all the 16 bit values for A thru H */ - lhui r20, r1, 44 /* A */ - lhui r21, r1, 46 /* B */ - lhui r22, r1, 48 /* C */ - lhui r23, r1, 50 /* D */ - lhui r24, r1, 52 /* E */ - lhui r25, r1, 54 /* F */ - lhui r26, r1, 56 /* G */ - lhui r27, r1, 58 /* H */ - -/* D * H ==> LSB of the result on stack ==> Store1 */ - mul r9, r23, r27 - swi r9, r1, 36 /* Pos2 and Pos3 */ - -/* Hi (Store1) + C * H + D * G ==> Store2 ==> Pos1 and Pos2 */ -/* Store the carry generated in position 2 for Pos 3 */ - lhui r11, r1, 36 /* Pos2 */ - mul r9, r22, r27 /* C * H */ - mul r10, r23, r26 /* D * G */ - add r9, r9, r10 - addc r12, r0, r0 - add r9, r9, r11 - addc r12, r12, r0 /* Store the Carry */ - shi r9, r1, 36 /* Store Pos2 */ - swi r9, r1, 32 - lhui r11, r1, 32 - shi r11, r1, 34 /* Store Pos1 */ - -/* Hi (Store2) + B * H + C * G + D * F ==> Store3 ==> Pos0 and Pos1 */ - mul r9, r21, r27 /* B * H */ - mul r10, r22, r26 /* C * G */ - mul r7, r23, r25 /* D * F */ - add r9, r9, r11 - add r9, r9, r10 - add r9, r9, r7 - swi r9, r1, 32 /* Pos0 and Pos1 */ - -/* Hi (Store3) + A * H + B * G + C * F + D * E ==> Store3 ==> Pos0 */ - lhui r11, r1, 32 /* Pos0 */ - mul r9, r20, r27 /* A * H */ - mul r10, r21, r26 /* B * G */ - mul r7, r22, r25 /* C * F */ - mul r8, r23, r24 /* D * E */ - add r9, r9, r11 - add r9, r9, r10 - add r9, r9, r7 - add r9, r9, r8 - sext16 r9, r9 /* Sign extend the MSB */ - shi r9, r1, 32 - -/* Move results to r3 and r4 */ - lhui r3, r1, 32 - add r3, r3, r12 - shi r3, r1, 32 - lwi r3, r1, 32 /* Hi Part */ - lwi r4, r1, 36 /* Lo Part */ - -/* Restore Callee saved registers */ - lw r20, r1, r0 - lwi r21, r1, 4 - lwi r22, r1, 8 - lwi r23, r1, 12 - lwi r24, r1, 16 - lwi r25, r1, 20 - lwi r26, r1, 24 - lwi r27, r1, 28 - -/* Restore Frame and return */ - rtsd r15, 8 - addi r1, r1, 40 - -.size __muldi3, . - __muldi3 -.end __muldi3 diff --git a/arch/microblaze/lib/muldi3.c b/arch/microblaze/lib/muldi3.c new file mode 100644 index 0000000..d4860e1 --- /dev/null +++ b/arch/microblaze/lib/muldi3.c @@ -0,0 +1,60 @@ +#include <linux/module.h> + +#include "libgcc.h" + +#define DWtype long long +#define UWtype unsigned long +#define UHWtype unsigned short + +#define W_TYPE_SIZE 32 + +#define __ll_B ((UWtype) 1 << (W_TYPE_SIZE / 2)) +#define __ll_lowpart(t) ((UWtype) (t) & (__ll_B - 1)) +#define __ll_highpart(t) ((UWtype) (t) >> (W_TYPE_SIZE / 2)) + +/* If we still don't have umul_ppmm, define it using plain C. */ +#if !defined(umul_ppmm) +#define umul_ppmm(w1, w0, u, v) \ + do { \ + UWtype __x0, __x1, __x2, __x3; \ + UHWtype __ul, __vl, __uh, __vh; \ + \ + __ul = __ll_lowpart(u); \ + __uh = __ll_highpart(u); \ + __vl = __ll_lowpart(v); \ + __vh = __ll_highpart(v); \ + \ + __x0 = (UWtype) __ul * __vl; \ + __x1 = (UWtype) __ul * __vh; \ + __x2 = (UWtype) __uh * __vl; \ + __x3 = (UWtype) __uh * __vh; \ + \ + __x1 += __ll_highpart(__x0); /* this can't give carry */\ + __x1 += __x2; /* but this indeed can */ \ + if (__x1 < __x2) /* did we get it? */ \ + __x3 += __ll_B; /* yes, add it in the proper pos */ \ + \ + (w1) = __x3 + __ll_highpart(__x1); \ + (w0) = __ll_lowpart(__x1) * __ll_B + __ll_lowpart(__x0);\ + } while (0) +#endif + +#if !defined(__umulsidi3) +#define __umulsidi3(u, v) ({ \ + DWunion __w; \ + umul_ppmm(__w.s.high, __w.s.low, u, v); \ + __w.ll; \ + }) +#endif + +DWtype __muldi3(DWtype u, DWtype v) +{ + const DWunion uu = {.ll = u}; + const DWunion vv = {.ll = v}; + DWunion w = {.ll = __umulsidi3(uu.s.low, vv.s.low)}; + + w.s.high += ((UWtype) uu.s.low * (UWtype) vv.s.high + + (UWtype) uu.s.high * (UWtype) vv.s.low); + + return w.ll; +} diff --git a/arch/mips/alchemy/common/power.c b/arch/mips/alchemy/common/power.c index e5916a5..647e518 100644 --- a/arch/mips/alchemy/common/power.c +++ b/arch/mips/alchemy/common/power.c @@ -130,7 +130,7 @@ static void restore_core_regs(void) au_writel(sleep_usb[1], USBD_ENABLE); au_sync(); #else - /* enable accces to OTG memory */ + /* enable access to OTG memory */ au_writel(au_readl(USB_MSR_BASE + 4) | (1 << 6), USB_MSR_BASE + 4); au_sync(); diff --git a/arch/mips/alchemy/devboards/pm.c b/arch/mips/alchemy/devboards/pm.c index 4bbd313..acaf91b 100644 --- a/arch/mips/alchemy/devboards/pm.c +++ b/arch/mips/alchemy/devboards/pm.c @@ -110,7 +110,7 @@ static void db1x_pm_end(void) } -static struct platform_suspend_ops db1x_pm_ops = { +static const struct platform_suspend_ops db1x_pm_ops = { .valid = suspend_valid_only_mem, .begin = db1x_pm_begin, .enter = db1x_pm_enter, diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c index 02f505f..ea57f39 100644 --- a/arch/mips/dec/time.c +++ b/arch/mips/dec/time.c @@ -104,7 +104,7 @@ int rtc_mips_set_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/mips/include/asm/mach-powertv/ioremap.h b/arch/mips/include/asm/mach-powertv/ioremap.h index 076f2ee..c86ef09 100644 --- a/arch/mips/include/asm/mach-powertv/ioremap.h +++ b/arch/mips/include/asm/mach-powertv/ioremap.h @@ -88,7 +88,7 @@ static inline dma_addr_t _dma_to_phys_offset_raw(dma_addr_t dma) } /* These are not portable and should not be used in drivers. Drivers should - * be using ioremap() and friends to map physical addreses to virtual + * be using ioremap() and friends to map physical addresses to virtual * addresses and dma_map*() and friends to map virtual addresses into DMA * addresses and back. */ diff --git a/arch/mips/include/asm/mc146818-time.h b/arch/mips/include/asm/mc146818-time.h index 199b457..4a08dbe 100644 --- a/arch/mips/include/asm/mc146818-time.h +++ b/arch/mips/include/asm/mc146818-time.h @@ -66,7 +66,7 @@ static inline int mc146818_set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/mips/jz4740/board-qi_lb60.c b/arch/mips/jz4740/board-qi_lb60.c index 5c0a357..2c0e107 100644 --- a/arch/mips/jz4740/board-qi_lb60.c +++ b/arch/mips/jz4740/board-qi_lb60.c @@ -65,7 +65,7 @@ static struct nand_ecclayout qi_lb60_ecclayout_1gb = { /* Early prototypes of the QI LB60 had only 1GB of NAND. * In order to support these devices aswell the partition and ecc layout is - * initalized depending on the NAND size */ + * initialized depending on the NAND size */ static struct mtd_partition qi_lb60_partitions_1gb[] = { { .name = "NAND BOOT partition", @@ -464,7 +464,7 @@ static int __init qi_lb60_board_setup(void) board_gpio_setup(); if (qi_lb60_init_platform_devices()) - panic("Failed to initalize platform devices\n"); + panic("Failed to initialize platform devices\n"); return 0; } diff --git a/arch/mips/jz4740/gpio.c b/arch/mips/jz4740/gpio.c index 38f60f3..88e6aed 100644 --- a/arch/mips/jz4740/gpio.c +++ b/arch/mips/jz4740/gpio.c @@ -546,7 +546,7 @@ static int __init jz4740_gpio_init(void) for (i = 0; i < ARRAY_SIZE(jz4740_gpio_chips); ++i) jz4740_gpio_chip_init(&jz4740_gpio_chips[i], i); - printk(KERN_INFO "JZ4740 GPIO initalized\n"); + printk(KERN_INFO "JZ4740 GPIO initialized\n"); return 0; } diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c index a999458..902d5b5 100644 --- a/arch/mips/jz4740/pm.c +++ b/arch/mips/jz4740/pm.c @@ -42,7 +42,7 @@ static int jz4740_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops jz4740_pm_ops = { +static const struct platform_suspend_ops jz4740_pm_ops = { .valid = suspend_valid_only_mem, .enter = jz4740_pm_enter, }; diff --git a/arch/mips/loongson/common/pm.c b/arch/mips/loongson/common/pm.c index 6c1fd90..f55e07a 100644 --- a/arch/mips/loongson/common/pm.c +++ b/arch/mips/loongson/common/pm.c @@ -147,7 +147,7 @@ static int loongson_pm_valid_state(suspend_state_t state) } } -static struct platform_suspend_ops loongson_pm_ops = { +static const struct platform_suspend_ops loongson_pm_ops = { .valid = loongson_pm_valid_state, .enter = loongson_pm_enter, }; diff --git a/arch/mips/mti-malta/malta-memory.c b/arch/mips/mti-malta/malta-memory.c index b27419c..a96d281 100644 --- a/arch/mips/mti-malta/malta-memory.c +++ b/arch/mips/mti-malta/malta-memory.c @@ -43,7 +43,7 @@ static struct prom_pmemblock mdesc[PROM_MAX_PMEMBLOCKS]; static char *mtypes[3] = { "Dont use memory", "YAMON PROM memory", - "Free memmory", + "Free memory", }; #endif diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c index 385f035..0583c463 100644 --- a/arch/mips/pci/pcie-octeon.c +++ b/arch/mips/pci/pcie-octeon.c @@ -900,7 +900,7 @@ static int cvmx_pcie_rc_initialize(int pcie_port) mem_access_subid.s.ror = 0; /* Disable Relaxed Ordering for Writes. */ mem_access_subid.s.row = 0; - /* PCIe Adddress Bits <63:34>. */ + /* PCIe Address Bits <63:34>. */ mem_access_subid.s.ba = 0; /* diff --git a/arch/mips/powertv/memory.c b/arch/mips/powertv/memory.c index 73880ad..fb3d296 100644 --- a/arch/mips/powertv/memory.c +++ b/arch/mips/powertv/memory.c @@ -57,7 +57,7 @@ unsigned long ptv_memsize; /* - * struct low_mem_reserved - Items in low memmory that are reserved + * struct low_mem_reserved - Items in low memory that are reserved * @start: Physical address of item * @size: Size, in bytes, of this item * @is_aliased: True if this is RAM aliased from another location. If false, diff --git a/arch/mips/txx9/generic/pci.c b/arch/mips/txx9/generic/pci.c index 9a0be81..96e69a0 100644 --- a/arch/mips/txx9/generic/pci.c +++ b/arch/mips/txx9/generic/pci.c @@ -107,7 +107,7 @@ int txx9_pci_mem_high __initdata; /* * allocate pci_controller and resources. - * mem_base, io_base: physical addresss. 0 for auto assignment. + * mem_base, io_base: physical address. 0 for auto assignment. * mem_size and io_size means max size on auto assignment. * pcic must be &txx9_primary_pcic or NULL. */ diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index 41ba385..8ed41cf 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -203,6 +203,7 @@ endmenu config SMP bool "Symmetric multi-processing support" default y + select USE_GENERIC_SMP_HELPERS depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050 ---help--- This enables support for systems with more than one CPU. If you have @@ -226,11 +227,6 @@ config NR_CPUS depends on SMP default "2" -config USE_GENERIC_SMP_HELPERS - bool - depends on SMP - default y - source "kernel/Kconfig.preempt" config MN10300_CURRENT_IN_E2 diff --git a/arch/mn10300/kernel/rtc.c b/arch/mn10300/kernel/rtc.c index e9e20f9..48d7058 100644 --- a/arch/mn10300/kernel/rtc.c +++ b/arch/mn10300/kernel/rtc.c @@ -89,7 +89,7 @@ static int set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 48fb479..959f38c 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -20,6 +20,9 @@ config WORD_SIZE config ARCH_PHYS_ADDR_T_64BIT def_bool PPC64 || PHYS_64BIT +config ARCH_DMA_ADDR_T_64BIT + def_bool ARCH_PHYS_ADDR_T_64BIT + config MMU bool default y @@ -209,7 +212,7 @@ config ARCH_HIBERNATION_POSSIBLE config ARCH_SUSPEND_POSSIBLE def_bool y depends on ADB_PMU || PPC_EFIKA || PPC_LITE5200 || PPC_83xx || \ - PPC_85xx || PPC_86xx || PPC_PSERIES + PPC_85xx || PPC_86xx || PPC_PSERIES || 44x || 40x config PPC_DCR_NATIVE bool @@ -595,13 +598,11 @@ config EXTRA_TARGETS If unsure, leave blank -if !44x || BROKEN config ARCH_WANTS_FREEZER_CONTROL def_bool y depends on ADB_PMU source kernel/power/Kconfig -endif config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" @@ -682,6 +683,15 @@ config FSL_PMC Freescale MPC85xx/MPC86xx power management controller support (suspend/resume). For MPC83xx see platforms/83xx/suspend.c +config PPC4xx_CPM + bool + default y + depends on SUSPEND && (44x || 40x) + help + PPC4xx Clock Power Management (CPM) support (suspend/resume). + It also enables support for two different idle states (idle-wait + and idle-doze). + config 4xx_SOC bool diff --git a/arch/powerpc/boot/dts/canyonlands.dts b/arch/powerpc/boot/dts/canyonlands.dts index a303703..5b27a4b 100644 --- a/arch/powerpc/boot/dts/canyonlands.dts +++ b/arch/powerpc/boot/dts/canyonlands.dts @@ -105,6 +105,15 @@ dcr-reg = <0x00c 0x002>; }; + CPM0: cpm { + compatible = "ibm,cpm"; + dcr-access-method = "native"; + dcr-reg = <0x160 0x003>; + unused-units = <0x00000100>; + idle-doze = <0x02000000>; + standby = <0xfeff791d>; + }; + L2C0: l2c { compatible = "ibm,l2-cache-460ex", "ibm,l2-cache"; dcr-reg = <0x020 0x008 /* Internal SRAM DCR's */ @@ -270,28 +279,6 @@ interrupts = <0x1 0x4>; }; - UART2: serial@ef600500 { - device_type = "serial"; - compatible = "ns16550"; - reg = <0xef600500 0x00000008>; - virtual-reg = <0xef600500>; - clock-frequency = <0>; /* Filled in by U-Boot */ - current-speed = <0>; /* Filled in by U-Boot */ - interrupt-parent = <&UIC1>; - interrupts = <28 0x4>; - }; - - UART3: serial@ef600600 { - device_type = "serial"; - compatible = "ns16550"; - reg = <0xef600600 0x00000008>; - virtual-reg = <0xef600600>; - clock-frequency = <0>; /* Filled in by U-Boot */ - current-speed = <0>; /* Filled in by U-Boot */ - interrupt-parent = <&UIC1>; - interrupts = <29 0x4>; - }; - IIC0: i2c@ef600700 { compatible = "ibm,iic-460ex", "ibm,iic"; reg = <0xef600700 0x00000014>; diff --git a/arch/powerpc/boot/dts/kilauea.dts b/arch/powerpc/boot/dts/kilauea.dts index 083e68e..89edb16 100644 --- a/arch/powerpc/boot/dts/kilauea.dts +++ b/arch/powerpc/boot/dts/kilauea.dts @@ -82,6 +82,15 @@ interrupt-parent = <&UIC0>; }; + CPM0: cpm { + compatible = "ibm,cpm"; + dcr-access-method = "native"; + dcr-reg = <0x0b0 0x003>; + unused-units = <0x00000000>; + idle-doze = <0x02000000>; + standby = <0xe3e74800>; + }; + plb { compatible = "ibm,plb-405ex", "ibm,plb4"; #address-cells = <1>; diff --git a/arch/powerpc/boot/dts/mpc8308_p1m.dts b/arch/powerpc/boot/dts/mpc8308_p1m.dts index 05a76cc..697b3f6 100644 --- a/arch/powerpc/boot/dts/mpc8308_p1m.dts +++ b/arch/powerpc/boot/dts/mpc8308_p1m.dts @@ -297,6 +297,14 @@ interrupt-parent = < &ipic >; }; + dma@2c000 { + compatible = "fsl,mpc8308-dma", "fsl,mpc5121-dma"; + reg = <0x2c000 0x1800>; + interrupts = <3 0x8 + 94 0x8>; + interrupt-parent = < &ipic >; + }; + }; pci0: pcie@e0009000 { diff --git a/arch/powerpc/boot/dts/mpc8308rdb.dts b/arch/powerpc/boot/dts/mpc8308rdb.dts index a97eb2d..d3db02f 100644 --- a/arch/powerpc/boot/dts/mpc8308rdb.dts +++ b/arch/powerpc/boot/dts/mpc8308rdb.dts @@ -265,6 +265,14 @@ interrupt-parent = < &ipic >; }; + dma@2c000 { + compatible = "fsl,mpc8308-dma", "fsl,mpc5121-dma"; + reg = <0x2c000 0x1800>; + interrupts = <3 0x8 + 94 0x8>; + interrupt-parent = < &ipic >; + }; + }; pci0: pcie@e0009000 { diff --git a/arch/powerpc/configs/40x/kilauea_defconfig b/arch/powerpc/configs/40x/kilauea_defconfig index 4e19ee7..34b8c1a 100644 --- a/arch/powerpc/configs/40x/kilauea_defconfig +++ b/arch/powerpc/configs/40x/kilauea_defconfig @@ -12,6 +12,8 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_KILAUEA=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y # CONFIG_WALNUT is not set CONFIG_SPARSE_IRQ=y CONFIG_PCI=y @@ -42,6 +44,9 @@ CONFIG_MTD_PHYSMAP_OF=y CONFIG_MTD_NAND=y CONFIG_MTD_NAND_NDFC=y CONFIG_PROC_DEVICETREE=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PPC4xx_CPM=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=35000 # CONFIG_MISC_DEVICES is not set diff --git a/arch/powerpc/configs/44x/canyonlands_defconfig b/arch/powerpc/configs/44x/canyonlands_defconfig index 45c64d8..17e4dd9 100644 --- a/arch/powerpc/configs/44x/canyonlands_defconfig +++ b/arch/powerpc/configs/44x/canyonlands_defconfig @@ -42,6 +42,9 @@ CONFIG_MTD_PHYSMAP_OF=y CONFIG_MTD_NAND=y CONFIG_MTD_NAND_NDFC=y CONFIG_PROC_DEVICETREE=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PPC4xx_CPM=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=35000 # CONFIG_MISC_DEVICES is not set diff --git a/arch/powerpc/include/asm/8xx_immap.h b/arch/powerpc/include/asm/8xx_immap.h index 4b0e152..6b6dc20 100644 --- a/arch/powerpc/include/asm/8xx_immap.h +++ b/arch/powerpc/include/asm/8xx_immap.h @@ -93,7 +93,7 @@ typedef struct mem_ctlr { } memctl8xx_t; /*----------------------------------------------------------------------- - * BR - Memory Controler: Base Register 16-9 + * BR - Memory Controller: Base Register 16-9 */ #define BR_BA_MSK 0xffff8000 /* Base Address Mask */ #define BR_AT_MSK 0x00007000 /* Address Type Mask */ @@ -110,7 +110,7 @@ typedef struct mem_ctlr { #define BR_V 0x00000001 /* Bank Valid */ /*----------------------------------------------------------------------- - * OR - Memory Controler: Option Register 16-11 + * OR - Memory Controller: Option Register 16-11 */ #define OR_AM_MSK 0xffff8000 /* Address Mask Mask */ #define OR_ATM_MSK 0x00007000 /* Address Type Mask Mask */ diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index 30964ae..8a7e9314 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -267,7 +267,16 @@ static __inline__ int fls64(__u64 x) #include <asm-generic/bitops/fls64.h> #endif /* __powerpc64__ */ +#ifdef CONFIG_PPC64 +unsigned int __arch_hweight8(unsigned int w); +unsigned int __arch_hweight16(unsigned int w); +unsigned int __arch_hweight32(unsigned int w); +unsigned long __arch_hweight64(__u64 w); +#include <asm-generic/bitops/const_hweight.h> +#else #include <asm-generic/bitops/hweight.h> +#endif + #include <asm-generic/bitops/find.h> /* Little-endian versions */ diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index f3a1fdd..f0a211d 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -199,6 +199,8 @@ extern const char *powerpc_base_platform; #define CPU_FTR_UNALIGNED_LD_STD LONG_ASM_CONST(0x0080000000000000) #define CPU_FTR_ASYM_SMT LONG_ASM_CONST(0x0100000000000000) #define CPU_FTR_STCX_CHECKS_ADDRESS LONG_ASM_CONST(0x0200000000000000) +#define CPU_FTR_POPCNTB LONG_ASM_CONST(0x0400000000000000) +#define CPU_FTR_POPCNTD LONG_ASM_CONST(0x0800000000000000) #ifndef __ASSEMBLY__ @@ -403,21 +405,22 @@ extern const char *powerpc_base_platform; CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ - CPU_FTR_PURR | CPU_FTR_STCX_CHECKS_ADDRESS) + CPU_FTR_PURR | CPU_FTR_STCX_CHECKS_ADDRESS | \ + CPU_FTR_POPCNTB) #define CPU_FTRS_POWER6 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD | \ - CPU_FTR_STCX_CHECKS_ADDRESS) + CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB) #define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \ - CPU_FTR_STCX_CHECKS_ADDRESS) + CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD) #define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h index a8e1844..f71bb4c 100644 --- a/arch/powerpc/include/asm/cputhreads.h +++ b/arch/powerpc/include/asm/cputhreads.h @@ -61,22 +61,25 @@ static inline cpumask_t cpu_online_cores_map(void) return cpu_thread_mask_to_cores(cpu_online_map); } -static inline int cpu_thread_to_core(int cpu) -{ - return cpu >> threads_shift; -} +#ifdef CONFIG_SMP +int cpu_core_index_of_thread(int cpu); +int cpu_first_thread_of_core(int core); +#else +static inline int cpu_core_index_of_thread(int cpu) { return cpu; } +static inline int cpu_first_thread_of_core(int core) { return core; } +#endif static inline int cpu_thread_in_core(int cpu) { return cpu & (threads_per_core - 1); } -static inline int cpu_first_thread_in_core(int cpu) +static inline int cpu_first_thread_sibling(int cpu) { return cpu & ~(threads_per_core - 1); } -static inline int cpu_last_thread_in_core(int cpu) +static inline int cpu_last_thread_sibling(int cpu) { return cpu | (threads_per_core - 1); } diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index a3954e4..16d25c0 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -9,6 +9,12 @@ struct dma_map_ops; struct device_node; +/* + * Arch extensions to struct device. + * + * When adding fields, consider macio_add_one_device in + * drivers/macintosh/macio_asic.c + */ struct dev_archdata { /* DMA operations on that device */ struct dma_map_ops *dma_ops; diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 20778a4..4ef662e 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -46,6 +46,7 @@ #define FW_FEATURE_PS3_LV1 ASM_CONST(0x0000000000800000) #define FW_FEATURE_BEAT ASM_CONST(0x0000000001000000) #define FW_FEATURE_CMO ASM_CONST(0x0000000002000000) +#define FW_FEATURE_VPHN ASM_CONST(0x0000000004000000) #ifndef __ASSEMBLY__ @@ -59,7 +60,7 @@ enum { FW_FEATURE_VIO | FW_FEATURE_RDMA | FW_FEATURE_LLAN | FW_FEATURE_BULK_REMOVE | FW_FEATURE_XDABR | FW_FEATURE_MULTITCE | FW_FEATURE_SPLPAR | FW_FEATURE_LPAR | - FW_FEATURE_CMO, + FW_FEATURE_CMO | FW_FEATURE_VPHN, FW_FEATURE_PSERIES_ALWAYS = 0, FW_FEATURE_ISERIES_POSSIBLE = FW_FEATURE_ISERIES | FW_FEATURE_LPAR, FW_FEATURE_ISERIES_ALWAYS = FW_FEATURE_ISERIES | FW_FEATURE_LPAR, diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index de03ca5..ec089ac 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -232,7 +232,9 @@ #define H_GET_EM_PARMS 0x2B8 #define H_SET_MPP 0x2D0 #define H_GET_MPP 0x2D4 -#define MAX_HCALL_OPCODE H_GET_MPP +#define H_HOME_NODE_ASSOCIATIVITY 0x2EC +#define H_BEST_ENERGY 0x2F4 +#define MAX_HCALL_OPCODE H_BEST_ENERGY #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/lppaca.h b/arch/powerpc/include/asm/lppaca.h index 7f5e0fe..380d48b 100644 --- a/arch/powerpc/include/asm/lppaca.h +++ b/arch/powerpc/include/asm/lppaca.h @@ -62,7 +62,10 @@ struct lppaca { volatile u32 dyn_pir; // Dynamic ProcIdReg value x20-x23 u32 dsei_data; // DSEI data x24-x27 u64 sprg3; // SPRG3 value x28-x2F - u8 reserved3[80]; // Reserved x30-x7F + u8 reserved3[40]; // Reserved x30-x57 + volatile u8 vphn_assoc_counts[8]; // Virtual processor home node + // associativity change counters x58-x5F + u8 reserved4[32]; // Reserved x60-x7F //============================================================================= // CACHE_LINE_2 0x0080 - 0x00FF Contains local read-write data diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index d045b01..8433d36 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -27,9 +27,7 @@ struct iommu_table; struct rtc_time; struct file; struct pci_controller; -#ifdef CONFIG_KEXEC struct kimage; -#endif #ifdef CONFIG_SMP struct smp_ops_t { @@ -72,7 +70,7 @@ struct machdep_calls { int psize, int ssize); void (*flush_hash_range)(unsigned long number, int local); - /* special for kexec, to be called in real mode, linar mapping is + /* special for kexec, to be called in real mode, linear mapping is * destroyed as well */ void (*hpte_clear_all)(void); @@ -324,8 +322,6 @@ extern sys_ctrler_t sys_ctrler; #endif /* CONFIG_PPC_PMAC */ -extern void setup_pci_ptrs(void); - #ifdef CONFIG_SMP /* Poor default implementations */ extern void __devinit smp_generic_give_timebase(void); diff --git a/arch/powerpc/include/asm/mmzone.h b/arch/powerpc/include/asm/mmzone.h index aac87cb..fd3fd58 100644 --- a/arch/powerpc/include/asm/mmzone.h +++ b/arch/powerpc/include/asm/mmzone.h @@ -33,6 +33,9 @@ extern int numa_cpu_lookup_table[]; extern cpumask_var_t node_to_cpumask_map[]; #ifdef CONFIG_MEMORY_HOTPLUG extern unsigned long max_pfn; +u64 memory_hotplug_max(void); +#else +#define memory_hotplug_max() memblock_end_of_DRAM() #endif /* @@ -42,6 +45,8 @@ extern unsigned long max_pfn; #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) #define node_end_pfn(nid) (NODE_DATA(nid)->node_end_pfn) +#else +#define memory_hotplug_max() memblock_end_of_DRAM() #endif /* CONFIG_NEED_MULTIPLE_NODES */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/nvram.h b/arch/powerpc/include/asm/nvram.h index 850b72f..92efe67 100644 --- a/arch/powerpc/include/asm/nvram.h +++ b/arch/powerpc/include/asm/nvram.h @@ -10,31 +10,7 @@ #ifndef _ASM_POWERPC_NVRAM_H #define _ASM_POWERPC_NVRAM_H -#include <linux/errno.h> - -#define NVRW_CNT 0x20 -#define NVRAM_HEADER_LEN 16 /* sizeof(struct nvram_header) */ -#define NVRAM_BLOCK_LEN 16 -#define NVRAM_MAX_REQ (2080/NVRAM_BLOCK_LEN) -#define NVRAM_MIN_REQ (1056/NVRAM_BLOCK_LEN) - -#define NVRAM_AS0 0x74 -#define NVRAM_AS1 0x75 -#define NVRAM_DATA 0x77 - - -/* RTC Offsets */ - -#define MOTO_RTC_SECONDS 0x1FF9 -#define MOTO_RTC_MINUTES 0x1FFA -#define MOTO_RTC_HOURS 0x1FFB -#define MOTO_RTC_DAY_OF_WEEK 0x1FFC -#define MOTO_RTC_DAY_OF_MONTH 0x1FFD -#define MOTO_RTC_MONTH 0x1FFE -#define MOTO_RTC_YEAR 0x1FFF -#define MOTO_RTC_CONTROLA 0x1FF8 -#define MOTO_RTC_CONTROLB 0x1FF9 - +/* Signatures for nvram partitions */ #define NVRAM_SIG_SP 0x02 /* support processor */ #define NVRAM_SIG_OF 0x50 /* open firmware config */ #define NVRAM_SIG_FW 0x51 /* general firmware */ @@ -49,32 +25,19 @@ #define NVRAM_SIG_OS 0xa0 /* OS defined */ #define NVRAM_SIG_PANIC 0xa1 /* Apple OSX "panic" */ -/* If change this size, then change the size of NVNAME_LEN */ -struct nvram_header { - unsigned char signature; - unsigned char checksum; - unsigned short length; - char name[12]; -}; - #ifdef __KERNEL__ +#include <linux/errno.h> #include <linux/list.h> -struct nvram_partition { - struct list_head partition; - struct nvram_header header; - unsigned int index; -}; - - +#ifdef CONFIG_PPC_PSERIES extern int nvram_write_error_log(char * buff, int length, unsigned int err_type, unsigned int err_seq); extern int nvram_read_error_log(char * buff, int length, unsigned int * err_type, unsigned int *err_seq); extern int nvram_clear_error_log(void); - extern int pSeries_nvram_init(void); +#endif /* CONFIG_PPC_PSERIES */ #ifdef CONFIG_MMIO_NVRAM extern int mmio_nvram_init(void); @@ -85,6 +48,13 @@ static inline int mmio_nvram_init(void) } #endif +extern int __init nvram_scan_partitions(void); +extern loff_t nvram_create_partition(const char *name, int sig, + int req_size, int min_size); +extern int nvram_remove_partition(const char *name, int sig); +extern int nvram_get_partition_size(loff_t data_index); +extern loff_t nvram_find_partition(const char *name, int sig, int *out_size); + #endif /* __KERNEL__ */ /* PowerMac specific nvram stuffs */ diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 43adc8b..1255569 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -36,6 +36,8 @@ #define PPC_INST_NOP 0x60000000 #define PPC_INST_POPCNTB 0x7c0000f4 #define PPC_INST_POPCNTB_MASK 0xfc0007fe +#define PPC_INST_POPCNTD 0x7c0003f4 +#define PPC_INST_POPCNTW 0x7c0002f4 #define PPC_INST_RFCI 0x4c000066 #define PPC_INST_RFDI 0x4c00004e #define PPC_INST_RFMCI 0x4c00004c @@ -88,6 +90,12 @@ __PPC_RB(b) | __PPC_EH(eh)) #define PPC_MSGSND(b) stringify_in_c(.long PPC_INST_MSGSND | \ __PPC_RB(b)) +#define PPC_POPCNTB(a, s) stringify_in_c(.long PPC_INST_POPCNTB | \ + __PPC_RA(a) | __PPC_RS(s)) +#define PPC_POPCNTD(a, s) stringify_in_c(.long PPC_INST_POPCNTD | \ + __PPC_RA(a) | __PPC_RS(s)) +#define PPC_POPCNTW(a, s) stringify_in_c(.long PPC_INST_POPCNTW | \ + __PPC_RA(a) | __PPC_RS(s)) #define PPC_RFCI stringify_in_c(.long PPC_INST_RFCI) #define PPC_RFDI stringify_in_c(.long PPC_INST_RFDI) #define PPC_RFMCI stringify_in_c(.long PPC_INST_RFMCI) diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index 4c14187..de1967a 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -122,7 +122,6 @@ extern struct task_struct *last_task_used_spe; TASK_UNMAPPED_BASE_USER32 : TASK_UNMAPPED_BASE_USER64 ) #endif -#ifdef __KERNEL__ #ifdef __powerpc64__ #define STACK_TOP_USER64 TASK_SIZE_USER64 @@ -139,7 +138,6 @@ extern struct task_struct *last_task_used_spe; #define STACK_TOP_MAX STACK_TOP #endif /* __powerpc64__ */ -#endif /* __KERNEL__ */ typedef struct { unsigned long seg; diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h index afe4aaa..7ef0d90 100644 --- a/arch/powerpc/include/asm/topology.h +++ b/arch/powerpc/include/asm/topology.h @@ -106,9 +106,22 @@ static inline void sysfs_remove_device_from_node(struct sys_device *dev, int nid) { } - #endif /* CONFIG_NUMA */ +#if defined(CONFIG_NUMA) && defined(CONFIG_PPC_SPLPAR) +extern int start_topology_update(void); +extern int stop_topology_update(void); +#else +static inline int start_topology_update(void) +{ + return 0; +} +static inline int stop_topology_update(void) +{ + return 0; +} +#endif /* CONFIG_NUMA && CONFIG_PPC_SPLPAR */ + #include <asm-generic/topology.h> #ifdef CONFIG_SMP diff --git a/arch/powerpc/include/asm/vdso_datapage.h b/arch/powerpc/include/asm/vdso_datapage.h index 08679c5..25e3922 100644 --- a/arch/powerpc/include/asm/vdso_datapage.h +++ b/arch/powerpc/include/asm/vdso_datapage.h @@ -116,9 +116,7 @@ struct vdso_data { #endif /* CONFIG_PPC64 */ -#ifdef __KERNEL__ extern struct vdso_data *vdso_data; -#endif #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 36c30f3..3bb2a3e 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -29,8 +29,10 @@ endif obj-y := cputable.o ptrace.o syscalls.o \ irq.o align.o signal_32.o pmc.o vdso.o \ init_task.o process.o systbl.o idle.o \ - signal.o sysfs.o cacheinfo.o -obj-y += vdso32/ + signal.o sysfs.o cacheinfo.o time.o \ + prom.o traps.o setup-common.o \ + udbg.o misc.o io.o dma.o \ + misc_$(CONFIG_WORD_SIZE).o vdso32/ obj-$(CONFIG_PPC64) += setup_64.o sys_ppc32.o \ signal_64.o ptrace32.o \ paca.o nvram_64.o firmware.o @@ -80,9 +82,6 @@ extra-$(CONFIG_FSL_BOOKE) := head_fsl_booke.o extra-$(CONFIG_8xx) := head_8xx.o extra-y += vmlinux.lds -obj-y += time.o prom.o traps.o setup-common.o \ - udbg.o misc.o io.o dma.o \ - misc_$(CONFIG_WORD_SIZE).o obj-$(CONFIG_PPC32) += entry_32.o setup_32.o obj-$(CONFIG_PPC64) += dma-iommu.o iommu.o obj-$(CONFIG_KGDB) += kgdb.o diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index bd0df2e..23e6a93 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -209,7 +209,6 @@ int main(void) DEFINE(RTASENTRY, offsetof(struct rtas_t, entry)); /* Interrupt register frame */ - DEFINE(STACK_FRAME_OVERHEAD, STACK_FRAME_OVERHEAD); DEFINE(INT_FRAME_SIZE, STACK_INT_FRAME_SIZE); DEFINE(SWITCH_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs)); #ifdef CONFIG_PPC64 diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index 96a908f..be5ab18 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -457,16 +457,26 @@ static struct cpu_spec __initdata cpu_specs[] = { .dcache_bsize = 128, .num_pmcs = 6, .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power7, - .cpu_restore = __restore_cpu_power7, .oprofile_cpu_type = "ppc64/power7", .oprofile_type = PPC_OPROFILE_POWER4, - .oprofile_mmcra_sihv = POWER6_MMCRA_SIHV, - .oprofile_mmcra_sipr = POWER6_MMCRA_SIPR, - .oprofile_mmcra_clear = POWER6_MMCRA_THRM | - POWER6_MMCRA_OTHER, .platform = "power7", }, + { /* Power7+ */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004A0000, + .cpu_name = "POWER7+ (raw)", + .cpu_features = CPU_FTRS_POWER7, + .cpu_user_features = COMMON_USER_POWER7, + .mmu_features = MMU_FTR_HPTE_TABLE | + MMU_FTR_TLBIE_206, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .oprofile_cpu_type = "ppc64/power7", + .oprofile_type = PPC_OPROFILE_POWER4, + .platform = "power7+", + }, { /* Cell Broadband Engine */ .pvr_mask = 0xffff0000, .pvr_value = 0x00700000, diff --git a/arch/powerpc/kernel/crash_dump.c b/arch/powerpc/kernel/crash_dump.c index 8e05c16..0a2af50 100644 --- a/arch/powerpc/kernel/crash_dump.c +++ b/arch/powerpc/kernel/crash_dump.c @@ -19,6 +19,7 @@ #include <asm/prom.h> #include <asm/firmware.h> #include <asm/uaccess.h> +#include <asm/rtas.h> #ifdef DEBUG #include <asm/udbg.h> @@ -141,3 +142,35 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf, return csize; } + +#ifdef CONFIG_PPC_RTAS +/* + * The crashkernel region will almost always overlap the RTAS region, so + * we have to be careful when shrinking the crashkernel region. + */ +void crash_free_reserved_phys_range(unsigned long begin, unsigned long end) +{ + unsigned long addr; + const u32 *basep, *sizep; + unsigned int rtas_start = 0, rtas_end = 0; + + basep = of_get_property(rtas.dev, "linux,rtas-base", NULL); + sizep = of_get_property(rtas.dev, "rtas-size", NULL); + + if (basep && sizep) { + rtas_start = *basep; + rtas_end = *basep + *sizep; + } + + for (addr = begin; addr < end; addr += PAGE_SIZE) { + /* Does this page overlap with the RTAS region? */ + if (addr <= rtas_end && ((addr + PAGE_SIZE) > rtas_start)) + continue; + + ClearPageReserved(pfn_to_page(addr >> PAGE_SHIFT)); + init_page_count(pfn_to_page(addr >> PAGE_SHIFT)); + free_page((unsigned long)__va(addr)); + totalram_pages++; + } +} +#endif diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 6e54a0f..e755415 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -19,7 +19,7 @@ static void *dma_iommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) { return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size, - dma_handle, device_to_mask(dev), flag, + dma_handle, dev->coherent_dma_mask, flag, dev_to_node(dev)); } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index ed4aeb9..c22dc1e 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -31,6 +31,7 @@ #include <asm/asm-offsets.h> #include <asm/unistd.h> #include <asm/ftrace.h> +#include <asm/ptrace.h> #undef SHOW_SYSCALLS #undef SHOW_SYSCALLS_TASK diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 9f8b01d..8a81799 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -13,6 +13,7 @@ */ #include <asm/exception-64s.h> +#include <asm/ptrace.h> /* * We layout physical memory as follows: diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S index e86c040..de36955 100644 --- a/arch/powerpc/kernel/fpu.S +++ b/arch/powerpc/kernel/fpu.S @@ -23,6 +23,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> #ifdef CONFIG_VSX #define REST_32FPVSRS(n,c,base) \ diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index 8278e8b..9dd21a8 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -40,6 +40,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> /* As with the other PowerPC ports, it is expected that when code * execution begins here, the following registers contain valid, yet diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 562305b..cbb3436 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -37,6 +37,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> #include <asm/synch.h> #include "head_booke.h" diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index f0dd577..782f23d 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -38,6 +38,7 @@ #include <asm/page_64.h> #include <asm/irqflags.h> #include <asm/kvm_book3s_asm.h> +#include <asm/ptrace.h> /* The physical memory is layed out such that the secondary processor * spin code sits at 0x0000...0x00ff. On server, the vectors follow @@ -96,7 +97,7 @@ __secondary_hold_acknowledge: .llong hvReleaseData-KERNELBASE #endif /* CONFIG_PPC_ISERIES */ -#ifdef CONFIG_CRASH_DUMP +#ifdef CONFIG_RELOCATABLE /* This flag is set to 1 by a loader if the kernel should run * at the loaded address instead of the linked address. This * is used by kexec-tools to keep the the kdump kernel in the @@ -384,12 +385,10 @@ _STATIC(__after_prom_start) /* process relocations for the final address of the kernel */ lis r25,PAGE_OFFSET@highest /* compute virtual base of kernel */ sldi r25,r25,32 -#ifdef CONFIG_CRASH_DUMP lwz r7,__run_at_load-_stext(r26) - cmplwi cr0,r7,1 /* kdump kernel ? - stay where we are */ + cmplwi cr0,r7,1 /* flagged to stay where we are ? */ bne 1f add r25,r25,r26 -#endif 1: mr r3,r25 bl .relocate #endif diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 1f1a04b..1cbf64e 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -29,6 +29,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> /* Macro to make the code more readable. */ #ifdef CONFIG_8xx_CPU6 diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index 529b817..3e02710 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -41,6 +41,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/cache.h> +#include <asm/ptrace.h> #include "head_booke.h" /* As with the other PowerPC ports, it is expected that when code diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index d583917..961bb03 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -311,8 +311,9 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl, /* Handle failure */ if (unlikely(entry == DMA_ERROR_CODE)) { if (printk_ratelimit()) - printk(KERN_INFO "iommu_alloc failed, tbl %p vaddr %lx" - " npages %lx\n", tbl, vaddr, npages); + dev_info(dev, "iommu_alloc failed, tbl %p " + "vaddr %lx npages %lu\n", tbl, vaddr, + npages); goto failure; } @@ -579,9 +580,9 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, attrs); if (dma_handle == DMA_ERROR_CODE) { if (printk_ratelimit()) { - printk(KERN_INFO "iommu_alloc failed, " - "tbl %p vaddr %p npages %d\n", - tbl, vaddr, npages); + dev_info(dev, "iommu_alloc failed, tbl %p " + "vaddr %p npages %d\n", tbl, vaddr, + npages); } } else dma_handle |= (uaddr & ~IOMMU_PAGE_MASK); @@ -627,7 +628,8 @@ void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, * the tce tables. */ if (order >= IOMAP_MAX_ORDER) { - printk("iommu_alloc_consistent size too large: 0x%lx\n", size); + dev_info(dev, "iommu_alloc_consistent size too large: 0x%lx\n", + size); return NULL; } diff --git a/arch/powerpc/kernel/misc.S b/arch/powerpc/kernel/misc.S index 2d29752..b69463e 100644 --- a/arch/powerpc/kernel/misc.S +++ b/arch/powerpc/kernel/misc.S @@ -122,8 +122,3 @@ _GLOBAL(longjmp) mtlr r0 mr r3,r4 blr - -_GLOBAL(__setup_cpu_power7) -_GLOBAL(__restore_cpu_power7) - /* place holder */ - blr diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index a7a570d..094bd98 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -30,6 +30,7 @@ #include <asm/processor.h> #include <asm/kexec.h> #include <asm/bug.h> +#include <asm/ptrace.h> .text diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index e514490..206a321 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -25,6 +25,7 @@ #include <asm/cputable.h> #include <asm/thread_info.h> #include <asm/kexec.h> +#include <asm/ptrace.h> .text diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 9cf197f..bb12b32 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -34,15 +34,26 @@ #undef DEBUG_NVRAM -static struct nvram_partition * nvram_part; -static long nvram_error_log_index = -1; -static long nvram_error_log_size = 0; +#define NVRAM_HEADER_LEN sizeof(struct nvram_header) +#define NVRAM_BLOCK_LEN NVRAM_HEADER_LEN + +/* If change this size, then change the size of NVNAME_LEN */ +struct nvram_header { + unsigned char signature; + unsigned char checksum; + unsigned short length; + /* Terminating null required only for names < 12 chars. */ + char name[12]; +}; -struct err_log_info { - int error_type; - unsigned int seq_num; +struct nvram_partition { + struct list_head partition; + struct nvram_header header; + unsigned int index; }; +static LIST_HEAD(nvram_partitions); + static loff_t dev_nvram_llseek(struct file *file, loff_t offset, int origin) { int size; @@ -186,14 +197,12 @@ static struct miscdevice nvram_dev = { #ifdef DEBUG_NVRAM static void __init nvram_print_partitions(char * label) { - struct list_head * p; struct nvram_partition * tmp_part; printk(KERN_WARNING "--------%s---------\n", label); printk(KERN_WARNING "indx\t\tsig\tchks\tlen\tname\n"); - list_for_each(p, &nvram_part->partition) { - tmp_part = list_entry(p, struct nvram_partition, partition); - printk(KERN_WARNING "%4d \t%02x\t%02x\t%d\t%s\n", + list_for_each_entry(tmp_part, &nvram_partitions, partition) { + printk(KERN_WARNING "%4d \t%02x\t%02x\t%d\t%12s\n", tmp_part->index, tmp_part->header.signature, tmp_part->header.checksum, tmp_part->header.length, tmp_part->header.name); @@ -228,95 +237,113 @@ static unsigned char __init nvram_checksum(struct nvram_header *p) return c_sum; } -static int __init nvram_remove_os_partition(void) +/** + * nvram_remove_partition - Remove one or more partitions in nvram + * @name: name of the partition to remove, or NULL for a + * signature only match + * @sig: signature of the partition(s) to remove + */ + +int __init nvram_remove_partition(const char *name, int sig) { - struct list_head *i; - struct list_head *j; - struct nvram_partition * part; - struct nvram_partition * cur_part; + struct nvram_partition *part, *prev, *tmp; int rc; - list_for_each(i, &nvram_part->partition) { - part = list_entry(i, struct nvram_partition, partition); - if (part->header.signature != NVRAM_SIG_OS) + list_for_each_entry(part, &nvram_partitions, partition) { + if (part->header.signature != sig) continue; - - /* Make os partition a free partition */ + if (name && strncmp(name, part->header.name, 12)) + continue; + + /* Make partition a free partition */ part->header.signature = NVRAM_SIG_FREE; - sprintf(part->header.name, "wwwwwwwwwwww"); + strncpy(part->header.name, "wwwwwwwwwwww", 12); part->header.checksum = nvram_checksum(&part->header); - - /* Merge contiguous free partitions backwards */ - list_for_each_prev(j, &part->partition) { - cur_part = list_entry(j, struct nvram_partition, partition); - if (cur_part == nvram_part || cur_part->header.signature != NVRAM_SIG_FREE) { - break; - } - - part->header.length += cur_part->header.length; - part->header.checksum = nvram_checksum(&part->header); - part->index = cur_part->index; - - list_del(&cur_part->partition); - kfree(cur_part); - j = &part->partition; /* fixup our loop */ - } - - /* Merge contiguous free partitions forwards */ - list_for_each(j, &part->partition) { - cur_part = list_entry(j, struct nvram_partition, partition); - if (cur_part == nvram_part || cur_part->header.signature != NVRAM_SIG_FREE) { - break; - } - - part->header.length += cur_part->header.length; - part->header.checksum = nvram_checksum(&part->header); - - list_del(&cur_part->partition); - kfree(cur_part); - j = &part->partition; /* fixup our loop */ - } - rc = nvram_write_header(part); if (rc <= 0) { - printk(KERN_ERR "nvram_remove_os_partition: nvram_write failed (%d)\n", rc); + printk(KERN_ERR "nvram_remove_partition: nvram_write failed (%d)\n", rc); return rc; } + } + /* Merge contiguous ones */ + prev = NULL; + list_for_each_entry_safe(part, tmp, &nvram_partitions, partition) { + if (part->header.signature != NVRAM_SIG_FREE) { + prev = NULL; + continue; + } + if (prev) { + prev->header.length += part->header.length; + prev->header.checksum = nvram_checksum(&part->header); + rc = nvram_write_header(part); + if (rc <= 0) { + printk(KERN_ERR "nvram_remove_partition: nvram_write failed (%d)\n", rc); + return rc; + } + list_del(&part->partition); + kfree(part); + } else + prev = part; } return 0; } -/* nvram_create_os_partition +/** + * nvram_create_partition - Create a partition in nvram + * @name: name of the partition to create + * @sig: signature of the partition to create + * @req_size: size of data to allocate in bytes + * @min_size: minimum acceptable size (0 means req_size) * - * Create a OS linux partition to buffer error logs. - * Will create a partition starting at the first free - * space found if space has enough room. + * Returns a negative error code or a positive nvram index + * of the beginning of the data area of the newly created + * partition. If you provided a min_size smaller than req_size + * you need to query for the actual size yourself after the + * call using nvram_partition_get_size(). */ -static int __init nvram_create_os_partition(void) +loff_t __init nvram_create_partition(const char *name, int sig, + int req_size, int min_size) { struct nvram_partition *part; struct nvram_partition *new_part; struct nvram_partition *free_part = NULL; - int seq_init[2] = { 0, 0 }; + static char nv_init_vals[16]; loff_t tmp_index; long size = 0; int rc; - + + /* Convert sizes from bytes to blocks */ + req_size = _ALIGN_UP(req_size, NVRAM_BLOCK_LEN) / NVRAM_BLOCK_LEN; + min_size = _ALIGN_UP(min_size, NVRAM_BLOCK_LEN) / NVRAM_BLOCK_LEN; + + /* If no minimum size specified, make it the same as the + * requested size + */ + if (min_size == 0) + min_size = req_size; + if (min_size > req_size) + return -EINVAL; + + /* Now add one block to each for the header */ + req_size += 1; + min_size += 1; + /* Find a free partition that will give us the maximum needed size If can't find one that will give us the minimum size needed */ - list_for_each_entry(part, &nvram_part->partition, partition) { + list_for_each_entry(part, &nvram_partitions, partition) { if (part->header.signature != NVRAM_SIG_FREE) continue; - if (part->header.length >= NVRAM_MAX_REQ) { - size = NVRAM_MAX_REQ; + if (part->header.length >= req_size) { + size = req_size; free_part = part; break; } - if (!size && part->header.length >= NVRAM_MIN_REQ) { - size = NVRAM_MIN_REQ; + if (part->header.length > size && + part->header.length >= min_size) { + size = part->header.length; free_part = part; } } @@ -326,136 +353,95 @@ static int __init nvram_create_os_partition(void) /* Create our OS partition */ new_part = kmalloc(sizeof(*new_part), GFP_KERNEL); if (!new_part) { - printk(KERN_ERR "nvram_create_os_partition: kmalloc failed\n"); + pr_err("nvram_create_os_partition: kmalloc failed\n"); return -ENOMEM; } new_part->index = free_part->index; - new_part->header.signature = NVRAM_SIG_OS; + new_part->header.signature = sig; new_part->header.length = size; - strcpy(new_part->header.name, "ppc64,linux"); + strncpy(new_part->header.name, name, 12); new_part->header.checksum = nvram_checksum(&new_part->header); rc = nvram_write_header(new_part); if (rc <= 0) { - printk(KERN_ERR "nvram_create_os_partition: nvram_write_header " - "failed (%d)\n", rc); - return rc; - } - - /* make sure and initialize to zero the sequence number and the error - type logged */ - tmp_index = new_part->index + NVRAM_HEADER_LEN; - rc = ppc_md.nvram_write((char *)&seq_init, sizeof(seq_init), &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_create_os_partition: nvram_write " + pr_err("nvram_create_os_partition: nvram_write_header " "failed (%d)\n", rc); return rc; } - - nvram_error_log_index = new_part->index + NVRAM_HEADER_LEN; - nvram_error_log_size = ((part->header.length - 1) * - NVRAM_BLOCK_LEN) - sizeof(struct err_log_info); - list_add_tail(&new_part->partition, &free_part->partition); - if (free_part->header.length <= size) { + /* Adjust or remove the partition we stole the space from */ + if (free_part->header.length > size) { + free_part->index += size * NVRAM_BLOCK_LEN; + free_part->header.length -= size; + free_part->header.checksum = nvram_checksum(&free_part->header); + rc = nvram_write_header(free_part); + if (rc <= 0) { + pr_err("nvram_create_os_partition: nvram_write_header " + "failed (%d)\n", rc); + return rc; + } + } else { list_del(&free_part->partition); kfree(free_part); - return 0; } - /* Adjust the partition we stole the space from */ - free_part->index += size * NVRAM_BLOCK_LEN; - free_part->header.length -= size; - free_part->header.checksum = nvram_checksum(&free_part->header); - - rc = nvram_write_header(free_part); - if (rc <= 0) { - printk(KERN_ERR "nvram_create_os_partition: nvram_write_header " - "failed (%d)\n", rc); - return rc; + /* Clear the new partition */ + for (tmp_index = new_part->index + NVRAM_HEADER_LEN; + tmp_index < ((size - 1) * NVRAM_BLOCK_LEN); + tmp_index += NVRAM_BLOCK_LEN) { + rc = ppc_md.nvram_write(nv_init_vals, NVRAM_BLOCK_LEN, &tmp_index); + if (rc <= 0) { + pr_err("nvram_create_partition: nvram_write failed (%d)\n", rc); + return rc; + } } - - return 0; + + return new_part->index + NVRAM_HEADER_LEN; } - -/* nvram_setup_partition - * - * This will setup the partition we need for buffering the - * error logs and cleanup partitions if needed. - * - * The general strategy is the following: - * 1.) If there is ppc64,linux partition large enough then use it. - * 2.) If there is not a ppc64,linux partition large enough, search - * for a free partition that is large enough. - * 3.) If there is not a free partition large enough remove - * _all_ OS partitions and consolidate the space. - * 4.) Will first try getting a chunk that will satisfy the maximum - * error log size (NVRAM_MAX_REQ). - * 5.) If the max chunk cannot be allocated then try finding a chunk - * that will satisfy the minum needed (NVRAM_MIN_REQ). +/** + * nvram_get_partition_size - Get the data size of an nvram partition + * @data_index: This is the offset of the start of the data of + * the partition. The same value that is returned by + * nvram_create_partition(). */ -static int __init nvram_setup_partition(void) +int nvram_get_partition_size(loff_t data_index) { - struct list_head * p; - struct nvram_partition * part; - int rc; - - /* For now, we don't do any of this on pmac, until I - * have figured out if it's worth killing some unused stuffs - * in our nvram, as Apple defined partitions use pretty much - * all of the space - */ - if (machine_is(powermac)) - return -ENOSPC; - - /* see if we have an OS partition that meets our needs. - will try getting the max we need. If not we'll delete - partitions and try again. */ - list_for_each(p, &nvram_part->partition) { - part = list_entry(p, struct nvram_partition, partition); - if (part->header.signature != NVRAM_SIG_OS) - continue; + struct nvram_partition *part; + + list_for_each_entry(part, &nvram_partitions, partition) { + if (part->index + NVRAM_HEADER_LEN == data_index) + return (part->header.length - 1) * NVRAM_BLOCK_LEN; + } + return -1; +} - if (strcmp(part->header.name, "ppc64,linux")) - continue; - if (part->header.length >= NVRAM_MIN_REQ) { - /* found our partition */ - nvram_error_log_index = part->index + NVRAM_HEADER_LEN; - nvram_error_log_size = ((part->header.length - 1) * - NVRAM_BLOCK_LEN) - sizeof(struct err_log_info); - return 0; +/** + * nvram_find_partition - Find an nvram partition by signature and name + * @name: Name of the partition or NULL for any name + * @sig: Signature to test against + * @out_size: if non-NULL, returns the size of the data part of the partition + */ +loff_t nvram_find_partition(const char *name, int sig, int *out_size) +{ + struct nvram_partition *p; + + list_for_each_entry(p, &nvram_partitions, partition) { + if (p->header.signature == sig && + (!name || !strncmp(p->header.name, name, 12))) { + if (out_size) + *out_size = (p->header.length - 1) * + NVRAM_BLOCK_LEN; + return p->index + NVRAM_HEADER_LEN; } } - - /* try creating a partition with the free space we have */ - rc = nvram_create_os_partition(); - if (!rc) { - return 0; - } - - /* need to free up some space */ - rc = nvram_remove_os_partition(); - if (rc) { - return rc; - } - - /* create a partition in this new space */ - rc = nvram_create_os_partition(); - if (rc) { - printk(KERN_ERR "nvram_create_os_partition: Could not find a " - "NVRAM partition large enough\n"); - return rc; - } - return 0; } - -static int __init nvram_scan_partitions(void) +int __init nvram_scan_partitions(void) { loff_t cur_index = 0; struct nvram_header phead; @@ -465,7 +451,7 @@ static int __init nvram_scan_partitions(void) int total_size; int err; - if (ppc_md.nvram_size == NULL) + if (ppc_md.nvram_size == NULL || ppc_md.nvram_size() <= 0) return -ENODEV; total_size = ppc_md.nvram_size(); @@ -512,12 +498,16 @@ static int __init nvram_scan_partitions(void) memcpy(&tmp_part->header, &phead, NVRAM_HEADER_LEN); tmp_part->index = cur_index; - list_add_tail(&tmp_part->partition, &nvram_part->partition); + list_add_tail(&tmp_part->partition, &nvram_partitions); cur_index += phead.length * NVRAM_BLOCK_LEN; } err = 0; +#ifdef DEBUG_NVRAM + nvram_print_partitions("NVRAM Partitions"); +#endif + out: kfree(header); return err; @@ -525,9 +515,10 @@ static int __init nvram_scan_partitions(void) static int __init nvram_init(void) { - int error; int rc; + BUILD_BUG_ON(NVRAM_BLOCK_LEN != 16); + if (ppc_md.nvram_size == NULL || ppc_md.nvram_size() <= 0) return -ENODEV; @@ -537,29 +528,6 @@ static int __init nvram_init(void) return rc; } - /* initialize our anchor for the nvram partition list */ - nvram_part = kmalloc(sizeof(struct nvram_partition), GFP_KERNEL); - if (!nvram_part) { - printk(KERN_ERR "nvram_init: Failed kmalloc\n"); - return -ENOMEM; - } - INIT_LIST_HEAD(&nvram_part->partition); - - /* Get all the NVRAM partitions */ - error = nvram_scan_partitions(); - if (error) { - printk(KERN_ERR "nvram_init: Failed nvram_scan_partitions\n"); - return error; - } - - if(nvram_setup_partition()) - printk(KERN_WARNING "nvram_init: Could not find nvram partition" - " for nvram buffered error logging.\n"); - -#ifdef DEBUG_NVRAM - nvram_print_partitions("NVRAM Partitions"); -#endif - return rc; } @@ -568,135 +536,6 @@ void __exit nvram_cleanup(void) misc_deregister( &nvram_dev ); } - -#ifdef CONFIG_PPC_PSERIES - -/* nvram_write_error_log - * - * We need to buffer the error logs into nvram to ensure that we have - * the failure information to decode. If we have a severe error there - * is no way to guarantee that the OS or the machine is in a state to - * get back to user land and write the error to disk. For example if - * the SCSI device driver causes a Machine Check by writing to a bad - * IO address, there is no way of guaranteeing that the device driver - * is in any state that is would also be able to write the error data - * captured to disk, thus we buffer it in NVRAM for analysis on the - * next boot. - * - * In NVRAM the partition containing the error log buffer will looks like: - * Header (in bytes): - * +-----------+----------+--------+------------+------------------+ - * | signature | checksum | length | name | data | - * |0 |1 |2 3|4 15|16 length-1| - * +-----------+----------+--------+------------+------------------+ - * - * The 'data' section would look like (in bytes): - * +--------------+------------+-----------------------------------+ - * | event_logged | sequence # | error log | - * |0 3|4 7|8 nvram_error_log_size-1| - * +--------------+------------+-----------------------------------+ - * - * event_logged: 0 if event has not been logged to syslog, 1 if it has - * sequence #: The unique sequence # for each event. (until it wraps) - * error log: The error log from event_scan - */ -int nvram_write_error_log(char * buff, int length, - unsigned int err_type, unsigned int error_log_cnt) -{ - int rc; - loff_t tmp_index; - struct err_log_info info; - - if (nvram_error_log_index == -1) { - return -ESPIPE; - } - - if (length > nvram_error_log_size) { - length = nvram_error_log_size; - } - - info.error_type = err_type; - info.seq_num = error_log_cnt; - - tmp_index = nvram_error_log_index; - - rc = ppc_md.nvram_write((char *)&info, sizeof(struct err_log_info), &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_write_error_log: Failed nvram_write (%d)\n", rc); - return rc; - } - - rc = ppc_md.nvram_write(buff, length, &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_write_error_log: Failed nvram_write (%d)\n", rc); - return rc; - } - - return 0; -} - -/* nvram_read_error_log - * - * Reads nvram for error log for at most 'length' - */ -int nvram_read_error_log(char * buff, int length, - unsigned int * err_type, unsigned int * error_log_cnt) -{ - int rc; - loff_t tmp_index; - struct err_log_info info; - - if (nvram_error_log_index == -1) - return -1; - - if (length > nvram_error_log_size) - length = nvram_error_log_size; - - tmp_index = nvram_error_log_index; - - rc = ppc_md.nvram_read((char *)&info, sizeof(struct err_log_info), &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_read_error_log: Failed nvram_read (%d)\n", rc); - return rc; - } - - rc = ppc_md.nvram_read(buff, length, &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_read_error_log: Failed nvram_read (%d)\n", rc); - return rc; - } - - *error_log_cnt = info.seq_num; - *err_type = info.error_type; - - return 0; -} - -/* This doesn't actually zero anything, but it sets the event_logged - * word to tell that this event is safely in syslog. - */ -int nvram_clear_error_log(void) -{ - loff_t tmp_index; - int clear_word = ERR_FLAG_ALREADY_LOGGED; - int rc; - - if (nvram_error_log_index == -1) - return -1; - - tmp_index = nvram_error_log_index; - - rc = ppc_md.nvram_write((char *)&clear_word, sizeof(int), &tmp_index); - if (rc <= 0) { - printk(KERN_ERR "nvram_clear_error_log: Failed nvram_write (%d)\n", rc); - return rc; - } - - return 0; -} - -#endif /* CONFIG_PPC_PSERIES */ - module_init(nvram_init); module_exit(nvram_cleanup); MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/kernel/pci_64.c b/arch/powerpc/kernel/pci_64.c index d43fc65..8515776 100644 --- a/arch/powerpc/kernel/pci_64.c +++ b/arch/powerpc/kernel/pci_64.c @@ -193,8 +193,7 @@ int __devinit pcibios_map_io_space(struct pci_bus *bus) hose->io_resource.start += io_virt_offset; hose->io_resource.end += io_virt_offset; - pr_debug(" hose->io_resource=0x%016llx...0x%016llx\n", - hose->io_resource.start, hose->io_resource.end); + pr_debug(" hose->io_resource=%pR\n", &hose->io_resource); return 0; } diff --git a/arch/powerpc/kernel/ppc_ksyms.c b/arch/powerpc/kernel/ppc_ksyms.c index ab3e392..ef3ef56 100644 --- a/arch/powerpc/kernel/ppc_ksyms.c +++ b/arch/powerpc/kernel/ppc_ksyms.c @@ -186,3 +186,10 @@ EXPORT_SYMBOL(__mtdcr); EXPORT_SYMBOL(__mfdcr); #endif EXPORT_SYMBOL(empty_zero_page); + +#ifdef CONFIG_PPC64 +EXPORT_SYMBOL(__arch_hweight8); +EXPORT_SYMBOL(__arch_hweight16); +EXPORT_SYMBOL(__arch_hweight32); +EXPORT_SYMBOL(__arch_hweight64); +#endif diff --git a/arch/powerpc/kernel/ppc_save_regs.S b/arch/powerpc/kernel/ppc_save_regs.S index 5113bd2..e83ba3f 100644 --- a/arch/powerpc/kernel/ppc_save_regs.S +++ b/arch/powerpc/kernel/ppc_save_regs.S @@ -11,6 +11,7 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> /* * Grab the register values as they are now. diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index a9b3296..9065369 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -1316,6 +1316,10 @@ static int set_dac_range(struct task_struct *child, static long ppc_set_hwdebug(struct task_struct *child, struct ppc_hw_breakpoint *bp_info) { +#ifndef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dabr; +#endif + if (bp_info->version != 1) return -ENOTSUPP; #ifdef CONFIG_PPC_ADV_DEBUG_REGS @@ -1353,11 +1357,10 @@ static long ppc_set_hwdebug(struct task_struct *child, /* * We only support one data breakpoint */ - if (((bp_info->trigger_type & PPC_BREAKPOINT_TRIGGER_RW) == 0) || - ((bp_info->trigger_type & ~PPC_BREAKPOINT_TRIGGER_RW) != 0) || - (bp_info->trigger_type != PPC_BREAKPOINT_TRIGGER_WRITE) || - (bp_info->addr_mode != PPC_BREAKPOINT_MODE_EXACT) || - (bp_info->condition_mode != PPC_BREAKPOINT_CONDITION_NONE)) + if ((bp_info->trigger_type & PPC_BREAKPOINT_TRIGGER_RW) == 0 || + (bp_info->trigger_type & ~PPC_BREAKPOINT_TRIGGER_RW) != 0 || + bp_info->addr_mode != PPC_BREAKPOINT_MODE_EXACT || + bp_info->condition_mode != PPC_BREAKPOINT_CONDITION_NONE) return -EINVAL; if (child->thread.dabr) @@ -1366,7 +1369,14 @@ static long ppc_set_hwdebug(struct task_struct *child, if ((unsigned long)bp_info->addr >= TASK_SIZE) return -EIO; - child->thread.dabr = (unsigned long)bp_info->addr; + dabr = (unsigned long)bp_info->addr & ~7UL; + dabr |= DABR_TRANSLATION; + if (bp_info->trigger_type & PPC_BREAKPOINT_TRIGGER_READ) + dabr |= DABR_DATA_READ; + if (bp_info->trigger_type & PPC_BREAKPOINT_TRIGGER_WRITE) + dabr |= DABR_DATA_WRITE; + + child->thread.dabr = dabr; return 1; #endif /* !CONFIG_PPC_ADV_DEBUG_DVCS */ diff --git a/arch/powerpc/kernel/ptrace32.c b/arch/powerpc/kernel/ptrace32.c index 8a6daf4..69c4be9 100644 --- a/arch/powerpc/kernel/ptrace32.c +++ b/arch/powerpc/kernel/ptrace32.c @@ -280,7 +280,11 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, /* We only support one DABR and no IABRS at the moment */ if (addr > 0) break; +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + ret = put_user(child->thread.dac1, (u32 __user *)data); +#else ret = put_user(child->thread.dabr, (u32 __user *)data); +#endif break; } @@ -312,6 +316,9 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case PTRACE_SET_DEBUGREG: case PTRACE_SYSCALL: case PTRACE_CONT: + case PPC_PTRACE_GETHWDBGINFO: + case PPC_PTRACE_SETHWDEBUG: + case PPC_PTRACE_DELHWDEBUG: ret = arch_ptrace(child, request, addr, data); break; diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 8fe8bc6..2097f2b 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -41,6 +41,7 @@ #include <asm/atomic.h> #include <asm/time.h> #include <asm/mmu.h> +#include <asm/topology.h> struct rtas_t rtas = { .lock = __ARCH_SPIN_LOCK_UNLOCKED @@ -713,6 +714,7 @@ static int __rtas_suspend_last_cpu(struct rtas_suspend_me_data *data, int wake_w int cpu; slb_set_size(SLB_MIN_SIZE); + stop_topology_update(); printk(KERN_DEBUG "calling ibm,suspend-me on cpu %i\n", smp_processor_id()); while (rc == H_MULTI_THREADS_ACTIVE && !atomic_read(&data->done) && @@ -728,6 +730,7 @@ static int __rtas_suspend_last_cpu(struct rtas_suspend_me_data *data, int wake_w rc = atomic_read(&data->error); atomic_set(&data->error, rc); + start_topology_update(); if (wake_when_done) { atomic_set(&data->done, 1); diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index ce6f61c..5a0401f 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -437,8 +437,8 @@ static void __init irqstack_early_init(void) unsigned int i; /* - * interrupt stacks must be under 256MB, we cannot afford to take - * SLB misses on them. + * Interrupt stacks must be in the first segment since we + * cannot afford to take SLB misses on them. */ for_each_possible_cpu(i) { softirq_ctx[i] = (struct thread_info *) diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 68034bb..9813605 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -466,7 +466,20 @@ out: return id; } -/* Must be called when no change can occur to cpu_present_mask, +/* Helper routines for cpu to core mapping */ +int cpu_core_index_of_thread(int cpu) +{ + return cpu >> threads_shift; +} +EXPORT_SYMBOL_GPL(cpu_core_index_of_thread); + +int cpu_first_thread_of_core(int core) +{ + return core << threads_shift; +} +EXPORT_SYMBOL_GPL(cpu_first_thread_of_core); + +/* Must be called when no change can occur to cpu_present_map, * i.e. during cpu online or offline. */ static struct device_node *cpu_to_l2cache(int cpu) @@ -514,7 +527,7 @@ int __devinit start_secondary(void *unused) notify_cpu_starting(cpu); set_cpu_online(cpu, true); /* Update sibling maps */ - base = cpu_first_thread_in_core(cpu); + base = cpu_first_thread_sibling(cpu); for (i = 0; i < threads_per_core; i++) { if (cpu_is_offline(base + i)) continue; @@ -600,7 +613,7 @@ int __cpu_disable(void) return err; /* Update sibling maps */ - base = cpu_first_thread_in_core(cpu); + base = cpu_first_thread_sibling(cpu); for (i = 0; i < threads_per_core; i++) { cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i)); cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu)); diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 0104069..09e4dea 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -155,7 +155,7 @@ EXPORT_SYMBOL_GPL(rtc_lock); static u64 tb_to_ns_scale __read_mostly; static unsigned tb_to_ns_shift __read_mostly; -static unsigned long boot_tb __read_mostly; +static u64 boot_tb __read_mostly; extern struct timezone sys_tz; static long timezone_offset; diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S index fe46048..9de6f39 100644 --- a/arch/powerpc/kernel/vector.S +++ b/arch/powerpc/kernel/vector.S @@ -5,6 +5,7 @@ #include <asm/cputable.h> #include <asm/thread_info.h> #include <asm/page.h> +#include <asm/ptrace.h> /* * load_up_altivec(unused, unused, tsk) diff --git a/arch/powerpc/kernel/vio.c b/arch/powerpc/kernel/vio.c index 441d2a7..1b695fd 100644 --- a/arch/powerpc/kernel/vio.c +++ b/arch/powerpc/kernel/vio.c @@ -600,6 +600,11 @@ static void vio_dma_iommu_unmap_sg(struct device *dev, vio_cmo_dealloc(viodev, alloc_size); } +static int vio_dma_iommu_dma_supported(struct device *dev, u64 mask) +{ + return dma_iommu_ops.dma_supported(dev, mask); +} + struct dma_map_ops vio_dma_mapping_ops = { .alloc_coherent = vio_dma_iommu_alloc_coherent, .free_coherent = vio_dma_iommu_free_coherent, @@ -607,6 +612,7 @@ struct dma_map_ops vio_dma_mapping_ops = { .unmap_sg = vio_dma_iommu_unmap_sg, .map_page = vio_dma_iommu_map_page, .unmap_page = vio_dma_iommu_unmap_page, + .dma_supported = vio_dma_iommu_dma_supported, }; @@ -858,8 +864,7 @@ static void vio_cmo_bus_remove(struct vio_dev *viodev) static void vio_cmo_set_dma_ops(struct vio_dev *viodev) { - vio_dma_mapping_ops.dma_supported = dma_iommu_ops.dma_supported; - viodev->dev.archdata.dma_ops = &vio_dma_mapping_ops; + set_dma_ops(&viodev->dev, &vio_dma_mapping_ops); } /** @@ -1244,7 +1249,7 @@ struct vio_dev *vio_register_device_node(struct device_node *of_node) if (firmware_has_feature(FW_FEATURE_CMO)) vio_cmo_set_dma_ops(viodev); else - viodev->dev.archdata.dma_ops = &dma_iommu_ops; + set_dma_ops(&viodev->dev, &dma_iommu_ops); set_iommu_table_base(&viodev->dev, vio_build_iommu_table(viodev)); set_dev_node(&viodev->dev, of_node_to_nid(of_node)); @@ -1252,6 +1257,10 @@ struct vio_dev *vio_register_device_node(struct device_node *of_node) viodev->dev.parent = &vio_bus_device.dev; viodev->dev.bus = &vio_bus_type; viodev->dev.release = vio_dev_release; + /* needed to ensure proper operation of coherent allocations + * later, in case driver doesn't set it explicitly */ + dma_set_mask(&viodev->dev, DMA_BIT_MASK(64)); + dma_set_coherent_mask(&viodev->dev, DMA_BIT_MASK(64)); /* register with generic device framework */ if (device_register(&viodev->dev)) { diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index e316847..badc983 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -1307,12 +1307,10 @@ struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id) int err = -ENOMEM; unsigned long p; - vcpu_book3s = vmalloc(sizeof(struct kvmppc_vcpu_book3s)); + vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); if (!vcpu_book3s) goto out; - memset(vcpu_book3s, 0, sizeof(struct kvmppc_vcpu_book3s)); - vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *) kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL); if (!vcpu_book3s->shadow_vcpu) diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 38f756f..9975846 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -145,18 +145,12 @@ void kvm_arch_check_processor_compat(void *rtn) *(int *)rtn = kvmppc_core_check_processor_compat(); } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm; - - kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - if (!kvm) - return ERR_PTR(-ENOMEM); - - return kvm; + return 0; } -static void kvmppc_free_vcpus(struct kvm *kvm) +void kvm_arch_destroy_vm(struct kvm *kvm) { unsigned int i; struct kvm_vcpu *vcpu; @@ -176,14 +170,6 @@ void kvm_arch_sync_events(struct kvm *kvm) { } -void kvm_arch_destroy_vm(struct kvm *kvm) -{ - kvmppc_free_vcpus(kvm); - kvm_free_physmem(kvm); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); -} - int kvm_dev_ioctl_check_extension(long ext) { int r; diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 889f2bc..166a6a0 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -16,7 +16,7 @@ obj-$(CONFIG_HAS_IOMEM) += devres.o obj-$(CONFIG_PPC64) += copypage_64.o copyuser_64.o \ memcpy_64.o usercopy_64.o mem_64.o string.o \ - checksum_wrappers_64.o + checksum_wrappers_64.o hweight_64.o obj-$(CONFIG_XMON) += sstep.o ldstfp.o obj-$(CONFIG_KPROBES) += sstep.o ldstfp.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += sstep.o ldstfp.o diff --git a/arch/powerpc/lib/hweight_64.S b/arch/powerpc/lib/hweight_64.S new file mode 100644 index 0000000..fda2786 --- /dev/null +++ b/arch/powerpc/lib/hweight_64.S @@ -0,0 +1,110 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) IBM Corporation, 2010 + * + * Author: Anton Blanchard <anton@au.ibm.com> + */ +#include <asm/processor.h> +#include <asm/ppc_asm.h> + +/* Note: This code relies on -mminimal-toc */ + +_GLOBAL(__arch_hweight8) +BEGIN_FTR_SECTION + b .__sw_hweight8 + nop + nop +FTR_SECTION_ELSE + PPC_POPCNTB(r3,r3) + clrldi r3,r3,64-8 + blr +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_POPCNTB) + +_GLOBAL(__arch_hweight16) +BEGIN_FTR_SECTION + b .__sw_hweight16 + nop + nop + nop + nop +FTR_SECTION_ELSE + BEGIN_FTR_SECTION_NESTED(50) + PPC_POPCNTB(r3,r3) + srdi r4,r3,8 + add r3,r4,r3 + clrldi r3,r3,64-8 + blr + FTR_SECTION_ELSE_NESTED(50) + clrlwi r3,r3,16 + PPC_POPCNTW(r3,r3) + clrldi r3,r3,64-8 + blr + ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_POPCNTD, 50) +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_POPCNTB) + +_GLOBAL(__arch_hweight32) +BEGIN_FTR_SECTION + b .__sw_hweight32 + nop + nop + nop + nop + nop + nop +FTR_SECTION_ELSE + BEGIN_FTR_SECTION_NESTED(51) + PPC_POPCNTB(r3,r3) + srdi r4,r3,16 + add r3,r4,r3 + srdi r4,r3,8 + add r3,r4,r3 + clrldi r3,r3,64-8 + blr + FTR_SECTION_ELSE_NESTED(51) + PPC_POPCNTW(r3,r3) + clrldi r3,r3,64-8 + blr + ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_POPCNTD, 51) +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_POPCNTB) + +_GLOBAL(__arch_hweight64) +BEGIN_FTR_SECTION + b .__sw_hweight64 + nop + nop + nop + nop + nop + nop + nop + nop +FTR_SECTION_ELSE + BEGIN_FTR_SECTION_NESTED(52) + PPC_POPCNTB(r3,r3) + srdi r4,r3,32 + add r3,r4,r3 + srdi r4,r3,16 + add r3,r4,r3 + srdi r4,r3,8 + add r3,r4,r3 + clrldi r3,r3,64-8 + blr + FTR_SECTION_ELSE_NESTED(52) + PPC_POPCNTD(r3,r3) + clrldi r3,r3,64-8 + blr + ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_POPCNTD, 52) +ALT_FTR_SECTION_END_IFCLR(CPU_FTR_POPCNTB) diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 5e95844..a5991fa 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -1070,7 +1070,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, unsigned long access, unsigned long trap) { unsigned long vsid; - void *pgdir; + pgd_t *pgdir; pte_t *ptep; unsigned long flags; int rc, ssize, local = 0; diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index 5ce9984..c0aab52 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -111,8 +111,8 @@ static unsigned int steal_context_smp(unsigned int id) * a core map instead but this will do for now. */ for_each_cpu(cpu, mm_cpumask(mm)) { - for (i = cpu_first_thread_in_core(cpu); - i <= cpu_last_thread_in_core(cpu); i++) + for (i = cpu_first_thread_sibling(cpu); + i <= cpu_last_thread_sibling(cpu); i++) __set_bit(id, stale_map[i]); cpu = i - 1; } @@ -264,14 +264,14 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) */ if (test_bit(id, stale_map[cpu])) { pr_hardcont(" | stale flush %d [%d..%d]", - id, cpu_first_thread_in_core(cpu), - cpu_last_thread_in_core(cpu)); + id, cpu_first_thread_sibling(cpu), + cpu_last_thread_sibling(cpu)); local_flush_tlb_mm(next); /* XXX This clear should ultimately be part of local_flush_tlb_mm */ - for (i = cpu_first_thread_in_core(cpu); - i <= cpu_last_thread_in_core(cpu); i++) { + for (i = cpu_first_thread_sibling(cpu); + i <= cpu_last_thread_sibling(cpu); i++) { __clear_bit(id, stale_map[i]); } } diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 74505b2..bf5cb91 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -20,10 +20,15 @@ #include <linux/memblock.h> #include <linux/of.h> #include <linux/pfn.h> +#include <linux/cpuset.h> +#include <linux/node.h> #include <asm/sparsemem.h> #include <asm/prom.h> #include <asm/system.h> #include <asm/smp.h> +#include <asm/firmware.h> +#include <asm/paca.h> +#include <asm/hvcall.h> static int numa_enabled = 1; @@ -163,7 +168,7 @@ static void __init get_node_active_region(unsigned long start_pfn, work_with_active_regions(nid, get_active_region_work_fn, node_ar); } -static void __cpuinit map_cpu_to_node(int cpu, int node) +static void map_cpu_to_node(int cpu, int node) { numa_cpu_lookup_table[cpu] = node; @@ -173,7 +178,7 @@ static void __cpuinit map_cpu_to_node(int cpu, int node) cpumask_set_cpu(cpu, node_to_cpumask_map[node]); } -#ifdef CONFIG_HOTPLUG_CPU +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR) static void unmap_cpu_from_node(unsigned long cpu) { int node = numa_cpu_lookup_table[cpu]; @@ -187,7 +192,7 @@ static void unmap_cpu_from_node(unsigned long cpu) cpu, node); } } -#endif /* CONFIG_HOTPLUG_CPU */ +#endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */ /* must hold reference to node during call */ static const int *of_get_associativity(struct device_node *dev) @@ -246,32 +251,41 @@ static void initialize_distance_lookup_table(int nid, /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa * info is found. */ -static int of_node_to_nid_single(struct device_node *device) +static int associativity_to_nid(const unsigned int *associativity) { int nid = -1; - const unsigned int *tmp; if (min_common_depth == -1) goto out; - tmp = of_get_associativity(device); - if (!tmp) - goto out; - - if (tmp[0] >= min_common_depth) - nid = tmp[min_common_depth]; + if (associativity[0] >= min_common_depth) + nid = associativity[min_common_depth]; /* POWER4 LPAR uses 0xffff as invalid node */ if (nid == 0xffff || nid >= MAX_NUMNODES) nid = -1; - if (nid > 0 && tmp[0] >= distance_ref_points_depth) - initialize_distance_lookup_table(nid, tmp); + if (nid > 0 && associativity[0] >= distance_ref_points_depth) + initialize_distance_lookup_table(nid, associativity); out: return nid; } +/* Returns the nid associated with the given device tree node, + * or -1 if not found. + */ +static int of_node_to_nid_single(struct device_node *device) +{ + int nid = -1; + const unsigned int *tmp; + + tmp = of_get_associativity(device); + if (tmp) + nid = associativity_to_nid(tmp); + return nid; +} + /* Walk the device tree upwards, looking for an associativity id */ int of_node_to_nid(struct device_node *device) { @@ -1247,4 +1261,275 @@ int hot_add_scn_to_nid(unsigned long scn_addr) return nid; } +static u64 hot_add_drconf_memory_max(void) +{ + struct device_node *memory = NULL; + unsigned int drconf_cell_cnt = 0; + u64 lmb_size = 0; + const u32 *dm = 0; + + memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); + if (memory) { + drconf_cell_cnt = of_get_drconf_memory(memory, &dm); + lmb_size = of_get_lmb_size(memory); + of_node_put(memory); + } + return lmb_size * drconf_cell_cnt; +} + +/* + * memory_hotplug_max - return max address of memory that may be added + * + * This is currently only used on systems that support drconfig memory + * hotplug. + */ +u64 memory_hotplug_max(void) +{ + return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM()); +} #endif /* CONFIG_MEMORY_HOTPLUG */ + +/* Vrtual Processor Home Node (VPHN) support */ +#ifdef CONFIG_PPC_SPLPAR +#define VPHN_NR_CHANGE_CTRS (8) +static u8 vphn_cpu_change_counts[NR_CPUS][VPHN_NR_CHANGE_CTRS]; +static cpumask_t cpu_associativity_changes_mask; +static int vphn_enabled; +static void set_topology_timer(void); + +/* + * Store the current values of the associativity change counters in the + * hypervisor. + */ +static void setup_cpu_associativity_change_counters(void) +{ + int cpu = 0; + + for_each_possible_cpu(cpu) { + int i = 0; + u8 *counts = vphn_cpu_change_counts[cpu]; + volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; + + for (i = 0; i < VPHN_NR_CHANGE_CTRS; i++) { + counts[i] = hypervisor_counts[i]; + } + } +} + +/* + * The hypervisor maintains a set of 8 associativity change counters in + * the VPA of each cpu that correspond to the associativity levels in the + * ibm,associativity-reference-points property. When an associativity + * level changes, the corresponding counter is incremented. + * + * Set a bit in cpu_associativity_changes_mask for each cpu whose home + * node associativity levels have changed. + * + * Returns the number of cpus with unhandled associativity changes. + */ +static int update_cpu_associativity_changes_mask(void) +{ + int cpu = 0, nr_cpus = 0; + cpumask_t *changes = &cpu_associativity_changes_mask; + + cpumask_clear(changes); + + for_each_possible_cpu(cpu) { + int i, changed = 0; + u8 *counts = vphn_cpu_change_counts[cpu]; + volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; + + for (i = 0; i < VPHN_NR_CHANGE_CTRS; i++) { + if (hypervisor_counts[i] > counts[i]) { + counts[i] = hypervisor_counts[i]; + changed = 1; + } + } + if (changed) { + cpumask_set_cpu(cpu, changes); + nr_cpus++; + } + } + + return nr_cpus; +} + +/* 6 64-bit registers unpacked into 12 32-bit associativity values */ +#define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32)) + +/* + * Convert the associativity domain numbers returned from the hypervisor + * to the sequence they would appear in the ibm,associativity property. + */ +static int vphn_unpack_associativity(const long *packed, unsigned int *unpacked) +{ + int i = 0; + int nr_assoc_doms = 0; + const u16 *field = (const u16*) packed; + +#define VPHN_FIELD_UNUSED (0xffff) +#define VPHN_FIELD_MSB (0x8000) +#define VPHN_FIELD_MASK (~VPHN_FIELD_MSB) + + for (i = 0; i < VPHN_ASSOC_BUFSIZE; i++) { + if (*field == VPHN_FIELD_UNUSED) { + /* All significant fields processed, and remaining + * fields contain the reserved value of all 1's. + * Just store them. + */ + unpacked[i] = *((u32*)field); + field += 2; + } + else if (*field & VPHN_FIELD_MSB) { + /* Data is in the lower 15 bits of this field */ + unpacked[i] = *field & VPHN_FIELD_MASK; + field++; + nr_assoc_doms++; + } + else { + /* Data is in the lower 15 bits of this field + * concatenated with the next 16 bit field + */ + unpacked[i] = *((u32*)field); + field += 2; + nr_assoc_doms++; + } + } + + return nr_assoc_doms; +} + +/* + * Retrieve the new associativity information for a virtual processor's + * home node. + */ +static long hcall_vphn(unsigned long cpu, unsigned int *associativity) +{ + long rc = 0; + long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; + u64 flags = 1; + int hwcpu = get_hard_smp_processor_id(cpu); + + rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu); + vphn_unpack_associativity(retbuf, associativity); + + return rc; +} + +static long vphn_get_associativity(unsigned long cpu, + unsigned int *associativity) +{ + long rc = 0; + + rc = hcall_vphn(cpu, associativity); + + switch (rc) { + case H_FUNCTION: + printk(KERN_INFO + "VPHN is not supported. Disabling polling...\n"); + stop_topology_update(); + break; + case H_HARDWARE: + printk(KERN_ERR + "hcall_vphn() experienced a hardware fault " + "preventing VPHN. Disabling polling...\n"); + stop_topology_update(); + } + + return rc; +} + +/* + * Update the node maps and sysfs entries for each cpu whose home node + * has changed. + */ +int arch_update_cpu_topology(void) +{ + int cpu = 0, nid = 0, old_nid = 0; + unsigned int associativity[VPHN_ASSOC_BUFSIZE] = {0}; + struct sys_device *sysdev = NULL; + + for_each_cpu_mask(cpu, cpu_associativity_changes_mask) { + vphn_get_associativity(cpu, associativity); + nid = associativity_to_nid(associativity); + + if (nid < 0 || !node_online(nid)) + nid = first_online_node; + + old_nid = numa_cpu_lookup_table[cpu]; + + /* Disable hotplug while we update the cpu + * masks and sysfs. + */ + get_online_cpus(); + unregister_cpu_under_node(cpu, old_nid); + unmap_cpu_from_node(cpu); + map_cpu_to_node(cpu, nid); + register_cpu_under_node(cpu, nid); + put_online_cpus(); + + sysdev = get_cpu_sysdev(cpu); + if (sysdev) + kobject_uevent(&sysdev->kobj, KOBJ_CHANGE); + } + + return 1; +} + +static void topology_work_fn(struct work_struct *work) +{ + rebuild_sched_domains(); +} +static DECLARE_WORK(topology_work, topology_work_fn); + +void topology_schedule_update(void) +{ + schedule_work(&topology_work); +} + +static void topology_timer_fn(unsigned long ignored) +{ + if (!vphn_enabled) + return; + if (update_cpu_associativity_changes_mask() > 0) + topology_schedule_update(); + set_topology_timer(); +} +static struct timer_list topology_timer = + TIMER_INITIALIZER(topology_timer_fn, 0, 0); + +static void set_topology_timer(void) +{ + topology_timer.data = 0; + topology_timer.expires = jiffies + 60 * HZ; + add_timer(&topology_timer); +} + +/* + * Start polling for VPHN associativity changes. + */ +int start_topology_update(void) +{ + int rc = 0; + + if (firmware_has_feature(FW_FEATURE_VPHN)) { + vphn_enabled = 1; + setup_cpu_associativity_change_counters(); + init_timer_deferrable(&topology_timer); + set_topology_timer(); + rc = 1; + } + + return rc; +} +__initcall(start_topology_update); + +/* + * Disable polling for VPHN associativity changes. + */ +int stop_topology_update(void) +{ + vphn_enabled = 0; + return del_timer_sync(&topology_timer); +} +#endif /* CONFIG_PPC_SPLPAR */ diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index a87ead0..8dc41c0 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -78,7 +78,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) /* pgdir take page or two with 4K pages and a page fraction otherwise */ #ifndef CONFIG_PPC_4K_PAGES - ret = (pgd_t *)kzalloc(1 << PGDIR_ORDER, GFP_KERNEL); + ret = kzalloc(1 << PGDIR_ORDER, GFP_KERNEL); #else ret = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, PGDIR_ORDER - PAGE_SHIFT); @@ -230,6 +230,7 @@ __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, area = get_vm_area_caller(size, VM_IOREMAP, caller); if (area == 0) return NULL; + area->phys_addr = p; v = (unsigned long) area->addr; } else { v = (ioremap_bot -= size); diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 21d6dfa..88927a0 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -223,6 +223,8 @@ void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size, caller); if (area == NULL) return NULL; + + area->phys_addr = paligned; ret = __ioremap_at(paligned, area->addr, size, flags); if (!ret) vunmap(area->addr); diff --git a/arch/powerpc/oprofile/op_model_cell.c b/arch/powerpc/oprofile/op_model_cell.c index 7fd90d0..c4d2b71 100644 --- a/arch/powerpc/oprofile/op_model_cell.c +++ b/arch/powerpc/oprofile/op_model_cell.c @@ -1469,7 +1469,7 @@ static int cell_global_start(struct op_counter_config *ctr) * The pm_interval register is setup to write the SPU PC value into the * trace buffer at the maximum rate possible. The trace buffer is configured * to store the PCs, wrapping when it is full. The performance counter is - * intialized to the max hardware count minus the number of events, N, between + * initialized to the max hardware count minus the number of events, N, between * samples. Once the N events have occured, a HW counter overflow occurs * causing the generation of a HW counter interrupt which also stops the * writing of the SPU PC values to the trace buffer. Hence the last PC diff --git a/arch/powerpc/platforms/44x/Makefile b/arch/powerpc/platforms/44x/Makefile index 82ff326..c04d16d 100644 --- a/arch/powerpc/platforms/44x/Makefile +++ b/arch/powerpc/platforms/44x/Makefile @@ -1,4 +1,7 @@ -obj-$(CONFIG_44x) := misc_44x.o idle.o +obj-$(CONFIG_44x) += misc_44x.o +ifneq ($(CONFIG_PPC4xx_CPM),y) +obj-$(CONFIG_44x) += idle.o +endif obj-$(CONFIG_PPC44x_SIMPLE) += ppc44x_simple.o obj-$(CONFIG_EBONY) += ebony.o obj-$(CONFIG_SAM440EP) += sam440ep.o diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c index 80234e5..eda0fc2 100644 --- a/arch/powerpc/platforms/52xx/lite5200_pm.c +++ b/arch/powerpc/platforms/52xx/lite5200_pm.c @@ -232,7 +232,7 @@ static void lite5200_pm_end(void) lite5200_pm_target_state = PM_SUSPEND_ON; } -static struct platform_suspend_ops lite5200_pm_ops = { +static const struct platform_suspend_ops lite5200_pm_ops = { .valid = lite5200_pm_valid, .begin = lite5200_pm_begin, .prepare = lite5200_pm_prepare, diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c index 568cef6..8310e8b 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c @@ -186,7 +186,7 @@ void mpc52xx_pm_finish(void) iounmap(mbar); } -static struct platform_suspend_ops mpc52xx_pm_ops = { +static const struct platform_suspend_ops mpc52xx_pm_ops = { .valid = mpc52xx_pm_valid, .prepare = mpc52xx_pm_prepare, .enter = mpc52xx_pm_enter, diff --git a/arch/powerpc/platforms/83xx/suspend-asm.S b/arch/powerpc/platforms/83xx/suspend-asm.S index 1930543..3d1ecd2 100644 --- a/arch/powerpc/platforms/83xx/suspend-asm.S +++ b/arch/powerpc/platforms/83xx/suspend-asm.S @@ -231,7 +231,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) ori r4, r4, 0x002a mtspr SPRN_DBAT0L, r4 lis r8, TMP_VIRT_IMMR@h - ori r4, r8, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r8, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT0U, r4 isync @@ -241,7 +241,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) ori r4, r4, 0x002a mtspr SPRN_DBAT1L, r4 lis r9, (TMP_VIRT_IMMR + 0x01000000)@h - ori r4, r9, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r9, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT1U, r4 isync @@ -253,7 +253,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) li r4, 0x0002 mtspr SPRN_DBAT2L, r4 lis r4, KERNELBASE@h - ori r4, r4, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r4, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT2U, r4 isync diff --git a/arch/powerpc/platforms/83xx/suspend.c b/arch/powerpc/platforms/83xx/suspend.c index 75ae77f..fd4f2f2 100644 --- a/arch/powerpc/platforms/83xx/suspend.c +++ b/arch/powerpc/platforms/83xx/suspend.c @@ -311,7 +311,7 @@ static int mpc83xx_is_pci_agent(void) return ret; } -static struct platform_suspend_ops mpc83xx_suspend_ops = { +static const struct platform_suspend_ops mpc83xx_suspend_ops = { .valid = mpc83xx_suspend_valid, .begin = mpc83xx_suspend_begin, .enter = mpc83xx_suspend_enter, diff --git a/arch/powerpc/platforms/85xx/mpc85xx_mds.c b/arch/powerpc/platforms/85xx/mpc85xx_mds.c index aa34cac..747d1ee 100644 --- a/arch/powerpc/platforms/85xx/mpc85xx_mds.c +++ b/arch/powerpc/platforms/85xx/mpc85xx_mds.c @@ -309,7 +309,7 @@ static void __init mpc85xx_mds_qe_init(void) /* P1021 has pins muxed for QE and other functions. To * enable QE UEC mode, we need to set bit QE0 for UCC1 * in Eth mode, QE0 and QE3 for UCC5 in Eth mode, QE9 - * and QE12 for QE MII management singals in PMUXCR + * and QE12 for QE MII management signals in PMUXCR * register. */ setbits32(pmuxcr, MPC85xx_PMUXCR_QE0 | diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig index 956154f..2057682 100644 --- a/arch/powerpc/platforms/Kconfig +++ b/arch/powerpc/platforms/Kconfig @@ -313,13 +313,14 @@ config OF_RTC source "arch/powerpc/sysdev/bestcomm/Kconfig" config MPC8xxx_GPIO - bool "MPC8xxx GPIO support" - depends on PPC_MPC831x || PPC_MPC834x || PPC_MPC837x || FSL_SOC_BOOKE || PPC_86xx + bool "MPC512x/MPC8xxx GPIO support" + depends on PPC_MPC512x || PPC_MPC831x || PPC_MPC834x || PPC_MPC837x || \ + FSL_SOC_BOOKE || PPC_86xx select GENERIC_GPIO select ARCH_REQUIRE_GPIOLIB help Say Y here if you're going to use hardware that connects to the - MPC831x/834x/837x/8572/8610 GPIOs. + MPC512x/831x/834x/837x/8572/8610 GPIOs. config SIMPLE_GPIO bool "Support for simple, memory-mapped GPIO controllers" diff --git a/arch/powerpc/platforms/cell/beat_iommu.c b/arch/powerpc/platforms/cell/beat_iommu.c index beec405..3ce6855 100644 --- a/arch/powerpc/platforms/cell/beat_iommu.c +++ b/arch/powerpc/platforms/cell/beat_iommu.c @@ -76,7 +76,7 @@ static void __init celleb_init_direct_mapping(void) static void celleb_dma_dev_setup(struct device *dev) { - dev->archdata.dma_ops = get_pci_dma_ops(); + set_dma_ops(dev, &dma_direct_ops); set_dma_offset(dev, celleb_dma_direct_offset); } @@ -106,7 +106,6 @@ static struct notifier_block celleb_of_bus_notifier = { static int __init celleb_init_iommu(void) { celleb_init_direct_mapping(); - set_pci_dma_ops(&dma_direct_ops); ppc_md.pci_dma_dev_setup = celleb_pci_dma_dev_setup; bus_register_notifier(&platform_bus_type, &celleb_of_bus_notifier); diff --git a/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c b/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c index a101abf..3b894f5 100644 --- a/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c +++ b/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c @@ -36,10 +36,9 @@ static int spu_alloc_lscsa_std(struct spu_state *csa) struct spu_lscsa *lscsa; unsigned char *p; - lscsa = vmalloc(sizeof(struct spu_lscsa)); + lscsa = vzalloc(sizeof(struct spu_lscsa)); if (!lscsa) return -ENOMEM; - memset(lscsa, 0, sizeof(struct spu_lscsa)); csa->lscsa = lscsa; /* Set LS pages reserved to allow for user-space mapping. */ diff --git a/arch/powerpc/platforms/chrp/time.c b/arch/powerpc/platforms/chrp/time.c index 054dfe5..f803f4b 100644 --- a/arch/powerpc/platforms/chrp/time.c +++ b/arch/powerpc/platforms/chrp/time.c @@ -29,6 +29,10 @@ extern spinlock_t rtc_lock; +#define NVRAM_AS0 0x74 +#define NVRAM_AS1 0x75 +#define NVRAM_DATA 0x77 + static int nvram_as1 = NVRAM_AS1; static int nvram_as0 = NVRAM_AS0; static int nvram_data = NVRAM_DATA; diff --git a/arch/powerpc/platforms/iseries/mf.c b/arch/powerpc/platforms/iseries/mf.c index 42d0a88..b5e026b 100644 --- a/arch/powerpc/platforms/iseries/mf.c +++ b/arch/powerpc/platforms/iseries/mf.c @@ -1045,71 +1045,9 @@ static const struct file_operations mf_side_proc_fops = { .write = mf_side_proc_write, }; -#if 0 -static void mf_getSrcHistory(char *buffer, int size) -{ - struct IplTypeReturnStuff return_stuff; - struct pending_event *ev = new_pending_event(); - int rc = 0; - char *pages[4]; - - pages[0] = kmalloc(4096, GFP_ATOMIC); - pages[1] = kmalloc(4096, GFP_ATOMIC); - pages[2] = kmalloc(4096, GFP_ATOMIC); - pages[3] = kmalloc(4096, GFP_ATOMIC); - if ((ev == NULL) || (pages[0] == NULL) || (pages[1] == NULL) - || (pages[2] == NULL) || (pages[3] == NULL)) - return -ENOMEM; - - return_stuff.xType = 0; - return_stuff.xRc = 0; - return_stuff.xDone = 0; - ev->event.hp_lp_event.xSubtype = 6; - ev->event.hp_lp_event.x.xSubtypeData = - subtype_data('M', 'F', 'V', 'I'); - ev->event.data.vsp_cmd.xEvent = &return_stuff; - ev->event.data.vsp_cmd.cmd = 4; - ev->event.data.vsp_cmd.lp_index = HvLpConfig_getLpIndex(); - ev->event.data.vsp_cmd.result_code = 0xFF; - ev->event.data.vsp_cmd.reserved = 0; - ev->event.data.vsp_cmd.sub_data.page[0] = iseries_hv_addr(pages[0]); - ev->event.data.vsp_cmd.sub_data.page[1] = iseries_hv_addr(pages[1]); - ev->event.data.vsp_cmd.sub_data.page[2] = iseries_hv_addr(pages[2]); - ev->event.data.vsp_cmd.sub_data.page[3] = iseries_hv_addr(pages[3]); - mb(); - if (signal_event(ev) != 0) - return; - - while (return_stuff.xDone != 1) - udelay(10); - if (return_stuff.xRc == 0) - memcpy(buffer, pages[0], size); - kfree(pages[0]); - kfree(pages[1]); - kfree(pages[2]); - kfree(pages[3]); -} -#endif - static int mf_src_proc_show(struct seq_file *m, void *v) { -#if 0 - int len; - - mf_getSrcHistory(page, count); - len = count; - len -= off; - if (len < count) { - *eof = 1; - if (len <= 0) - return 0; - } else - len = count; - *start = page + off; - return len; -#else return 0; -#endif } static int mf_src_proc_open(struct inode *inode, struct file *file) diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c index 1f9fb2c..14943ef 100644 --- a/arch/powerpc/platforms/pasemi/iommu.c +++ b/arch/powerpc/platforms/pasemi/iommu.c @@ -156,20 +156,12 @@ static void iommu_table_iobmap_setup(void) static void pci_dma_bus_setup_pasemi(struct pci_bus *bus) { - struct device_node *dn; - pr_debug("pci_dma_bus_setup, bus %p, bus->self %p\n", bus, bus->self); if (!iommu_table_iobmap_inited) { iommu_table_iobmap_inited = 1; iommu_table_iobmap_setup(); } - - dn = pci_bus_to_OF_node(bus); - - if (dn) - PCI_DN(dn)->iommu_table = &iommu_table_iobmap; - } @@ -192,9 +184,6 @@ static void pci_dma_dev_setup_pasemi(struct pci_dev *dev) set_iommu_table_base(&dev->dev, &iommu_table_iobmap); } -static void pci_dma_bus_setup_null(struct pci_bus *b) { } -static void pci_dma_dev_setup_null(struct pci_dev *d) { } - int __init iob_init(struct device_node *dn) { unsigned long tmp; @@ -251,14 +240,8 @@ void __init iommu_init_early_pasemi(void) iommu_off = of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL); #endif - if (iommu_off) { - /* Direct I/O, IOMMU off */ - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_null; - ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_null; - set_pci_dma_ops(&dma_direct_ops); - + if (iommu_off) return; - } iob_init(NULL); diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index 9deb274..d5aceb7 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -506,6 +506,15 @@ static int __init pmac_declare_of_platform_devices(void) of_platform_device_create(np, "smu", NULL); of_node_put(np); } + np = of_find_node_by_type(NULL, "fcu"); + if (np == NULL) { + /* Some machines have strangely broken device-tree */ + np = of_find_node_by_path("/u3@0,f8000000/i2c@f8001000/fan@15e"); + } + if (np) { + of_platform_device_create(np, "temperature", NULL); + of_node_put(np); + } return 0; } diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c index b341018..6c4b583 100644 --- a/arch/powerpc/platforms/ps3/device-init.c +++ b/arch/powerpc/platforms/ps3/device-init.c @@ -566,10 +566,10 @@ static int ps3_setup_dynamic_device(const struct ps3_repository_device *repo) case PS3_DEV_TYPE_STOR_DISK: result = ps3_setup_storage_dev(repo, PS3_MATCH_ID_STOR_DISK); - /* Some devices are not accessable from the Other OS lpar. */ + /* Some devices are not accessible from the Other OS lpar. */ if (result == -ENODEV) { result = 0; - pr_debug("%s:%u: not accessable\n", __func__, + pr_debug("%s:%u: not accessible\n", __func__, __LINE__); } diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index 59d9712..92290ff 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -44,7 +44,7 @@ * @lock: * @ipi_debug_brk_mask: * - * The HV mantains per SMT thread mappings of HV outlet to HV plug on + * The HV maintains per SMT thread mappings of HV outlet to HV plug on * behalf of the guest. These mappings are implemented as 256 bit guest * supplied bitmaps indexed by plug number. The addresses of the bitmaps * are registered with the HV through lv1_configure_irq_state_bitmap(). diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index 3139814..5d1b743 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -33,6 +33,16 @@ config PSERIES_MSI depends on PCI_MSI && EEH default y +config PSERIES_ENERGY + tristate "pSeries energy management capabilities driver" + depends on PPC_PSERIES + default y + help + Provides interface to platform energy management capabilities + on supported PSERIES platforms. + Provides: /sys/devices/system/cpu/pseries_(de)activation_hint_list + and /sys/devices/system/cpu/cpuN/pseries_(de)activation_hint + config SCANLOG tristate "Scanlog dump interface" depends on RTAS_PROC && PPC_PSERIES diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile index 59eb8bd..fc52378 100644 --- a/arch/powerpc/platforms/pseries/Makefile +++ b/arch/powerpc/platforms/pseries/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_EEH) += eeh.o eeh_cache.o eeh_driver.o eeh_event.o eeh_sysfs.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_PCI) += pci.o pci_dlpar.o obj-$(CONFIG_PSERIES_MSI) += msi.o +obj-$(CONFIG_PSERIES_ENERGY) += pseries_energy.o obj-$(CONFIG_HOTPLUG_CPU) += hotplug-cpu.o obj-$(CONFIG_MEMORY_HOTPLUG) += hotplug-memory.o diff --git a/arch/powerpc/platforms/pseries/firmware.c b/arch/powerpc/platforms/pseries/firmware.c index 0a14d8c..0b0eff0 100644 --- a/arch/powerpc/platforms/pseries/firmware.c +++ b/arch/powerpc/platforms/pseries/firmware.c @@ -55,6 +55,7 @@ firmware_features_table[FIRMWARE_MAX_FEATURES] = { {FW_FEATURE_XDABR, "hcall-xdabr"}, {FW_FEATURE_MULTITCE, "hcall-multi-tce"}, {FW_FEATURE_SPLPAR, "hcall-splpar"}, + {FW_FEATURE_VPHN, "hcall-vphn"}, }; /* Build up the firmware features bitmask using the contents of diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S index 48d2057..fd05fde 100644 --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -11,6 +11,7 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/ptrace.h> #define STK_PARM(i) (48 + ((i)-3)*8) diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c index e19ff02..f106662 100644 --- a/arch/powerpc/platforms/pseries/hvCall_inst.c +++ b/arch/powerpc/platforms/pseries/hvCall_inst.c @@ -55,7 +55,7 @@ static void hc_stop(struct seq_file *m, void *p) static int hc_show(struct seq_file *m, void *p) { unsigned long h_num = (unsigned long)p; - struct hcall_stats *hs = (struct hcall_stats *)m->private; + struct hcall_stats *hs = m->private; if (hs[h_num].num_calls) { if (cpu_has_feature(CPU_FTR_PURR)) diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index a77bcae..edea60b 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -140,7 +140,7 @@ static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum, return ret; } -static DEFINE_PER_CPU(u64 *, tce_page) = NULL; +static DEFINE_PER_CPU(u64 *, tce_page); static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages, unsigned long uaddr, @@ -323,14 +323,13 @@ static void iommu_table_setparms(struct pci_controller *phb, static void iommu_table_setparms_lpar(struct pci_controller *phb, struct device_node *dn, struct iommu_table *tbl, - const void *dma_window, - int bussubno) + const void *dma_window) { unsigned long offset, size; - tbl->it_busno = bussubno; of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size); + tbl->it_busno = phb->bus->number; tbl->it_base = 0; tbl->it_blocksize = 16; tbl->it_type = TCE_PCI; @@ -450,14 +449,10 @@ static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) if (!ppci->iommu_table) { tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, ppci->phb->node); - iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window, - bus->number); + iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window); ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node); pr_debug(" created table: %p\n", ppci->iommu_table); } - - if (pdn != dn) - PCI_DN(dn)->iommu_table = ppci->iommu_table; } @@ -533,21 +528,11 @@ static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev) } pr_debug(" parent is %s\n", pdn->full_name); - /* Check for parent == NULL so we don't try to setup the empty EADS - * slots on POWER4 machines. - */ - if (dma_window == NULL || pdn->parent == NULL) { - pr_debug(" no dma window for device, linking to parent\n"); - set_iommu_table_base(&dev->dev, PCI_DN(pdn)->iommu_table); - return; - } - pci = PCI_DN(pdn); if (!pci->iommu_table) { tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, pci->phb->node); - iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window, - pci->phb->bus->number); + iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window); pci->iommu_table = iommu_init_table(tbl, pci->phb->node); pr_debug(" created table: %p\n", pci->iommu_table); } else { @@ -571,8 +556,7 @@ static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long acti switch (action) { case PSERIES_RECONFIG_REMOVE: - if (pci && pci->iommu_table && - of_get_property(np, "ibm,dma-window", NULL)) + if (pci && pci->iommu_table) iommu_free_table(pci->iommu_table, np->full_name); break; default: @@ -589,13 +573,8 @@ static struct notifier_block iommu_reconfig_nb = { /* These are called very early. */ void iommu_init_early_pSeries(void) { - if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) { - /* Direct I/O, IOMMU off */ - ppc_md.pci_dma_dev_setup = NULL; - ppc_md.pci_dma_bus_setup = NULL; - set_pci_dma_ops(&dma_direct_ops); + if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL)) return; - } if (firmware_has_feature(FW_FEATURE_LPAR)) { if (firmware_has_feature(FW_FEATURE_MULTITCE)) { @@ -622,3 +601,17 @@ void iommu_init_early_pSeries(void) set_pci_dma_ops(&dma_iommu_ops); } +static int __init disable_multitce(char *str) +{ + if (strcmp(str, "off") == 0 && + firmware_has_feature(FW_FEATURE_LPAR) && + firmware_has_feature(FW_FEATURE_MULTITCE)) { + printk(KERN_INFO "Disabling MULTITCE firmware feature\n"); + ppc_md.tce_build = tce_build_pSeriesLP; + ppc_md.tce_free = tce_free_pSeriesLP; + powerpc_firmware_features &= ~FW_FEATURE_MULTITCE; + } + return 1; +} + +__setup("multitce=", disable_multitce); diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index f129040..5d3ea9f 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -627,6 +627,18 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local) spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags); } +static int __init disable_bulk_remove(char *str) +{ + if (strcmp(str, "off") == 0 && + firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { + printk(KERN_INFO "Disabling BULK_REMOVE firmware feature"); + powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; + } + return 1; +} + +__setup("bulk_remove=", disable_bulk_remove); + void __init hpte_init_lpar(void) { ppc_md.hpte_invalidate = pSeries_lpar_hpte_invalidate; diff --git a/arch/powerpc/platforms/pseries/nvram.c b/arch/powerpc/platforms/pseries/nvram.c index bc3c7f2..7e828ba 100644 --- a/arch/powerpc/platforms/pseries/nvram.c +++ b/arch/powerpc/platforms/pseries/nvram.c @@ -22,11 +22,25 @@ #include <asm/prom.h> #include <asm/machdep.h> +/* Max bytes to read/write in one go */ +#define NVRW_CNT 0x20 + static unsigned int nvram_size; static int nvram_fetch, nvram_store; static char nvram_buf[NVRW_CNT]; /* assume this is in the first 4GB */ static DEFINE_SPINLOCK(nvram_lock); +static long nvram_error_log_index = -1; +static long nvram_error_log_size = 0; + +struct err_log_info { + int error_type; + unsigned int seq_num; +}; +#define NVRAM_MAX_REQ 2079 +#define NVRAM_MIN_REQ 1055 + +#define NVRAM_LOG_PART_NAME "ibm,rtas-log" static ssize_t pSeries_nvram_read(char *buf, size_t count, loff_t *index) { @@ -119,6 +133,197 @@ static ssize_t pSeries_nvram_get_size(void) return nvram_size ? nvram_size : -ENODEV; } + +/* nvram_write_error_log + * + * We need to buffer the error logs into nvram to ensure that we have + * the failure information to decode. If we have a severe error there + * is no way to guarantee that the OS or the machine is in a state to + * get back to user land and write the error to disk. For example if + * the SCSI device driver causes a Machine Check by writing to a bad + * IO address, there is no way of guaranteeing that the device driver + * is in any state that is would also be able to write the error data + * captured to disk, thus we buffer it in NVRAM for analysis on the + * next boot. + * + * In NVRAM the partition containing the error log buffer will looks like: + * Header (in bytes): + * +-----------+----------+--------+------------+------------------+ + * | signature | checksum | length | name | data | + * |0 |1 |2 3|4 15|16 length-1| + * +-----------+----------+--------+------------+------------------+ + * + * The 'data' section would look like (in bytes): + * +--------------+------------+-----------------------------------+ + * | event_logged | sequence # | error log | + * |0 3|4 7|8 nvram_error_log_size-1| + * +--------------+------------+-----------------------------------+ + * + * event_logged: 0 if event has not been logged to syslog, 1 if it has + * sequence #: The unique sequence # for each event. (until it wraps) + * error log: The error log from event_scan + */ +int nvram_write_error_log(char * buff, int length, + unsigned int err_type, unsigned int error_log_cnt) +{ + int rc; + loff_t tmp_index; + struct err_log_info info; + + if (nvram_error_log_index == -1) { + return -ESPIPE; + } + + if (length > nvram_error_log_size) { + length = nvram_error_log_size; + } + + info.error_type = err_type; + info.seq_num = error_log_cnt; + + tmp_index = nvram_error_log_index; + + rc = ppc_md.nvram_write((char *)&info, sizeof(struct err_log_info), &tmp_index); + if (rc <= 0) { + printk(KERN_ERR "nvram_write_error_log: Failed nvram_write (%d)\n", rc); + return rc; + } + + rc = ppc_md.nvram_write(buff, length, &tmp_index); + if (rc <= 0) { + printk(KERN_ERR "nvram_write_error_log: Failed nvram_write (%d)\n", rc); + return rc; + } + + return 0; +} + +/* nvram_read_error_log + * + * Reads nvram for error log for at most 'length' + */ +int nvram_read_error_log(char * buff, int length, + unsigned int * err_type, unsigned int * error_log_cnt) +{ + int rc; + loff_t tmp_index; + struct err_log_info info; + + if (nvram_error_log_index == -1) + return -1; + + if (length > nvram_error_log_size) + length = nvram_error_log_size; + + tmp_index = nvram_error_log_index; + + rc = ppc_md.nvram_read((char *)&info, sizeof(struct err_log_info), &tmp_index); + if (rc <= 0) { + printk(KERN_ERR "nvram_read_error_log: Failed nvram_read (%d)\n", rc); + return rc; + } + + rc = ppc_md.nvram_read(buff, length, &tmp_index); + if (rc <= 0) { + printk(KERN_ERR "nvram_read_error_log: Failed nvram_read (%d)\n", rc); + return rc; + } + + *error_log_cnt = info.seq_num; + *err_type = info.error_type; + + return 0; +} + +/* This doesn't actually zero anything, but it sets the event_logged + * word to tell that this event is safely in syslog. + */ +int nvram_clear_error_log(void) +{ + loff_t tmp_index; + int clear_word = ERR_FLAG_ALREADY_LOGGED; + int rc; + + if (nvram_error_log_index == -1) + return -1; + + tmp_index = nvram_error_log_index; + + rc = ppc_md.nvram_write((char *)&clear_word, sizeof(int), &tmp_index); + if (rc <= 0) { + printk(KERN_ERR "nvram_clear_error_log: Failed nvram_write (%d)\n", rc); + return rc; + } + + return 0; +} + +/* pseries_nvram_init_log_partition + * + * This will setup the partition we need for buffering the + * error logs and cleanup partitions if needed. + * + * The general strategy is the following: + * 1.) If there is log partition large enough then use it. + * 2.) If there is none large enough, search + * for a free partition that is large enough. + * 3.) If there is not a free partition large enough remove + * _all_ OS partitions and consolidate the space. + * 4.) Will first try getting a chunk that will satisfy the maximum + * error log size (NVRAM_MAX_REQ). + * 5.) If the max chunk cannot be allocated then try finding a chunk + * that will satisfy the minum needed (NVRAM_MIN_REQ). + */ +static int __init pseries_nvram_init_log_partition(void) +{ + loff_t p; + int size; + + /* Scan nvram for partitions */ + nvram_scan_partitions(); + + /* Lookg for ours */ + p = nvram_find_partition(NVRAM_LOG_PART_NAME, NVRAM_SIG_OS, &size); + + /* Found one but too small, remove it */ + if (p && size < NVRAM_MIN_REQ) { + pr_info("nvram: Found too small "NVRAM_LOG_PART_NAME" partition" + ",removing it..."); + nvram_remove_partition(NVRAM_LOG_PART_NAME, NVRAM_SIG_OS); + p = 0; + } + + /* Create one if we didn't find */ + if (!p) { + p = nvram_create_partition(NVRAM_LOG_PART_NAME, NVRAM_SIG_OS, + NVRAM_MAX_REQ, NVRAM_MIN_REQ); + /* No room for it, try to get rid of any OS partition + * and try again + */ + if (p == -ENOSPC) { + pr_info("nvram: No room to create "NVRAM_LOG_PART_NAME + " partition, deleting all OS partitions..."); + nvram_remove_partition(NULL, NVRAM_SIG_OS); + p = nvram_create_partition(NVRAM_LOG_PART_NAME, + NVRAM_SIG_OS, NVRAM_MAX_REQ, + NVRAM_MIN_REQ); + } + } + + if (p <= 0) { + pr_err("nvram: Failed to find or create "NVRAM_LOG_PART_NAME + " partition, err %d\n", (int)p); + return 0; + } + + nvram_error_log_index = p; + nvram_error_log_size = nvram_get_partition_size(p) - + sizeof(struct err_log_info); + + return 0; +} +machine_arch_initcall(pseries, pseries_nvram_init_log_partition); + int __init pSeries_nvram_init(void) { struct device_node *nvram; diff --git a/arch/powerpc/platforms/pseries/pseries_energy.c b/arch/powerpc/platforms/pseries/pseries_energy.c new file mode 100644 index 0000000..c8b3c69 --- /dev/null +++ b/arch/powerpc/platforms/pseries/pseries_energy.c @@ -0,0 +1,326 @@ +/* + * POWER platform energy management driver + * Copyright (C) 2010 IBM Corporation + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This pseries platform device driver provides access to + * platform energy management capabilities. + */ + +#include <linux/module.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/seq_file.h> +#include <linux/sysdev.h> +#include <linux/cpu.h> +#include <linux/of.h> +#include <asm/cputhreads.h> +#include <asm/page.h> +#include <asm/hvcall.h> + + +#define MODULE_VERS "1.0" +#define MODULE_NAME "pseries_energy" + +/* Driver flags */ + +static int sysfs_entries; + +/* Helper routines */ + +/* + * Routine to detect firmware support for hcall + * return 1 if H_BEST_ENERGY is supported + * else return 0 + */ + +static int check_for_h_best_energy(void) +{ + struct device_node *rtas = NULL; + const char *hypertas, *s; + int length; + int rc = 0; + + rtas = of_find_node_by_path("/rtas"); + if (!rtas) + return 0; + + hypertas = of_get_property(rtas, "ibm,hypertas-functions", &length); + if (!hypertas) { + of_node_put(rtas); + return 0; + } + + /* hypertas will have list of strings with hcall names */ + for (s = hypertas; s < hypertas + length; s += strlen(s) + 1) { + if (!strncmp("hcall-best-energy-1", s, 19)) { + rc = 1; /* Found the string */ + break; + } + } + of_node_put(rtas); + return rc; +} + +/* Helper Routines to convert between drc_index to cpu numbers */ + +static u32 cpu_to_drc_index(int cpu) +{ + struct device_node *dn = NULL; + const int *indexes; + int i; + int rc = 1; + u32 ret = 0; + + dn = of_find_node_by_path("/cpus"); + if (dn == NULL) + goto err; + indexes = of_get_property(dn, "ibm,drc-indexes", NULL); + if (indexes == NULL) + goto err_of_node_put; + /* Convert logical cpu number to core number */ + i = cpu_core_index_of_thread(cpu); + /* + * The first element indexes[0] is the number of drc_indexes + * returned in the list. Hence i+1 will get the drc_index + * corresponding to core number i. + */ + WARN_ON(i > indexes[0]); + ret = indexes[i + 1]; + rc = 0; + +err_of_node_put: + of_node_put(dn); +err: + if (rc) + printk(KERN_WARNING "cpu_to_drc_index(%d) failed", cpu); + return ret; +} + +static int drc_index_to_cpu(u32 drc_index) +{ + struct device_node *dn = NULL; + const int *indexes; + int i, cpu = 0; + int rc = 1; + + dn = of_find_node_by_path("/cpus"); + if (dn == NULL) + goto err; + indexes = of_get_property(dn, "ibm,drc-indexes", NULL); + if (indexes == NULL) + goto err_of_node_put; + /* + * First element in the array is the number of drc_indexes + * returned. Search through the list to find the matching + * drc_index and get the core number + */ + for (i = 0; i < indexes[0]; i++) { + if (indexes[i + 1] == drc_index) + break; + } + /* Convert core number to logical cpu number */ + cpu = cpu_first_thread_of_core(i); + rc = 0; + +err_of_node_put: + of_node_put(dn); +err: + if (rc) + printk(KERN_WARNING "drc_index_to_cpu(%d) failed", drc_index); + return cpu; +} + +/* + * pseries hypervisor call H_BEST_ENERGY provides hints to OS on + * preferred logical cpus to activate or deactivate for optimized + * energy consumption. + */ + +#define FLAGS_MODE1 0x004E200000080E01 +#define FLAGS_MODE2 0x004E200000080401 +#define FLAGS_ACTIVATE 0x100 + +static ssize_t get_best_energy_list(char *page, int activate) +{ + int rc, cnt, i, cpu; + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; + unsigned long flags = 0; + u32 *buf_page; + char *s = page; + + buf_page = (u32 *) get_zeroed_page(GFP_KERNEL); + if (!buf_page) + return -ENOMEM; + + flags = FLAGS_MODE1; + if (activate) + flags |= FLAGS_ACTIVATE; + + rc = plpar_hcall9(H_BEST_ENERGY, retbuf, flags, 0, __pa(buf_page), + 0, 0, 0, 0, 0, 0); + if (rc != H_SUCCESS) { + free_page((unsigned long) buf_page); + return -EINVAL; + } + + cnt = retbuf[0]; + for (i = 0; i < cnt; i++) { + cpu = drc_index_to_cpu(buf_page[2*i+1]); + if ((cpu_online(cpu) && !activate) || + (!cpu_online(cpu) && activate)) + s += sprintf(s, "%d,", cpu); + } + if (s > page) { /* Something to show */ + s--; /* Suppress last comma */ + s += sprintf(s, "\n"); + } + + free_page((unsigned long) buf_page); + return s-page; +} + +static ssize_t get_best_energy_data(struct sys_device *dev, + char *page, int activate) +{ + int rc; + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; + unsigned long flags = 0; + + flags = FLAGS_MODE2; + if (activate) + flags |= FLAGS_ACTIVATE; + + rc = plpar_hcall9(H_BEST_ENERGY, retbuf, flags, + cpu_to_drc_index(dev->id), + 0, 0, 0, 0, 0, 0, 0); + + if (rc != H_SUCCESS) + return -EINVAL; + + return sprintf(page, "%lu\n", retbuf[1] >> 32); +} + +/* Wrapper functions */ + +static ssize_t cpu_activate_hint_list_show(struct sysdev_class *class, + struct sysdev_class_attribute *attr, char *page) +{ + return get_best_energy_list(page, 1); +} + +static ssize_t cpu_deactivate_hint_list_show(struct sysdev_class *class, + struct sysdev_class_attribute *attr, char *page) +{ + return get_best_energy_list(page, 0); +} + +static ssize_t percpu_activate_hint_show(struct sys_device *dev, + struct sysdev_attribute *attr, char *page) +{ + return get_best_energy_data(dev, page, 1); +} + +static ssize_t percpu_deactivate_hint_show(struct sys_device *dev, + struct sysdev_attribute *attr, char *page) +{ + return get_best_energy_data(dev, page, 0); +} + +/* + * Create sysfs interface: + * /sys/devices/system/cpu/pseries_activate_hint_list + * /sys/devices/system/cpu/pseries_deactivate_hint_list + * Comma separated list of cpus to activate or deactivate + * /sys/devices/system/cpu/cpuN/pseries_activate_hint + * /sys/devices/system/cpu/cpuN/pseries_deactivate_hint + * Per-cpu value of the hint + */ + +struct sysdev_class_attribute attr_cpu_activate_hint_list = + _SYSDEV_CLASS_ATTR(pseries_activate_hint_list, 0444, + cpu_activate_hint_list_show, NULL); + +struct sysdev_class_attribute attr_cpu_deactivate_hint_list = + _SYSDEV_CLASS_ATTR(pseries_deactivate_hint_list, 0444, + cpu_deactivate_hint_list_show, NULL); + +struct sysdev_attribute attr_percpu_activate_hint = + _SYSDEV_ATTR(pseries_activate_hint, 0444, + percpu_activate_hint_show, NULL); + +struct sysdev_attribute attr_percpu_deactivate_hint = + _SYSDEV_ATTR(pseries_deactivate_hint, 0444, + percpu_deactivate_hint_show, NULL); + +static int __init pseries_energy_init(void) +{ + int cpu, err; + struct sys_device *cpu_sys_dev; + + if (!check_for_h_best_energy()) { + printk(KERN_INFO "Hypercall H_BEST_ENERGY not supported\n"); + return 0; + } + /* Create the sysfs files */ + err = sysfs_create_file(&cpu_sysdev_class.kset.kobj, + &attr_cpu_activate_hint_list.attr); + if (!err) + err = sysfs_create_file(&cpu_sysdev_class.kset.kobj, + &attr_cpu_deactivate_hint_list.attr); + + if (err) + return err; + for_each_possible_cpu(cpu) { + cpu_sys_dev = get_cpu_sysdev(cpu); + err = sysfs_create_file(&cpu_sys_dev->kobj, + &attr_percpu_activate_hint.attr); + if (err) + break; + err = sysfs_create_file(&cpu_sys_dev->kobj, + &attr_percpu_deactivate_hint.attr); + if (err) + break; + } + + if (err) + return err; + + sysfs_entries = 1; /* Removed entries on cleanup */ + return 0; + +} + +static void __exit pseries_energy_cleanup(void) +{ + int cpu; + struct sys_device *cpu_sys_dev; + + if (!sysfs_entries) + return; + + /* Remove the sysfs files */ + sysfs_remove_file(&cpu_sysdev_class.kset.kobj, + &attr_cpu_activate_hint_list.attr); + + sysfs_remove_file(&cpu_sysdev_class.kset.kobj, + &attr_cpu_deactivate_hint_list.attr); + + for_each_possible_cpu(cpu) { + cpu_sys_dev = get_cpu_sysdev(cpu); + sysfs_remove_file(&cpu_sys_dev->kobj, + &attr_percpu_activate_hint.attr); + sysfs_remove_file(&cpu_sys_dev->kobj, + &attr_percpu_deactivate_hint.attr); + } +} + +module_init(pseries_energy_init); +module_exit(pseries_energy_cleanup); +MODULE_DESCRIPTION("Driver for pSeries platform energy management"); +MODULE_AUTHOR("Vaidyanathan Srinivasan"); +MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c index ed72098..a8ca289 100644 --- a/arch/powerpc/platforms/pseries/suspend.c +++ b/arch/powerpc/platforms/pseries/suspend.c @@ -153,7 +153,7 @@ static struct sysdev_class suspend_sysdev_class = { .name = "power", }; -static struct platform_suspend_ops pseries_suspend_ops = { +static const struct platform_suspend_ops pseries_suspend_ops = { .valid = suspend_valid_only_mem, .begin = pseries_suspend_begin, .prepare_late = pseries_prepare_late, diff --git a/arch/powerpc/sysdev/Makefile b/arch/powerpc/sysdev/Makefile index 0bef9da..9c29734 100644 --- a/arch/powerpc/sysdev/Makefile +++ b/arch/powerpc/sysdev/Makefile @@ -41,6 +41,7 @@ obj-$(CONFIG_OF_RTC) += of_rtc.o ifeq ($(CONFIG_PCI),y) obj-$(CONFIG_4xx) += ppc4xx_pci.o endif +obj-$(CONFIG_PPC4xx_CPM) += ppc4xx_cpm.o obj-$(CONFIG_PPC4xx_GPIO) += ppc4xx_gpio.o obj-$(CONFIG_CPM) += cpm_common.o diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 17cf15e..8e9e06a 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -312,17 +312,10 @@ static void pci_dma_dev_setup_dart(struct pci_dev *dev) static void pci_dma_bus_setup_dart(struct pci_bus *bus) { - struct device_node *dn; - if (!iommu_table_dart_inited) { iommu_table_dart_inited = 1; iommu_table_dart_setup(); } - - dn = pci_bus_to_OF_node(bus); - - if (dn) - PCI_DN(dn)->iommu_table = &iommu_table_dart; } static bool dart_device_on_pcie(struct device *dev) @@ -373,7 +366,7 @@ void __init iommu_init_early_dart(void) if (dn == NULL) { dn = of_find_compatible_node(NULL, "dart", "u4-dart"); if (dn == NULL) - goto bail; + return; /* use default direct_dma_ops */ dart_is_u4 = 1; } diff --git a/arch/powerpc/sysdev/fsl_pmc.c b/arch/powerpc/sysdev/fsl_pmc.c index 44de855..e9381bf 100644 --- a/arch/powerpc/sysdev/fsl_pmc.c +++ b/arch/powerpc/sysdev/fsl_pmc.c @@ -53,7 +53,7 @@ static int pmc_suspend_valid(suspend_state_t state) return 1; } -static struct platform_suspend_ops pmc_suspend_ops = { +static const struct platform_suspend_ops pmc_suspend_ops = { .valid = pmc_suspend_valid, .enter = pmc_suspend_enter, }; diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index 9725369..9f99bef 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -973,7 +973,6 @@ fsl_rio_dbell_handler(int irq, void *dev_instance) if (dsr & DOORBELL_DSR_QFI) { pr_info("RIO: doorbell queue full\n"); out_be32(&priv->msg_regs->dsr, DOORBELL_DSR_QFI); - goto out; } /* XXX Need to check/dispatch until queue empty */ diff --git a/arch/powerpc/sysdev/mpc8xxx_gpio.c b/arch/powerpc/sysdev/mpc8xxx_gpio.c index c0ea05e..c48cd81 100644 --- a/arch/powerpc/sysdev/mpc8xxx_gpio.c +++ b/arch/powerpc/sysdev/mpc8xxx_gpio.c @@ -1,5 +1,5 @@ /* - * GPIOs on MPC8349/8572/8610 and compatible + * GPIOs on MPC512x/8349/8572/8610 and compatible * * Copyright (C) 2008 Peter Korsgaard <jacmet@sunsite.dk> * @@ -26,6 +26,7 @@ #define GPIO_IER 0x0c #define GPIO_IMR 0x10 #define GPIO_ICR 0x14 +#define GPIO_ICR2 0x18 struct mpc8xxx_gpio_chip { struct of_mm_gpio_chip mm_gc; @@ -37,6 +38,7 @@ struct mpc8xxx_gpio_chip { */ u32 data; struct irq_host *irq; + void *of_dev_id_data; }; static inline u32 mpc8xxx_gpio2mask(unsigned int gpio) @@ -215,6 +217,51 @@ static int mpc8xxx_irq_set_type(unsigned int virq, unsigned int flow_type) return 0; } +static int mpc512x_irq_set_type(unsigned int virq, unsigned int flow_type) +{ + struct mpc8xxx_gpio_chip *mpc8xxx_gc = get_irq_chip_data(virq); + struct of_mm_gpio_chip *mm = &mpc8xxx_gc->mm_gc; + unsigned long gpio = virq_to_hw(virq); + void __iomem *reg; + unsigned int shift; + unsigned long flags; + + if (gpio < 16) { + reg = mm->regs + GPIO_ICR; + shift = (15 - gpio) * 2; + } else { + reg = mm->regs + GPIO_ICR2; + shift = (15 - (gpio % 16)) * 2; + } + + switch (flow_type) { + case IRQ_TYPE_EDGE_FALLING: + case IRQ_TYPE_LEVEL_LOW: + spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + clrsetbits_be32(reg, 3 << shift, 2 << shift); + spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + break; + + case IRQ_TYPE_EDGE_RISING: + case IRQ_TYPE_LEVEL_HIGH: + spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + clrsetbits_be32(reg, 3 << shift, 1 << shift); + spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + break; + + case IRQ_TYPE_EDGE_BOTH: + spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + clrbits32(reg, 3 << shift); + spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + break; + + default: + return -EINVAL; + } + + return 0; +} + static struct irq_chip mpc8xxx_irq_chip = { .name = "mpc8xxx-gpio", .unmask = mpc8xxx_irq_unmask, @@ -226,6 +273,11 @@ static struct irq_chip mpc8xxx_irq_chip = { static int mpc8xxx_gpio_irq_map(struct irq_host *h, unsigned int virq, irq_hw_number_t hw) { + struct mpc8xxx_gpio_chip *mpc8xxx_gc = h->host_data; + + if (mpc8xxx_gc->of_dev_id_data) + mpc8xxx_irq_chip.set_type = mpc8xxx_gc->of_dev_id_data; + set_irq_chip_data(virq, h->host_data); set_irq_chip_and_handler(virq, &mpc8xxx_irq_chip, handle_level_irq); set_irq_type(virq, IRQ_TYPE_NONE); @@ -253,11 +305,20 @@ static struct irq_host_ops mpc8xxx_gpio_irq_ops = { .xlate = mpc8xxx_gpio_irq_xlate, }; +static struct of_device_id mpc8xxx_gpio_ids[] __initdata = { + { .compatible = "fsl,mpc8349-gpio", }, + { .compatible = "fsl,mpc8572-gpio", }, + { .compatible = "fsl,mpc8610-gpio", }, + { .compatible = "fsl,mpc5121-gpio", .data = mpc512x_irq_set_type, }, + {} +}; + static void __init mpc8xxx_add_controller(struct device_node *np) { struct mpc8xxx_gpio_chip *mpc8xxx_gc; struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; + const struct of_device_id *id; unsigned hwirq; int ret; @@ -297,6 +358,10 @@ static void __init mpc8xxx_add_controller(struct device_node *np) if (!mpc8xxx_gc->irq) goto skip_irq; + id = of_match_node(mpc8xxx_gpio_ids, np); + if (id) + mpc8xxx_gc->of_dev_id_data = id->data; + mpc8xxx_gc->irq->host_data = mpc8xxx_gc; /* ack and mask all irqs */ @@ -321,13 +386,7 @@ static int __init mpc8xxx_add_gpiochips(void) { struct device_node *np; - for_each_compatible_node(np, NULL, "fsl,mpc8349-gpio") - mpc8xxx_add_controller(np); - - for_each_compatible_node(np, NULL, "fsl,mpc8572-gpio") - mpc8xxx_add_controller(np); - - for_each_compatible_node(np, NULL, "fsl,mpc8610-gpio") + for_each_matching_node(np, mpc8xxx_gpio_ids) mpc8xxx_add_controller(np); for_each_compatible_node(np, NULL, "fsl,qoriq-gpio") diff --git a/arch/powerpc/sysdev/ppc4xx_cpm.c b/arch/powerpc/sysdev/ppc4xx_cpm.c new file mode 100644 index 0000000..73b86cc --- /dev/null +++ b/arch/powerpc/sysdev/ppc4xx_cpm.c @@ -0,0 +1,346 @@ +/* + * PowerPC 4xx Clock and Power Management + * + * Copyright (C) 2010, Applied Micro Circuits Corporation + * Victor Gallardo (vgallardo@apm.com) + * + * Based on arch/powerpc/platforms/44x/idle.c: + * Jerone Young <jyoung5@us.ibm.com> + * Copyright 2008 IBM Corp. + * + * Based on arch/powerpc/sysdev/fsl_pmc.c: + * Anton Vorontsov <avorontsov@ru.mvista.com> + * Copyright 2009 MontaVista Software, Inc. + * + * See file CREDITS for list of people who contributed to this + * project. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, + * MA 02111-1307 USA + */ + +#include <linux/kernel.h> +#include <linux/of_platform.h> +#include <linux/sysfs.h> +#include <linux/cpu.h> +#include <linux/suspend.h> +#include <asm/dcr.h> +#include <asm/dcr-native.h> +#include <asm/machdep.h> + +#define CPM_ER 0 +#define CPM_FR 1 +#define CPM_SR 2 + +#define CPM_IDLE_WAIT 0 +#define CPM_IDLE_DOZE 1 + +struct cpm { + dcr_host_t dcr_host; + unsigned int dcr_offset[3]; + unsigned int powersave_off; + unsigned int unused; + unsigned int idle_doze; + unsigned int standby; + unsigned int suspend; +}; + +static struct cpm cpm; + +struct cpm_idle_mode { + unsigned int enabled; + const char *name; +}; + +static struct cpm_idle_mode idle_mode[] = { + [CPM_IDLE_WAIT] = { 1, "wait" }, /* default */ + [CPM_IDLE_DOZE] = { 0, "doze" }, +}; + +static unsigned int cpm_set(unsigned int cpm_reg, unsigned int mask) +{ + unsigned int value; + + /* CPM controller supports 3 different types of sleep interface + * known as class 1, 2 and 3. For class 1 units, they are + * unconditionally put to sleep when the corresponding CPM bit is + * set. For class 2 and 3 units this is not case; if they can be + * put to to sleep, they will. Here we do not verify, we just + * set them and expect them to eventually go off when they can. + */ + value = dcr_read(cpm.dcr_host, cpm.dcr_offset[cpm_reg]); + dcr_write(cpm.dcr_host, cpm.dcr_offset[cpm_reg], value | mask); + + /* return old state, to restore later if needed */ + return value; +} + +static void cpm_idle_wait(void) +{ + unsigned long msr_save; + + /* save off initial state */ + msr_save = mfmsr(); + /* sync required when CPM0_ER[CPU] is set */ + mb(); + /* set wait state MSR */ + mtmsr(msr_save|MSR_WE|MSR_EE|MSR_CE|MSR_DE); + isync(); + /* return to initial state */ + mtmsr(msr_save); + isync(); +} + +static void cpm_idle_sleep(unsigned int mask) +{ + unsigned int er_save; + + /* update CPM_ER state */ + er_save = cpm_set(CPM_ER, mask); + + /* go to wait state so that CPM0_ER[CPU] can take effect */ + cpm_idle_wait(); + + /* restore CPM_ER state */ + dcr_write(cpm.dcr_host, cpm.dcr_offset[CPM_ER], er_save); +} + +static void cpm_idle_doze(void) +{ + cpm_idle_sleep(cpm.idle_doze); +} + +static void cpm_idle_config(int mode) +{ + int i; + + if (idle_mode[mode].enabled) + return; + + for (i = 0; i < ARRAY_SIZE(idle_mode); i++) + idle_mode[i].enabled = 0; + + idle_mode[mode].enabled = 1; +} + +static ssize_t cpm_idle_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + char *s = buf; + int i; + + for (i = 0; i < ARRAY_SIZE(idle_mode); i++) { + if (idle_mode[i].enabled) + s += sprintf(s, "[%s] ", idle_mode[i].name); + else + s += sprintf(s, "%s ", idle_mode[i].name); + } + + *(s-1) = '\n'; /* convert the last space to a newline */ + + return s - buf; +} + +static ssize_t cpm_idle_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + int i; + char *p; + int len; + + p = memchr(buf, '\n', n); + len = p ? p - buf : n; + + for (i = 0; i < ARRAY_SIZE(idle_mode); i++) { + if (strncmp(buf, idle_mode[i].name, len) == 0) { + cpm_idle_config(i); + return n; + } + } + + return -EINVAL; +} + +static struct kobj_attribute cpm_idle_attr = + __ATTR(idle, 0644, cpm_idle_show, cpm_idle_store); + +static void cpm_idle_config_sysfs(void) +{ + struct sys_device *sys_dev; + unsigned long ret; + + sys_dev = get_cpu_sysdev(0); + + ret = sysfs_create_file(&sys_dev->kobj, + &cpm_idle_attr.attr); + if (ret) + printk(KERN_WARNING + "cpm: failed to create idle sysfs entry\n"); +} + +static void cpm_idle(void) +{ + if (idle_mode[CPM_IDLE_DOZE].enabled) + cpm_idle_doze(); + else + cpm_idle_wait(); +} + +static int cpm_suspend_valid(suspend_state_t state) +{ + switch (state) { + case PM_SUSPEND_STANDBY: + return !!cpm.standby; + case PM_SUSPEND_MEM: + return !!cpm.suspend; + default: + return 0; + } +} + +static void cpm_suspend_standby(unsigned int mask) +{ + unsigned long tcr_save; + + /* disable decrement interrupt */ + tcr_save = mfspr(SPRN_TCR); + mtspr(SPRN_TCR, tcr_save & ~TCR_DIE); + + /* go to sleep state */ + cpm_idle_sleep(mask); + + /* restore decrement interrupt */ + mtspr(SPRN_TCR, tcr_save); +} + +static int cpm_suspend_enter(suspend_state_t state) +{ + switch (state) { + case PM_SUSPEND_STANDBY: + cpm_suspend_standby(cpm.standby); + break; + case PM_SUSPEND_MEM: + cpm_suspend_standby(cpm.suspend); + break; + } + + return 0; +} + +static struct platform_suspend_ops cpm_suspend_ops = { + .valid = cpm_suspend_valid, + .enter = cpm_suspend_enter, +}; + +static int cpm_get_uint_property(struct device_node *np, + const char *name) +{ + int len; + const unsigned int *prop = of_get_property(np, name, &len); + + if (prop == NULL || len < sizeof(u32)) + return 0; + + return *prop; +} + +static int __init cpm_init(void) +{ + struct device_node *np; + int dcr_base, dcr_len; + int ret = 0; + + if (!cpm.powersave_off) { + cpm_idle_config(CPM_IDLE_WAIT); + ppc_md.power_save = &cpm_idle; + } + + np = of_find_compatible_node(NULL, NULL, "ibm,cpm"); + if (!np) { + ret = -EINVAL; + goto out; + } + + dcr_base = dcr_resource_start(np, 0); + dcr_len = dcr_resource_len(np, 0); + + if (dcr_base == 0 || dcr_len == 0) { + printk(KERN_ERR "cpm: could not parse dcr property for %s\n", + np->full_name); + ret = -EINVAL; + goto out; + } + + cpm.dcr_host = dcr_map(np, dcr_base, dcr_len); + + if (!DCR_MAP_OK(cpm.dcr_host)) { + printk(KERN_ERR "cpm: failed to map dcr property for %s\n", + np->full_name); + ret = -EINVAL; + goto out; + } + + /* All 4xx SoCs with a CPM controller have one of two + * different order for the CPM registers. Some have the + * CPM registers in the following order (ER,FR,SR). The + * others have them in the following order (SR,ER,FR). + */ + + if (cpm_get_uint_property(np, "er-offset") == 0) { + cpm.dcr_offset[CPM_ER] = 0; + cpm.dcr_offset[CPM_FR] = 1; + cpm.dcr_offset[CPM_SR] = 2; + } else { + cpm.dcr_offset[CPM_ER] = 1; + cpm.dcr_offset[CPM_FR] = 2; + cpm.dcr_offset[CPM_SR] = 0; + } + + /* Now let's see what IPs to turn off for the following modes */ + + cpm.unused = cpm_get_uint_property(np, "unused-units"); + cpm.idle_doze = cpm_get_uint_property(np, "idle-doze"); + cpm.standby = cpm_get_uint_property(np, "standby"); + cpm.suspend = cpm_get_uint_property(np, "suspend"); + + /* If some IPs are unused let's turn them off now */ + + if (cpm.unused) { + cpm_set(CPM_ER, cpm.unused); + cpm_set(CPM_FR, cpm.unused); + } + + /* Now let's export interfaces */ + + if (!cpm.powersave_off && cpm.idle_doze) + cpm_idle_config_sysfs(); + + if (cpm.standby || cpm.suspend) + suspend_set_ops(&cpm_suspend_ops); +out: + if (np) + of_node_put(np); + return ret; +} + +late_initcall(cpm_init); + +static int __init cpm_powersave_off(char *arg) +{ + cpm.powersave_off = 1; + return 0; +} +__setup("powersave=off", cpm_powersave_off); diff --git a/arch/powerpc/sysdev/tsi108_dev.c b/arch/powerpc/sysdev/tsi108_dev.c index c2d675b..ee05680 100644 --- a/arch/powerpc/sysdev/tsi108_dev.c +++ b/arch/powerpc/sysdev/tsi108_dev.c @@ -84,8 +84,8 @@ static int __init tsi108_eth_of_init(void) memset(&tsi_eth_data, 0, sizeof(tsi_eth_data)); ret = of_address_to_resource(np, 0, &r[0]); - DBG("%s: name:start->end = %s:0x%lx-> 0x%lx\n", - __func__,r[0].name, r[0].start, r[0].end); + DBG("%s: name:start->end = %s:%pR\n", + __func__, r[0].name, &r[0]); if (ret) goto err; @@ -93,8 +93,8 @@ static int __init tsi108_eth_of_init(void) r[1].start = irq_of_parse_and_map(np, 0); r[1].end = irq_of_parse_and_map(np, 0); r[1].flags = IORESOURCE_IRQ; - DBG("%s: name:start->end = %s:0x%lx-> 0x%lx\n", - __func__,r[1].name, r[1].start, r[1].end); + DBG("%s: name:start->end = %s:%pR\n", + __func__, r[1].name, &r[1]); tsi_eth_dev = platform_device_register_simple("tsi-ethernet", i++, &r[0], diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 985d825..bade533 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -164,24 +164,18 @@ long kvm_arch_vm_ioctl(struct file *filp, return r; } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm; int rc; char debug_name[16]; rc = s390_enable_sie(); if (rc) - goto out_nokvm; - - rc = -ENOMEM; - kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - if (!kvm) - goto out_nokvm; + goto out_err; kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); if (!kvm->arch.sca) - goto out_nosca; + goto out_err; sprintf(debug_name, "kvm-%u", current->pid); @@ -195,13 +189,11 @@ struct kvm *kvm_arch_create_vm(void) debug_register_view(kvm->arch.dbf, &debug_sprintf_view); VM_EVENT(kvm, 3, "%s", "vm created"); - return kvm; + return 0; out_nodbf: free_page((unsigned long)(kvm->arch.sca)); -out_nosca: - kfree(kvm); -out_nokvm: - return ERR_PTR(rc); +out_err: + return rc; } void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) @@ -240,11 +232,8 @@ void kvm_arch_sync_events(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); free_page((unsigned long)(kvm->arch.sca)); debug_unregister(kvm->arch.dbf); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); } /* Section: vcpu related */ diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig index 2018c7e..d893411 100644 --- a/arch/sh/boards/Kconfig +++ b/arch/sh/boards/Kconfig @@ -3,6 +3,9 @@ menu "Board support" config SOLUTION_ENGINE bool +config SH_ALPHA_BOARD + bool + config SH_SOLUTION_ENGINE bool "SolutionEngine" select SOLUTION_ENGINE @@ -320,6 +323,21 @@ config SH_SH2007 Compact Flash socket, two serial ports and PC-104 bus. More information at <http://sh2000.sh-linux.org>. +config SH_APSH4A3A + bool "AP-SH4A-3A" + select SH_ALPHA_BOARD + depends on CPU_SUBTYPE_SH7785 + help + Select AP-SH4A-3A if configuring for an ALPHAPROJECT AP-SH4A-3A. + +config SH_APSH4AD0A + bool "AP-SH4AD-0A" + select SH_ALPHA_BOARD + select SYS_SUPPORTS_PCI + depends on CPU_SUBTYPE_SH7786 + help + Select AP-SH4AD-0A if configuring for an ALPHAPROJECT AP-SH4AD-0A. + endmenu source "arch/sh/boards/mach-r2d/Kconfig" diff --git a/arch/sh/boards/Makefile b/arch/sh/boards/Makefile index be7d11d..975a0f6 100644 --- a/arch/sh/boards/Makefile +++ b/arch/sh/boards/Makefile @@ -13,3 +13,5 @@ obj-$(CONFIG_SH_ESPT) += board-espt.o obj-$(CONFIG_SH_POLARIS) += board-polaris.o obj-$(CONFIG_SH_TITAN) += board-titan.o obj-$(CONFIG_SH_SH7757LCR) += board-sh7757lcr.o +obj-$(CONFIG_SH_APSH4A3A) += board-apsh4a3a.o +obj-$(CONFIG_SH_APSH4AD0A) += board-apsh4ad0a.o diff --git a/arch/sh/boards/board-apsh4a3a.c b/arch/sh/boards/board-apsh4a3a.c new file mode 100644 index 0000000..8e2a270 --- /dev/null +++ b/arch/sh/boards/board-apsh4a3a.c @@ -0,0 +1,175 @@ +/* + * ALPHAPROJECT AP-SH4A-3A Support. + * + * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd. + * Copyright (C) 2008 Yoshihiro Shimoda + * Copyright (C) 2009 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/mtd/physmap.h> +#include <linux/smsc911x.h> +#include <linux/irq.h> +#include <linux/clk.h> +#include <asm/machvec.h> +#include <asm/sizes.h> +#include <asm/clock.h> + +static struct mtd_partition nor_flash_partitions[] = { + { + .name = "loader", + .offset = 0x00000000, + .size = 512 * 1024, + }, + { + .name = "bootenv", + .offset = MTDPART_OFS_APPEND, + .size = 512 * 1024, + }, + { + .name = "kernel", + .offset = MTDPART_OFS_APPEND, + .size = 4 * 1024 * 1024, + }, + { + .name = "data", + .offset = MTDPART_OFS_APPEND, + .size = MTDPART_SIZ_FULL, + }, +}; + +static struct physmap_flash_data nor_flash_data = { + .width = 4, + .parts = nor_flash_partitions, + .nr_parts = ARRAY_SIZE(nor_flash_partitions), +}; + +static struct resource nor_flash_resources[] = { + [0] = { + .start = 0x00000000, + .end = 0x01000000 - 1, + .flags = IORESOURCE_MEM, + } +}; + +static struct platform_device nor_flash_device = { + .name = "physmap-flash", + .dev = { + .platform_data = &nor_flash_data, + }, + .num_resources = ARRAY_SIZE(nor_flash_resources), + .resource = nor_flash_resources, +}; + +static struct resource smsc911x_resources[] = { + [0] = { + .name = "smsc911x-memory", + .start = 0xA4000000, + .end = 0xA4000000 + SZ_256 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .name = "smsc911x-irq", + .start = evt2irq(0x200), + .end = evt2irq(0x200), + .flags = IORESOURCE_IRQ, + }, +}; + +static struct smsc911x_platform_config smsc911x_config = { + .irq_polarity = SMSC911X_IRQ_POLARITY_ACTIVE_LOW, + .irq_type = SMSC911X_IRQ_TYPE_OPEN_DRAIN, + .flags = SMSC911X_USE_16BIT, + .phy_interface = PHY_INTERFACE_MODE_MII, +}; + +static struct platform_device smsc911x_device = { + .name = "smsc911x", + .id = -1, + .num_resources = ARRAY_SIZE(smsc911x_resources), + .resource = smsc911x_resources, + .dev = { + .platform_data = &smsc911x_config, + }, +}; + +static struct platform_device *apsh4a3a_devices[] __initdata = { + &nor_flash_device, + &smsc911x_device, +}; + +static int __init apsh4a3a_devices_setup(void) +{ + return platform_add_devices(apsh4a3a_devices, + ARRAY_SIZE(apsh4a3a_devices)); +} +device_initcall(apsh4a3a_devices_setup); + +static int apsh4a3a_clk_init(void) +{ + struct clk *clk; + int ret; + + clk = clk_get(NULL, "extal"); + if (!clk || IS_ERR(clk)) + return PTR_ERR(clk); + ret = clk_set_rate(clk, 33333000); + clk_put(clk); + + return ret; +} + +/* Initialize the board */ +static void __init apsh4a3a_setup(char **cmdline_p) +{ + printk(KERN_INFO "Alpha Project AP-SH4A-3A support:\n"); +} + +static void __init apsh4a3a_init_irq(void) +{ + plat_irq_setup_pins(IRQ_MODE_IRQ7654); +} + +/* Return the board specific boot mode pin configuration */ +static int apsh4a3a_mode_pins(void) +{ + int value = 0; + + /* These are the factory default settings of SW1 and SW2. + * If you change these dip switches then you will need to + * adjust the values below as well. + */ + value &= ~MODE_PIN0; /* Clock Mode 16 */ + value &= ~MODE_PIN1; + value &= ~MODE_PIN2; + value &= ~MODE_PIN3; + value |= MODE_PIN4; + value &= ~MODE_PIN5; /* 16-bit Area0 bus width */ + value |= MODE_PIN6; /* Area 0 SRAM interface */ + value |= MODE_PIN7; + value |= MODE_PIN8; /* Little Endian */ + value |= MODE_PIN9; /* Master Mode */ + value |= MODE_PIN10; /* Crystal resonator */ + value |= MODE_PIN11; /* Display Unit */ + value |= MODE_PIN12; + value &= ~MODE_PIN13; /* 29-bit address mode */ + value |= MODE_PIN14; /* No PLL step-up */ + + return value; +} + +/* + * The Machine Vector + */ +static struct sh_machine_vector mv_apsh4a3a __initmv = { + .mv_name = "AP-SH4A-3A", + .mv_setup = apsh4a3a_setup, + .mv_clk_init = apsh4a3a_clk_init, + .mv_init_irq = apsh4a3a_init_irq, + .mv_mode_pins = apsh4a3a_mode_pins, +}; diff --git a/arch/sh/boards/board-apsh4ad0a.c b/arch/sh/boards/board-apsh4ad0a.c new file mode 100644 index 0000000..e2bd218 --- /dev/null +++ b/arch/sh/boards/board-apsh4ad0a.c @@ -0,0 +1,125 @@ +/* + * ALPHAPROJECT AP-SH4AD-0A Support. + * + * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd. + * Copyright (C) 2010 Matt Fleming + * Copyright (C) 2010 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/smsc911x.h> +#include <linux/irq.h> +#include <linux/clk.h> +#include <asm/machvec.h> +#include <asm/sizes.h> + +static struct resource smsc911x_resources[] = { + [0] = { + .name = "smsc911x-memory", + .start = 0xA4000000, + .end = 0xA4000000 + SZ_256 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .name = "smsc911x-irq", + .start = evt2irq(0x200), + .end = evt2irq(0x200), + .flags = IORESOURCE_IRQ, + }, +}; + +static struct smsc911x_platform_config smsc911x_config = { + .irq_polarity = SMSC911X_IRQ_POLARITY_ACTIVE_LOW, + .irq_type = SMSC911X_IRQ_TYPE_OPEN_DRAIN, + .flags = SMSC911X_USE_16BIT, + .phy_interface = PHY_INTERFACE_MODE_MII, +}; + +static struct platform_device smsc911x_device = { + .name = "smsc911x", + .id = -1, + .num_resources = ARRAY_SIZE(smsc911x_resources), + .resource = smsc911x_resources, + .dev = { + .platform_data = &smsc911x_config, + }, +}; + +static struct platform_device *apsh4ad0a_devices[] __initdata = { + &smsc911x_device, +}; + +static int __init apsh4ad0a_devices_setup(void) +{ + return platform_add_devices(apsh4ad0a_devices, + ARRAY_SIZE(apsh4ad0a_devices)); +} +device_initcall(apsh4ad0a_devices_setup); + +static int apsh4ad0a_mode_pins(void) +{ + int value = 0; + + /* These are the factory default settings of SW1 and SW2. + * If you change these dip switches then you will need to + * adjust the values below as well. + */ + value |= MODE_PIN0; /* Clock Mode 3 */ + value |= MODE_PIN1; + value &= ~MODE_PIN2; + value &= ~MODE_PIN3; + value &= ~MODE_PIN4; /* 16-bit Area0 bus width */ + value |= MODE_PIN5; + value |= MODE_PIN6; + value |= MODE_PIN7; /* Normal mode */ + value |= MODE_PIN8; /* Little Endian */ + value |= MODE_PIN9; /* Crystal resonator */ + value &= ~MODE_PIN10; /* 29-bit address mode */ + value &= ~MODE_PIN11; /* PCI-E Root port */ + value &= ~MODE_PIN12; /* 4 lane + 1 lane */ + value |= MODE_PIN13; /* AUD Enable */ + value &= ~MODE_PIN14; /* Normal Operation */ + + return value; +} + +static int apsh4ad0a_clk_init(void) +{ + struct clk *clk; + int ret; + + clk = clk_get(NULL, "extal"); + if (!clk || IS_ERR(clk)) + return PTR_ERR(clk); + ret = clk_set_rate(clk, 33333000); + clk_put(clk); + + return ret; +} + +/* Initialize the board */ +static void __init apsh4ad0a_setup(char **cmdline_p) +{ + pr_info("Alpha Project AP-SH4AD-0A support:\n"); +} + +static void __init apsh4ad0a_init_irq(void) +{ + plat_irq_setup_pins(IRQ_MODE_IRQ3210); +} + +/* + * The Machine Vector + */ +static struct sh_machine_vector mv_apsh4ad0a __initmv = { + .mv_name = "AP-SH4AD-0A", + .mv_setup = apsh4ad0a_setup, + .mv_mode_pins = apsh4ad0a_mode_pins, + .mv_clk_init = apsh4ad0a_clk_init, + .mv_init_irq = apsh4ad0a_init_irq, +}; diff --git a/arch/sh/boards/board-edosk7705.c b/arch/sh/boards/board-edosk7705.c index 4cb3bb7..541d8a2 100644 --- a/arch/sh/boards/board-edosk7705.c +++ b/arch/sh/boards/board-edosk7705.c @@ -66,7 +66,7 @@ static int __init init_edosk7705_devices(void) return platform_add_devices(edosk7705_devices, ARRAY_SIZE(edosk7705_devices)); } -__initcall(init_edosk7705_devices); +device_initcall(init_edosk7705_devices); /* * The Machine Vector diff --git a/arch/sh/boards/board-edosk7760.c b/arch/sh/boards/board-edosk7760.c index 35dc099..f47ac82 100644 --- a/arch/sh/boards/board-edosk7760.c +++ b/arch/sh/boards/board-edosk7760.c @@ -182,7 +182,7 @@ static int __init init_edosk7760_devices(void) return platform_add_devices(edosk7760_devices, ARRAY_SIZE(edosk7760_devices)); } -__initcall(init_edosk7760_devices); +device_initcall(init_edosk7760_devices); /* * The Machine Vector diff --git a/arch/sh/boards/board-sh7785lcr.c b/arch/sh/boards/board-sh7785lcr.c index fe7e686..ee65ff0 100644 --- a/arch/sh/boards/board-sh7785lcr.c +++ b/arch/sh/boards/board-sh7785lcr.c @@ -284,7 +284,7 @@ static int __init sh7785lcr_devices_setup(void) return platform_add_devices(sh7785lcr_devices, ARRAY_SIZE(sh7785lcr_devices)); } -__initcall(sh7785lcr_devices_setup); +device_initcall(sh7785lcr_devices_setup); /* Initialize IRQ setting */ void __init init_sh7785lcr_IRQ(void) diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c index 07ea908..3e5fc3b 100644 --- a/arch/sh/boards/mach-ap325rxa/setup.c +++ b/arch/sh/boards/mach-ap325rxa/setup.c @@ -14,6 +14,8 @@ #include <linux/device.h> #include <linux/interrupt.h> #include <linux/platform_device.h> +#include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/mtd/sh_flctl.h> #include <linux/delay.h> @@ -430,11 +432,18 @@ static struct resource sdhi0_cn3_resources[] = { }, }; +static struct sh_mobile_sdhi_info sdhi0_cn3_data = { + .tmio_caps = MMC_CAP_SDIO_IRQ, +}; + static struct platform_device sdhi0_cn3_device = { .name = "sh_mobile_sdhi", .id = 0, /* "sdhi0" clock */ .num_resources = ARRAY_SIZE(sdhi0_cn3_resources), .resource = sdhi0_cn3_resources, + .dev = { + .platform_data = &sdhi0_cn3_data, + }, .archdata = { .hwblk_id = HWBLK_SDHI0, }, @@ -453,11 +462,18 @@ static struct resource sdhi1_cn7_resources[] = { }, }; +static struct sh_mobile_sdhi_info sdhi1_cn7_data = { + .tmio_caps = MMC_CAP_SDIO_IRQ, +}; + static struct platform_device sdhi1_cn7_device = { .name = "sh_mobile_sdhi", .id = 1, /* "sdhi1" clock */ .num_resources = ARRAY_SIZE(sdhi1_cn7_resources), .resource = sdhi1_cn7_resources, + .dev = { + .platform_data = &sdhi1_cn7_data, + }, .archdata = { .hwblk_id = HWBLK_SDHI1, }, diff --git a/arch/sh/boards/mach-cayman/setup.c b/arch/sh/boards/mach-cayman/setup.c index 7e8216a..e89e8e1 100644 --- a/arch/sh/boards/mach-cayman/setup.c +++ b/arch/sh/boards/mach-cayman/setup.c @@ -165,7 +165,7 @@ static int __init smsc_superio_setup(void) return 0; } -__initcall(smsc_superio_setup); +device_initcall(smsc_superio_setup); static void __iomem *cayman_ioport_map(unsigned long port, unsigned int len) { diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c index f48c492..33b6629 100644 --- a/arch/sh/boards/mach-ecovec24/setup.c +++ b/arch/sh/boards/mach-ecovec24/setup.c @@ -473,6 +473,7 @@ static struct sh_mobile_sdhi_info sdhi0_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, .set_pwr = sdhi0_set_pwr, + .tmio_caps = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD, }; static struct resource sdhi0_resources[] = { @@ -511,6 +512,7 @@ static void sdhi1_set_pwr(struct platform_device *pdev, int state) static struct sh_mobile_sdhi_info sdhi1_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI1_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD, .set_pwr = sdhi1_set_pwr, }; diff --git a/arch/sh/boards/mach-hp6xx/pm.c b/arch/sh/boards/mach-hp6xx/pm.c index 4499a37..adc9b4b 100644 --- a/arch/sh/boards/mach-hp6xx/pm.c +++ b/arch/sh/boards/mach-hp6xx/pm.c @@ -143,7 +143,7 @@ static int hp6x0_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops hp6x0_pm_ops = { +static const struct platform_suspend_ops hp6x0_pm_ops = { .enter = hp6x0_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c index 9b60eaa..7504daa 100644 --- a/arch/sh/boards/mach-kfr2r09/setup.c +++ b/arch/sh/boards/mach-kfr2r09/setup.c @@ -11,6 +11,7 @@ #include <linux/platform_device.h> #include <linux/interrupt.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mfd/tmio.h> #include <linux/mtd/physmap.h> #include <linux/mtd/onenand.h> @@ -366,6 +367,7 @@ static struct sh_mobile_sdhi_info sh7724_sdhi0_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, .tmio_flags = TMIO_MMC_WRPROTECT_DISABLE, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device kfr2r09_sh_sdhi0_device = { diff --git a/arch/sh/boards/mach-landisk/irq.c b/arch/sh/boards/mach-landisk/irq.c index e79412a..c00ace3 100644 --- a/arch/sh/boards/mach-landisk/irq.c +++ b/arch/sh/boards/mach-landisk/irq.c @@ -1,9 +1,10 @@ /* - * arch/sh/boards/landisk/irq.c + * arch/sh/boards/mach-landisk/irq.c * * I-O DATA Device, Inc. LANDISK Support * * Copyright (C) 2005-2007 kogiidena + * Copyright (C) 2011 Nobuhiro Iwamatsu * * Copyright (C) 2001 Ian da Silva, Jeremy Siegel * Based largely on io_se.c. @@ -12,44 +13,54 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. */ + #include <linux/init.h> #include <linux/irq.h> #include <linux/interrupt.h> #include <linux/io.h> #include <mach-landisk/mach/iodata_landisk.h> -static void disable_landisk_irq(struct irq_data *data) -{ - unsigned char mask = 0xff ^ (0x01 << (data->irq - 5)); +enum { + UNUSED = 0, - __raw_writeb(__raw_readb(PA_IMASK) & mask, PA_IMASK); -} - -static void enable_landisk_irq(struct irq_data *data) -{ - unsigned char value = (0x01 << (data->irq - 5)); + PCI_INTA, /* PCI int A */ + PCI_INTB, /* PCI int B */ + PCI_INTC, /* PCI int C */ + PCI_INTD, /* PCI int D */ + ATA, /* ATA */ + FATA, /* CF */ + POWER, /* Power swtich */ + BUTTON, /* Button swtich */ +}; - __raw_writeb(__raw_readb(PA_IMASK) | value, PA_IMASK); -} +/* Vectors for LANDISK */ +static struct intc_vect vectors_landisk[] __initdata = { + INTC_IRQ(PCI_INTA, IRQ_PCIINTA), + INTC_IRQ(PCI_INTB, IRQ_PCIINTB), + INTC_IRQ(PCI_INTC, IRQ_PCIINTC), + INTC_IRQ(PCI_INTD, IRQ_PCIINTD), + INTC_IRQ(ATA, IRQ_ATA), + INTC_IRQ(FATA, IRQ_FATA), + INTC_IRQ(POWER, IRQ_POWER), + INTC_IRQ(BUTTON, IRQ_BUTTON), +}; -static struct irq_chip landisk_irq_chip __read_mostly = { - .name = "LANDISK", - .irq_mask = disable_landisk_irq, - .irq_unmask = enable_landisk_irq, +/* IRLMSK mask register layout for LANDISK */ +static struct intc_mask_reg mask_registers_landisk[] __initdata = { + { PA_IMASK, 0, 8, /* IRLMSK */ + { BUTTON, POWER, FATA, ATA, + PCI_INTD, PCI_INTC, PCI_INTB, PCI_INTA, + } + }, }; +static DECLARE_INTC_DESC(intc_desc_landisk, "landisk", vectors_landisk, NULL, + mask_registers_landisk, NULL, NULL); /* * Initialize IRQ setting */ void __init init_landisk_IRQ(void) { - int i; - - for (i = 5; i < 14; i++) { - disable_irq_nosync(i); - set_irq_chip_and_handler_name(i, &landisk_irq_chip, - handle_level_irq, "level"); - enable_landisk_irq(irq_get_irq_data(i)); - } + register_intc_controller(&intc_desc_landisk); __raw_writeb(0x00, PA_PWRINT_CLR); } diff --git a/arch/sh/boards/mach-landisk/setup.c b/arch/sh/boards/mach-landisk/setup.c index 50337acc..94186cf 100644 --- a/arch/sh/boards/mach-landisk/setup.c +++ b/arch/sh/boards/mach-landisk/setup.c @@ -21,8 +21,6 @@ #include <mach-landisk/mach/iodata_landisk.h> #include <asm/io.h> -void init_landisk_IRQ(void); - static void landisk_power_off(void) { __raw_writeb(0x01, PA_SHUTDOWN); @@ -83,7 +81,7 @@ static int __init landisk_devices_setup(void) ARRAY_SIZE(landisk_devices)); } -__initcall(landisk_devices_setup); +device_initcall(landisk_devices_setup); static void __init landisk_setup(char **cmdline_p) { diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index c8acfec..03a7ffe7 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -13,6 +13,7 @@ #include <linux/input.h> #include <linux/input/sh_keysc.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/mtd/nand.h> #include <linux/i2c.h> @@ -410,6 +411,7 @@ static struct resource sdhi_cn9_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi_cn9_device = { diff --git a/arch/sh/boards/mach-r2d/setup.c b/arch/sh/boards/mach-r2d/setup.c index b84df6a..4b98a52 100644 --- a/arch/sh/boards/mach-r2d/setup.c +++ b/arch/sh/boards/mach-r2d/setup.c @@ -258,7 +258,7 @@ static int __init rts7751r2d_devices_setup(void) return platform_add_devices(rts7751r2d_devices, ARRAY_SIZE(rts7751r2d_devices)); } -__initcall(rts7751r2d_devices_setup); +device_initcall(rts7751r2d_devices_setup); static void rts7751r2d_power_off(void) { diff --git a/arch/sh/boards/mach-sdk7786/setup.c b/arch/sh/boards/mach-sdk7786/setup.c index 75e4ddb..1521aa7 100644 --- a/arch/sh/boards/mach-sdk7786/setup.c +++ b/arch/sh/boards/mach-sdk7786/setup.c @@ -15,13 +15,13 @@ #include <linux/i2c.h> #include <linux/irq.h> #include <linux/clk.h> +#include <linux/clkdev.h> #include <mach/fpga.h> #include <mach/irq.h> #include <asm/machvec.h> #include <asm/heartbeat.h> #include <asm/sizes.h> #include <asm/clock.h> -#include <asm/clkdev.h> #include <asm/reboot.h> #include <asm/smp-ops.h> @@ -135,7 +135,7 @@ static int __init sdk7786_devices_setup(void) return sdk7786_i2c_setup(); } -__initcall(sdk7786_devices_setup); +device_initcall(sdk7786_devices_setup); static int sdk7786_mode_pins(void) { diff --git a/arch/sh/boards/mach-se/7206/setup.c b/arch/sh/boards/mach-se/7206/setup.c index 33039e0..8ab8330 100644 --- a/arch/sh/boards/mach-se/7206/setup.c +++ b/arch/sh/boards/mach-se/7206/setup.c @@ -77,7 +77,7 @@ static int __init se7206_devices_setup(void) { return platform_add_devices(se7206_devices, ARRAY_SIZE(se7206_devices)); } -__initcall(se7206_devices_setup); +device_initcall(se7206_devices_setup); static int se7206_mode_pins(void) { diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c index 527a0cd..5276793 100644 --- a/arch/sh/boards/mach-se/7724/setup.c +++ b/arch/sh/boards/mach-se/7724/setup.c @@ -15,6 +15,7 @@ #include <linux/interrupt.h> #include <linux/platform_device.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/delay.h> #include <linux/smc91x.h> @@ -318,6 +319,10 @@ static struct platform_device fsi_device = { }, }; +static struct platform_device fsi_ak4642_device = { + .name = "sh_fsi_a_ak4642", +}; + /* KEYSC in SoC (Needs SW33-2 set to ON) */ static struct sh_keysc_info keysc_info = { .mode = SH_KEYSC_MODE_1, @@ -467,6 +472,7 @@ static struct resource sdhi0_cn7_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi0_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi0_cn7_device = { @@ -498,6 +504,7 @@ static struct resource sdhi1_cn8_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi1_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI1_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi1_cn8_device = { @@ -590,6 +597,7 @@ static struct platform_device *ms7724se_devices[] __initdata = { &sh7724_usb0_host_device, &sh7724_usb1_gadget_device, &fsi_device, + &fsi_ak4642_device, &sdhi0_cn7_device, &sdhi1_cn8_device, &irda_device, diff --git a/arch/sh/boards/mach-se/7751/setup.c b/arch/sh/boards/mach-se/7751/setup.c index 9fbc51b..4ed60c5 100644 --- a/arch/sh/boards/mach-se/7751/setup.c +++ b/arch/sh/boards/mach-se/7751/setup.c @@ -48,7 +48,7 @@ static int __init se7751_devices_setup(void) { return platform_add_devices(se7751_devices, ARRAY_SIZE(se7751_devices)); } -__initcall(se7751_devices_setup); +device_initcall(se7751_devices_setup); /* * The Machine Vector diff --git a/arch/sh/boards/mach-sh03/rtc.c b/arch/sh/boards/mach-sh03/rtc.c index 1b20099..f83ac79 100644 --- a/arch/sh/boards/mach-sh03/rtc.c +++ b/arch/sh/boards/mach-sh03/rtc.c @@ -108,7 +108,7 @@ static int set_rtc_mmss(unsigned long nowtime) __raw_writeb(real_minutes % 10, RTC_MIN1); __raw_writeb(real_minutes / 10, RTC_MIN10); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/sh/boards/mach-sh03/setup.c b/arch/sh/boards/mach-sh03/setup.c index af4a0c0..d4f79b2 100644 --- a/arch/sh/boards/mach-sh03/setup.c +++ b/arch/sh/boards/mach-sh03/setup.c @@ -96,7 +96,7 @@ static int __init sh03_devices_setup(void) return platform_add_devices(sh03_devices, ARRAY_SIZE(sh03_devices)); } -__initcall(sh03_devices_setup); +device_initcall(sh03_devices_setup); static struct sh_machine_vector mv_sh03 __initmv = { .mv_name = "Interface (CTP/PCI-SH03)", diff --git a/arch/sh/boot/romimage/mmcif-sh7724.c b/arch/sh/boot/romimage/mmcif-sh7724.c index 14863d7..c84e783 100644 --- a/arch/sh/boot/romimage/mmcif-sh7724.c +++ b/arch/sh/boot/romimage/mmcif-sh7724.c @@ -21,9 +21,6 @@ #define HIZCRC 0xa405015c #define DRVCRA 0xa405018a -enum { MMCIF_PROGRESS_ENTER, MMCIF_PROGRESS_INIT, - MMCIF_PROGRESS_LOAD, MMCIF_PROGRESS_DONE }; - /* SH7724 specific MMCIF loader * * loads the romImage from an MMC card starting from block 512 @@ -63,7 +60,9 @@ asmlinkage void mmcif_loader(unsigned char *buf, unsigned long no_bytes) mmcif_update_progress(MMCIF_PROGRESS_LOAD); /* load kernel via MMCIF interface */ - sh_mmcif_boot_slurp(MMCIF_BASE, buf, no_bytes); + sh_mmcif_boot_do_read(MMCIF_BASE, 512, + (no_bytes + SH_MMCIF_BBS - 1) / SH_MMCIF_BBS, + buf); /* disable clock to the MMCIF hardware block */ __raw_writel(__raw_readl(MSTPCR2) | 0x20000000, MSTPCR2); diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig new file mode 100644 index 0000000..6cb3279 --- /dev/null +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -0,0 +1,102 @@ +CONFIG_EXPERIMENTAL=y +CONFIG_SYSVIPC=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_SYSFS_DEPRECATED=y +CONFIG_SYSFS_DEPRECATED_V2=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +# CONFIG_BLK_DEV_BSG is not set +CONFIG_CPU_SUBTYPE_SH7785=y +CONFIG_MEMORY_START=0x0C000000 +CONFIG_FLATMEM_MANUAL=y +CONFIG_SH_STORE_QUEUES=y +CONFIG_SH_APSH4A3A=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_KEXEC=y +CONFIG_PREEMPT=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +# CONFIG_INET_LRO is not set +# CONFIG_IPV6 is not set +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +# CONFIG_FW_LOADER is not set +CONFIG_MTD=y +CONFIG_MTD_CONCAT=y +CONFIG_MTD_PARTITIONS=y +CONFIG_MTD_CHAR=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_PHYSMAP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_NETDEVICES=y +CONFIG_NET_ETHERNET=y +CONFIG_SMSC911X=y +# CONFIG_NETDEV_1000 is not set +# CONFIG_NETDEV_10000 is not set +# CONFIG_WLAN is not set +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO is not set +CONFIG_VT_HW_CONSOLE_BINDING=y +CONFIG_SERIAL_SH_SCI=y +CONFIG_SERIAL_SH_SCI_NR_UARTS=6 +CONFIG_SERIAL_SH_SCI_CONSOLE=y +CONFIG_HW_RANDOM=y +# CONFIG_HWMON is not set +CONFIG_FB=y +CONFIG_FB_SH7785FB=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +CONFIG_LOGO=y +# CONFIG_HID_SUPPORT is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_JFFS2_FS=y +CONFIG_CRAMFS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V4=y +CONFIG_CIFS=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_932=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +# CONFIG_ENABLE_WARN_DEPRECATED is not set +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_KERNEL=y +# CONFIG_DEBUG_PREEMPT is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO=y +# CONFIG_RCU_CPU_STALL_DETECTOR is not set +# CONFIG_FTRACE is not set +# CONFIG_CRYPTO_ANSI_CPRNG is not set +# CONFIG_CRYPTO_HW is not set diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig new file mode 100644 index 0000000..e71a531 --- /dev/null +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -0,0 +1,133 @@ +CONFIG_EXPERIMENTAL=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_RCU_TRACE=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUPS=y +CONFIG_CGROUP_NS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_RESOURCE_COUNTERS=y +CONFIG_CGROUP_MEM_RES_CTLR=y +CONFIG_BLK_CGROUP=y +CONFIG_NAMESPACES=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +# CONFIG_LBDAF is not set +# CONFIG_BLK_DEV_BSG is not set +CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_CPU_SUBTYPE_SH7786=y +CONFIG_MEMORY_SIZE=0x10000000 +CONFIG_HUGETLB_PAGE_SIZE_1MB=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_KSM=y +CONFIG_SH_STORE_QUEUES=y +CONFIG_SH_APSH4AD0A=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=m +CONFIG_CPU_FREQ_GOV_USERSPACE=m +CONFIG_CPU_FREQ_GOV_ONDEMAND=m +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m +CONFIG_SH_CPU_FREQ=y +CONFIG_KEXEC=y +CONFIG_SECCOMP=y +CONFIG_PREEMPT=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_PM=y +CONFIG_PM_DEBUG=y +CONFIG_PM_VERBOSE=y +CONFIG_PM_RUNTIME=y +CONFIG_CPU_IDLE=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_NET_KEY=y +CONFIG_INET=y +# CONFIG_INET_LRO is not set +# CONFIG_IPV6 is not set +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +# CONFIG_FW_LOADER is not set +CONFIG_MTD=y +CONFIG_MTD_CFI=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_SCSI_MULTI_LUN=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_NETDEVICES=y +CONFIG_MDIO_BITBANG=y +CONFIG_NET_ETHERNET=y +CONFIG_SMSC911X=y +# CONFIG_NETDEV_1000 is not set +# CONFIG_NETDEV_10000 is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO is not set +CONFIG_SERIAL_SH_SCI=y +CONFIG_SERIAL_SH_SCI_NR_UARTS=6 +CONFIG_SERIAL_SH_SCI_CONSOLE=y +# CONFIG_LEGACY_PTYS is not set +# CONFIG_HW_RANDOM is not set +# CONFIG_HWMON is not set +CONFIG_VIDEO_OUTPUT_CONTROL=y +CONFIG_FB=y +CONFIG_FB_SH7785FB=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_DEBUG=y +CONFIG_USB_MON=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_HUGETLBFS=y +CONFIG_JFFS2_FS=y +CONFIG_CRAMFS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V4=y +CONFIG_CIFS=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_932=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_SHIRQ=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_VM=y +# CONFIG_RCU_CPU_STALL_DETECTOR is not set +CONFIG_DWARF_UNWINDER=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig index 273f3fa..5f7f667 100644 --- a/arch/sh/configs/sh7757lcr_defconfig +++ b/arch/sh/configs/sh7757lcr_defconfig @@ -39,21 +39,15 @@ CONFIG_IPV6=y CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_RAM=y -# CONFIG_MISC_DEVICES is not set CONFIG_NETDEVICES=y -CONFIG_PHYLIB=y CONFIG_VITESSE_PHY=y -CONFIG_MDIO_BITBANG=y CONFIG_NET_ETHERNET=y -CONFIG_MII=y +CONFIG_SH_ETH=y # CONFIG_NETDEV_10000 is not set # CONFIG_WLAN is not set # CONFIG_KEYBOARD_ATKBD is not set # CONFIG_MOUSE_PS2 is not set # CONFIG_SERIO is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=2 CONFIG_SERIAL_SH_SCI=y CONFIG_SERIAL_SH_SCI_NR_UARTS=3 CONFIG_SERIAL_SH_SCI_CONSOLE=y @@ -63,7 +57,6 @@ CONFIG_SERIAL_SH_SCI_CONSOLE=y # CONFIG_USB_SUPPORT is not set CONFIG_EXT2_FS=y CONFIG_EXT3_FS=y -CONFIG_INOTIFY=y CONFIG_ISO9660_FS=y CONFIG_VFAT_FS=y CONFIG_PROC_KCORE=y @@ -76,10 +69,8 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y CONFIG_DEBUG_KERNEL=y -# CONFIG_DETECT_SOFTLOCKUP is not set # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_BUGVERBOSE is not set CONFIG_DEBUG_INFO=y -# CONFIG_RCU_CPU_STALL_DETECTOR is not set # CONFIG_FTRACE is not set # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/drivers/pci/fixups-landisk.c b/arch/sh/drivers/pci/fixups-landisk.c index bb1a6bb..95c6e2d 100644 --- a/arch/sh/drivers/pci/fixups-landisk.c +++ b/arch/sh/drivers/pci/fixups-landisk.c @@ -1,9 +1,10 @@ /* - * arch/sh/drivers/pci/ops-landisk.c + * arch/sh/drivers/pci/fixups-landisk.c * * PCI initialization for the I-O DATA Device, Inc. LANDISK board * * Copyright (C) 2006 kogiidena + * Copyright (C) 2010 Nobuhiro Iwamatsu * * May be copied or modified under the terms of the GNU General Public * License. See linux/COPYING for more information. @@ -15,6 +16,9 @@ #include <linux/pci.h> #include "pci-sh4.h" +#define PCIMCR_MRSET_OFF 0xBFFFFFFF +#define PCIMCR_RFSH_OFF 0xFFFFFFFB + int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin) { /* @@ -26,9 +30,29 @@ int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin) int irq = ((slot + pin - 1) & 0x3) + 5; if ((slot | (pin - 1)) > 0x3) { - printk("PCI: Bad IRQ mapping request for slot %d pin %c\n", + printk(KERN_WARNING "PCI: Bad IRQ mapping request for slot %d pin %c\n", slot, pin - 1 + 'A'); return -1; } return irq; } + +int pci_fixup_pcic(struct pci_channel *chan) +{ + unsigned long bcr1, mcr; + + bcr1 = __raw_readl(SH7751_BCR1); + bcr1 |= 0x40080000; /* Enable Bit 19 BREQEN, set PCIC to slave */ + pci_write_reg(chan, bcr1, SH4_PCIBCR1); + + mcr = __raw_readl(SH7751_MCR); + mcr = (mcr & PCIMCR_MRSET_OFF) & PCIMCR_RFSH_OFF; + pci_write_reg(chan, mcr, SH4_PCIMCR); + + pci_write_reg(chan, 0x0c000000, SH7751_PCICONF5); + pci_write_reg(chan, 0xd0000000, SH7751_PCICONF6); + pci_write_reg(chan, 0x0c000000, SH4_PCILAR0); + pci_write_reg(chan, 0x00000000, SH4_PCILAR1); + + return 0; +} diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index 89ab2c5..28c5aa5 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h @@ -11,11 +11,6 @@ * * While read{b,w,l,q} and write{b,w,l,q} contain memory barriers * automatically, there are also __raw versions, which do not. - * - * Historically, we have also had ctrl_in{b,w,l,q}/ctrl_out{b,w,l,q} for - * SuperH specific I/O (raw I/O to on-chip CPU peripherals). In practice - * these have the same semantics as the __raw variants, and as such, all - * new code should be using the __raw versions. */ #include <linux/errno.h> #include <asm/cache.h> @@ -231,52 +226,6 @@ __BUILD_IOPORT_STRING(q, u64) #endif -/* - * Legacy SuperH on-chip I/O functions - * - * These are all deprecated, all new (and especially cross-platform) code - * should be using the __raw_xxx() routines directly. - */ -static inline u8 __deprecated ctrl_inb(unsigned long addr) -{ - return __raw_readb(addr); -} - -static inline u16 __deprecated ctrl_inw(unsigned long addr) -{ - return __raw_readw(addr); -} - -static inline u32 __deprecated ctrl_inl(unsigned long addr) -{ - return __raw_readl(addr); -} - -static inline u64 __deprecated ctrl_inq(unsigned long addr) -{ - return __raw_readq(addr); -} - -static inline void __deprecated ctrl_outb(u8 v, unsigned long addr) -{ - __raw_writeb(v, addr); -} - -static inline void __deprecated ctrl_outw(u16 v, unsigned long addr) -{ - __raw_writew(v, addr); -} - -static inline void __deprecated ctrl_outl(u32 v, unsigned long addr) -{ - __raw_writel(v, addr); -} - -static inline void __deprecated ctrl_outq(u64 v, unsigned long addr) -{ - __raw_writeq(v, addr); -} - #define IO_SPACE_LIMIT 0xffffffff /* synco on SH-4A, otherwise a nop */ @@ -341,7 +290,15 @@ __ioremap_29bit(phys_addr_t offset, unsigned long size, pgprot_t prot) * mapping must be done by the PMB or by using page tables. */ if (likely(PXSEG(offset) < P3SEG && PXSEG(last_addr) < P3SEG)) { - if (unlikely(pgprot_val(prot) & _PAGE_CACHABLE)) + u64 flags = pgprot_val(prot); + + /* + * Anything using the legacy PTEA space attributes needs + * to be kicked down to page table mappings. + */ + if (unlikely(flags & _PAGE_PCC_MASK)) + return NULL; + if (unlikely(flags & _PAGE_CACHABLE)) return (void __iomem *)P1SEGADDR(offset); return (void __iomem *)P2SEGADDR(offset); diff --git a/arch/sh/include/asm/machvec.h b/arch/sh/include/asm/machvec.h index dd5d6e5..57c5c3d 100644 --- a/arch/sh/include/asm/machvec.h +++ b/arch/sh/include/asm/machvec.h @@ -31,6 +31,7 @@ struct sh_machine_vector { int (*mv_mode_pins)(void); void (*mv_mem_init)(void); + void (*mv_mem_reserve)(void); }; extern struct sh_machine_vector sh_mv; diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 43528ec..b799fe7 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -76,6 +76,10 @@ /* Wrapper for extended mode pgprot twiddling */ #define _PAGE_EXT(x) ((unsigned long long)(x) << 32) +#ifdef CONFIG_X2TLB +#define _PAGE_PCC_MASK 0x00000000 /* No legacy PTEA support */ +#else + /* software: moves to PTEA.TC (Timing Control) */ #define _PAGE_PCC_AREA5 0x00000000 /* use BSC registers for area5 */ #define _PAGE_PCC_AREA6 0x80000000 /* use BSC registers for area6 */ @@ -89,7 +93,8 @@ #define _PAGE_PCC_ATR8 0x60000000 /* Attribute Memory space, 8 bit bus */ #define _PAGE_PCC_ATR16 0x60000001 /* Attribute Memory space, 6 bit bus */ -#ifndef CONFIG_X2TLB +#define _PAGE_PCC_MASK 0xe0000001 + /* copy the ptea attributes */ static inline unsigned long copy_ptea_attributes(unsigned long x) { @@ -231,13 +236,7 @@ static inline unsigned long copy_ptea_attributes(unsigned long x) _PAGE_EXT_KERN_EXEC)) #define PAGE_KERNEL_PCC(slot, type) \ - __pgprot(_PAGE_PRESENT | _PAGE_DIRTY | \ - _PAGE_ACCESSED | _PAGE_FLAGS_HARD | \ - _PAGE_EXT(_PAGE_EXT_KERN_READ | \ - _PAGE_EXT_KERN_WRITE | \ - _PAGE_EXT_KERN_EXEC) \ - (slot ? _PAGE_PCC_AREA5 : _PAGE_PCC_AREA6) | \ - (type)) + __pgprot(0) #elif defined(CONFIG_MMU) /* SH-X TLB */ #define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_CACHABLE | \ diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h index c9e7cbc..9c7bdfc 100644 --- a/arch/sh/include/asm/processor.h +++ b/arch/sh/include/asm/processor.h @@ -35,7 +35,7 @@ enum cpu_type { CPU_SH7723, CPU_SH7724, CPU_SH7757, CPU_SHX3, /* SH4AL-DSP types */ - CPU_SH7343, CPU_SH7722, CPU_SH7366, + CPU_SH7343, CPU_SH7722, CPU_SH7366, CPU_SH7372, /* SH-5 types */ CPU_SH5_101, CPU_SH5_103, diff --git a/arch/sh/include/asm/processor_32.h b/arch/sh/include/asm/processor_32.h index e3c73cd..900f8d7 100644 --- a/arch/sh/include/asm/processor_32.h +++ b/arch/sh/include/asm/processor_32.h @@ -194,15 +194,17 @@ extern unsigned long get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[15]) #if defined(CONFIG_CPU_SH2A) || defined(CONFIG_CPU_SH4) + #define PREFETCH_STRIDE L1_CACHE_BYTES #define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW -static inline void prefetch(void *x) + +static inline void prefetch(const void *x) { __builtin_prefetch(x, 0, 3); } -static inline void prefetchw(void *x) +static inline void prefetchw(const void *x) { __builtin_prefetch(x, 1, 3); } diff --git a/arch/sh/include/mach-landisk/mach/iodata_landisk.h b/arch/sh/include/mach-landisk/mach/iodata_landisk.h index 6fb04ab..f432773 100644 --- a/arch/sh/include/mach-landisk/mach/iodata_landisk.h +++ b/arch/sh/include/mach-landisk/mach/iodata_landisk.h @@ -2,7 +2,7 @@ #define __ASM_SH_IODATA_LANDISK_H /* - * linux/include/asm-sh/landisk/iodata_landisk.h + * arch/sh/include/mach-landisk/mach/iodata_landisk.h * * Copyright (C) 2000 Atom Create Engineering Co., Ltd. * @@ -27,7 +27,7 @@ #define IRQ_PCIINTA 5 /* PCI INTA IRQ */ #define IRQ_PCIINTB 6 /* PCI INTB IRQ */ -#define IRQ_PCIINDC 7 /* PCI INTC IRQ */ +#define IRQ_PCIINTC 7 /* PCI INTC IRQ */ #define IRQ_PCIINTD 8 /* PCI INTD IRQ */ #define IRQ_ATA 9 /* ATA IRQ */ #define IRQ_FATA 10 /* FATA IRQ */ @@ -35,6 +35,8 @@ #define IRQ_BUTTON 12 /* USL-5P Button IRQ */ #define IRQ_FAULT 13 /* USL-5P Fault IRQ */ +void init_landisk_IRQ(void); + #define __IO_PREFIX landisk #include <asm/io_generic.h> diff --git a/arch/sh/kernel/cpu/proc.c b/arch/sh/kernel/cpu/proc.c index e80a936..f47be87 100644 --- a/arch/sh/kernel/cpu/proc.c +++ b/arch/sh/kernel/cpu/proc.c @@ -25,7 +25,7 @@ static const char *cpu_name[] = { [CPU_SH5_101] = "SH5-101", [CPU_SH5_103] = "SH5-103", [CPU_MXG] = "MX-G", [CPU_SH7723] = "SH7723", [CPU_SH7366] = "SH7366", [CPU_SH7724] = "SH7724", - [CPU_SH_NONE] = "Unknown" + [CPU_SH7372] = "SH7372", [CPU_SH_NONE] = "Unknown" }; const char *get_cpu_subtype(struct sh_cpuinfo *c) diff --git a/arch/sh/kernel/cpu/sh2/setup-sh7619.c b/arch/sh/kernel/cpu/sh2/setup-sh7619.c index c363851..0f8befc 100644 --- a/arch/sh/kernel/cpu/sh2/setup-sh7619.c +++ b/arch/sh/kernel/cpu/sh2/setup-sh7619.c @@ -62,6 +62,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7619", vectors, NULL, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xf8400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 88, 88, 88, 88 }, }; @@ -77,6 +79,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xf8410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 92, 92, 92, 92 }, }; @@ -92,6 +96,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xf8420000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 96, 96, 96, 96 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-mxg.c b/arch/sh/kernel/cpu/sh2a/setup-mxg.c index 6c96ea0..949bf2b 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-mxg.c +++ b/arch/sh/kernel/cpu/sh2a/setup-mxg.c @@ -201,6 +201,8 @@ static struct platform_device mtu2_2_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xff804000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 220, 220, 220, 220 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c index d08bf4c..9df558d 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c @@ -180,6 +180,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7201", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 180, 180, 180, 180 } }; @@ -195,6 +197,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 184, 184, 184, 184 } }; @@ -210,6 +214,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 188, 188, 188, 188 } }; @@ -225,6 +231,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 192, 192, 192, 192 } }; @@ -240,6 +248,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xfffea000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 196, 196, 196, 196 } }; @@ -255,6 +265,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xfffea800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 200, 200, 200, 200 } }; @@ -270,6 +282,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xfffeb000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 204, 204, 204, 204 } }; @@ -285,6 +299,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xfffeb800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 208, 208, 208, 208 } }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c index 832f401..a43124e 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c @@ -176,6 +176,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7203", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 192, 192, 192, 192 }, }; @@ -191,6 +193,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 196, 196, 196, 196 }, }; @@ -206,6 +210,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 200, 200, 200, 200 }, }; @@ -221,6 +227,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 204, 204, 204, 204 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c index dc47b04..5d14f84 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c @@ -136,6 +136,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7206", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 240, 240, 240, 240 }, }; @@ -151,6 +153,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 244, 244, 244, 244 }, }; @@ -166,6 +170,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 248, 248, 248, 248 }, }; @@ -181,6 +187,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 252, 252, 252, 252 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7705.c b/arch/sh/kernel/cpu/sh3/setup-sh7705.c index baadd7f..cd2e702 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7705.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7705.c @@ -70,6 +70,9 @@ static DECLARE_INTC_DESC(intc_desc, "sh7705", vectors, NULL, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TIE | SCSCR_RIE | SCSCR_TE | + SCSCR_RE | SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 56, 56, 56 }, }; @@ -85,6 +88,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TIE | SCSCR_RIE | SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 52, 52, 52 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh770x.c b/arch/sh/kernel/cpu/sh3/setup-sh770x.c index 3cf8c8e..4551ad6 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh770x.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh770x.c @@ -109,6 +109,8 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffffe80, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 23, 23, 23, 0 }, }; @@ -126,6 +128,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4000150, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 56, 56, 56, 56 }, }; @@ -143,6 +147,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xa4000140, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_IRDA, .irqs = { 52, 52, 52, 52 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7710.c b/arch/sh/kernel/cpu/sh3/setup-sh7710.c index b0c2fb4..78f6b01 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7710.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7710.c @@ -99,6 +99,9 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE | + SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 52, 52, 52 }, }; @@ -114,6 +117,9 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE | + SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 56, 56, 56, 56 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7720.c b/arch/sh/kernel/cpu/sh3/setup-sh7720.c index 24b1713..365b94a6 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7720.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7720.c @@ -1,5 +1,5 @@ /* - * SH7720 Setup + * Setup code for SH7720, SH7721. * * Copyright (C) 2007 Markus Brunner, Mark Jonas * Copyright (C) 2009 Paul Mundt @@ -51,6 +51,8 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4430000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -66,6 +68,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4438000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; diff --git a/arch/sh/kernel/cpu/sh4/probe.c b/arch/sh/kernel/cpu/sh4/probe.c index b93458f..971cf0f 100644 --- a/arch/sh/kernel/cpu/sh4/probe.c +++ b/arch/sh/kernel/cpu/sh4/probe.c @@ -151,8 +151,14 @@ void __cpuinit cpu_probe(void) boot_cpu_data.flags |= CPU_HAS_L2_CACHE; break; case 0x10: + case 0x11: boot_cpu_data.type = CPU_SH7757; break; + case 0xd0: + case 0x40: /* yon-ten-go */ + boot_cpu_data.type = CPU_SH7372; + break; + } break; case 0x4000: /* 1st cut */ diff --git a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c index e916b18..5b28331 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c @@ -18,6 +18,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7750.c b/arch/sh/kernel/cpu/sh4/setup-sh7750.c index 911d196..c2b0aaa 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh7750.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh7750.c @@ -14,6 +14,7 @@ #include <linux/io.h> #include <linux/sh_timer.h> #include <linux/serial_sci.h> +#include <asm/machtypes.h> static struct resource rtc_resources[] = { [0] = { @@ -35,33 +36,37 @@ static struct platform_device rtc_device = { .resource = rtc_resources, }; -static struct plat_sci_port scif0_platform_data = { +static struct plat_sci_port sci_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 23, 23, 23, 0 }, }; -static struct platform_device scif0_device = { +static struct platform_device sci_device = { .name = "sh-sci", .id = 0, .dev = { - .platform_data = &scif0_platform_data, + .platform_data = &sci_platform_data, }, }; -static struct plat_sci_port scif1_platform_data = { +static struct plat_sci_port scif_platform_data = { .mapbase = 0xffe80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; -static struct platform_device scif1_device = { +static struct platform_device scif_device = { .name = "sh-sci", .id = 1, .dev = { - .platform_data = &scif1_platform_data, + .platform_data = &scif_platform_data, }, }; @@ -210,8 +215,6 @@ static struct platform_device tmu4_device = { #endif static struct platform_device *sh7750_devices[] __initdata = { - &scif0_device, - &scif1_device, &rtc_device, &tmu0_device, &tmu1_device, @@ -226,14 +229,19 @@ static struct platform_device *sh7750_devices[] __initdata = { static int __init sh7750_devices_setup(void) { + if (mach_is_rts7751r2d()) { + platform_register_device(&scif_device); + } else { + platform_register_device(&sci_device); + platform_register_device(&scif_device); + } + return platform_add_devices(sh7750_devices, ARRAY_SIZE(sh7750_devices)); } arch_initcall(sh7750_devices_setup); static struct platform_device *sh7750_early_devices[] __initdata = { - &scif0_device, - &scif1_device, &tmu0_device, &tmu1_device, &tmu2_device, @@ -247,6 +255,14 @@ static struct platform_device *sh7750_early_devices[] __initdata = { void __init plat_early_device_setup(void) { + if (mach_is_rts7751r2d()) { + scif_platform_data.scscr |= SCSCR_CKE1; + early_platform_add_devices(&scif_device, 1); + } else { + early_platform_add_devices(&sci_device, 1); + early_platform_add_devices(&scif_device, 1); + } + early_platform_add_devices(sh7750_early_devices, ARRAY_SIZE(sh7750_early_devices)); } diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7760.c b/arch/sh/kernel/cpu/sh4/setup-sh7760.c index 48ea8fe..78bbf23 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh7760.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh7760.c @@ -129,6 +129,8 @@ static DECLARE_INTC_DESC(intc_desc_irq, "sh7760-irq", vectors_irq, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfe600000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 53, 55, 54 }, }; @@ -145,6 +147,8 @@ static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfe610000, .flags = UPF_BOOT_AUTOCONF, .type = PORT_SCIF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .irqs = { 72, 73, 75, 74 }, }; @@ -159,6 +163,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfe620000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 77, 79, 78 }, }; @@ -174,6 +180,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfe480000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 80, 81, 82, 0 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c index 3681cafd..1b88483 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -34,6 +36,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -49,6 +53,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -64,6 +70,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffe30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 83, 83, 83, 83 }, }; @@ -360,6 +368,8 @@ void __init plat_early_device_setup(void) enum { UNUSED = 0, + ENABLED, + DISABLED, /* interrupt sources */ IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7, @@ -375,15 +385,13 @@ enum { I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI, I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI, SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI, - IRDA, - SDHI0, SDHI1, SDHI2, SDHI3, - CMT, TSIF, SIU, + IRDA, SDHI, CMT, TSIF, SIU, TMU0, TMU1, TMU2, JPU, LCDC, /* interrupt groups */ - DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, SDHI, USB, + DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, USB, }; static struct intc_vect vectors[] __initdata = { @@ -412,8 +420,8 @@ static struct intc_vect vectors[] __initdata = { INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0), INTC_VECT(I2C0_ALI, 0xe00), INTC_VECT(I2C0_TACKI, 0xe20), INTC_VECT(I2C0_WAITI, 0xe40), INTC_VECT(I2C0_DTEI, 0xe60), - INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0), - INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0), + INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0), + INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0), INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20), INTC_VECT(SIU, 0xf80), INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420), @@ -431,7 +439,6 @@ static struct intc_group groups[] __initdata = { INTC_GROUP(I2C0, I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI), INTC_GROUP(I2C1, I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI), INTC_GROUP(SIM, SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI), - INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3), INTC_GROUP(USB, USBI0, USBI1), }; @@ -452,7 +459,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USBI1, USBI0 } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ @@ -488,9 +495,13 @@ static struct intc_mask_reg ack_registers[] __initdata = { { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } }, }; -static DECLARE_INTC_DESC_ACK(intc_desc, "sh7343", vectors, groups, - mask_registers, prio_registers, sense_registers, - ack_registers); +static struct intc_desc intc_desc __initdata = { + .name = "sh7343", + .force_enable = ENABLED, + .force_disable = DISABLED, + .hw = INTC_HW_DESC(vectors, groups, mask_registers, + prio_registers, sense_registers, ack_registers), +}; void __init plat_irq_setup(void) { diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c index 8dab9e1..82616af 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c @@ -21,6 +21,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -319,6 +321,8 @@ void __init plat_early_device_setup(void) enum { UNUSED=0, + ENABLED, + DISABLED, /* interrupt sources */ IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7, @@ -332,14 +336,13 @@ enum { DENC, MSIOF, FLCTL_FLSTEI, FLCTL_FLENDI, FLCTL_FLTREQ0I, FLCTL_FLTREQ1I, I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI, - SDHI0, SDHI1, SDHI2, SDHI3, - CMT, TSIF, SIU, + SDHI, CMT, TSIF, SIU, TMU0, TMU1, TMU2, VEU2, LCDC, /* interrupt groups */ - DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C, SDHI, + DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C, }; static struct intc_vect vectors[] __initdata = { @@ -364,8 +367,8 @@ static struct intc_vect vectors[] __initdata = { INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0), INTC_VECT(I2C_ALI, 0xe00), INTC_VECT(I2C_TACKI, 0xe20), INTC_VECT(I2C_WAITI, 0xe40), INTC_VECT(I2C_DTEI, 0xe60), - INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0), - INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0), + INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0), + INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0), INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20), INTC_VECT(SIU, 0xf80), INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420), @@ -381,7 +384,6 @@ static struct intc_group groups[] __initdata = { INTC_GROUP(FLCTL, FLCTL_FLSTEI, FLCTL_FLENDI, FLCTL_FLTREQ0I, FLCTL_FLTREQ1I), INTC_GROUP(I2C, I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI), - INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3), }; static struct intc_mask_reg mask_registers[] __initdata = { @@ -403,7 +405,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB, } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ @@ -441,9 +443,13 @@ static struct intc_mask_reg ack_registers[] __initdata = { { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } }, }; -static DECLARE_INTC_DESC_ACK(intc_desc, "sh7366", vectors, groups, - mask_registers, prio_registers, sense_registers, - ack_registers); +static struct intc_desc intc_desc __initdata = { + .name = "sh7366", + .force_enable = ENABLED, + .force_disable = DISABLED, + .hw = INTC_HW_DESC(vectors, groups, mask_registers, + prio_registers, sense_registers, ack_registers), +}; void __init plat_irq_setup(void) { diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c index d551ed8..5813d80 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c @@ -181,6 +181,8 @@ struct platform_device dma_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -196,6 +198,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -211,6 +215,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -699,7 +705,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { DISABLED, DISABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB_USBI1, USB_USBI0, } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c index 0eadefd..0723822 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c @@ -24,6 +24,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -39,6 +41,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -54,6 +58,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -69,6 +75,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xa4e30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 56, 56, 56, 56 }, }; @@ -84,6 +92,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xa4e40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 88, 88, 88, 88 }, }; @@ -99,6 +109,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xa4e50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 109, 109, 109, 109 }, }; @@ -719,7 +731,7 @@ static struct intc_group groups[] __initdata = { static struct intc_mask_reg mask_registers[] __initdata = { { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */ { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0, - 0, DISABLED, ENABLED, ENABLED } }, + 0, ENABLED, ENABLED, ENABLED } }, { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */ { VIO_VOUI, VIO_VEU2HI,VIO_BEUI,VIO_CEUI,DMAC0A_DEI3,DMAC0A_DEI2,DMAC0A_DEI1,DMAC0A_DEI0 } }, { 0xa4080088, 0xa40800c8, 8, /* IMR2 / IMCR2 */ @@ -736,7 +748,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { 0, DISABLED, ENABLED, ENABLED, + { 0, ENABLED, ENABLED, ENABLED, 0, 0, SCIFA_SCIFA2, SIU_SIUI } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT_CMTI, 0, 0, USB_USI0,0 } }, diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c index 828c965..0333fe9 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c @@ -257,6 +257,8 @@ static struct platform_device dma1_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -272,6 +274,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -287,6 +291,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -302,6 +308,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xa4e30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 56, 56, 56, 56 }, }; @@ -317,6 +325,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xa4e40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 88, 88, 88, 88 }, }; @@ -332,6 +342,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xa4e50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 109, 109, 109, 109 }, }; @@ -1144,7 +1156,7 @@ static struct intc_group groups[] __initdata = { static struct intc_mask_reg mask_registers[] __initdata = { { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */ { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0, - 0, DISABLED, ENABLED, ENABLED } }, + 0, ENABLED, ENABLED, ENABLED } }, { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */ { VIO_VOU, VIO_VEU1, VIO_BEU0, VIO_CEU0, DMAC0A_DEI3, DMAC0A_DEI2, DMAC0A_DEI1, DMAC0A_DEI0 } }, @@ -1166,7 +1178,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI, I2C1_DTEI, I2C1_WAITI, I2C1_TACKI, I2C1_ALI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, SCIFA5, FSI } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB1, USB0, 0 } }, diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c index 749c638..9c1de26 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c @@ -20,6 +20,8 @@ static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfe4b0000, /* SCIF2 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -35,6 +37,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfe4c0000, /* SCIF3 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -50,6 +54,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xfe4d0000, /* SCIF4 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 104, 104, 104, 104 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c index 5b5f6b0..593eca6 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -34,6 +36,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe08000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -49,6 +53,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 104, 104, 104, 104 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c index 7270d7f..2c6aa22 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c @@ -17,6 +17,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xff923000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 61, 61, 61, 61 }, }; @@ -32,6 +34,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xff924000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 62, 62, 62, 62 }, }; @@ -47,6 +51,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xff925000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 63, 63, 63, 63 }, }; @@ -62,6 +68,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xff926000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 64, 64, 64, 64 }, }; @@ -77,6 +85,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xff927000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 65, 65, 65, 65 }, }; @@ -92,6 +102,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xff928000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 66, 66, 66, 66 }, }; @@ -107,6 +119,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xff929000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 67, 67, 67, 67 }, }; @@ -122,6 +136,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xff92a000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 68, 68, 68, 68 }, }; @@ -137,6 +153,8 @@ static struct platform_device scif7_device = { static struct plat_sci_port scif8_platform_data = { .mapbase = 0xff92b000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 69, 69, 69, 69 }, }; @@ -152,6 +170,8 @@ static struct platform_device scif8_device = { static struct plat_sci_port scif9_platform_data = { .mapbase = 0xff92c000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 70, 70, 70, 70 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c index 0f41486..08add7f 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c @@ -20,6 +20,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -35,6 +37,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -379,6 +383,7 @@ static int __init sh7780_devices_setup(void) ARRAY_SIZE(sh7780_devices)); } arch_initcall(sh7780_devices_setup); + static struct platform_device *sh7780_early_devices[] __initdata = { &scif0_device, &scif1_device, @@ -392,6 +397,13 @@ static struct platform_device *sh7780_early_devices[] __initdata = { void __init plat_early_device_setup(void) { + if (mach_is_sh2007()) { + scif0_platform_data.scscr &= ~SCSCR_CKE1; + scif0_platform_data.scbrr_algo_id = SCBRR_ALGO_2; + scif1_platform_data.scscr &= ~SCSCR_CKE1; + scif1_platform_data.scbrr_algo_id = SCBRR_ALGO_2; + } + early_platform_add_devices(sh7780_early_devices, ARRAY_SIZE(sh7780_early_devices)); } diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c index c9a572b..18d8fc1 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c @@ -23,6 +23,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffea0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -38,6 +40,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffeb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 44, 44, 44, 44 }, }; @@ -53,6 +57,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffec0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 60, 60, 60, 60 }, }; @@ -68,6 +74,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffed0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 61, 61, 61, 61 }, }; @@ -83,6 +91,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xffee0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 62, 62, 62, 62 }, }; @@ -98,6 +108,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xffef0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 63, 63, 63, 63 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c index 0170dbd..1656b8c 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c @@ -29,6 +29,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffea0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; @@ -47,6 +49,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffeb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 44, 44, 44, 44 }, }; @@ -62,6 +66,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffec0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 50, 50, 50, 50 }, }; @@ -77,6 +83,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffed0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 51, 51, 51, 51 }, }; @@ -92,6 +100,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xffee0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 52, 52, 52, 52 }, }; @@ -107,6 +117,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xffef0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 53, 53, 53, 53 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-shx3.c b/arch/sh/kernel/cpu/sh4a/setup-shx3.c index 013f0b1..bb20880 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-shx3.c +++ b/arch/sh/kernel/cpu/sh4a/setup-shx3.c @@ -29,6 +29,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffc30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; @@ -44,6 +46,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffc40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 44, 45, 47, 46 }, }; @@ -59,6 +63,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffc60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 53, 55, 54 }, }; diff --git a/arch/sh/kernel/cpu/sh5/setup-sh5.c b/arch/sh/kernel/cpu/sh5/setup-sh5.c index d910666..18419f1 100644 --- a/arch/sh/kernel/cpu/sh5/setup-sh5.c +++ b/arch/sh/kernel/cpu/sh5/setup-sh5.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = PHYS_PERIPHERAL_BLOCK + 0x01030000, .flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 39, 40, 42, 0 }, }; diff --git a/arch/sh/kernel/cpu/shmobile/pm.c b/arch/sh/kernel/cpu/shmobile/pm.c index e559687..a6f95ae 100644 --- a/arch/sh/kernel/cpu/shmobile/pm.c +++ b/arch/sh/kernel/cpu/shmobile/pm.c @@ -141,7 +141,7 @@ static int sh_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops sh_pm_ops = { +static const struct platform_suspend_ops sh_pm_ops = { .enter = sh_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c index eb4cc4e..d1bffbc 100644 --- a/arch/sh/mm/cache-sh5.c +++ b/arch/sh/mm/cache-sh5.c @@ -568,7 +568,7 @@ static void sh5_flush_dcache_page(void *page) } /* - * Flush the range [start,end] of kernel virtual adddress space from + * Flush the range [start,end] of kernel virtual address space from * the I-cache. The corresponding range must be purged from the * D-cache also because the SH-5 doesn't have cache snooping between * the caches. The addresses will be visible through the superpage diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 3385b28..0d3f912 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -2,7 +2,7 @@ * linux/arch/sh/mm/init.c * * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2002 - 2010 Paul Mundt + * Copyright (C) 2002 - 2011 Paul Mundt * * Based on linux/arch/i386/mm/init.c: * Copyright (C) 1995 Linus Torvalds @@ -325,11 +325,17 @@ void __init paging_init(void) int nid; memblock_init(); - sh_mv.mv_mem_init(); early_reserve_mem(); + /* + * Once the early reservations are out of the way, give the + * platforms a chance to kick out some memory. + */ + if (sh_mv.mv_mem_reserve) + sh_mv.mv_mem_reserve(); + memblock_enforce_memory_limit(memory_limit); memblock_analyze(); diff --git a/arch/sh/tools/mach-types b/arch/sh/tools/mach-types index 0e68465..6dd56c4 100644 --- a/arch/sh/tools/mach-types +++ b/arch/sh/tools/mach-types @@ -9,6 +9,7 @@ SE SH_SOLUTION_ENGINE HIGHLANDER SH_HIGHLANDER RTS7751R2D SH_RTS7751R2D RSK SH_RSK +ALPHA_BOARD SH_ALPHA_BOARD # # List of companion chips / MFDs. @@ -61,3 +62,5 @@ ESPT SH_ESPT POLARIS SH_POLARIS KFR2R09 SH_KFR2R09 ECOVEC SH_ECOVEC +APSH4A3A SH_APSH4A3A +APSH4AD0A SH_APSH4AD0A diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c index e447938..0dc714f 100644 --- a/arch/sparc/kernel/cpu.c +++ b/arch/sparc/kernel/cpu.c @@ -375,5 +375,5 @@ static int __init cpu_type_probe(void) return 0; } -arch_initcall(cpu_type_probe); +early_initcall(cpu_type_probe); #endif diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c index b87873c..ae96cf5 100644 --- a/arch/sparc/kernel/pcr.c +++ b/arch/sparc/kernel/pcr.c @@ -168,4 +168,4 @@ out_unregister: return err; } -arch_initcall(pcr_arch_init); +early_initcall(pcr_arch_init); diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 42ad2ba..1e97709 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -622,7 +622,7 @@ static const char CHAFSR_PERR_msg[] = static const char CHAFSR_IERR_msg[] = "Internal processor error"; static const char CHAFSR_ISAP_msg[] = - "System request parity error on incoming addresss"; + "System request parity error on incoming address"; static const char CHAFSR_UCU_msg[] = "Uncorrectable E-cache ECC error for ifetch/data"; static const char CHAFSR_UCC_msg[] = diff --git a/arch/um/Kconfig.um b/arch/um/Kconfig.um index 50d6aa2..f8d1d0d 100644 --- a/arch/um/Kconfig.um +++ b/arch/um/Kconfig.um @@ -131,7 +131,7 @@ config NR_CPUS config HIGHMEM bool "Highmem support (EXPERIMENTAL)" - depends on !64BIT && EXPERIMENTAL + depends on !64BIT && BROKEN default n help This was used to allow UML to run with big amounts of memory. diff --git a/arch/um/drivers/line.c b/arch/um/drivers/line.c index 1664cce..050e4dd 100644 --- a/arch/um/drivers/line.c +++ b/arch/um/drivers/line.c @@ -821,12 +821,12 @@ void register_winch_irq(int fd, int tty_fd, int pid, struct tty_struct *tty, static void unregister_winch(struct tty_struct *tty) { - struct list_head *ele; + struct list_head *ele, *next; struct winch *winch; spin_lock(&winch_handler_lock); - list_for_each(ele, &winch_handlers) { + list_for_each_safe(ele, next, &winch_handlers) { winch = list_entry(ele, struct winch, list); if (winch->tty == tty) { free_winch(winch, 1); diff --git a/arch/um/drivers/mmapper_kern.c b/arch/um/drivers/mmapper_kern.c index 8501e7d..7e0619c 100644 --- a/arch/um/drivers/mmapper_kern.c +++ b/arch/um/drivers/mmapper_kern.c @@ -37,13 +37,7 @@ static ssize_t mmapper_write(struct file *file, const char __user *buf, if (*ppos > mmapper_size) return -EINVAL; - if (count > mmapper_size - *ppos) - count = mmapper_size - *ppos; - - if (copy_from_user(&v_buf[*ppos], buf, count)) - return -EFAULT; - - return count; + return simple_write_to_buffer(v_buf, mmapper_size, ppos, buf, count); } static long mmapper_ioctl(struct file *file, unsigned int cmd, unsigned long arg) @@ -137,3 +131,4 @@ module_exit(mmapper_exit); MODULE_AUTHOR("Greg Lonnon <glonnon@ridgerun.com>"); MODULE_DESCRIPTION("DSPLinux simulator mmapper driver"); +MODULE_LICENSE("GPL"); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b6fccb0..36ed2e2 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -51,6 +51,7 @@ config X86 select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA + select HAVE_KERNEL_XZ select HAVE_KERNEL_LZO select HAVE_HW_BREAKPOINT select HAVE_MIXED_BREAKPOINTS_REGS @@ -65,6 +66,7 @@ config X86 select HAVE_SPARSE_IRQ select GENERIC_IRQ_PROBE select GENERIC_PENDING_IRQ if SMP + select USE_GENERIC_SMP_HELPERS if SMP config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) @@ -203,10 +205,6 @@ config HAVE_INTEL_TXT def_bool y depends on EXPERIMENTAL && DMAR && ACPI -config USE_GENERIC_SMP_HELPERS - def_bool y - depends on SMP - config X86_32_SMP def_bool y depends on X86_32 && SMP @@ -2078,11 +2076,17 @@ config OLPC_OPENFIRMWARE bool "Support for OLPC's Open Firmware" depends on !X86_64 && !X86_PAE default n + select OF help This option adds support for the implementation of Open Firmware that is used on the OLPC XO-1 Children's Machine. If unsure, say N here. +config OLPC_OPENFIRMWARE_DT + bool + default y if OLPC_OPENFIRMWARE && PROC_DEVICETREE + select OF_PROMTREE + endif # X86_32 config AMD_NB diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 0c22955..09664ef 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -4,7 +4,7 @@ # create a compressed vmlinux image from the original vmlinux # -targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o +targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC @@ -49,12 +49,15 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE $(call if_changed,bzip2) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE $(call if_changed,lzma) +$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE + $(call if_changed,xzkern) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE $(call if_changed,lzo) suffix-$(CONFIG_KERNEL_GZIP) := gz suffix-$(CONFIG_KERNEL_BZIP2) := bz2 suffix-$(CONFIG_KERNEL_LZMA) := lzma +suffix-$(CONFIG_KERNEL_XZ) := xz suffix-$(CONFIG_KERNEL_LZO) := lzo quiet_cmd_mkpiggy = MKPIGGY $@ diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 325c052..3a19d04 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -139,6 +139,10 @@ static int lines, cols; #include "../../../../lib/decompress_unlzma.c" #endif +#ifdef CONFIG_KERNEL_XZ +#include "../../../../lib/decompress_unxz.c" +#endif + #ifdef CONFIG_KERNEL_LZO #include "../../../../lib/decompress_unlzo.c" #endif diff --git a/arch/x86/boot/compressed/mkpiggy.c b/arch/x86/boot/compressed/mkpiggy.c index 5c22812..646aa78 100644 --- a/arch/x86/boot/compressed/mkpiggy.c +++ b/arch/x86/boot/compressed/mkpiggy.c @@ -74,7 +74,7 @@ int main(int argc, char *argv[]) offs = (olen > ilen) ? olen - ilen : 0; offs += olen >> 12; /* Add 8 bytes for each 32K block */ - offs += 32*1024 + 18; /* Add 32K + 18 bytes slack */ + offs += 64*1024 + 128; /* Add 64K + 128 bytes slack */ offs = (offs+4095) & ~4095; /* Round to a 4K boundary */ printf(".section \".rodata..compressed\",\"a\",@progbits\n"); diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index ff16756..8fe2a49 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -9,6 +9,20 @@ * Vinodh Gopal <vinodh.gopal@intel.com> * Kahraman Akdemir * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Erdinc Ozturk (erdinc.ozturk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Adrian Hoban <adrian.hoban@intel.com> + * James Guilford (james.guilford@intel.com) + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Tadeusz Struk (tadeusz.struk@intel.com) + * Wajdi Feghali (wajdi.k.feghali@intel.com) + * Copyright (c) 2010, Intel Corporation. + * + * Ported x86_64 version to x86: + * Author: Mathias Krause <minipli@googlemail.com> + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -18,8 +32,62 @@ #include <linux/linkage.h> #include <asm/inst.h> +#ifdef __x86_64__ +.data +POLY: .octa 0xC2000000000000000000000000000001 +TWOONE: .octa 0x00000001000000000000000000000001 + +# order of these constants should not change. +# more specifically, ALL_F should follow SHIFT_MASK, +# and ZERO should follow ALL_F + +SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F +MASK1: .octa 0x0000000000000000ffffffffffffffff +MASK2: .octa 0xffffffffffffffff0000000000000000 +SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 +ALL_F: .octa 0xffffffffffffffffffffffffffffffff +ZERO: .octa 0x00000000000000000000000000000000 +ONE: .octa 0x00000000000000000000000000000001 +F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0 +dec: .octa 0x1 +enc: .octa 0x2 + + .text + +#define STACK_OFFSET 8*3 +#define HashKey 16*0 // store HashKey <<1 mod poly here +#define HashKey_2 16*1 // store HashKey^2 <<1 mod poly here +#define HashKey_3 16*2 // store HashKey^3 <<1 mod poly here +#define HashKey_4 16*3 // store HashKey^4 <<1 mod poly here +#define HashKey_k 16*4 // store XOR of High 64 bits and Low 64 + // bits of HashKey <<1 mod poly here + //(for Karatsuba purposes) +#define HashKey_2_k 16*5 // store XOR of High 64 bits and Low 64 + // bits of HashKey^2 <<1 mod poly here + // (for Karatsuba purposes) +#define HashKey_3_k 16*6 // store XOR of High 64 bits and Low 64 + // bits of HashKey^3 <<1 mod poly here + // (for Karatsuba purposes) +#define HashKey_4_k 16*7 // store XOR of High 64 bits and Low 64 + // bits of HashKey^4 <<1 mod poly here + // (for Karatsuba purposes) +#define VARIABLE_OFFSET 16*8 + +#define arg1 rdi +#define arg2 rsi +#define arg3 rdx +#define arg4 rcx +#define arg5 r8 +#define arg6 r9 +#define arg7 STACK_OFFSET+8(%r14) +#define arg8 STACK_OFFSET+16(%r14) +#define arg9 STACK_OFFSET+24(%r14) +#define arg10 STACK_OFFSET+32(%r14) +#endif + + #define STATE1 %xmm0 #define STATE2 %xmm4 #define STATE3 %xmm5 @@ -32,12 +100,16 @@ #define IN IN1 #define KEY %xmm2 #define IV %xmm3 + #define BSWAP_MASK %xmm10 #define CTR %xmm11 #define INC %xmm12 +#ifdef __x86_64__ +#define AREG %rax #define KEYP %rdi #define OUTP %rsi +#define UKEYP OUTP #define INP %rdx #define LEN %rcx #define IVP %r8 @@ -46,6 +118,1588 @@ #define TKEYP T1 #define T2 %r11 #define TCTR_LOW T2 +#else +#define AREG %eax +#define KEYP %edi +#define OUTP AREG +#define UKEYP OUTP +#define INP %edx +#define LEN %esi +#define IVP %ebp +#define KLEN %ebx +#define T1 %ecx +#define TKEYP T1 +#endif + + +#ifdef __x86_64__ +/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +* +* +* Input: A and B (128-bits each, bit-reflected) +* Output: C = A*B*x mod poly, (i.e. >>1 ) +* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +* +*/ +.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5 + movdqa \GH, \TMP1 + pshufd $78, \GH, \TMP2 + pshufd $78, \HK, \TMP3 + pxor \GH, \TMP2 # TMP2 = a1+a0 + pxor \HK, \TMP3 # TMP3 = b1+b0 + PCLMULQDQ 0x11, \HK, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \HK, \GH # GH = a0*b0 + PCLMULQDQ 0x00, \TMP3, \TMP2 # TMP2 = (a0+a1)*(b1+b0) + pxor \GH, \TMP2 + pxor \TMP1, \TMP2 # TMP2 = (a0*b0)+(a1*b0) + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \GH + pxor \TMP2, \TMP1 # TMP2:GH holds the result of GH*HK + + # first phase of the reduction + + movdqa \GH, \TMP2 + movdqa \GH, \TMP3 + movdqa \GH, \TMP4 # copy GH into TMP2,TMP3 and TMP4 + # in in order to perform + # independent shifts + pslld $31, \TMP2 # packed right shift <<31 + pslld $30, \TMP3 # packed right shift <<30 + pslld $25, \TMP4 # packed right shift <<25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift TMP5 1 DW + pslldq $12, \TMP2 # left shift TMP2 3 DWs + pxor \TMP2, \GH + + # second phase of the reduction + + movdqa \GH,\TMP2 # copy GH into TMP2,TMP3 and TMP4 + # in in order to perform + # independent shifts + movdqa \GH,\TMP3 + movdqa \GH,\TMP4 + psrld $1,\TMP2 # packed left shift >>1 + psrld $2,\TMP3 # packed left shift >>2 + psrld $7,\TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \GH + pxor \TMP1, \GH # result is in TMP1 +.endm + +/* +* if a = number of total plaintext bytes +* b = floor(a/16) +* num_initial_blocks = b mod 4 +* encrypt the initial num_initial_blocks blocks and apply ghash on +* the ciphertext +* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers +* are clobbered +* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified +*/ + + +.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ +XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + mov arg7, %r10 # %r10 = AAD + mov arg8, %r12 # %r12 = aadLen + mov %r12, %r11 + pxor %xmm\i, %xmm\i +_get_AAD_loop\num_initial_blocks\operation: + movd (%r10), \TMP1 + pslldq $12, \TMP1 + psrldq $4, %xmm\i + pxor \TMP1, %xmm\i + add $4, %r10 + sub $4, %r12 + jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 + je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 +_get_AAD_loop2\num_initial_blocks\operation: + psrldq $4, %xmm\i + sub $4, %r12 + cmp %r11, %r12 + jne _get_AAD_loop2\num_initial_blocks\operation +_get_AAD_loop2_done\num_initial_blocks\operation: + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + + xor %r11, %r11 # initialise the data pointer offset as zero + + # start AES for num_initial_blocks blocks + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), \XMM0 # XMM0 = Y0 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM0 + +.if (\i == 5) || (\i == 6) || (\i == 7) +.irpc index, \i_seq + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + +.endr +.irpc index, \i_seq + pxor 16*0(%arg1), %xmm\index +.endr +.irpc index, \i_seq + movaps 0x10(%rdi), \TMP1 + AESENC \TMP1, %xmm\index # Round 1 +.endr +.irpc index, \i_seq + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x30(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x40(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x50(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x60(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x70(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x80(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x90(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0xa0(%arg1), \TMP1 + AESENCLAST \TMP1, %xmm\index # Round 10 +.endr +.irpc index, \i_seq + movdqu (%arg3 , %r11, 1), \TMP1 + pxor \TMP1, %xmm\index + movdqu %xmm\index, (%arg2 , %r11, 1) + # write back plaintext/ciphertext for num_initial_blocks + add $16, %r11 + + movdqa \TMP1, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index + + # prepare plaintext/ciphertext for GHASH computation +.endr +.endif + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks + +.if \i == 5 + pxor %xmm5, %xmm6 + GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 6 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 7 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.endif + cmp $64, %r13 + jl _initial_blocks_done\num_initial_blocks\operation + # no need for precomputed values +/* +* +* Precomputations for HashKey parallel with encryption of first 4 blocks. +* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i +*/ + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM1 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM2 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM3 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM4 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + + pxor 16*0(%arg1), \XMM1 + pxor 16*0(%arg1), \XMM2 + pxor 16*0(%arg1), \XMM3 + pxor 16*0(%arg1), \XMM4 + movdqa \TMP3, \TMP5 + pshufd $78, \TMP3, \TMP1 + pxor \TMP3, \TMP1 + movdqa \TMP1, HashKey_k(%rsp) + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^2<<1 (mod poly) + movdqa \TMP5, HashKey_2(%rsp) +# HashKey_2 = HashKey^2<<1 (mod poly) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_2_k(%rsp) +.irpc index, 1234 # do 4 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_3(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_3_k(%rsp) +.irpc index, 56789 # do next 5 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_4(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_4_k(%rsp) + movaps 0xa0(%arg1), \TMP2 + AESENCLAST \TMP2, \XMM1 + AESENCLAST \TMP2, \XMM2 + AESENCLAST \TMP2, \XMM3 + AESENCLAST \TMP2, \XMM4 + movdqu 16*0(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM1 + movdqu \XMM1, 16*0(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM1 + movdqu 16*1(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM2 + movdqu \XMM2, 16*1(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM2 + movdqu 16*2(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM3 + movdqu \XMM3, 16*2(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM3 + movdqu 16*3(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM4 + movdqu \XMM4, 16*3(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM4 + add $64, %r11 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + pxor \XMMDst, \XMM1 +# combine GHASHed value with the corresponding ciphertext + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + +_initial_blocks_done\num_initial_blocks\operation: + +.endm + + +/* +* if a = number of total plaintext bytes +* b = floor(a/16) +* num_initial_blocks = b mod 4 +* encrypt the initial num_initial_blocks blocks and apply ghash on +* the ciphertext +* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers +* are clobbered +* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified +*/ + + +.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ +XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + mov arg7, %r10 # %r10 = AAD + mov arg8, %r12 # %r12 = aadLen + mov %r12, %r11 + pxor %xmm\i, %xmm\i +_get_AAD_loop\num_initial_blocks\operation: + movd (%r10), \TMP1 + pslldq $12, \TMP1 + psrldq $4, %xmm\i + pxor \TMP1, %xmm\i + add $4, %r10 + sub $4, %r12 + jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 + je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 +_get_AAD_loop2\num_initial_blocks\operation: + psrldq $4, %xmm\i + sub $4, %r12 + cmp %r11, %r12 + jne _get_AAD_loop2\num_initial_blocks\operation +_get_AAD_loop2_done\num_initial_blocks\operation: + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + + xor %r11, %r11 # initialise the data pointer offset as zero + + # start AES for num_initial_blocks blocks + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), \XMM0 # XMM0 = Y0 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM0 + +.if (\i == 5) || (\i == 6) || (\i == 7) +.irpc index, \i_seq + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + +.endr +.irpc index, \i_seq + pxor 16*0(%arg1), %xmm\index +.endr +.irpc index, \i_seq + movaps 0x10(%rdi), \TMP1 + AESENC \TMP1, %xmm\index # Round 1 +.endr +.irpc index, \i_seq + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x30(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x40(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x50(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x60(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x70(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x80(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x90(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0xa0(%arg1), \TMP1 + AESENCLAST \TMP1, %xmm\index # Round 10 +.endr +.irpc index, \i_seq + movdqu (%arg3 , %r11, 1), \TMP1 + pxor \TMP1, %xmm\index + movdqu %xmm\index, (%arg2 , %r11, 1) + # write back plaintext/ciphertext for num_initial_blocks + add $16, %r11 + + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index + + # prepare plaintext/ciphertext for GHASH computation +.endr +.endif + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks + +.if \i == 5 + pxor %xmm5, %xmm6 + GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 6 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 7 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.endif + cmp $64, %r13 + jl _initial_blocks_done\num_initial_blocks\operation + # no need for precomputed values +/* +* +* Precomputations for HashKey parallel with encryption of first 4 blocks. +* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i +*/ + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM1 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM2 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM3 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM4 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + + pxor 16*0(%arg1), \XMM1 + pxor 16*0(%arg1), \XMM2 + pxor 16*0(%arg1), \XMM3 + pxor 16*0(%arg1), \XMM4 + movdqa \TMP3, \TMP5 + pshufd $78, \TMP3, \TMP1 + pxor \TMP3, \TMP1 + movdqa \TMP1, HashKey_k(%rsp) + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^2<<1 (mod poly) + movdqa \TMP5, HashKey_2(%rsp) +# HashKey_2 = HashKey^2<<1 (mod poly) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_2_k(%rsp) +.irpc index, 1234 # do 4 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_3(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_3_k(%rsp) +.irpc index, 56789 # do next 5 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_4(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_4_k(%rsp) + movaps 0xa0(%arg1), \TMP2 + AESENCLAST \TMP2, \XMM1 + AESENCLAST \TMP2, \XMM2 + AESENCLAST \TMP2, \XMM3 + AESENCLAST \TMP2, \XMM4 + movdqu 16*0(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM1 + movdqu 16*1(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM2 + movdqu 16*2(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM3 + movdqu 16*3(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM4 + movdqu \XMM1, 16*0(%arg2 , %r11 , 1) + movdqu \XMM2, 16*1(%arg2 , %r11 , 1) + movdqu \XMM3, 16*2(%arg2 , %r11 , 1) + movdqu \XMM4, 16*3(%arg2 , %r11 , 1) + + add $64, %r11 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + pxor \XMMDst, \XMM1 +# combine GHASHed value with the corresponding ciphertext + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + +_initial_blocks_done\num_initial_blocks\operation: + +.endm + +/* +* encrypt 4 blocks at a time +* ghash the 4 previously encrypted ciphertext blocks +* arg1, %arg2, %arg3 are used as pointers only, not modified +* %r11 is the data offset value +*/ +.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \ +TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation + + movdqa \XMM1, \XMM5 + movdqa \XMM2, \XMM6 + movdqa \XMM3, \XMM7 + movdqa \XMM4, \XMM8 + + movdqa SHUF_MASK(%rip), %xmm15 + # multiply TMP5 * HashKey using karatsuba + + movdqa \XMM5, \TMP4 + pshufd $78, \XMM5, \TMP6 + pxor \XMM5, \TMP6 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 + movdqa \XMM0, \XMM1 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM2 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM3 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor (%arg1), \XMM1 + pxor (%arg1), \XMM2 + pxor (%arg1), \XMM3 + pxor (%arg1), \XMM4 + movdqa HashKey_4_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) + movaps 0x10(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 2 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movdqa \XMM6, \TMP1 + pshufd $78, \XMM6, \TMP2 + pxor \XMM6, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 + movaps 0x30(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 3 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 + movaps 0x40(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 4 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_3_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x50(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 5 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM6, \XMM5 + pxor \TMP2, \TMP6 + movdqa \XMM7, \TMP1 + pshufd $78, \XMM7, \TMP2 + pxor \XMM7, \TMP2 + movdqa HashKey_2(%rsp ), \TMP5 + + # Multiply TMP5 * HashKey using karatsuba + + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x60(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 6 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 + movaps 0x70(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 7 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_2_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x80(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 8 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM7, \XMM5 + pxor \TMP2, \TMP6 + + # Multiply XMM8 * HashKey + # XMM8 and TMP5 hold the values for the two operands + + movdqa \XMM8, \TMP1 + pshufd $78, \XMM8, \TMP2 + pxor \XMM8, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x90(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 9 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 + movaps 0xa0(%arg1), \TMP3 + AESENCLAST \TMP3, \XMM1 # Round 10 + AESENCLAST \TMP3, \XMM2 + AESENCLAST \TMP3, \XMM3 + AESENCLAST \TMP3, \XMM4 + movdqa HashKey_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqu (%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK + movdqu 16(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK + movdqu 32(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK + movdqu 48(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK + movdqu \XMM1, (%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM2, 16(%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM3, 32(%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM4, 48(%arg2,%r11,1) # Write to the ciphertext buffer + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor \TMP4, \TMP1 + pxor \XMM8, \XMM5 + pxor \TMP6, \TMP2 + pxor \TMP1, \TMP2 + pxor \XMM5, \TMP2 + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \XMM5 + pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 + + # first phase of reduction + + movdqa \XMM5, \TMP2 + movdqa \XMM5, \TMP3 + movdqa \XMM5, \TMP4 +# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently + pslld $31, \TMP2 # packed right shift << 31 + pslld $30, \TMP3 # packed right shift << 30 + pslld $25, \TMP4 # packed right shift << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift T5 1 DW + pslldq $12, \TMP2 # left shift T2 3 DWs + pxor \TMP2, \XMM5 + + # second phase of reduction + + movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 + movdqa \XMM5,\TMP3 + movdqa \XMM5,\TMP4 + psrld $1, \TMP2 # packed left shift >>1 + psrld $2, \TMP3 # packed left shift >>2 + psrld $7, \TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \XMM5 + pxor \TMP1, \XMM5 # result is in TMP1 + + pxor \XMM5, \XMM1 +.endm + +/* +* decrypt 4 blocks at a time +* ghash the 4 previously decrypted ciphertext blocks +* arg1, %arg2, %arg3 are used as pointers only, not modified +* %r11 is the data offset value +*/ +.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \ +TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation + + movdqa \XMM1, \XMM5 + movdqa \XMM2, \XMM6 + movdqa \XMM3, \XMM7 + movdqa \XMM4, \XMM8 + + movdqa SHUF_MASK(%rip), %xmm15 + # multiply TMP5 * HashKey using karatsuba + + movdqa \XMM5, \TMP4 + pshufd $78, \XMM5, \TMP6 + pxor \XMM5, \TMP6 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 + movdqa \XMM0, \XMM1 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM2 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM3 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor (%arg1), \XMM1 + pxor (%arg1), \XMM2 + pxor (%arg1), \XMM3 + pxor (%arg1), \XMM4 + movdqa HashKey_4_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) + movaps 0x10(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 2 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movdqa \XMM6, \TMP1 + pshufd $78, \XMM6, \TMP2 + pxor \XMM6, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 + movaps 0x30(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 3 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 + movaps 0x40(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 4 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_3_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x50(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 5 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM6, \XMM5 + pxor \TMP2, \TMP6 + movdqa \XMM7, \TMP1 + pshufd $78, \XMM7, \TMP2 + pxor \XMM7, \TMP2 + movdqa HashKey_2(%rsp ), \TMP5 + + # Multiply TMP5 * HashKey using karatsuba + + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x60(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 6 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 + movaps 0x70(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 7 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_2_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x80(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 8 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM7, \XMM5 + pxor \TMP2, \TMP6 + + # Multiply XMM8 * HashKey + # XMM8 and TMP5 hold the values for the two operands + + movdqa \XMM8, \TMP1 + pshufd $78, \XMM8, \TMP2 + pxor \XMM8, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x90(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 9 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 + movaps 0xa0(%arg1), \TMP3 + AESENCLAST \TMP3, \XMM1 # Round 10 + AESENCLAST \TMP3, \XMM2 + AESENCLAST \TMP3, \XMM3 + AESENCLAST \TMP3, \XMM4 + movdqa HashKey_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqu (%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK + movdqu \XMM1, (%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM1 + movdqu 16(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK + movdqu \XMM2, 16(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM2 + movdqu 32(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK + movdqu \XMM3, 32(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM3 + movdqu 48(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK + movdqu \XMM4, 48(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor \TMP4, \TMP1 + pxor \XMM8, \XMM5 + pxor \TMP6, \TMP2 + pxor \TMP1, \TMP2 + pxor \XMM5, \TMP2 + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \XMM5 + pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 + + # first phase of reduction + + movdqa \XMM5, \TMP2 + movdqa \XMM5, \TMP3 + movdqa \XMM5, \TMP4 +# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently + pslld $31, \TMP2 # packed right shift << 31 + pslld $30, \TMP3 # packed right shift << 30 + pslld $25, \TMP4 # packed right shift << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift T5 1 DW + pslldq $12, \TMP2 # left shift T2 3 DWs + pxor \TMP2, \XMM5 + + # second phase of reduction + + movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 + movdqa \XMM5,\TMP3 + movdqa \XMM5,\TMP4 + psrld $1, \TMP2 # packed left shift >>1 + psrld $2, \TMP3 # packed left shift >>2 + psrld $7, \TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \XMM5 + pxor \TMP1, \XMM5 # result is in TMP1 + + pxor \XMM5, \XMM1 +.endm + +/* GHASH the last 4 ciphertext blocks. */ +.macro GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \ +TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst + + # Multiply TMP6 * HashKey (using Karatsuba) + + movdqa \XMM1, \TMP6 + pshufd $78, \XMM1, \TMP2 + pxor \XMM1, \TMP2 + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP6 # TMP6 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM1 # XMM1 = a0*b0 + movdqa HashKey_4_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqa \XMM1, \XMMDst + movdqa \TMP2, \XMM1 # result in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + + movdqa \XMM2, \TMP1 + pshufd $78, \XMM2, \TMP2 + pxor \XMM2, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM2 # XMM2 = a0*b0 + movdqa HashKey_3_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM2, \XMMDst + pxor \TMP2, \XMM1 +# results accumulated in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + + movdqa \XMM3, \TMP1 + pshufd $78, \XMM3, \TMP2 + pxor \XMM3, \TMP2 + movdqa HashKey_2(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM3 # XMM3 = a0*b0 + movdqa HashKey_2_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM3, \XMMDst + pxor \TMP2, \XMM1 # results accumulated in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + movdqa \XMM4, \TMP1 + pshufd $78, \XMM4, \TMP2 + pxor \XMM4, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM4 # XMM4 = a0*b0 + movdqa HashKey_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM4, \XMMDst + pxor \XMM1, \TMP2 + pxor \TMP6, \TMP2 + pxor \XMMDst, \TMP2 + # middle section of the temp results combined as in karatsuba algorithm + movdqa \TMP2, \TMP4 + pslldq $8, \TMP4 # left shift TMP4 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP4, \XMMDst + pxor \TMP2, \TMP6 +# TMP6:XMMDst holds the result of the accumulated carry-less multiplications + # first phase of the reduction + movdqa \XMMDst, \TMP2 + movdqa \XMMDst, \TMP3 + movdqa \XMMDst, \TMP4 +# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently + pslld $31, \TMP2 # packed right shifting << 31 + pslld $30, \TMP3 # packed right shifting << 30 + pslld $25, \TMP4 # packed right shifting << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP7 + psrldq $4, \TMP7 # right shift TMP7 1 DW + pslldq $12, \TMP2 # left shift TMP2 3 DWs + pxor \TMP2, \XMMDst + + # second phase of the reduction + movdqa \XMMDst, \TMP2 + # make 3 copies of XMMDst for doing 3 shift operations + movdqa \XMMDst, \TMP3 + movdqa \XMMDst, \TMP4 + psrld $1, \TMP2 # packed left shift >> 1 + psrld $2, \TMP3 # packed left shift >> 2 + psrld $7, \TMP4 # packed left shift >> 7 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + pxor \TMP7, \TMP2 + pxor \TMP2, \XMMDst + pxor \TMP6, \XMMDst # reduced result is in XMMDst +.endm + +/* Encryption of a single block done*/ +.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 + + pxor (%arg1), \XMM0 + movaps 16(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 32(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 48(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 64(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 80(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 96(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 112(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 128(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 144(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 160(%arg1), \TMP1 + AESENCLAST \TMP1, \XMM0 +.endm + + +/***************************************************************************** +* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. +* u8 *out, // Plaintext output. Encrypt in-place is allowed. +* const u8 *in, // Ciphertext input +* u64 plaintext_len, // Length of data in bytes for decryption. +* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) +* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) +* // concatenated with 0x00000001. 16-byte aligned pointer. +* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. +* const u8 *aad, // Additional Authentication Data (AAD) +* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes +* u8 *auth_tag, // Authenticated Tag output. The driver will compare this to the +* // given authentication tag and only return the plaintext if they match. +* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 +* // (most likely), 12 or 8. +* +* Assumptions: +* +* keys: +* keys are pre-expanded and aligned to 16 bytes. we are using the first +* set of 11 keys in the data structure void *aes_ctx +* +* iv: +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Salt (From the SA) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Initialization Vector | +* | (This is the sequence number from IPSec header) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x1 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* +* +* AAD: +* AAD padded to 128 bits with 0 +* for example, assume AAD is a u32 vector +* +* if AAD is 8 bytes: +* AAD[3] = {A0, A1}; +* padded AAD in xmm register = {A1 A0 0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A1) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 32-bit Sequence Number (A0) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 32-bit Sequence Number +* +* if AAD is 12 bytes: +* AAD[3] = {A0, A1, A2}; +* padded AAD in xmm register = {A2 A1 A0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A2) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 64-bit Extended Sequence Number {A1,A0} | +* | | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 64-bit Extended Sequence Number +* +* aadLen: +* from the definition of the spec, aadLen can only be 8 or 12 bytes. +* The code supports 16 too but for other sizes, the code will fail. +* +* TLen: +* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +* For other sizes, the code will fail. +* +* poly = x^128 + x^127 + x^126 + x^121 + 1 +* +*****************************************************************************/ + +ENTRY(aesni_gcm_dec) + push %r12 + push %r13 + push %r14 + mov %rsp, %r14 +/* +* states of %xmm registers %xmm6:%xmm15 not saved +* all %xmm registers are clobbered +*/ + sub $VARIABLE_OFFSET, %rsp + and $~63, %rsp # align rsp to 64 bytes + mov %arg6, %r12 + movdqu (%r12), %xmm13 # %xmm13 = HashKey + movdqa SHUF_MASK(%rip), %xmm2 + PSHUFB_XMM %xmm2, %xmm13 + + +# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH) + + movdqa %xmm13, %xmm2 + psllq $1, %xmm13 + psrlq $63, %xmm2 + movdqa %xmm2, %xmm1 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + por %xmm2, %xmm13 + + # Reduction + + pshufd $0x24, %xmm1, %xmm2 + pcmpeqd TWOONE(%rip), %xmm2 + pand POLY(%rip), %xmm2 + pxor %xmm2, %xmm13 # %xmm13 holds the HashKey<<1 (mod poly) + + + # Decrypt first few blocks + + movdqa %xmm13, HashKey(%rsp) # store HashKey<<1 (mod poly) + mov %arg4, %r13 # save the number of bytes of plaintext/ciphertext + and $-16, %r13 # %r13 = %r13 - (%r13 mod 16) + mov %r13, %r12 + and $(3<<4), %r12 + jz _initial_num_blocks_is_0_decrypt + cmp $(2<<4), %r12 + jb _initial_num_blocks_is_1_decrypt + je _initial_num_blocks_is_2_decrypt +_initial_num_blocks_is_3_decrypt: + INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec + sub $48, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_2_decrypt: + INITIAL_BLOCKS_DEC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec + sub $32, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_1_decrypt: + INITIAL_BLOCKS_DEC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec + sub $16, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_0_decrypt: + INITIAL_BLOCKS_DEC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec +_initial_blocks_decrypted: + cmp $0, %r13 + je _zero_cipher_left_decrypt + sub $64, %r13 + je _four_cipher_left_decrypt +_decrypt_by_4: + GHASH_4_ENCRYPT_4_PARALLEL_DEC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \ +%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec + add $64, %r11 + sub $64, %r13 + jne _decrypt_by_4 +_four_cipher_left_decrypt: + GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \ +%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8 +_zero_cipher_left_decrypt: + mov %arg4, %r13 + and $15, %r13 # %r13 = arg4 (mod 16) + je _multiple_of_16_bytes_decrypt + + # Handle the last <16 byte block seperately + + paddd ONE(%rip), %xmm0 # increment CNT to get Yn + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm0 + + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Yn) + sub $16, %r11 + add %r13, %r11 + movdqu (%arg3,%r11,1), %xmm1 # recieve the last <16 byte block + lea SHIFT_MASK+16(%rip), %r12 + sub %r13, %r12 +# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes +# (%r13 is the number of bytes in plaintext mod 16) + movdqu (%r12), %xmm2 # get the appropriate shuffle mask + PSHUFB_XMM %xmm2, %xmm1 # right shift 16-%r13 butes + + movdqa %xmm1, %xmm2 + pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn) + movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0 + pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0 + pand %xmm1, %xmm2 + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10 ,%xmm2 + + pxor %xmm2, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # GHASH computation for the last <16 byte block + sub %r13, %r11 + add $16, %r11 + + # output %r13 bytes + MOVQ_R64_XMM %xmm0, %rax + cmp $8, %r13 + jle _less_than_8_bytes_left_decrypt + mov %rax, (%arg2 , %r11, 1) + add $8, %r11 + psrldq $8, %xmm0 + MOVQ_R64_XMM %xmm0, %rax + sub $8, %r13 +_less_than_8_bytes_left_decrypt: + mov %al, (%arg2, %r11, 1) + add $1, %r11 + shr $8, %rax + sub $1, %r13 + jne _less_than_8_bytes_left_decrypt +_multiple_of_16_bytes_decrypt: + mov arg8, %r12 # %r13 = aadLen (number of bytes) + shl $3, %r12 # convert into number of bits + movd %r12d, %xmm15 # len(A) in %xmm15 + shl $3, %arg4 # len(C) in bits (*128) + MOVQ_R64_XMM %arg4, %xmm1 + pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000 + pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C) + pxor %xmm15, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # final GHASH computation + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm8 + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), %xmm0 # %xmm0 = Y0 + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0) + pxor %xmm8, %xmm0 +_return_T_decrypt: + mov arg9, %r10 # %r10 = authTag + mov arg10, %r11 # %r11 = auth_tag_len + cmp $16, %r11 + je _T_16_decrypt + cmp $12, %r11 + je _T_12_decrypt +_T_8_decrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + jmp _return_T_done_decrypt +_T_12_decrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + psrldq $8, %xmm0 + movd %xmm0, %eax + mov %eax, 8(%r10) + jmp _return_T_done_decrypt +_T_16_decrypt: + movdqu %xmm0, (%r10) +_return_T_done_decrypt: + mov %r14, %rsp + pop %r14 + pop %r13 + pop %r12 + ret + + +/***************************************************************************** +* void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. +* u8 *out, // Ciphertext output. Encrypt in-place is allowed. +* const u8 *in, // Plaintext input +* u64 plaintext_len, // Length of data in bytes for encryption. +* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) +* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) +* // concatenated with 0x00000001. 16-byte aligned pointer. +* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. +* const u8 *aad, // Additional Authentication Data (AAD) +* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes +* u8 *auth_tag, // Authenticated Tag output. +* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely), +* // 12 or 8. +* +* Assumptions: +* +* keys: +* keys are pre-expanded and aligned to 16 bytes. we are using the +* first set of 11 keys in the data structure void *aes_ctx +* +* +* iv: +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Salt (From the SA) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Initialization Vector | +* | (This is the sequence number from IPSec header) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x1 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* +* +* AAD: +* AAD padded to 128 bits with 0 +* for example, assume AAD is a u32 vector +* +* if AAD is 8 bytes: +* AAD[3] = {A0, A1}; +* padded AAD in xmm register = {A1 A0 0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A1) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 32-bit Sequence Number (A0) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 32-bit Sequence Number +* +* if AAD is 12 bytes: +* AAD[3] = {A0, A1, A2}; +* padded AAD in xmm register = {A2 A1 A0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A2) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 64-bit Extended Sequence Number {A1,A0} | +* | | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 64-bit Extended Sequence Number +* +* aadLen: +* from the definition of the spec, aadLen can only be 8 or 12 bytes. +* The code supports 16 too but for other sizes, the code will fail. +* +* TLen: +* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +* For other sizes, the code will fail. +* +* poly = x^128 + x^127 + x^126 + x^121 + 1 +***************************************************************************/ +ENTRY(aesni_gcm_enc) + push %r12 + push %r13 + push %r14 + mov %rsp, %r14 +# +# states of %xmm registers %xmm6:%xmm15 not saved +# all %xmm registers are clobbered +# + sub $VARIABLE_OFFSET, %rsp + and $~63, %rsp + mov %arg6, %r12 + movdqu (%r12), %xmm13 + movdqa SHUF_MASK(%rip), %xmm2 + PSHUFB_XMM %xmm2, %xmm13 + + +# precompute HashKey<<1 mod poly from the HashKey (required for GHASH) + + movdqa %xmm13, %xmm2 + psllq $1, %xmm13 + psrlq $63, %xmm2 + movdqa %xmm2, %xmm1 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + por %xmm2, %xmm13 + + # reduce HashKey<<1 + + pshufd $0x24, %xmm1, %xmm2 + pcmpeqd TWOONE(%rip), %xmm2 + pand POLY(%rip), %xmm2 + pxor %xmm2, %xmm13 + movdqa %xmm13, HashKey(%rsp) + mov %arg4, %r13 # %xmm13 holds HashKey<<1 (mod poly) + and $-16, %r13 + mov %r13, %r12 + + # Encrypt first few blocks + + and $(3<<4), %r12 + jz _initial_num_blocks_is_0_encrypt + cmp $(2<<4), %r12 + jb _initial_num_blocks_is_1_encrypt + je _initial_num_blocks_is_2_encrypt +_initial_num_blocks_is_3_encrypt: + INITIAL_BLOCKS_ENC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc + sub $48, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_2_encrypt: + INITIAL_BLOCKS_ENC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc + sub $32, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_1_encrypt: + INITIAL_BLOCKS_ENC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc + sub $16, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_0_encrypt: + INITIAL_BLOCKS_ENC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc +_initial_blocks_encrypted: + + # Main loop - Encrypt remaining blocks + + cmp $0, %r13 + je _zero_cipher_left_encrypt + sub $64, %r13 + je _four_cipher_left_encrypt +_encrypt_by_4_encrypt: + GHASH_4_ENCRYPT_4_PARALLEL_ENC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \ +%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc + add $64, %r11 + sub $64, %r13 + jne _encrypt_by_4_encrypt +_four_cipher_left_encrypt: + GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \ +%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8 +_zero_cipher_left_encrypt: + mov %arg4, %r13 + and $15, %r13 # %r13 = arg4 (mod 16) + je _multiple_of_16_bytes_encrypt + + # Handle the last <16 Byte block seperately + paddd ONE(%rip), %xmm0 # INCR CNT to get Yn + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm0 + + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) + sub $16, %r11 + add %r13, %r11 + movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte blocks + lea SHIFT_MASK+16(%rip), %r12 + sub %r13, %r12 + # adjust the shuffle mask pointer to be able to shift 16-r13 bytes + # (%r13 is the number of bytes in plaintext mod 16) + movdqu (%r12), %xmm2 # get the appropriate shuffle mask + PSHUFB_XMM %xmm2, %xmm1 # shift right 16-r13 byte + pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn) + movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + # get the appropriate mask to mask out top 16-r13 bytes of xmm0 + pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0 + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10,%xmm0 + + pxor %xmm0, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # GHASH computation for the last <16 byte block + sub %r13, %r11 + add $16, %r11 + PSHUFB_XMM %xmm10, %xmm1 + + # shuffle xmm0 back to output as ciphertext + + # Output %r13 bytes + MOVQ_R64_XMM %xmm0, %rax + cmp $8, %r13 + jle _less_than_8_bytes_left_encrypt + mov %rax, (%arg2 , %r11, 1) + add $8, %r11 + psrldq $8, %xmm0 + MOVQ_R64_XMM %xmm0, %rax + sub $8, %r13 +_less_than_8_bytes_left_encrypt: + mov %al, (%arg2, %r11, 1) + add $1, %r11 + shr $8, %rax + sub $1, %r13 + jne _less_than_8_bytes_left_encrypt +_multiple_of_16_bytes_encrypt: + mov arg8, %r12 # %r12 = addLen (number of bytes) + shl $3, %r12 + movd %r12d, %xmm15 # len(A) in %xmm15 + shl $3, %arg4 # len(C) in bits (*128) + MOVQ_R64_XMM %arg4, %xmm1 + pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000 + pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C) + pxor %xmm15, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # final GHASH computation + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm8 # perform a 16 byte swap + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), %xmm0 # %xmm0 = Y0 + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm15 # Encrypt(K, Y0) + pxor %xmm8, %xmm0 +_return_T_encrypt: + mov arg9, %r10 # %r10 = authTag + mov arg10, %r11 # %r11 = auth_tag_len + cmp $16, %r11 + je _T_16_encrypt + cmp $12, %r11 + je _T_12_encrypt +_T_8_encrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + jmp _return_T_done_encrypt +_T_12_encrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + psrldq $8, %xmm0 + movd %xmm0, %eax + mov %eax, 8(%r10) + jmp _return_T_done_encrypt +_T_16_encrypt: + movdqu %xmm0, (%r10) +_return_T_done_encrypt: + mov %r14, %rsp + pop %r14 + pop %r13 + pop %r12 + ret + +#endif + _key_expansion_128: _key_expansion_256a: @@ -55,10 +1709,11 @@ _key_expansion_256a: shufps $0b10001100, %xmm0, %xmm4 pxor %xmm4, %xmm0 pxor %xmm1, %xmm0 - movaps %xmm0, (%rcx) - add $0x10, %rcx + movaps %xmm0, (TKEYP) + add $0x10, TKEYP ret +.align 4 _key_expansion_192a: pshufd $0b01010101, %xmm1, %xmm1 shufps $0b00010000, %xmm0, %xmm4 @@ -76,12 +1731,13 @@ _key_expansion_192a: movaps %xmm0, %xmm1 shufps $0b01000100, %xmm0, %xmm6 - movaps %xmm6, (%rcx) + movaps %xmm6, (TKEYP) shufps $0b01001110, %xmm2, %xmm1 - movaps %xmm1, 16(%rcx) - add $0x20, %rcx + movaps %xmm1, 0x10(TKEYP) + add $0x20, TKEYP ret +.align 4 _key_expansion_192b: pshufd $0b01010101, %xmm1, %xmm1 shufps $0b00010000, %xmm0, %xmm4 @@ -96,10 +1752,11 @@ _key_expansion_192b: pxor %xmm3, %xmm2 pxor %xmm5, %xmm2 - movaps %xmm0, (%rcx) - add $0x10, %rcx + movaps %xmm0, (TKEYP) + add $0x10, TKEYP ret +.align 4 _key_expansion_256b: pshufd $0b10101010, %xmm1, %xmm1 shufps $0b00010000, %xmm2, %xmm4 @@ -107,8 +1764,8 @@ _key_expansion_256b: shufps $0b10001100, %xmm2, %xmm4 pxor %xmm4, %xmm2 pxor %xmm1, %xmm2 - movaps %xmm2, (%rcx) - add $0x10, %rcx + movaps %xmm2, (TKEYP) + add $0x10, TKEYP ret /* @@ -116,17 +1773,23 @@ _key_expansion_256b: * unsigned int key_len) */ ENTRY(aesni_set_key) - movups (%rsi), %xmm0 # user key (first 16 bytes) - movaps %xmm0, (%rdi) - lea 0x10(%rdi), %rcx # key addr - movl %edx, 480(%rdi) +#ifndef __x86_64__ + pushl KEYP + movl 8(%esp), KEYP # ctx + movl 12(%esp), UKEYP # in_key + movl 16(%esp), %edx # key_len +#endif + movups (UKEYP), %xmm0 # user key (first 16 bytes) + movaps %xmm0, (KEYP) + lea 0x10(KEYP), TKEYP # key addr + movl %edx, 480(KEYP) pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x cmp $24, %dl jb .Lenc_key128 je .Lenc_key192 - movups 0x10(%rsi), %xmm2 # other user key - movaps %xmm2, (%rcx) - add $0x10, %rcx + movups 0x10(UKEYP), %xmm2 # other user key + movaps %xmm2, (TKEYP) + add $0x10, TKEYP AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 call _key_expansion_256a AESKEYGENASSIST 0x1 %xmm0 %xmm1 @@ -155,7 +1818,7 @@ ENTRY(aesni_set_key) call _key_expansion_256a jmp .Ldec_key .Lenc_key192: - movq 0x10(%rsi), %xmm2 # other user key + movq 0x10(UKEYP), %xmm2 # other user key AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 call _key_expansion_192a AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2 @@ -195,33 +1858,47 @@ ENTRY(aesni_set_key) AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10 call _key_expansion_128 .Ldec_key: - sub $0x10, %rcx - movaps (%rdi), %xmm0 - movaps (%rcx), %xmm1 - movaps %xmm0, 240(%rcx) - movaps %xmm1, 240(%rdi) - add $0x10, %rdi - lea 240-16(%rcx), %rsi + sub $0x10, TKEYP + movaps (KEYP), %xmm0 + movaps (TKEYP), %xmm1 + movaps %xmm0, 240(TKEYP) + movaps %xmm1, 240(KEYP) + add $0x10, KEYP + lea 240-16(TKEYP), UKEYP .align 4 .Ldec_key_loop: - movaps (%rdi), %xmm0 + movaps (KEYP), %xmm0 AESIMC %xmm0 %xmm1 - movaps %xmm1, (%rsi) - add $0x10, %rdi - sub $0x10, %rsi - cmp %rcx, %rdi + movaps %xmm1, (UKEYP) + add $0x10, KEYP + sub $0x10, UKEYP + cmp TKEYP, KEYP jb .Ldec_key_loop - xor %rax, %rax + xor AREG, AREG +#ifndef __x86_64__ + popl KEYP +#endif ret /* * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) */ ENTRY(aesni_enc) +#ifndef __x86_64__ + pushl KEYP + pushl KLEN + movl 12(%esp), KEYP + movl 16(%esp), OUTP + movl 20(%esp), INP +#endif movl 480(KEYP), KLEN # key length movups (INP), STATE # input call _aesni_enc1 movups STATE, (OUTP) # output +#ifndef __x86_64__ + popl KLEN + popl KEYP +#endif ret /* @@ -236,6 +1913,7 @@ ENTRY(aesni_enc) * KEY * TKEYP (T1) */ +.align 4 _aesni_enc1: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -298,6 +1976,7 @@ _aesni_enc1: * KEY * TKEYP (T1) */ +.align 4 _aesni_enc4: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -391,11 +2070,22 @@ _aesni_enc4: * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) */ ENTRY(aesni_dec) +#ifndef __x86_64__ + pushl KEYP + pushl KLEN + movl 12(%esp), KEYP + movl 16(%esp), OUTP + movl 20(%esp), INP +#endif mov 480(KEYP), KLEN # key length add $240, KEYP movups (INP), STATE # input call _aesni_dec1 movups STATE, (OUTP) #output +#ifndef __x86_64__ + popl KLEN + popl KEYP +#endif ret /* @@ -410,6 +2100,7 @@ ENTRY(aesni_dec) * KEY * TKEYP (T1) */ +.align 4 _aesni_dec1: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -472,6 +2163,7 @@ _aesni_dec1: * KEY * TKEYP (T1) */ +.align 4 _aesni_dec4: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -566,6 +2258,15 @@ _aesni_dec4: * size_t len) */ ENTRY(aesni_ecb_enc) +#ifndef __x86_64__ + pushl LEN + pushl KEYP + pushl KLEN + movl 16(%esp), KEYP + movl 20(%esp), OUTP + movl 24(%esp), INP + movl 28(%esp), LEN +#endif test LEN, LEN # check length jz .Lecb_enc_ret mov 480(KEYP), KLEN @@ -602,6 +2303,11 @@ ENTRY(aesni_ecb_enc) cmp $16, LEN jge .Lecb_enc_loop1 .Lecb_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN +#endif ret /* @@ -609,6 +2315,15 @@ ENTRY(aesni_ecb_enc) * size_t len); */ ENTRY(aesni_ecb_dec) +#ifndef __x86_64__ + pushl LEN + pushl KEYP + pushl KLEN + movl 16(%esp), KEYP + movl 20(%esp), OUTP + movl 24(%esp), INP + movl 28(%esp), LEN +#endif test LEN, LEN jz .Lecb_dec_ret mov 480(KEYP), KLEN @@ -646,6 +2361,11 @@ ENTRY(aesni_ecb_dec) cmp $16, LEN jge .Lecb_dec_loop1 .Lecb_dec_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN +#endif ret /* @@ -653,6 +2373,17 @@ ENTRY(aesni_ecb_dec) * size_t len, u8 *iv) */ ENTRY(aesni_cbc_enc) +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl 20(%esp), KEYP + movl 24(%esp), OUTP + movl 28(%esp), INP + movl 32(%esp), LEN + movl 36(%esp), IVP +#endif cmp $16, LEN jb .Lcbc_enc_ret mov 480(KEYP), KLEN @@ -670,6 +2401,12 @@ ENTRY(aesni_cbc_enc) jge .Lcbc_enc_loop movups STATE, (IVP) .Lcbc_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif ret /* @@ -677,6 +2414,17 @@ ENTRY(aesni_cbc_enc) * size_t len, u8 *iv) */ ENTRY(aesni_cbc_dec) +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl 20(%esp), KEYP + movl 24(%esp), OUTP + movl 28(%esp), INP + movl 32(%esp), LEN + movl 36(%esp), IVP +#endif cmp $16, LEN jb .Lcbc_dec_just_ret mov 480(KEYP), KLEN @@ -690,16 +2438,30 @@ ENTRY(aesni_cbc_dec) movaps IN1, STATE1 movups 0x10(INP), IN2 movaps IN2, STATE2 +#ifdef __x86_64__ movups 0x20(INP), IN3 movaps IN3, STATE3 movups 0x30(INP), IN4 movaps IN4, STATE4 +#else + movups 0x20(INP), IN1 + movaps IN1, STATE3 + movups 0x30(INP), IN2 + movaps IN2, STATE4 +#endif call _aesni_dec4 pxor IV, STATE1 +#ifdef __x86_64__ pxor IN1, STATE2 pxor IN2, STATE3 pxor IN3, STATE4 movaps IN4, IV +#else + pxor (INP), STATE2 + pxor 0x10(INP), STATE3 + pxor IN1, STATE4 + movaps IN2, IV +#endif movups STATE1, (OUTP) movups STATE2, 0x10(OUTP) movups STATE3, 0x20(OUTP) @@ -727,8 +2489,15 @@ ENTRY(aesni_cbc_dec) .Lcbc_dec_ret: movups IV, (IVP) .Lcbc_dec_just_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif ret +#ifdef __x86_64__ .align 16 .Lbswap_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 @@ -744,6 +2513,7 @@ ENTRY(aesni_cbc_dec) * INC: == 1, in little endian * BSWAP_MASK == endian swapping mask */ +.align 4 _aesni_inc_init: movaps .Lbswap_mask, BSWAP_MASK movaps IV, CTR @@ -768,6 +2538,7 @@ _aesni_inc_init: * CTR: == output IV, in little endian * TCTR_LOW: == lower qword of CTR */ +.align 4 _aesni_inc: paddq INC, CTR add $1, TCTR_LOW @@ -839,3 +2610,4 @@ ENTRY(aesni_ctr_enc) movups IV, (IVP) .Lctr_enc_just_ret: ret +#endif diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 2cb3dcc..e1e60c7 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -5,6 +5,14 @@ * Copyright (C) 2008, Intel Corp. * Author: Huang Ying <ying.huang@intel.com> * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Adrian Hoban <adrian.hoban@intel.com> + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Tadeusz Struk (tadeusz.struk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Copyright (c) 2010, Intel Corporation. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -21,6 +29,10 @@ #include <crypto/ctr.h> #include <asm/i387.h> #include <asm/aes.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/aead.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) #define HAS_CTR @@ -42,8 +54,31 @@ struct async_aes_ctx { struct cryptd_ablkcipher *cryptd_tfm; }; -#define AESNI_ALIGN 16 +/* This data is stored at the end of the crypto_tfm struct. + * It's a type of per "session" data storage location. + * This needs to be 16 byte aligned. + */ +struct aesni_rfc4106_gcm_ctx { + u8 hash_subkey[16]; + struct crypto_aes_ctx aes_key_expanded; + u8 nonce[4]; + struct cryptd_aead *cryptd_tfm; +}; + +struct aesni_gcm_set_hash_subkey_result { + int err; + struct completion completion; +}; + +struct aesni_hash_subkey_req_data { + u8 iv[16]; + struct aesni_gcm_set_hash_subkey_result result; + struct scatterlist sg; +}; + +#define AESNI_ALIGN (16) #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) +#define RFC4106_HASH_SUBKEY_SIZE 16 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); @@ -59,9 +94,62 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +#ifdef CONFIG_X86_64 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +/* asmlinkage void aesni_gcm_enc() + * void *ctx, AES Key schedule. Starts on a 16 byte boundary. + * u8 *out, Ciphertext output. Encrypt in-place is allowed. + * const u8 *in, Plaintext input + * unsigned long plaintext_len, Length of data in bytes for encryption. + * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) + * concatenated with 8 byte Initialisation Vector (from IPSec ESP + * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. + * const u8 *aad, Additional Authentication Data (AAD) + * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this + * is going to be 8 or 12 bytes + * u8 *auth_tag, Authenticated Tag output. + * unsigned long auth_tag_len), Authenticated Tag Length in bytes. + * Valid values are 16 (most likely), 12 or 8. + */ +asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, + const u8 *in, unsigned long plaintext_len, u8 *iv, + u8 *hash_subkey, const u8 *aad, unsigned long aad_len, + u8 *auth_tag, unsigned long auth_tag_len); + +/* asmlinkage void aesni_gcm_dec() + * void *ctx, AES Key schedule. Starts on a 16 byte boundary. + * u8 *out, Plaintext output. Decrypt in-place is allowed. + * const u8 *in, Ciphertext input + * unsigned long ciphertext_len, Length of data in bytes for decryption. + * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) + * concatenated with 8 byte Initialisation Vector (from IPSec ESP + * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. + * const u8 *aad, Additional Authentication Data (AAD) + * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going + * to be 8 or 12 bytes + * u8 *auth_tag, Authenticated Tag output. + * unsigned long auth_tag_len) Authenticated Tag Length in bytes. + * Valid values are 16 (most likely), 12 or 8. + */ +asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, + const u8 *in, unsigned long ciphertext_len, u8 *iv, + u8 *hash_subkey, const u8 *aad, unsigned long aad_len, + u8 *auth_tag, unsigned long auth_tag_len); + +static inline struct +aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) +{ + return + (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *) + crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); +} +#endif + static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) { unsigned long addr = (unsigned long)raw_ctx; @@ -324,6 +412,7 @@ static struct crypto_alg blk_cbc_alg = { }, }; +#ifdef CONFIG_X86_64 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, struct blkcipher_walk *walk) { @@ -389,6 +478,7 @@ static struct crypto_alg blk_ctr_alg = { }, }, }; +#endif static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) @@ -536,6 +626,7 @@ static struct crypto_alg ablk_cbc_alg = { }, }; +#ifdef CONFIG_X86_64 static int ablk_ctr_init(struct crypto_tfm *tfm) { struct cryptd_ablkcipher *cryptd_tfm; @@ -612,6 +703,7 @@ static struct crypto_alg ablk_rfc3686_ctr_alg = { }, }; #endif +#endif #ifdef HAS_LRW static int ablk_lrw_init(struct crypto_tfm *tfm) @@ -730,6 +822,424 @@ static struct crypto_alg ablk_xts_alg = { }; #endif +#ifdef CONFIG_X86_64 +static int rfc4106_init(struct crypto_tfm *tfm) +{ + struct cryptd_aead *cryptd_tfm; + struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); + cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + ctx->cryptd_tfm = cryptd_tfm; + tfm->crt_aead.reqsize = sizeof(struct aead_request) + + crypto_aead_reqsize(&cryptd_tfm->base); + return 0; +} + +static void rfc4106_exit(struct crypto_tfm *tfm) +{ + struct aesni_rfc4106_gcm_ctx *ctx = + (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); + if (!IS_ERR(ctx->cryptd_tfm)) + cryptd_free_aead(ctx->cryptd_tfm); + return; +} + +static void +rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) +{ + struct aesni_gcm_set_hash_subkey_result *result = req->data; + + if (err == -EINPROGRESS) + return; + result->err = err; + complete(&result->completion); +} + +static int +rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) +{ + struct crypto_ablkcipher *ctr_tfm; + struct ablkcipher_request *req; + int ret = -EINVAL; + struct aesni_hash_subkey_req_data *req_data; + + ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); + if (IS_ERR(ctr_tfm)) + return PTR_ERR(ctr_tfm); + + crypto_ablkcipher_clear_flags(ctr_tfm, ~0); + + ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); + if (ret) { + crypto_free_ablkcipher(ctr_tfm); + return ret; + } + + req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); + if (!req) { + crypto_free_ablkcipher(ctr_tfm); + return -EINVAL; + } + + req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); + if (!req_data) { + crypto_free_ablkcipher(ctr_tfm); + return -ENOMEM; + } + memset(req_data->iv, 0, sizeof(req_data->iv)); + + /* Clear the data in the hash sub key container to zero.*/ + /* We want to cipher all zeros to create the hash sub key. */ + memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); + + init_completion(&req_data->result.completion); + sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); + ablkcipher_request_set_tfm(req, ctr_tfm); + ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + rfc4106_set_hash_subkey_done, + &req_data->result); + + ablkcipher_request_set_crypt(req, &req_data->sg, + &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); + + ret = crypto_ablkcipher_encrypt(req); + if (ret == -EINPROGRESS || ret == -EBUSY) { + ret = wait_for_completion_interruptible + (&req_data->result.completion); + if (!ret) + ret = req_data->result.err; + } + ablkcipher_request_free(req); + kfree(req_data); + crypto_free_ablkcipher(ctr_tfm); + return ret; +} + +static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, + unsigned int key_len) +{ + int ret = 0; + struct crypto_tfm *tfm = crypto_aead_tfm(parent); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); + u8 *new_key_mem = NULL; + + if (key_len < 4) { + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + /*Account for 4 byte nonce at the end.*/ + key_len -= 4; + if (key_len != AES_KEYSIZE_128) { + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); + /*This must be on a 16 byte boundary!*/ + if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) + return -EINVAL; + + if ((unsigned long)key % AESNI_ALIGN) { + /*key is not aligned: use an auxuliar aligned pointer*/ + new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); + if (!new_key_mem) + return -ENOMEM; + + new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); + memcpy(new_key_mem, key, key_len); + key = new_key_mem; + } + + if (!irq_fpu_usable()) + ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), + key, key_len); + else { + kernel_fpu_begin(); + ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); + kernel_fpu_end(); + } + /*This must be on a 16 byte boundary!*/ + if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { + ret = -EINVAL; + goto exit; + } + ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); +exit: + kfree(new_key_mem); + return ret; +} + +/* This is the Integrity Check Value (aka the authentication tag length and can + * be 8, 12 or 16 bytes long. */ +static int rfc4106_set_authsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + crypto_aead_crt(parent)->authsize = authsize; + crypto_aead_crt(cryptd_child)->authsize = authsize; + return 0; +} + +static int rfc4106_encrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + return crypto_aead_encrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = cryptd_child->base.crt_aead.encrypt(req); + kernel_fpu_end(); + return ret; + } +} + +static int rfc4106_decrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + return crypto_aead_decrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = cryptd_child->base.crt_aead.decrypt(req); + kernel_fpu_end(); + return ret; + } +} + +static struct crypto_alg rfc4106_alg = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, + .cra_alignmask = 0, + .cra_type = &crypto_nivaead_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list), + .cra_init = rfc4106_init, + .cra_exit = rfc4106_exit, + .cra_u = { + .aead = { + .setkey = rfc4106_set_key, + .setauthsize = rfc4106_set_authsize, + .encrypt = rfc4106_encrypt, + .decrypt = rfc4106_decrypt, + .geniv = "seqiv", + .ivsize = 8, + .maxauthsize = 16, + }, + }, +}; + +static int __driver_rfc4106_encrypt(struct aead_request *req) +{ + u8 one_entry_in_sg = 0; + u8 *src, *dst, *assoc; + __be32 counter = cpu_to_be32(1); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 iv_tab[16+AESNI_ALIGN]; + u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); + struct scatter_walk src_sg_walk; + struct scatter_walk assoc_sg_walk; + struct scatter_walk dst_sg_walk; + unsigned int i; + + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length equal */ + /* to 8 or 12 bytes */ + if (unlikely(req->assoclen != 8 && req->assoclen != 12)) + return -EINVAL; + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { + one_entry_in_sg = 1; + scatterwalk_start(&src_sg_walk, req->src); + scatterwalk_start(&assoc_sg_walk, req->assoc); + src = scatterwalk_map(&src_sg_walk, 0); + assoc = scatterwalk_map(&assoc_sg_walk, 0); + dst = src; + if (unlikely(req->src != req->dst)) { + scatterwalk_start(&dst_sg_walk, req->dst); + dst = scatterwalk_map(&dst_sg_walk, 0); + } + + } else { + /* Allocate memory for src, dst, assoc */ + src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, + GFP_ATOMIC); + if (unlikely(!src)) + return -ENOMEM; + assoc = (src + req->cryptlen + auth_tag_len); + scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); + scatterwalk_map_and_copy(assoc, req->assoc, 0, + req->assoclen, 0); + dst = src; + } + + aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, + ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst + + ((unsigned long)req->cryptlen), auth_tag_len); + + /* The authTag (aka the Integrity Check Value) needs to be written + * back to the packet. */ + if (one_entry_in_sg) { + if (unlikely(req->src != req->dst)) { + scatterwalk_unmap(dst, 0); + scatterwalk_done(&dst_sg_walk, 0, 0); + } + scatterwalk_unmap(src, 0); + scatterwalk_unmap(assoc, 0); + scatterwalk_done(&src_sg_walk, 0, 0); + scatterwalk_done(&assoc_sg_walk, 0, 0); + } else { + scatterwalk_map_and_copy(dst, req->dst, 0, + req->cryptlen + auth_tag_len, 1); + kfree(src); + } + return 0; +} + +static int __driver_rfc4106_decrypt(struct aead_request *req) +{ + u8 one_entry_in_sg = 0; + u8 *src, *dst, *assoc; + unsigned long tempCipherLen = 0; + __be32 counter = cpu_to_be32(1); + int retval = 0; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 iv_and_authTag[32+AESNI_ALIGN]; + u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); + u8 *authTag = iv + 16; + struct scatter_walk src_sg_walk; + struct scatter_walk assoc_sg_walk; + struct scatter_walk dst_sg_walk; + unsigned int i; + + if (unlikely((req->cryptlen < auth_tag_len) || + (req->assoclen != 8 && req->assoclen != 12))) + return -EINVAL; + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length */ + /* equal to 8 or 12 bytes */ + + tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { + one_entry_in_sg = 1; + scatterwalk_start(&src_sg_walk, req->src); + scatterwalk_start(&assoc_sg_walk, req->assoc); + src = scatterwalk_map(&src_sg_walk, 0); + assoc = scatterwalk_map(&assoc_sg_walk, 0); + dst = src; + if (unlikely(req->src != req->dst)) { + scatterwalk_start(&dst_sg_walk, req->dst); + dst = scatterwalk_map(&dst_sg_walk, 0); + } + + } else { + /* Allocate memory for src, dst, assoc */ + src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); + if (!src) + return -ENOMEM; + assoc = (src + req->cryptlen + auth_tag_len); + scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); + scatterwalk_map_and_copy(assoc, req->assoc, 0, + req->assoclen, 0); + dst = src; + } + + aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, + ctx->hash_subkey, assoc, (unsigned long)req->assoclen, + authTag, auth_tag_len); + + /* Compare generated tag with passed in tag. */ + retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? + -EBADMSG : 0; + + if (one_entry_in_sg) { + if (unlikely(req->src != req->dst)) { + scatterwalk_unmap(dst, 0); + scatterwalk_done(&dst_sg_walk, 0, 0); + } + scatterwalk_unmap(src, 0); + scatterwalk_unmap(assoc, 0); + scatterwalk_done(&src_sg_walk, 0, 0); + scatterwalk_done(&assoc_sg_walk, 0, 0); + } else { + scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); + kfree(src); + } + return retval; +} + +static struct crypto_alg __rfc4106_alg = { + .cra_name = "__gcm-aes-aesni", + .cra_driver_name = "__driver-gcm-aes-aesni", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_AEAD, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, + .cra_alignmask = 0, + .cra_type = &crypto_aead_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list), + .cra_u = { + .aead = { + .encrypt = __driver_rfc4106_encrypt, + .decrypt = __driver_rfc4106_decrypt, + }, + }, +}; +#endif + static int __init aesni_init(void) { int err; @@ -738,6 +1248,7 @@ static int __init aesni_init(void) printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); return -ENODEV; } + if ((err = crypto_register_alg(&aesni_alg))) goto aes_err; if ((err = crypto_register_alg(&__aesni_alg))) @@ -746,18 +1257,24 @@ static int __init aesni_init(void) goto blk_ecb_err; if ((err = crypto_register_alg(&blk_cbc_alg))) goto blk_cbc_err; - if ((err = crypto_register_alg(&blk_ctr_alg))) - goto blk_ctr_err; if ((err = crypto_register_alg(&ablk_ecb_alg))) goto ablk_ecb_err; if ((err = crypto_register_alg(&ablk_cbc_alg))) goto ablk_cbc_err; +#ifdef CONFIG_X86_64 + if ((err = crypto_register_alg(&blk_ctr_alg))) + goto blk_ctr_err; if ((err = crypto_register_alg(&ablk_ctr_alg))) goto ablk_ctr_err; + if ((err = crypto_register_alg(&__rfc4106_alg))) + goto __aead_gcm_err; + if ((err = crypto_register_alg(&rfc4106_alg))) + goto aead_gcm_err; #ifdef HAS_CTR if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) goto ablk_rfc3686_ctr_err; #endif +#endif #ifdef HAS_LRW if ((err = crypto_register_alg(&ablk_lrw_alg))) goto ablk_lrw_err; @@ -770,7 +1287,6 @@ static int __init aesni_init(void) if ((err = crypto_register_alg(&ablk_xts_alg))) goto ablk_xts_err; #endif - return err; #ifdef HAS_XTS @@ -784,18 +1300,24 @@ ablk_pcbc_err: crypto_unregister_alg(&ablk_lrw_alg); ablk_lrw_err: #endif +#ifdef CONFIG_X86_64 #ifdef HAS_CTR crypto_unregister_alg(&ablk_rfc3686_ctr_alg); ablk_rfc3686_ctr_err: #endif + crypto_unregister_alg(&rfc4106_alg); +aead_gcm_err: + crypto_unregister_alg(&__rfc4106_alg); +__aead_gcm_err: crypto_unregister_alg(&ablk_ctr_alg); ablk_ctr_err: + crypto_unregister_alg(&blk_ctr_alg); +blk_ctr_err: +#endif crypto_unregister_alg(&ablk_cbc_alg); ablk_cbc_err: crypto_unregister_alg(&ablk_ecb_alg); ablk_ecb_err: - crypto_unregister_alg(&blk_ctr_alg); -blk_ctr_err: crypto_unregister_alg(&blk_cbc_alg); blk_cbc_err: crypto_unregister_alg(&blk_ecb_alg); @@ -818,13 +1340,17 @@ static void __exit aesni_exit(void) #ifdef HAS_LRW crypto_unregister_alg(&ablk_lrw_alg); #endif +#ifdef CONFIG_X86_64 #ifdef HAS_CTR crypto_unregister_alg(&ablk_rfc3686_ctr_alg); #endif + crypto_unregister_alg(&rfc4106_alg); + crypto_unregister_alg(&__rfc4106_alg); crypto_unregister_alg(&ablk_ctr_alg); + crypto_unregister_alg(&blk_ctr_alg); +#endif crypto_unregister_alg(&ablk_cbc_alg); crypto_unregister_alg(&ablk_ecb_alg); - crypto_unregister_alg(&blk_ctr_alg); crypto_unregister_alg(&blk_cbc_alg); crypto_unregister_alg(&blk_ecb_alg); crypto_unregister_alg(&__aesni_alg); diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h index 55d106b..211ca3f 100644 --- a/arch/x86/include/asm/acpi.h +++ b/arch/x86/include/asm/acpi.h @@ -185,17 +185,16 @@ struct bootnode; #ifdef CONFIG_ACPI_NUMA extern int acpi_numa; -extern int acpi_get_nodes(struct bootnode *physnodes); +extern void acpi_get_nodes(struct bootnode *physnodes, unsigned long start, + unsigned long end); extern int acpi_scan_nodes(unsigned long start, unsigned long end); #define NR_NODE_MEMBLKS (MAX_NUMNODES*2) + +#ifdef CONFIG_NUMA_EMU extern void acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes); -#else -static inline void acpi_fake_nodes(const struct bootnode *fake_nodes, - int num_nodes) -{ -} #endif +#endif /* CONFIG_ACPI_NUMA */ #define acpi_unlazy_tlb(x) leave_mm(x) diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index 6aee50d..64dc82e 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h @@ -3,16 +3,27 @@ #include <linux/pci.h> +struct amd_nb_bus_dev_range { + u8 bus; + u8 dev_base; + u8 dev_limit; +}; + extern struct pci_device_id amd_nb_misc_ids[]; +extern const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[]; struct bootnode; extern int early_is_amd_nb(u32 value); extern int amd_cache_northbridges(void); extern void amd_flush_garts(void); -extern int amd_get_nodes(struct bootnode *nodes); extern int amd_numa_init(unsigned long start_pfn, unsigned long end_pfn); extern int amd_scan_nodes(void); +#ifdef CONFIG_NUMA_EMU +extern void amd_fake_nodes(const struct bootnode *nodes, int nr_nodes); +extern void amd_get_nodes(struct bootnode *nodes); +#endif + struct amd_northbridge { struct pci_dev *misc; }; diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h index 3b62ab5..5e1a2ee 100644 --- a/arch/x86/include/asm/boot.h +++ b/arch/x86/include/asm/boot.h @@ -32,11 +32,7 @@ #define BOOT_HEAP_SIZE 0x400000 #else /* !CONFIG_KERNEL_BZIP2 */ -#ifdef CONFIG_X86_64 -#define BOOT_HEAP_SIZE 0x7000 -#else -#define BOOT_HEAP_SIZE 0x4000 -#endif +#define BOOT_HEAP_SIZE 0x8000 #endif /* !CONFIG_KERNEL_BZIP2 */ diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 0141b23..4729b2b 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h @@ -116,11 +116,11 @@ enum fixed_addresses { #endif FIX_TEXT_POKE1, /* reserve 2 pages for text_poke() */ FIX_TEXT_POKE0, /* first page is last, because allocation is backward */ - __end_of_permanent_fixed_addresses, - #ifdef CONFIG_X86_MRST FIX_LNW_VRTC, #endif + __end_of_permanent_fixed_addresses, + /* * 256 temporary boot-time mappings, used by early_ioremap(), * before ioremap() is functional. diff --git a/arch/x86/include/asm/gpio.h b/arch/x86/include/asm/gpio.h index 49dbfdf..91d915a 100644 --- a/arch/x86/include/asm/gpio.h +++ b/arch/x86/include/asm/gpio.h @@ -38,12 +38,9 @@ static inline int gpio_cansleep(unsigned int gpio) return __gpio_cansleep(gpio); } -/* - * Not implemented, yet. - */ static inline int gpio_to_irq(unsigned int gpio) { - return -ENOSYS; + return __gpio_to_irq(gpio); } static inline int irq_to_gpio(unsigned int irq) diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index ba870bb..c704b38 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -10,6 +10,9 @@ #include <asm/apicdef.h> #include <asm/irq_vectors.h> +/* Even though we don't support this, supply it to appease OF */ +static inline void irq_dispose_mapping(unsigned int virq) { } + static inline int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); diff --git a/arch/x86/include/asm/kdebug.h b/arch/x86/include/asm/kdebug.h index f23eb25..ca242d3 100644 --- a/arch/x86/include/asm/kdebug.h +++ b/arch/x86/include/asm/kdebug.h @@ -18,7 +18,6 @@ enum die_val { DIE_TRAP, DIE_GPF, DIE_CALL, - DIE_NMI_IPI, DIE_PAGE_FAULT, DIE_NMIUNKNOWN, }; diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h index b36c6b3..8e37deb 100644 --- a/arch/x86/include/asm/kvm_emulate.h +++ b/arch/x86/include/asm/kvm_emulate.h @@ -15,6 +15,14 @@ struct x86_emulate_ctxt; +struct x86_exception { + u8 vector; + bool error_code_valid; + u16 error_code; + bool nested_page_fault; + u64 address; /* cr2 or nested page fault gpa */ +}; + /* * x86_emulate_ops: * @@ -64,7 +72,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to read from memory. */ int (*read_std)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * write_std: Write bytes of standard (non-emulated/special) memory. @@ -74,7 +83,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to write to memory. */ int (*write_std)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * fetch: Read bytes of standard (non-emulated/special) memory. * Used for instruction fetch. @@ -83,7 +93,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to read from memory. */ int (*fetch)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * read_emulated: Read bytes from emulated/special memory area. @@ -94,7 +105,7 @@ struct x86_emulate_ops { int (*read_emulated)(unsigned long addr, void *val, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); /* @@ -107,7 +118,7 @@ struct x86_emulate_ops { int (*write_emulated)(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); /* @@ -122,7 +133,7 @@ struct x86_emulate_ops { const void *old, const void *new, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); int (*pio_in_emulated)(int size, unsigned short port, void *val, @@ -159,7 +170,10 @@ struct operand { }; union { unsigned long *reg; - unsigned long mem; + struct segmented_address { + ulong ea; + unsigned seg; + } mem; } addr; union { unsigned long val; @@ -226,9 +240,8 @@ struct x86_emulate_ctxt { bool perm_ok; /* do not check permissions if true */ - int exception; /* exception that happens during emulation or -1 */ - u32 error_code; /* error code for exception */ - bool error_code_valid; + bool have_exception; + struct x86_exception exception; /* decode cache */ struct decode_cache decode; @@ -252,7 +265,7 @@ struct x86_emulate_ctxt { #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 #endif -int x86_decode_insn(struct x86_emulate_ctxt *ctxt); +int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len); #define EMULATION_FAILED -1 #define EMULATION_OK 0 #define EMULATION_RESTART 1 diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f702f82..aa75f21 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -83,11 +83,14 @@ #define KVM_NR_FIXED_MTRR_REGION 88 #define KVM_NR_VAR_MTRR 8 +#define ASYNC_PF_PER_VCPU 64 + extern spinlock_t kvm_lock; extern struct list_head vm_list; struct kvm_vcpu; struct kvm; +struct kvm_async_pf; enum kvm_reg { VCPU_REGS_RAX = 0, @@ -114,6 +117,7 @@ enum kvm_reg { enum kvm_reg_ex { VCPU_EXREG_PDPTR = NR_VCPU_REGS, + VCPU_EXREG_CR3, }; enum { @@ -238,16 +242,18 @@ struct kvm_mmu { void (*new_cr3)(struct kvm_vcpu *vcpu); void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root); unsigned long (*get_cr3)(struct kvm_vcpu *vcpu); - int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*inject_page_fault)(struct kvm_vcpu *vcpu); + int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err, + bool prefault); + void (*inject_page_fault)(struct kvm_vcpu *vcpu, + struct x86_exception *fault); void (*free)(struct kvm_vcpu *vcpu); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access, - u32 *error); + struct x86_exception *exception); gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access); void (*prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page); int (*sync_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, bool clear_unsync); + struct kvm_mmu_page *sp); void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva); hpa_t root_hpa; int root_level; @@ -315,16 +321,6 @@ struct kvm_vcpu_arch { */ struct kvm_mmu *walk_mmu; - /* - * This struct is filled with the necessary information to propagate a - * page fault into the guest - */ - struct { - u64 address; - unsigned error_code; - bool nested; - } fault; - /* only needed in kvm_pv_mmu_op() path, but it's hot so * put it here to avoid allocation */ struct kvm_pv_mmu_op_buffer mmu_op_buffer; @@ -412,6 +408,15 @@ struct kvm_vcpu_arch { u64 hv_vapic; cpumask_var_t wbinvd_dirty_mask; + + struct { + bool halted; + gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)]; + struct gfn_to_hva_cache data; + u64 msr_val; + u32 id; + bool send_user_only; + } apf; }; struct kvm_arch { @@ -456,6 +461,10 @@ struct kvm_arch { /* fields used by HYPER-V emulation */ u64 hv_guest_os_id; u64 hv_hypercall; + + #ifdef CONFIG_KVM_MMU_AUDIT + int audit_point; + #endif }; struct kvm_vm_stat { @@ -529,6 +538,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); + void (*decache_cr3)(struct kvm_vcpu *vcpu); void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); @@ -582,9 +592,17 @@ struct kvm_x86_ops { void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); + void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2); const struct trace_print_flags *exit_reasons_str; }; +struct kvm_arch_async_pf { + u32 token; + gfn_t gfn; + unsigned long cr3; + bool direct_map; +}; + extern struct kvm_x86_ops *kvm_x86_ops; int kvm_mmu_module_init(void); @@ -594,7 +612,6 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu); int kvm_mmu_create(struct kvm_vcpu *vcpu); int kvm_mmu_setup(struct kvm_vcpu *vcpu); void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); -void kvm_mmu_set_base_ptes(u64 base_pte); void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask); @@ -623,8 +640,15 @@ enum emulation_result { #define EMULTYPE_NO_DECODE (1 << 0) #define EMULTYPE_TRAP_UD (1 << 1) #define EMULTYPE_SKIP (1 << 2) -int emulate_instruction(struct kvm_vcpu *vcpu, - unsigned long cr2, u16 error_code, int emulation_type); +int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2, + int emulation_type, void *insn, int insn_len); + +static inline int emulate_instruction(struct kvm_vcpu *vcpu, + int emulation_type) +{ + return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); +} + void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); @@ -650,7 +674,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason, int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); -void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); @@ -668,11 +692,11 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gfn_t gfn, void *data, int offset, int len, u32 access); -void kvm_propagate_fault(struct kvm_vcpu *vcpu); +void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); int kvm_pic_set_irq(void *opaque, int irq, int level); @@ -690,16 +714,21 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu); -gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); +gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); int kvm_fix_hypercall(struct kvm_vcpu *vcpu); -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code, + void *insn, int insn_len); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_enable_tdp(void); @@ -766,20 +795,25 @@ enum { #define HF_VINTR_MASK (1 << 2) #define HF_NMI_MASK (1 << 3) #define HF_IRET_MASK (1 << 4) +#define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ /* * Hardware virtualization extension instructions may fault if a * reboot turns off virtualization while processes are running. * Trap the fault and ignore the instruction if that happens. */ -asmlinkage void kvm_handle_fault_on_reboot(void); +asmlinkage void kvm_spurious_fault(void); +extern bool kvm_rebooting; #define __kvm_handle_fault_on_reboot(insn) \ "666: " insn "\n\t" \ + "668: \n\t" \ ".pushsection .fixup, \"ax\" \n" \ "667: \n\t" \ + "cmpb $0, kvm_rebooting \n\t" \ + "jne 668b \n\t" \ __ASM_SIZE(push) " $666b \n\t" \ - "jmp kvm_handle_fault_on_reboot \n\t" \ + "call kvm_spurious_fault \n\t" \ ".popsection \n\t" \ ".pushsection __ex_table, \"a\" \n\t" \ _ASM_PTR " 666b, 667b \n\t" \ @@ -799,4 +833,15 @@ void kvm_set_shared_msr(unsigned index, u64 val, u64 mask); bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); +void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); +extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); + +void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); + #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 7b562b6..a427bf7 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -20,6 +20,7 @@ * are available. The use of 0x11 and 0x12 is deprecated */ #define KVM_FEATURE_CLOCKSOURCE2 3 +#define KVM_FEATURE_ASYNC_PF 4 /* The last 8 bits are used to indicate how to interpret the flags field * in pvclock structure. If no bits are set, all flags are ignored. @@ -32,9 +33,13 @@ /* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */ #define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01 +#define MSR_KVM_ASYNC_PF_EN 0x4b564d02 #define KVM_MAX_MMU_OP_BATCH 32 +#define KVM_ASYNC_PF_ENABLED (1 << 0) +#define KVM_ASYNC_PF_SEND_ALWAYS (1 << 1) + /* Operations for KVM_HC_MMU_OP */ #define KVM_MMU_OP_WRITE_PTE 1 #define KVM_MMU_OP_FLUSH_TLB 2 @@ -61,10 +66,20 @@ struct kvm_mmu_op_release_pt { __u64 pt_phys; }; +#define KVM_PV_REASON_PAGE_NOT_PRESENT 1 +#define KVM_PV_REASON_PAGE_READY 2 + +struct kvm_vcpu_pv_apf_data { + __u32 reason; + __u8 pad[60]; + __u32 enabled; +}; + #ifdef __KERNEL__ #include <asm/processor.h> extern void kvmclock_init(void); +extern int kvm_register_clock(char *txt); /* This instruction is vmcall. On non-VT architectures, it will generate a @@ -160,8 +175,17 @@ static inline unsigned int kvm_arch_para_features(void) #ifdef CONFIG_KVM_GUEST void __init kvm_guest_init(void); +void kvm_async_pf_task_wait(u32 token); +void kvm_async_pf_task_wake(u32 token); +u32 kvm_read_and_reset_pf_reason(void); #else #define kvm_guest_init() do { } while (0) +#define kvm_async_pf_task_wait(T) do {} while(0) +#define kvm_async_pf_task_wake(T) do {} while(0) +static inline u32 kvm_read_and_reset_pf_reason(void) +{ + return 0; +} #endif #endif /* __KERNEL__ */ diff --git a/arch/x86/include/asm/mach_traps.h b/arch/x86/include/asm/mach_traps.h index f792060..72a8b52 100644 --- a/arch/x86/include/asm/mach_traps.h +++ b/arch/x86/include/asm/mach_traps.h @@ -7,9 +7,19 @@ #include <asm/mc146818rtc.h> +#define NMI_REASON_PORT 0x61 + +#define NMI_REASON_SERR 0x80 +#define NMI_REASON_IOCHK 0x40 +#define NMI_REASON_MASK (NMI_REASON_SERR | NMI_REASON_IOCHK) + +#define NMI_REASON_CLEAR_SERR 0x04 +#define NMI_REASON_CLEAR_IOCHK 0x08 +#define NMI_REASON_CLEAR_MASK 0x0f + static inline unsigned char get_nmi_reason(void) { - return inb(0x61); + return inb(NMI_REASON_PORT); } static inline void reassert_nmi(void) diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h index c4021b9..c76f5b9 100644 --- a/arch/x86/include/asm/nmi.h +++ b/arch/x86/include/asm/nmi.h @@ -23,6 +23,26 @@ void arch_trigger_all_cpu_backtrace(void); #define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace #endif +/* + * Define some priorities for the nmi notifier call chain. + * + * Create a local nmi bit that has a higher priority than + * external nmis, because the local ones are more frequent. + * + * Also setup some default high/normal/low settings for + * subsystems to registers with. Using 4 bits to seperate + * the priorities. This can go alot higher if needed be. + */ + +#define NMI_LOCAL_SHIFT 16 /* randomly picked */ +#define NMI_LOCAL_BIT (1ULL << NMI_LOCAL_SHIFT) +#define NMI_HIGH_PRIOR (1ULL << 8) +#define NMI_NORMAL_PRIOR (1ULL << 4) +#define NMI_LOW_PRIOR (1ULL << 0) +#define NMI_LOCAL_HIGH_PRIOR (NMI_LOCAL_BIT | NMI_HIGH_PRIOR) +#define NMI_LOCAL_NORMAL_PRIOR (NMI_LOCAL_BIT | NMI_NORMAL_PRIOR) +#define NMI_LOCAL_LOW_PRIOR (NMI_LOCAL_BIT | NMI_LOW_PRIOR) + void stop_nmi(void); void restart_nmi(void); diff --git a/arch/x86/include/asm/numa_64.h b/arch/x86/include/asm/numa_64.h index 823e070..5ae8728 100644 --- a/arch/x86/include/asm/numa_64.h +++ b/arch/x86/include/asm/numa_64.h @@ -38,7 +38,7 @@ extern void __cpuinit numa_add_cpu(int cpu); extern void __cpuinit numa_remove_cpu(int cpu); #ifdef CONFIG_NUMA_EMU -#define FAKE_NODE_MIN_SIZE ((u64)64 << 20) +#define FAKE_NODE_MIN_SIZE ((u64)32 << 20) #define FAKE_NODE_MIN_HASH_MASK (~(FAKE_NODE_MIN_SIZE - 1UL)) #endif /* CONFIG_NUMA_EMU */ #else diff --git a/arch/x86/include/asm/olpc.h b/arch/x86/include/asm/olpc.h index 42a978c..f482010 100644 --- a/arch/x86/include/asm/olpc.h +++ b/arch/x86/include/asm/olpc.h @@ -107,10 +107,14 @@ extern int olpc_ec_mask_unset(uint8_t bits); /* GPIO assignments */ #define OLPC_GPIO_MIC_AC 1 -#define OLPC_GPIO_DCON_IRQ geode_gpio(7) +#define OLPC_GPIO_DCON_STAT0 5 +#define OLPC_GPIO_DCON_STAT1 6 +#define OLPC_GPIO_DCON_IRQ 7 #define OLPC_GPIO_THRM_ALRM geode_gpio(10) -#define OLPC_GPIO_SMB_CLK geode_gpio(14) -#define OLPC_GPIO_SMB_DATA geode_gpio(15) +#define OLPC_GPIO_DCON_LOAD 11 +#define OLPC_GPIO_DCON_BLANK 12 +#define OLPC_GPIO_SMB_CLK 14 +#define OLPC_GPIO_SMB_DATA 15 #define OLPC_GPIO_WORKAUX geode_gpio(24) #define OLPC_GPIO_LID geode_gpio(26) #define OLPC_GPIO_ECSCI geode_gpio(27) diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h index 2a84781..641988e 100644 --- a/arch/x86/include/asm/olpc_ofw.h +++ b/arch/x86/include/asm/olpc_ofw.h @@ -8,6 +8,8 @@ #ifdef CONFIG_OLPC_OPENFIRMWARE +extern bool olpc_ofw_is_installed(void); + /* run an OFW command by calling into the firmware */ #define olpc_ofw(name, args, res) \ __olpc_ofw((name), ARRAY_SIZE(args), args, ARRAY_SIZE(res), res) @@ -26,10 +28,17 @@ extern bool olpc_ofw_present(void); #else /* !CONFIG_OLPC_OPENFIRMWARE */ +static inline bool olpc_ofw_is_installed(void) { return false; } static inline void olpc_ofw_detect(void) { } static inline void setup_olpc_ofw_pgd(void) { } static inline bool olpc_ofw_present(void) { return false; } #endif /* !CONFIG_OLPC_OPENFIRMWARE */ +#ifdef CONFIG_OLPC_OPENFIRMWARE_DT +extern void olpc_dt_build_devicetree(void); +#else +static inline void olpc_dt_build_devicetree(void) { } +#endif /* CONFIG_OLPC_OPENFIRMWARE_DT */ + #endif /* _ASM_X86_OLPC_OFW_H */ diff --git a/arch/x86/include/asm/perf_event_p4.h b/arch/x86/include/asm/perf_event_p4.h index 295e2ff..e2f6a99 100644 --- a/arch/x86/include/asm/perf_event_p4.h +++ b/arch/x86/include/asm/perf_event_p4.h @@ -20,6 +20,9 @@ #define ARCH_P4_MAX_ESCR (ARCH_P4_TOTAL_ESCR - ARCH_P4_RESERVED_ESCR) #define ARCH_P4_MAX_CCCR (18) +#define ARCH_P4_CNTRVAL_BITS (40) +#define ARCH_P4_CNTRVAL_MASK ((1ULL << ARCH_P4_CNTRVAL_BITS) - 1) + #define P4_ESCR_EVENT_MASK 0x7e000000U #define P4_ESCR_EVENT_SHIFT 25 #define P4_ESCR_EVENTMASK_MASK 0x01fffe00U diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index 271de94..b4389a4 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -92,7 +92,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, - unsigned long adddress) + unsigned long address) { ___pmd_free_tlb(tlb, pmd); } diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c6efecf..53fd1d5 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -901,7 +901,7 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); /* * The below -8 is to reserve 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs" - * is accessable even if the CPU haven't stored the SS/ESP registers + * is accessible even if the CPU haven't stored the SS/ESP registers * on the stack (interrupt gate does not save these registers * when switching to the same priv ring). * Therefore beware: accessing the ss/esp fields of the diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h new file mode 100644 index 0000000..b4ec95f --- /dev/null +++ b/arch/x86/include/asm/prom.h @@ -0,0 +1 @@ +/* dummy prom.h; here to make linux/of.h's #includes happy */ diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 0e83105..f2b83bc 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -47,14 +47,13 @@ enum { INTERCEPT_MONITOR, INTERCEPT_MWAIT, INTERCEPT_MWAIT_COND, + INTERCEPT_XSETBV, }; struct __attribute__ ((__packed__)) vmcb_control_area { - u16 intercept_cr_read; - u16 intercept_cr_write; - u16 intercept_dr_read; - u16 intercept_dr_write; + u32 intercept_cr; + u32 intercept_dr; u32 intercept_exceptions; u64 intercept; u8 reserved_1[42]; @@ -81,14 +80,19 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u32 event_inj_err; u64 nested_cr3; u64 lbr_ctl; - u64 reserved_5; + u32 clean; + u32 reserved_5; u64 next_rip; - u8 reserved_6[816]; + u8 insn_len; + u8 insn_bytes[15]; + u8 reserved_6[800]; }; #define TLB_CONTROL_DO_NOTHING 0 #define TLB_CONTROL_FLUSH_ALL_ASID 1 +#define TLB_CONTROL_FLUSH_ASID 3 +#define TLB_CONTROL_FLUSH_ASID_LOCAL 7 #define V_TPR_MASK 0x0f @@ -204,19 +208,31 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK #define SVM_SELECTOR_CODE_MASK (1 << 3) -#define INTERCEPT_CR0_MASK 1 -#define INTERCEPT_CR3_MASK (1 << 3) -#define INTERCEPT_CR4_MASK (1 << 4) -#define INTERCEPT_CR8_MASK (1 << 8) - -#define INTERCEPT_DR0_MASK 1 -#define INTERCEPT_DR1_MASK (1 << 1) -#define INTERCEPT_DR2_MASK (1 << 2) -#define INTERCEPT_DR3_MASK (1 << 3) -#define INTERCEPT_DR4_MASK (1 << 4) -#define INTERCEPT_DR5_MASK (1 << 5) -#define INTERCEPT_DR6_MASK (1 << 6) -#define INTERCEPT_DR7_MASK (1 << 7) +#define INTERCEPT_CR0_READ 0 +#define INTERCEPT_CR3_READ 3 +#define INTERCEPT_CR4_READ 4 +#define INTERCEPT_CR8_READ 8 +#define INTERCEPT_CR0_WRITE (16 + 0) +#define INTERCEPT_CR3_WRITE (16 + 3) +#define INTERCEPT_CR4_WRITE (16 + 4) +#define INTERCEPT_CR8_WRITE (16 + 8) + +#define INTERCEPT_DR0_READ 0 +#define INTERCEPT_DR1_READ 1 +#define INTERCEPT_DR2_READ 2 +#define INTERCEPT_DR3_READ 3 +#define INTERCEPT_DR4_READ 4 +#define INTERCEPT_DR5_READ 5 +#define INTERCEPT_DR6_READ 6 +#define INTERCEPT_DR7_READ 7 +#define INTERCEPT_DR0_WRITE (16 + 0) +#define INTERCEPT_DR1_WRITE (16 + 1) +#define INTERCEPT_DR2_WRITE (16 + 2) +#define INTERCEPT_DR3_WRITE (16 + 3) +#define INTERCEPT_DR4_WRITE (16 + 4) +#define INTERCEPT_DR5_WRITE (16 + 5) +#define INTERCEPT_DR6_WRITE (16 + 6) +#define INTERCEPT_DR7_WRITE (16 + 7) #define SVM_EVTINJ_VEC_MASK 0xff @@ -246,6 +262,8 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXITINFOSHIFT_TS_REASON_JMP 38 #define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44 +#define SVM_EXITINFO_REG_MASK 0x0F + #define SVM_EXIT_READ_CR0 0x000 #define SVM_EXIT_READ_CR3 0x003 #define SVM_EXIT_READ_CR4 0x004 @@ -316,6 +334,7 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXIT_MONITOR 0x08a #define SVM_EXIT_MWAIT 0x08b #define SVM_EXIT_MWAIT_COND 0x08c +#define SVM_EXIT_XSETBV 0x08d #define SVM_EXIT_NPF 0x400 #define SVM_EXIT_ERR -1 diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index f66cda5..0310da6 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -30,6 +30,7 @@ asmlinkage void segment_not_present(void); asmlinkage void stack_segment(void); asmlinkage void general_protection(void); asmlinkage void page_fault(void); +asmlinkage void async_page_fault(void); asmlinkage void spurious_interrupt_bug(void); asmlinkage void coprocessor_error(void); asmlinkage void alignment_check(void); diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 9f0cbd9..84471b8 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -66,15 +66,23 @@ #define PIN_BASED_NMI_EXITING 0x00000008 #define PIN_BASED_VIRTUAL_NMIS 0x00000020 +#define VM_EXIT_SAVE_DEBUG_CONTROLS 0x00000002 #define VM_EXIT_HOST_ADDR_SPACE_SIZE 0x00000200 +#define VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL 0x00001000 #define VM_EXIT_ACK_INTR_ON_EXIT 0x00008000 #define VM_EXIT_SAVE_IA32_PAT 0x00040000 #define VM_EXIT_LOAD_IA32_PAT 0x00080000 +#define VM_EXIT_SAVE_IA32_EFER 0x00100000 +#define VM_EXIT_LOAD_IA32_EFER 0x00200000 +#define VM_EXIT_SAVE_VMX_PREEMPTION_TIMER 0x00400000 +#define VM_ENTRY_LOAD_DEBUG_CONTROLS 0x00000002 #define VM_ENTRY_IA32E_MODE 0x00000200 #define VM_ENTRY_SMM 0x00000400 #define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 +#define VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL 0x00002000 #define VM_ENTRY_LOAD_IA32_PAT 0x00004000 +#define VM_ENTRY_LOAD_IA32_EFER 0x00008000 /* VMCS Encodings */ enum vmcs_field { @@ -239,6 +247,7 @@ enum vmcs_field { #define EXIT_REASON_TASK_SWITCH 9 #define EXIT_REASON_CPUID 10 #define EXIT_REASON_HLT 12 +#define EXIT_REASON_INVD 13 #define EXIT_REASON_INVLPG 14 #define EXIT_REASON_RDPMC 15 #define EXIT_REASON_RDTSC 16 @@ -296,6 +305,12 @@ enum vmcs_field { #define GUEST_INTR_STATE_SMI 0x00000004 #define GUEST_INTR_STATE_NMI 0x00000008 +/* GUEST_ACTIVITY_STATE flags */ +#define GUEST_ACTIVITY_ACTIVE 0 +#define GUEST_ACTIVITY_HLT 1 +#define GUEST_ACTIVITY_SHUTDOWN 2 +#define GUEST_ACTIVITY_WAIT_SIPI 3 + /* * Exit Qualifications for MOV for Control Register Access */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index d2fdb08..57ca777 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1086,7 +1086,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, dma_dom->aperture_size += APERTURE_RANGE_SIZE; - /* Intialize the exclusion range if necessary */ + /* Initialize the exclusion range if necessary */ for_each_iommu(iommu) { if (iommu->exclusion_start && iommu->exclusion_start >= dma_dom->aperture[index]->offset @@ -1353,7 +1353,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) /* * Allocates a new protection domain usable for the dma_ops functions. - * It also intializes the page table and the address allocator data + * It also initializes the page table and the address allocator data * structures required for the dma_ops interface */ static struct dma_ops_domain *dma_ops_domain_alloc(void) diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index affacb5..0a99f71 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -20,6 +20,13 @@ struct pci_device_id amd_nb_misc_ids[] = { }; EXPORT_SYMBOL(amd_nb_misc_ids); +const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[] __initconst = { + { 0x00, 0x18, 0x20 }, + { 0xff, 0x00, 0x20 }, + { 0xfe, 0x00, 0x20 }, + { } +}; + struct amd_northbridge_info amd_northbridges; EXPORT_SYMBOL(amd_northbridges); diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index dcd7c83..5955a78 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c @@ -39,18 +39,6 @@ int fallback_aper_force __initdata; int fix_aperture __initdata = 1; -struct bus_dev_range { - int bus; - int dev_base; - int dev_limit; -}; - -static struct bus_dev_range bus_dev_ranges[] __initdata = { - { 0x00, 0x18, 0x20}, - { 0xff, 0x00, 0x20}, - { 0xfe, 0x00, 0x20} -}; - static struct resource gart_resource = { .name = "GART", .flags = IORESOURCE_MEM, @@ -294,13 +282,13 @@ void __init early_gart_iommu_check(void) search_agp_bridge(&agp_aper_order, &valid_agp); fix = 0; - for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { + for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { int bus; int dev_base, dev_limit; - bus = bus_dev_ranges[i].bus; - dev_base = bus_dev_ranges[i].dev_base; - dev_limit = bus_dev_ranges[i].dev_limit; + bus = amd_nb_bus_dev_ranges[i].bus; + dev_base = amd_nb_bus_dev_ranges[i].dev_base; + dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; for (slot = dev_base; slot < dev_limit; slot++) { if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) @@ -349,13 +337,13 @@ void __init early_gart_iommu_check(void) return; /* disable them all at first */ - for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { + for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { int bus; int dev_base, dev_limit; - bus = bus_dev_ranges[i].bus; - dev_base = bus_dev_ranges[i].dev_base; - dev_limit = bus_dev_ranges[i].dev_limit; + bus = amd_nb_bus_dev_ranges[i].bus; + dev_base = amd_nb_bus_dev_ranges[i].dev_base; + dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; for (slot = dev_base; slot < dev_limit; slot++) { if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) @@ -390,14 +378,14 @@ int __init gart_iommu_hole_init(void) fix = 0; node = 0; - for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { + for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { int bus; int dev_base, dev_limit; u32 ctl; - bus = bus_dev_ranges[i].bus; - dev_base = bus_dev_ranges[i].dev_base; - dev_limit = bus_dev_ranges[i].dev_limit; + bus = amd_nb_bus_dev_ranges[i].bus; + dev_base = amd_nb_bus_dev_ranges[i].dev_base; + dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; for (slot = dev_base; slot < dev_limit; slot++) { if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) @@ -505,7 +493,7 @@ out: } /* Fix up the north bridges */ - for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) { + for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { int bus, dev_base, dev_limit; /* @@ -514,9 +502,9 @@ out: */ u32 ctl = DISTLBWALKPRB | aper_order << 1; - bus = bus_dev_ranges[i].bus; - dev_base = bus_dev_ranges[i].dev_base; - dev_limit = bus_dev_ranges[i].dev_limit; + bus = amd_nb_bus_dev_ranges[i].bus; + dev_base = amd_nb_bus_dev_ranges[i].dev_base; + dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; for (slot = dev_base; slot < dev_limit; slot++) { if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) continue; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index a51345b..06c196d 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -684,7 +684,7 @@ static int __init calibrate_APIC_clock(void) lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS, lapic_clockevent.shift); lapic_clockevent.max_delta_ns = - clockevent_delta2ns(0x7FFFFF, &lapic_clockevent); + clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent); lapic_clockevent.min_delta_ns = clockevent_delta2ns(0xF, &lapic_clockevent); diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index 72ec29e..79fd43c 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -68,7 +68,6 @@ arch_trigger_all_cpu_backtrace_handler(struct notifier_block *self, switch (cmd) { case DIE_NMI: - case DIE_NMI_IPI: break; default: @@ -96,7 +95,7 @@ arch_trigger_all_cpu_backtrace_handler(struct notifier_block *self, static __read_mostly struct notifier_block backtrace_notifier = { .notifier_call = arch_trigger_all_cpu_backtrace_handler, .next = NULL, - .priority = 1 + .priority = NMI_LOCAL_LOW_PRIOR, }; static int __init register_trigger_all_cpu_backtrace(void) diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index ecca5f4..bd16b58 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -378,7 +378,7 @@ struct apic __refdata apic_x2apic_uv_x = { static __cpuinit void set_x2apic_extra_bits(int pnode) { - __this_cpu_write(x2apic_extra_bits, (pnode << 6)); + __this_cpu_write(x2apic_extra_bits, pnode << uvh_apicid.s.pnode_shift); } /* @@ -641,7 +641,7 @@ void __cpuinit uv_cpu_init(void) */ int uv_handle_nmi(struct notifier_block *self, unsigned long reason, void *data) { - if (reason != DIE_NMI_IPI) + if (reason != DIE_NMIUNKNOWN) return NOTIFY_OK; if (in_crash_kexec) diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c index e7dbde7..a779719 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-inject.c +++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c @@ -25,6 +25,7 @@ #include <linux/gfp.h> #include <asm/mce.h> #include <asm/apic.h> +#include <asm/nmi.h> /* Update fake mce registers on current CPU. */ static void inject_mce(struct mce *m) @@ -83,7 +84,7 @@ static int mce_raise_notify(struct notifier_block *self, struct die_args *args = (struct die_args *)data; int cpu = smp_processor_id(); struct mce *m = &__get_cpu_var(injectm); - if (val != DIE_NMI_IPI || !cpumask_test_cpu(cpu, mce_inject_cpumask)) + if (val != DIE_NMI || !cpumask_test_cpu(cpu, mce_inject_cpumask)) return NOTIFY_DONE; cpumask_clear_cpu(cpu, mce_inject_cpumask); if (m->inject_flags & MCJ_EXCEPTION) @@ -95,7 +96,7 @@ static int mce_raise_notify(struct notifier_block *self, static struct notifier_block mce_raise_nb = { .notifier_call = mce_raise_notify, - .priority = 1000, + .priority = NMI_LOCAL_NORMAL_PRIOR, }; /* Inject mce on current CPU */ diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 0492101..9d977a2 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1267,7 +1267,6 @@ perf_event_nmi_handler(struct notifier_block *self, switch (cmd) { case DIE_NMI: - case DIE_NMI_IPI: break; case DIE_NMIUNKNOWN: this_nmi = percpu_read(irq_stat.__nmi_count); @@ -1317,7 +1316,7 @@ perf_event_nmi_handler(struct notifier_block *self, static __read_mostly struct notifier_block perf_event_nmi_notifier = { .notifier_call = perf_event_nmi_handler, .next = NULL, - .priority = 1 + .priority = NMI_LOCAL_LOW_PRIOR, }; static struct event_constraint unconstrained; diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index 81400b9..e56b9bf 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -753,19 +753,21 @@ out: static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc) { - int overflow = 0; - u32 low, high; + u64 v; - rdmsr(hwc->config_base + hwc->idx, low, high); - - /* we need to check high bit for unflagged overflows */ - if ((low & P4_CCCR_OVF) || !(high & (1 << 31))) { - overflow = 1; - (void)checking_wrmsrl(hwc->config_base + hwc->idx, - ((u64)low) & ~P4_CCCR_OVF); + /* an official way for overflow indication */ + rdmsrl(hwc->config_base + hwc->idx, v); + if (v & P4_CCCR_OVF) { + wrmsrl(hwc->config_base + hwc->idx, v & ~P4_CCCR_OVF); + return 1; } - return overflow; + /* it might be unflagged overflow */ + rdmsrl(hwc->event_base + hwc->idx, v); + if (!(v & ARCH_P4_CNTRVAL_MASK)) + return 1; + + return 0; } static void p4_pmu_disable_pebs(void) @@ -1152,9 +1154,9 @@ static __initconst const struct x86_pmu p4_pmu = { */ .num_counters = ARCH_P4_MAX_CCCR, .apic = 1, - .cntval_bits = 40, - .cntval_mask = (1ULL << 40) - 1, - .max_period = (1ULL << 39) - 1, + .cntval_bits = ARCH_P4_CNTRVAL_BITS, + .cntval_mask = ARCH_P4_CNTRVAL_MASK, + .max_period = (1ULL << (ARCH_P4_CNTRVAL_BITS - 1)) - 1, .hw_config = p4_hw_config, .schedule_events = p4_pmu_schedule_events, /* diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 8474c99..d6fb146 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -197,14 +197,8 @@ void show_stack(struct task_struct *task, unsigned long *sp) */ void dump_stack(void) { - unsigned long bp = 0; unsigned long stack; -#ifdef CONFIG_FRAME_POINTER - if (!bp) - get_bp(bp); -#endif - printk("Pid: %d, comm: %.20s %s %s %.*s\n", current->pid, current->comm, print_tainted(), init_utsname()->release, diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 591e601..c8b4efa 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S @@ -1406,6 +1406,16 @@ ENTRY(general_protection) CFI_ENDPROC END(general_protection) +#ifdef CONFIG_KVM_GUEST +ENTRY(async_page_fault) + RING0_EC_FRAME + pushl $do_async_page_fault + CFI_ADJUST_CFA_OFFSET 4 + jmp error_code + CFI_ENDPROC +END(apf_page_fault) +#endif + /* * End of kprobes section */ diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index e3ba417..aed1ffb 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -299,17 +299,21 @@ ENDPROC(native_usergs_sysret64) ENTRY(save_args) XCPT_FRAME cld - movq_cfi rdi, RDI+16-ARGOFFSET - movq_cfi rsi, RSI+16-ARGOFFSET - movq_cfi rdx, RDX+16-ARGOFFSET - movq_cfi rcx, RCX+16-ARGOFFSET - movq_cfi rax, RAX+16-ARGOFFSET - movq_cfi r8, R8+16-ARGOFFSET - movq_cfi r9, R9+16-ARGOFFSET - movq_cfi r10, R10+16-ARGOFFSET - movq_cfi r11, R11+16-ARGOFFSET - - leaq -ARGOFFSET+16(%rsp),%rdi /* arg1 for handler */ + /* + * start from rbp in pt_regs and jump over + * return address. + */ + movq_cfi rdi, RDI+8-RBP + movq_cfi rsi, RSI+8-RBP + movq_cfi rdx, RDX+8-RBP + movq_cfi rcx, RCX+8-RBP + movq_cfi rax, RAX+8-RBP + movq_cfi r8, R8+8-RBP + movq_cfi r9, R9+8-RBP + movq_cfi r10, R10+8-RBP + movq_cfi r11, R11+8-RBP + + leaq -RBP+8(%rsp),%rdi /* arg1 for handler */ movq_cfi rbp, 8 /* push %rbp */ leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ testl $3, CS(%rdi) @@ -782,8 +786,9 @@ END(interrupt) /* 0(%rsp): ~(interrupt number) */ .macro interrupt func - subq $ORIG_RAX-ARGOFFSET+8, %rsp - CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8 + /* reserve pt_regs for scratch regs and rbp */ + subq $ORIG_RAX-RBP, %rsp + CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP call save_args PARTIAL_FRAME 0 call \func @@ -808,9 +813,14 @@ ret_from_intr: TRACE_IRQS_OFF decl PER_CPU_VAR(irq_count) leaveq + CFI_RESTORE rbp CFI_DEF_CFA_REGISTER rsp CFI_ADJUST_CFA_OFFSET -8 + + /* we did not save rbx, restore only from ARGOFFSET */ + addq $8, %rsp + CFI_ADJUST_CFA_OFFSET -8 exit_intr: GET_THREAD_INFO(%rcx) testl $3,CS-ARGOFFSET(%rsp) @@ -1319,6 +1329,9 @@ errorentry xen_stack_segment do_stack_segment #endif errorentry general_protection do_general_protection errorentry page_fault do_page_fault +#ifdef CONFIG_KVM_GUEST +errorentry async_page_fault do_async_page_fault +#endif #ifdef CONFIG_X86_MCE paranoidzeroentry machine_check *machine_check_vector(%rip) #endif diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 9f54b20..fc293dc 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -126,7 +126,7 @@ ENTRY(startup_32) movsl movl pa(boot_params) + NEW_CL_POINTER,%esi andl %esi,%esi - jz 1f # No comand line + jz 1f # No command line movl $pa(boot_command_line),%edi movl $(COMMAND_LINE_SIZE/4),%ecx rep diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index 58bb239..e60c38c 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -169,6 +169,7 @@ int init_fpu(struct task_struct *tsk) set_stopped_child_used_math(tsk); return 0; } +EXPORT_SYMBOL_GPL(init_fpu); /* * The xstateregs_active() routine is the same as the fpregs_active() routine, diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 3a43caa..52945da 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -4,6 +4,7 @@ #include <linux/cpu.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> +#include <linux/of.h> #include <linux/seq_file.h> #include <linux/smp.h> #include <linux/ftrace.h> @@ -275,6 +276,15 @@ void smp_x86_platform_ipi(struct pt_regs *regs) EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); +#ifdef CONFIG_OF +unsigned int irq_create_of_mapping(struct device_node *controller, + const u32 *intspec, unsigned int intsize) +{ + return intspec[0]; +} +EXPORT_SYMBOL_GPL(irq_create_of_mapping); +#endif + #ifdef CONFIG_HOTPLUG_CPU /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ void fixup_irqs(void) diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c index cd21b65..a413000 100644 --- a/arch/x86/kernel/kgdb.c +++ b/arch/x86/kernel/kgdb.c @@ -48,6 +48,7 @@ #include <asm/apicdef.h> #include <asm/system.h> #include <asm/apic.h> +#include <asm/nmi.h> struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { @@ -525,10 +526,6 @@ static int __kgdb_notify(struct die_args *args, unsigned long cmd) } return NOTIFY_DONE; - case DIE_NMI_IPI: - /* Just ignore, we will handle the roundup on DIE_NMI. */ - return NOTIFY_DONE; - case DIE_NMIUNKNOWN: if (was_in_debug_nmi[raw_smp_processor_id()]) { was_in_debug_nmi[raw_smp_processor_id()] = 0; @@ -606,7 +603,7 @@ static struct notifier_block kgdb_notifier = { /* * Lowest-prio notifier priority, we want to be notified last: */ - .priority = -INT_MAX, + .priority = NMI_LOCAL_LOW_PRIOR, }; /** diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 63b0ec8..8dc4466 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -27,16 +27,37 @@ #include <linux/mm.h> #include <linux/highmem.h> #include <linux/hardirq.h> +#include <linux/notifier.h> +#include <linux/reboot.h> +#include <linux/hash.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/kprobes.h> #include <asm/timer.h> +#include <asm/cpu.h> +#include <asm/traps.h> +#include <asm/desc.h> +#include <asm/tlbflush.h> #define MMU_QUEUE_SIZE 1024 +static int kvmapf = 1; + +static int parse_no_kvmapf(char *arg) +{ + kvmapf = 0; + return 0; +} + +early_param("no-kvmapf", parse_no_kvmapf); + struct kvm_para_state { u8 mmu_queue[MMU_QUEUE_SIZE]; int mmu_queue_len; }; static DEFINE_PER_CPU(struct kvm_para_state, para_state); +static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); static struct kvm_para_state *kvm_para_state(void) { @@ -50,6 +71,195 @@ static void kvm_io_delay(void) { } +#define KVM_TASK_SLEEP_HASHBITS 8 +#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS) + +struct kvm_task_sleep_node { + struct hlist_node link; + wait_queue_head_t wq; + u32 token; + int cpu; + bool halted; + struct mm_struct *mm; +}; + +static struct kvm_task_sleep_head { + spinlock_t lock; + struct hlist_head list; +} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE]; + +static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, + u32 token) +{ + struct hlist_node *p; + + hlist_for_each(p, &b->list) { + struct kvm_task_sleep_node *n = + hlist_entry(p, typeof(*n), link); + if (n->token == token) + return n; + } + + return NULL; +} + +void kvm_async_pf_task_wait(u32 token) +{ + u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); + struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; + struct kvm_task_sleep_node n, *e; + DEFINE_WAIT(wait); + int cpu, idle; + + cpu = get_cpu(); + idle = idle_cpu(cpu); + put_cpu(); + + spin_lock(&b->lock); + e = _find_apf_task(b, token); + if (e) { + /* dummy entry exist -> wake up was delivered ahead of PF */ + hlist_del(&e->link); + kfree(e); + spin_unlock(&b->lock); + return; + } + + n.token = token; + n.cpu = smp_processor_id(); + n.mm = current->active_mm; + n.halted = idle || preempt_count() > 1; + atomic_inc(&n.mm->mm_count); + init_waitqueue_head(&n.wq); + hlist_add_head(&n.link, &b->list); + spin_unlock(&b->lock); + + for (;;) { + if (!n.halted) + prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE); + if (hlist_unhashed(&n.link)) + break; + + if (!n.halted) { + local_irq_enable(); + schedule(); + local_irq_disable(); + } else { + /* + * We cannot reschedule. So halt. + */ + native_safe_halt(); + local_irq_disable(); + } + } + if (!n.halted) + finish_wait(&n.wq, &wait); + + return; +} +EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait); + +static void apf_task_wake_one(struct kvm_task_sleep_node *n) +{ + hlist_del_init(&n->link); + if (!n->mm) + return; + mmdrop(n->mm); + if (n->halted) + smp_send_reschedule(n->cpu); + else if (waitqueue_active(&n->wq)) + wake_up(&n->wq); +} + +static void apf_task_wake_all(void) +{ + int i; + + for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { + struct hlist_node *p, *next; + struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; + spin_lock(&b->lock); + hlist_for_each_safe(p, next, &b->list) { + struct kvm_task_sleep_node *n = + hlist_entry(p, typeof(*n), link); + if (n->cpu == smp_processor_id()) + apf_task_wake_one(n); + } + spin_unlock(&b->lock); + } +} + +void kvm_async_pf_task_wake(u32 token) +{ + u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); + struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; + struct kvm_task_sleep_node *n; + + if (token == ~0) { + apf_task_wake_all(); + return; + } + +again: + spin_lock(&b->lock); + n = _find_apf_task(b, token); + if (!n) { + /* + * async PF was not yet handled. + * Add dummy entry for the token. + */ + n = kmalloc(sizeof(*n), GFP_ATOMIC); + if (!n) { + /* + * Allocation failed! Busy wait while other cpu + * handles async PF. + */ + spin_unlock(&b->lock); + cpu_relax(); + goto again; + } + n->token = token; + n->cpu = smp_processor_id(); + n->mm = NULL; + init_waitqueue_head(&n->wq); + hlist_add_head(&n->link, &b->list); + } else + apf_task_wake_one(n); + spin_unlock(&b->lock); + return; +} +EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); + +u32 kvm_read_and_reset_pf_reason(void) +{ + u32 reason = 0; + + if (__get_cpu_var(apf_reason).enabled) { + reason = __get_cpu_var(apf_reason).reason; + __get_cpu_var(apf_reason).reason = 0; + } + + return reason; +} +EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); + +dotraplinkage void __kprobes +do_async_page_fault(struct pt_regs *regs, unsigned long error_code) +{ + switch (kvm_read_and_reset_pf_reason()) { + default: + do_page_fault(regs, error_code); + break; + case KVM_PV_REASON_PAGE_NOT_PRESENT: + /* page is swapped out by the host. */ + kvm_async_pf_task_wait((u32)read_cr2()); + break; + case KVM_PV_REASON_PAGE_READY: + kvm_async_pf_task_wake((u32)read_cr2()); + break; + } +} + static void kvm_mmu_op(void *buffer, unsigned len) { int r; @@ -231,10 +441,117 @@ static void __init paravirt_ops_setup(void) #endif } +void __cpuinit kvm_guest_cpu_init(void) +{ + if (!kvm_para_available()) + return; + + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) { + u64 pa = __pa(&__get_cpu_var(apf_reason)); + +#ifdef CONFIG_PREEMPT + pa |= KVM_ASYNC_PF_SEND_ALWAYS; +#endif + wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED); + __get_cpu_var(apf_reason).enabled = 1; + printk(KERN_INFO"KVM setup async PF for cpu %d\n", + smp_processor_id()); + } +} + +static void kvm_pv_disable_apf(void *unused) +{ + if (!__get_cpu_var(apf_reason).enabled) + return; + + wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); + __get_cpu_var(apf_reason).enabled = 0; + + printk(KERN_INFO"Unregister pv shared memory for cpu %d\n", + smp_processor_id()); +} + +static int kvm_pv_reboot_notify(struct notifier_block *nb, + unsigned long code, void *unused) +{ + if (code == SYS_RESTART) + on_each_cpu(kvm_pv_disable_apf, NULL, 1); + return NOTIFY_DONE; +} + +static struct notifier_block kvm_pv_reboot_nb = { + .notifier_call = kvm_pv_reboot_notify, +}; + +#ifdef CONFIG_SMP +static void __init kvm_smp_prepare_boot_cpu(void) +{ +#ifdef CONFIG_KVM_CLOCK + WARN_ON(kvm_register_clock("primary cpu clock")); +#endif + kvm_guest_cpu_init(); + native_smp_prepare_boot_cpu(); +} + +static void kvm_guest_cpu_online(void *dummy) +{ + kvm_guest_cpu_init(); +} + +static void kvm_guest_cpu_offline(void *dummy) +{ + kvm_pv_disable_apf(NULL); + apf_task_wake_all(); +} + +static int __cpuinit kvm_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + int cpu = (unsigned long)hcpu; + switch (action) { + case CPU_ONLINE: + case CPU_DOWN_FAILED: + case CPU_ONLINE_FROZEN: + smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0); + break; + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata kvm_cpu_notifier = { + .notifier_call = kvm_cpu_notify, +}; +#endif + +static void __init kvm_apf_trap_init(void) +{ + set_intr_gate(14, &async_page_fault); +} + void __init kvm_guest_init(void) { + int i; + if (!kvm_para_available()) return; paravirt_ops_setup(); + register_reboot_notifier(&kvm_pv_reboot_nb); + for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) + spin_lock_init(&async_pf_sleepers[i].lock); + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF)) + x86_init.irqs.trap_init = kvm_apf_trap_init; + +#ifdef CONFIG_SMP + smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; + register_cpu_notifier(&kvm_cpu_notifier); +#else + kvm_guest_cpu_init(); +#endif } diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index ca43ce3..f98d3ea 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -125,7 +125,7 @@ static struct clocksource kvm_clock = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static int kvm_register_clock(char *txt) +int kvm_register_clock(char *txt) { int cpu = smp_processor_id(); int low, high, ret; @@ -152,14 +152,6 @@ static void __cpuinit kvm_setup_secondary_clock(void) } #endif -#ifdef CONFIG_SMP -static void __init kvm_smp_prepare_boot_cpu(void) -{ - WARN_ON(kvm_register_clock("primary cpu clock")); - native_smp_prepare_boot_cpu(); -} -#endif - /* * After the clock is registered, the host will keep writing to the * registered memory location. If the guest happens to shutdown, this memory @@ -206,9 +198,6 @@ void __init kvmclock_init(void) x86_cpuinit.setup_percpu_clockev = kvm_setup_secondary_clock; #endif -#ifdef CONFIG_SMP - smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; -#endif machine_ops.shutdown = kvm_shutdown; #ifdef CONFIG_KEXEC machine_ops.crash_shutdown = kvm_crash_shutdown; diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index c495aa8..fc7aae1 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -18,6 +18,7 @@ #include <asm/pci_x86.h> #include <asm/virtext.h> #include <asm/cpu.h> +#include <asm/nmi.h> #ifdef CONFIG_X86_32 # include <linux/ctype.h> @@ -747,7 +748,7 @@ static int crash_nmi_callback(struct notifier_block *self, { int cpu; - if (val != DIE_NMI_IPI) + if (val != DIE_NMI) return NOTIFY_OK; cpu = raw_smp_processor_id(); @@ -778,6 +779,8 @@ static void smp_send_nmi_allbutself(void) static struct notifier_block crash_nmi_nb = { .notifier_call = crash_nmi_callback, + /* we want to be the first one called */ + .priority = NMI_LOCAL_HIGH_PRIOR+1, }; /* Halt all other CPUs, calling the specified function on each of them diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 1cfbbfc..6f39cab 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -76,7 +76,7 @@ int mach_set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index c7149c9..763df77 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -97,12 +97,12 @@ static DEFINE_PER_CPU(struct task_struct *, idle_thread_array); */ static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex); -void cpu_hotplug_driver_lock() +void cpu_hotplug_driver_lock(void) { mutex_lock(&x86_cpu_hotplug_driver_mutex); } -void cpu_hotplug_driver_unlock() +void cpu_hotplug_driver_unlock(void) { mutex_unlock(&x86_cpu_hotplug_driver_mutex); } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index c76aaca..b9b6716 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -84,6 +84,11 @@ EXPORT_SYMBOL_GPL(used_vectors); static int ignore_nmis; int unknown_nmi_panic; +/* + * Prevent NMI reason port (0x61) being accessed simultaneously, can + * only be used in NMI handler. + */ +static DEFINE_RAW_SPINLOCK(nmi_reason_lock); static inline void conditional_sti(struct pt_regs *regs) { @@ -310,15 +315,15 @@ static int __init setup_unknown_nmi_panic(char *str) __setup("unknown_nmi_panic", setup_unknown_nmi_panic); static notrace __kprobes void -mem_parity_error(unsigned char reason, struct pt_regs *regs) +pci_serr_error(unsigned char reason, struct pt_regs *regs) { - printk(KERN_EMERG - "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n", - reason, smp_processor_id()); - - printk(KERN_EMERG - "You have some hardware problem, likely on the PCI bus.\n"); + pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n", + reason, smp_processor_id()); + /* + * On some machines, PCI SERR line is used to report memory + * errors. EDAC makes use of it. + */ #if defined(CONFIG_EDAC) if (edac_handler_set()) { edac_atomic_assert_error(); @@ -329,11 +334,11 @@ mem_parity_error(unsigned char reason, struct pt_regs *regs) if (panic_on_unrecovered_nmi) panic("NMI: Not continuing"); - printk(KERN_EMERG "Dazed and confused, but trying to continue\n"); + pr_emerg("Dazed and confused, but trying to continue\n"); - /* Clear and disable the memory parity error line. */ - reason = (reason & 0xf) | 4; - outb(reason, 0x61); + /* Clear and disable the PCI SERR error line. */ + reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR; + outb(reason, NMI_REASON_PORT); } static notrace __kprobes void @@ -341,15 +346,17 @@ io_check_error(unsigned char reason, struct pt_regs *regs) { unsigned long i; - printk(KERN_EMERG "NMI: IOCK error (debug interrupt?)\n"); + pr_emerg( + "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n", + reason, smp_processor_id()); show_registers(regs); if (panic_on_io_nmi) panic("NMI IOCK error: Not continuing"); /* Re-enable the IOCK line, wait for a few seconds */ - reason = (reason & 0xf) | 8; - outb(reason, 0x61); + reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK; + outb(reason, NMI_REASON_PORT); i = 20000; while (--i) { @@ -357,8 +364,8 @@ io_check_error(unsigned char reason, struct pt_regs *regs) udelay(100); } - reason &= ~8; - outb(reason, 0x61); + reason &= ~NMI_REASON_CLEAR_IOCHK; + outb(reason, NMI_REASON_PORT); } static notrace __kprobes void @@ -377,57 +384,50 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) return; } #endif - printk(KERN_EMERG - "Uhhuh. NMI received for unknown reason %02x on CPU %d.\n", - reason, smp_processor_id()); + pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n", + reason, smp_processor_id()); - printk(KERN_EMERG "Do you have a strange power saving mode enabled?\n"); + pr_emerg("Do you have a strange power saving mode enabled?\n"); if (unknown_nmi_panic || panic_on_unrecovered_nmi) panic("NMI: Not continuing"); - printk(KERN_EMERG "Dazed and confused, but trying to continue\n"); + pr_emerg("Dazed and confused, but trying to continue\n"); } static notrace __kprobes void default_do_nmi(struct pt_regs *regs) { unsigned char reason = 0; - int cpu; - cpu = smp_processor_id(); - - /* Only the BSP gets external NMIs from the system. */ - if (!cpu) - reason = get_nmi_reason(); + /* + * CPU-specific NMI must be processed before non-CPU-specific + * NMI, otherwise we may lose it, because the CPU-specific + * NMI can not be detected/processed on other CPUs. + */ + if (notify_die(DIE_NMI, "nmi", regs, 0, 2, SIGINT) == NOTIFY_STOP) + return; - if (!(reason & 0xc0)) { - if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT) - == NOTIFY_STOP) - return; + /* Non-CPU-specific NMI: NMI sources can be processed on any CPU */ + raw_spin_lock(&nmi_reason_lock); + reason = get_nmi_reason(); -#ifdef CONFIG_X86_LOCAL_APIC - if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) - == NOTIFY_STOP) - return; + if (reason & NMI_REASON_MASK) { + if (reason & NMI_REASON_SERR) + pci_serr_error(reason, regs); + else if (reason & NMI_REASON_IOCHK) + io_check_error(reason, regs); +#ifdef CONFIG_X86_32 + /* + * Reassert NMI in case it became active + * meanwhile as it's edge-triggered: + */ + reassert_nmi(); #endif - unknown_nmi_error(reason, regs); - + raw_spin_unlock(&nmi_reason_lock); return; } - if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) - return; + raw_spin_unlock(&nmi_reason_lock); - /* AK: following checks seem to be broken on modern chipsets. FIXME */ - if (reason & 0x80) - mem_parity_error(reason, regs); - if (reason & 0x40) - io_check_error(reason, regs); -#ifdef CONFIG_X86_32 - /* - * Reassert NMI in case it became active meanwhile - * as it's edge-triggered: - */ - reassert_nmi(); -#endif + unknown_nmi_error(reason, regs); } dotraplinkage notrace __kprobes void diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 03d2ea8..823f79a 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -965,7 +965,7 @@ out: static int __init init_tsc_clocksource(void) { - if (!cpu_has_tsc || tsc_disabled > 0) + if (!cpu_has_tsc || tsc_disabled > 0 || !tsc_khz) return 0; if (tsc_clocksource_reliable) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index ddc131f..50f6364 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -28,6 +28,7 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_EVENTFD select KVM_APIC_ARCHITECTURE + select KVM_ASYNC_PF select USER_RETURN_NOTIFIER select KVM_MMIO ---help--- diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 31a7035..f15501f43 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,5 +1,5 @@ -EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm +ccflags-y += -Ivirt/kvm -Iarch/x86/kvm CFLAGS_x86.o := -I. CFLAGS_svm.o := -I. @@ -9,6 +9,7 @@ kvm-y += $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \ coalesced_mmio.o irq_comm.o eventfd.o \ assigned-dev.o) kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o) +kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o) kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ i8254.o timer.o diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 38b6e8d..caf9667 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -20,16 +20,8 @@ * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 */ -#ifndef __KERNEL__ -#include <stdio.h> -#include <stdint.h> -#include <public/xen.h> -#define DPRINTF(_f, _a ...) printf(_f , ## _a) -#else #include <linux/kvm_host.h> #include "kvm_cache_regs.h" -#define DPRINTF(x...) do {} while (0) -#endif #include <linux/module.h> #include <asm/kvm_emulate.h> @@ -418,9 +410,9 @@ address_mask(struct decode_cache *c, unsigned long reg) } static inline unsigned long -register_address(struct decode_cache *c, unsigned long base, unsigned long reg) +register_address(struct decode_cache *c, unsigned long reg) { - return base + address_mask(c, reg); + return address_mask(c, reg); } static inline void @@ -452,60 +444,55 @@ static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, return ops->get_cached_segment_base(seg, ctxt->vcpu); } -static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - struct decode_cache *c) +static unsigned seg_override(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + struct decode_cache *c) { if (!c->has_seg_override) return 0; - return seg_base(ctxt, ops, c->seg_override); + return c->seg_override; } -static unsigned long es_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) +static ulong linear(struct x86_emulate_ctxt *ctxt, + struct segmented_address addr) { - return seg_base(ctxt, ops, VCPU_SREG_ES); -} - -static unsigned long ss_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - return seg_base(ctxt, ops, VCPU_SREG_SS); -} + struct decode_cache *c = &ctxt->decode; + ulong la; -static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, - u32 error, bool valid) -{ - ctxt->exception = vec; - ctxt->error_code = error; - ctxt->error_code_valid = valid; + la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea; + if (c->ad_bytes != 8) + la &= (u32)-1; + return la; } -static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err) +static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, + u32 error, bool valid) { - emulate_exception(ctxt, GP_VECTOR, err, true); + ctxt->exception.vector = vec; + ctxt->exception.error_code = error; + ctxt->exception.error_code_valid = valid; + return X86EMUL_PROPAGATE_FAULT; } -static void emulate_pf(struct x86_emulate_ctxt *ctxt) +static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err) { - emulate_exception(ctxt, PF_VECTOR, 0, true); + return emulate_exception(ctxt, GP_VECTOR, err, true); } -static void emulate_ud(struct x86_emulate_ctxt *ctxt) +static int emulate_ud(struct x86_emulate_ctxt *ctxt) { - emulate_exception(ctxt, UD_VECTOR, 0, false); + return emulate_exception(ctxt, UD_VECTOR, 0, false); } -static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err) +static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err) { - emulate_exception(ctxt, TS_VECTOR, err, true); + return emulate_exception(ctxt, TS_VECTOR, err, true); } static int emulate_de(struct x86_emulate_ctxt *ctxt) { - emulate_exception(ctxt, DE_VECTOR, 0, false); - return X86EMUL_PROPAGATE_FAULT; + return emulate_exception(ctxt, DE_VECTOR, 0, false); } static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, @@ -520,7 +507,7 @@ static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, cur_size = fc->end - fc->start; size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip)); rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size, - size, ctxt->vcpu, NULL); + size, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; fc->end += size; @@ -564,7 +551,7 @@ static void *decode_register(u8 modrm_reg, unsigned long *regs, static int read_descriptor(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - ulong addr, + struct segmented_address addr, u16 *size, unsigned long *address, int op_bytes) { int rc; @@ -572,10 +559,13 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt, if (op_bytes == 2) op_bytes = 3; *address = 0; - rc = ops->read_std(addr, (unsigned long *)size, 2, ctxt->vcpu, NULL); + rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2, + ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; - rc = ops->read_std(addr + 2, address, op_bytes, ctxt->vcpu, NULL); + addr.ea += 2; + rc = ops->read_std(linear(ctxt, addr), address, op_bytes, + ctxt->vcpu, &ctxt->exception); return rc; } @@ -768,7 +758,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, break; } } - op->addr.mem = modrm_ea; + op->addr.mem.ea = modrm_ea; done: return rc; } @@ -783,13 +773,13 @@ static int decode_abs(struct x86_emulate_ctxt *ctxt, op->type = OP_MEM; switch (c->ad_bytes) { case 2: - op->addr.mem = insn_fetch(u16, 2, c->eip); + op->addr.mem.ea = insn_fetch(u16, 2, c->eip); break; case 4: - op->addr.mem = insn_fetch(u32, 4, c->eip); + op->addr.mem.ea = insn_fetch(u32, 4, c->eip); break; case 8: - op->addr.mem = insn_fetch(u64, 8, c->eip); + op->addr.mem.ea = insn_fetch(u64, 8, c->eip); break; } done: @@ -808,7 +798,7 @@ static void fetch_bit_operand(struct decode_cache *c) else if (c->src.bytes == 4) sv = (s32)c->src.val & (s32)mask; - c->dst.addr.mem += (sv >> 3); + c->dst.addr.mem.ea += (sv >> 3); } /* only subword offset */ @@ -821,7 +811,6 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, { int rc; struct read_cache *mc = &ctxt->decode.mem_read; - u32 err; while (size) { int n = min(size, 8u); @@ -829,10 +818,8 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, if (mc->pos < mc->end) goto read_cached; - rc = ops->read_emulated(addr, mc->data + mc->end, n, &err, - ctxt->vcpu); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + rc = ops->read_emulated(addr, mc->data + mc->end, n, + &ctxt->exception, ctxt->vcpu); if (rc != X86EMUL_CONTINUE) return rc; mc->end += n; @@ -907,19 +894,15 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt, struct desc_ptr dt; u16 index = selector >> 3; int ret; - u32 err; ulong addr; get_descriptor_table_ptr(ctxt, ops, selector, &dt); - if (dt.size < index * 8 + 7) { - emulate_gp(ctxt, selector & 0xfffc); - return X86EMUL_PROPAGATE_FAULT; - } + if (dt.size < index * 8 + 7) + return emulate_gp(ctxt, selector & 0xfffc); addr = dt.address + index * 8; - ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, + &ctxt->exception); return ret; } @@ -931,21 +914,17 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt, { struct desc_ptr dt; u16 index = selector >> 3; - u32 err; ulong addr; int ret; get_descriptor_table_ptr(ctxt, ops, selector, &dt); - if (dt.size < index * 8 + 7) { - emulate_gp(ctxt, selector & 0xfffc); - return X86EMUL_PROPAGATE_FAULT; - } + if (dt.size < index * 8 + 7) + return emulate_gp(ctxt, selector & 0xfffc); addr = dt.address + index * 8; - ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, + &ctxt->exception); return ret; } @@ -1092,7 +1071,6 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, { int rc; struct decode_cache *c = &ctxt->decode; - u32 err; switch (c->dst.type) { case OP_REG: @@ -1101,21 +1079,19 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, case OP_MEM: if (c->lock_prefix) rc = ops->cmpxchg_emulated( - c->dst.addr.mem, + linear(ctxt, c->dst.addr.mem), &c->dst.orig_val, &c->dst.val, c->dst.bytes, - &err, + &ctxt->exception, ctxt->vcpu); else rc = ops->write_emulated( - c->dst.addr.mem, + linear(ctxt, c->dst.addr.mem), &c->dst.val, c->dst.bytes, - &err, + &ctxt->exception, ctxt->vcpu); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); if (rc != X86EMUL_CONTINUE) return rc; break; @@ -1137,8 +1113,8 @@ static inline void emulate_push(struct x86_emulate_ctxt *ctxt, c->dst.bytes = c->op_bytes; c->dst.val = c->src.val; register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes); - c->dst.addr.mem = register_address(c, ss_base(ctxt, ops), - c->regs[VCPU_REGS_RSP]); + c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]); + c->dst.addr.mem.seg = VCPU_SREG_SS; } static int emulate_pop(struct x86_emulate_ctxt *ctxt, @@ -1147,10 +1123,11 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt, { struct decode_cache *c = &ctxt->decode; int rc; + struct segmented_address addr; - rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops), - c->regs[VCPU_REGS_RSP]), - dest, len); + addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]); + addr.seg = VCPU_SREG_SS; + rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len); if (rc != X86EMUL_CONTINUE) return rc; @@ -1184,10 +1161,8 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt, change_mask |= EFLG_IF; break; case X86EMUL_MODE_VM86: - if (iopl < 3) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (iopl < 3) + return emulate_gp(ctxt, 0); change_mask |= EFLG_IF; break; default: /* real mode */ @@ -1198,9 +1173,6 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt, *(unsigned long *)dest = (ctxt->eflags & ~change_mask) | (val & change_mask); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); - return rc; } @@ -1287,7 +1259,6 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, gva_t cs_addr; gva_t eip_addr; u16 cs, eip; - u32 err; /* TODO: Add limit checks */ c->src.val = ctxt->eflags; @@ -1317,11 +1288,11 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, eip_addr = dt.address + (irq << 2); cs_addr = dt.address + (irq << 2) + 2; - rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &err); + rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; - rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &err); + rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; @@ -1370,10 +1341,8 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt, if (rc != X86EMUL_CONTINUE) return rc; - if (temp_eip & ~0xffff) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (temp_eip & ~0xffff) + return emulate_gp(ctxt, 0); rc = emulate_pop(ctxt, ops, &cs, c->op_bytes); @@ -1624,10 +1593,8 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* syscall is not available in real mode */ if (ctxt->mode == X86EMUL_MODE_REAL || - ctxt->mode == X86EMUL_MODE_VM86) { - emulate_ud(ctxt); - return X86EMUL_PROPAGATE_FAULT; - } + ctxt->mode == X86EMUL_MODE_VM86) + return emulate_ud(ctxt); setup_syscalls_segments(ctxt, ops, &cs, &ss); @@ -1678,34 +1645,26 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) u16 cs_sel, ss_sel; /* inject #GP if in real mode */ - if (ctxt->mode == X86EMUL_MODE_REAL) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (ctxt->mode == X86EMUL_MODE_REAL) + return emulate_gp(ctxt, 0); /* XXX sysenter/sysexit have not been tested in 64bit mode. * Therefore, we inject an #UD. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) { - emulate_ud(ctxt); - return X86EMUL_PROPAGATE_FAULT; - } + if (ctxt->mode == X86EMUL_MODE_PROT64) + return emulate_ud(ctxt); setup_syscalls_segments(ctxt, ops, &cs, &ss); ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data); switch (ctxt->mode) { case X86EMUL_MODE_PROT32: - if ((msr_data & 0xfffc) == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if ((msr_data & 0xfffc) == 0x0) + return emulate_gp(ctxt, 0); break; case X86EMUL_MODE_PROT64: - if (msr_data == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (msr_data == 0x0) + return emulate_gp(ctxt, 0); break; } @@ -1745,10 +1704,8 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* inject #GP if in real mode or Virtual 8086 mode */ if (ctxt->mode == X86EMUL_MODE_REAL || - ctxt->mode == X86EMUL_MODE_VM86) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + ctxt->mode == X86EMUL_MODE_VM86) + return emulate_gp(ctxt, 0); setup_syscalls_segments(ctxt, ops, &cs, &ss); @@ -1763,18 +1720,14 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (usermode) { case X86EMUL_MODE_PROT32: cs_sel = (u16)(msr_data + 16); - if ((msr_data & 0xfffc) == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if ((msr_data & 0xfffc) == 0x0) + return emulate_gp(ctxt, 0); ss_sel = (u16)(msr_data + 24); break; case X86EMUL_MODE_PROT64: cs_sel = (u16)(msr_data + 32); - if (msr_data == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (msr_data == 0x0) + return emulate_gp(ctxt, 0); ss_sel = cs_sel + 8; cs.d = 0; cs.l = 1; @@ -1934,33 +1887,27 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt, { struct tss_segment_16 tss_seg; int ret; - u32 err, new_tss_base = get_desc_base(new_desc); + u32 new_tss_base = get_desc_base(new_desc); ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } save_state_to_tss16(ctxt, ops, &tss_seg); ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } if (old_tss_sel != 0xffff) { tss_seg.prev_task_link = old_tss_sel; @@ -1968,12 +1915,10 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt, ret = ops->write_std(new_tss_base, &tss_seg.prev_task_link, sizeof tss_seg.prev_task_link, - ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + ctxt->vcpu, &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } } return load_state_from_tss16(ctxt, ops, &tss_seg); @@ -2013,10 +1958,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt, struct decode_cache *c = &ctxt->decode; int ret; - if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) + return emulate_gp(ctxt, 0); c->eip = tss->eip; ctxt->eflags = tss->eflags | 2; c->regs[VCPU_REGS_RAX] = tss->eax; @@ -2076,33 +2019,27 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt, { struct tss_segment_32 tss_seg; int ret; - u32 err, new_tss_base = get_desc_base(new_desc); + u32 new_tss_base = get_desc_base(new_desc); ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } save_state_to_tss32(ctxt, ops, &tss_seg); ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } if (old_tss_sel != 0xffff) { tss_seg.prev_task_link = old_tss_sel; @@ -2110,12 +2047,10 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt, ret = ops->write_std(new_tss_base, &tss_seg.prev_task_link, sizeof tss_seg.prev_task_link, - ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + ctxt->vcpu, &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } } return load_state_from_tss32(ctxt, ops, &tss_seg); @@ -2146,10 +2081,8 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt, if (reason != TASK_SWITCH_IRET) { if ((tss_selector & 3) > next_tss_desc.dpl || - ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) + return emulate_gp(ctxt, 0); } desc_limit = desc_limit_scaled(&next_tss_desc); @@ -2231,14 +2164,15 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt, return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; } -static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base, +static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg, int reg, struct operand *op) { struct decode_cache *c = &ctxt->decode; int df = (ctxt->eflags & EFLG_DF) ? -1 : 1; register_address_increment(c, &c->regs[reg], df * op->bytes); - op->addr.mem = register_address(c, base, c->regs[reg]); + op->addr.mem.ea = register_address(c, c->regs[reg]); + op->addr.mem.seg = seg; } static int em_push(struct x86_emulate_ctxt *ctxt) @@ -2369,10 +2303,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt) struct decode_cache *c = &ctxt->decode; u64 tsc = 0; - if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) + return emulate_gp(ctxt, 0); ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc); c->regs[VCPU_REGS_RAX] = (u32)tsc; c->regs[VCPU_REGS_RDX] = tsc >> 32; @@ -2647,7 +2579,7 @@ static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op, op->type = OP_IMM; op->bytes = size; - op->addr.mem = c->eip; + op->addr.mem.ea = c->eip; /* NB. Immediates are sign-extended as necessary. */ switch (op->bytes) { case 1: @@ -2678,7 +2610,7 @@ done: } int -x86_decode_insn(struct x86_emulate_ctxt *ctxt) +x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len) { struct x86_emulate_ops *ops = ctxt->ops; struct decode_cache *c = &ctxt->decode; @@ -2689,7 +2621,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt) struct operand memop = { .type = OP_NONE }; c->eip = ctxt->eip; - c->fetch.start = c->fetch.end = c->eip; + c->fetch.start = c->eip; + c->fetch.end = c->fetch.start + insn_len; + if (insn_len > 0) + memcpy(c->fetch.data, insn, insn_len); ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS); switch (mode) { @@ -2803,10 +2738,8 @@ done_prefixes: c->execute = opcode.u.execute; /* Unrecognised? */ - if (c->d == 0 || (c->d & Undefined)) { - DPRINTF("Cannot emulate %02x\n", c->b); + if (c->d == 0 || (c->d & Undefined)) return -1; - } if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) c->op_bytes = 8; @@ -2831,14 +2764,13 @@ done_prefixes: if (!c->has_seg_override) set_seg_override(c, VCPU_SREG_DS); - if (memop.type == OP_MEM && !(!c->twobyte && c->b == 0x8d)) - memop.addr.mem += seg_override_base(ctxt, ops, c); + memop.addr.mem.seg = seg_override(ctxt, ops, c); if (memop.type == OP_MEM && c->ad_bytes != 8) - memop.addr.mem = (u32)memop.addr.mem; + memop.addr.mem.ea = (u32)memop.addr.mem.ea; if (memop.type == OP_MEM && c->rip_relative) - memop.addr.mem += c->eip; + memop.addr.mem.ea += c->eip; /* * Decode and fetch the source operand: register, memory @@ -2890,14 +2822,14 @@ done_prefixes: case SrcSI: c->src.type = OP_MEM; c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->src.addr.mem = - register_address(c, seg_override_base(ctxt, ops, c), - c->regs[VCPU_REGS_RSI]); + c->src.addr.mem.ea = + register_address(c, c->regs[VCPU_REGS_RSI]); + c->src.addr.mem.seg = seg_override(ctxt, ops, c), c->src.val = 0; break; case SrcImmFAddr: c->src.type = OP_IMM; - c->src.addr.mem = c->eip; + c->src.addr.mem.ea = c->eip; c->src.bytes = c->op_bytes + 2; insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip); break; @@ -2944,7 +2876,7 @@ done_prefixes: break; case DstImmUByte: c->dst.type = OP_IMM; - c->dst.addr.mem = c->eip; + c->dst.addr.mem.ea = c->eip; c->dst.bytes = 1; c->dst.val = insn_fetch(u8, 1, c->eip); break; @@ -2969,9 +2901,9 @@ done_prefixes: case DstDI: c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.addr.mem = - register_address(c, es_base(ctxt, ops), - c->regs[VCPU_REGS_RDI]); + c->dst.addr.mem.ea = + register_address(c, c->regs[VCPU_REGS_RDI]); + c->dst.addr.mem.seg = VCPU_SREG_ES; c->dst.val = 0; break; case ImplicitOps: @@ -3020,24 +2952,24 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) ctxt->decode.mem_read.pos = 0; if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } /* LOCK prefix is allowed only with some instructions */ if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } /* Privileged instruction can be executed only in CPL=0 */ if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } @@ -3050,7 +2982,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) } if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) { - rc = read_emulated(ctxt, ops, c->src.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem), c->src.valptr, c->src.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3058,7 +2990,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) } if (c->src2.type == OP_MEM) { - rc = read_emulated(ctxt, ops, c->src2.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem), &c->src2.val, c->src2.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3070,7 +3002,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) if ((c->dst.type == OP_MEM) && !(c->d & Mov)) { /* optimisation - avoid slow emulated read if Mov */ - rc = read_emulated(ctxt, ops, c->dst.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem), &c->dst.val, c->dst.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3215,13 +3147,13 @@ special_insn: break; case 0x8c: /* mov r/m, sreg */ if (c->modrm_reg > VCPU_SREG_GS) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu); break; case 0x8d: /* lea r16/r32, m */ - c->dst.val = c->src.addr.mem; + c->dst.val = c->src.addr.mem.ea; break; case 0x8e: { /* mov seg, r/m16 */ uint16_t sel; @@ -3230,7 +3162,7 @@ special_insn: if (c->modrm_reg == VCPU_SREG_CS || c->modrm_reg > VCPU_SREG_GS) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } @@ -3268,7 +3200,6 @@ special_insn: break; case 0xa6 ... 0xa7: /* cmps */ c->dst.type = OP_NONE; /* Disable writeback. */ - DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.addr.mem, c->dst.addr.mem); goto cmp; case 0xa8 ... 0xa9: /* test ax, imm */ goto test; @@ -3363,7 +3294,7 @@ special_insn: do_io_in: c->dst.bytes = min(c->dst.bytes, 4u); if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val, @@ -3377,7 +3308,7 @@ special_insn: c->src.bytes = min(c->src.bytes, 4u); if (!emulator_io_permited(ctxt, ops, c->dst.val, c->src.bytes)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } ops->pio_out_emulated(c->src.bytes, c->dst.val, @@ -3402,14 +3333,14 @@ special_insn: break; case 0xfa: /* cli */ if (emulator_bad_iopl(ctxt, ops)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } else ctxt->eflags &= ~X86_EFLAGS_IF; break; case 0xfb: /* sti */ if (emulator_bad_iopl(ctxt, ops)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } else { ctxt->interruptibility = KVM_X86_SHADOW_INT_STI; @@ -3449,11 +3380,11 @@ writeback: c->dst.type = saved_dst_type; if ((c->d & SrcMask) == SrcSI) - string_addr_inc(ctxt, seg_override_base(ctxt, ops, c), + string_addr_inc(ctxt, seg_override(ctxt, ops, c), VCPU_REGS_RSI, &c->src); if ((c->d & DstMask) == DstDI) - string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI, + string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI, &c->dst); if (c->rep_prefix && (c->d & String)) { @@ -3482,6 +3413,8 @@ writeback: ctxt->eip = c->eip; done: + if (rc == X86EMUL_PROPAGATE_FAULT) + ctxt->have_exception = true; return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; twobyte_insn: @@ -3544,9 +3477,11 @@ twobyte_insn: break; case 5: /* not defined */ emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; case 7: /* invlpg*/ - emulate_invlpg(ctxt->vcpu, c->src.addr.mem); + emulate_invlpg(ctxt->vcpu, + linear(ctxt, c->src.addr.mem)); /* Disable writeback. */ c->dst.type = OP_NONE; break; @@ -3573,6 +3508,7 @@ twobyte_insn: case 5 ... 7: case 9 ... 15: emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu); @@ -3581,6 +3517,7 @@ twobyte_insn: if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) && (c->modrm_reg == 4 || c->modrm_reg == 5)) { emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu); @@ -3588,6 +3525,7 @@ twobyte_insn: case 0x22: /* mov reg, cr */ if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } c->dst.type = OP_NONE; @@ -3596,6 +3534,7 @@ twobyte_insn: if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) && (c->modrm_reg == 4 || c->modrm_reg == 5)) { emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } @@ -3604,6 +3543,7 @@ twobyte_insn: ~0ULL : ~0U), ctxt->vcpu) < 0) { /* #UD condition is already handled by the code above */ emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } @@ -3615,6 +3555,7 @@ twobyte_insn: | ((u64)c->regs[VCPU_REGS_RDX] << 32); if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } rc = X86EMUL_CONTINUE; @@ -3623,6 +3564,7 @@ twobyte_insn: /* rdmsr */ if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; @@ -3785,6 +3727,5 @@ twobyte_insn: goto writeback; cannot_emulate: - DPRINTF("Cannot emulate %02x\n", c->b); return -1; } diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 975bb45..3377d53 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -73,6 +73,13 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) return vcpu->arch.cr4 & mask; } +static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu) +{ + if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) + kvm_x86_ops->decache_cr3(vcpu); + return vcpu->arch.cr3; +} + static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu) { return kvm_read_cr4_bits(vcpu, ~0UL); @@ -84,4 +91,19 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32); } +static inline void enter_guest_mode(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hflags |= HF_GUEST_MASK; +} + +static inline void leave_guest_mode(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hflags &= ~HF_GUEST_MASK; +} + +static inline bool is_guest_mode(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.hflags & HF_GUEST_MASK; +} + #endif diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 413f897..93cf9d0 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -277,7 +277,8 @@ static void apic_update_ppr(struct kvm_lapic *apic) if (old_ppr != ppr) { apic_set_reg(apic, APIC_PROCPRI, ppr); - kvm_make_request(KVM_REQ_EVENT, apic->vcpu); + if (ppr < old_ppr) + kvm_make_request(KVM_REQ_EVENT, apic->vcpu); } } diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index fbb04ae..9cafbb4 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -18,9 +18,11 @@ * */ +#include "irq.h" #include "mmu.h" #include "x86.h" #include "kvm_cache_regs.h" +#include "x86.h" #include <linux/kvm_host.h> #include <linux/types.h> @@ -194,7 +196,6 @@ static struct percpu_counter kvm_total_used_mmu_pages; static u64 __read_mostly shadow_trap_nonpresent_pte; static u64 __read_mostly shadow_notrap_nonpresent_pte; -static u64 __read_mostly shadow_base_present_pte; static u64 __read_mostly shadow_nx_mask; static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ static u64 __read_mostly shadow_user_mask; @@ -213,12 +214,6 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) } EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); -void kvm_mmu_set_base_ptes(u64 base_pte) -{ - shadow_base_present_pte = base_pte; -} -EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes); - void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask) { @@ -482,46 +477,46 @@ static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) } /* - * Return the pointer to the largepage write count for a given - * gfn, handling slots that are not large page aligned. + * Return the pointer to the large page information for a given gfn, + * handling slots that are not large page aligned. */ -static int *slot_largepage_idx(gfn_t gfn, - struct kvm_memory_slot *slot, - int level) +static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, + struct kvm_memory_slot *slot, + int level) { unsigned long idx; idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); - return &slot->lpage_info[level - 2][idx].write_count; + return &slot->lpage_info[level - 2][idx]; } static void account_shadowed(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; - int *write_count; + struct kvm_lpage_info *linfo; int i; slot = gfn_to_memslot(kvm, gfn); for (i = PT_DIRECTORY_LEVEL; i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) { - write_count = slot_largepage_idx(gfn, slot, i); - *write_count += 1; + linfo = lpage_info_slot(gfn, slot, i); + linfo->write_count += 1; } } static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; - int *write_count; + struct kvm_lpage_info *linfo; int i; slot = gfn_to_memslot(kvm, gfn); for (i = PT_DIRECTORY_LEVEL; i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) { - write_count = slot_largepage_idx(gfn, slot, i); - *write_count -= 1; - WARN_ON(*write_count < 0); + linfo = lpage_info_slot(gfn, slot, i); + linfo->write_count -= 1; + WARN_ON(linfo->write_count < 0); } } @@ -530,12 +525,12 @@ static int has_wrprotected_page(struct kvm *kvm, int level) { struct kvm_memory_slot *slot; - int *largepage_idx; + struct kvm_lpage_info *linfo; slot = gfn_to_memslot(kvm, gfn); if (slot) { - largepage_idx = slot_largepage_idx(gfn, slot, level); - return *largepage_idx; + linfo = lpage_info_slot(gfn, slot, level); + return linfo->write_count; } return 1; @@ -590,16 +585,15 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn) static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level) { struct kvm_memory_slot *slot; - unsigned long idx; + struct kvm_lpage_info *linfo; slot = gfn_to_memslot(kvm, gfn); if (likely(level == PT_PAGE_TABLE_LEVEL)) return &slot->rmap[gfn - slot->base_gfn]; - idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - - (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); + linfo = lpage_info_slot(gfn, slot, level); - return &slot->lpage_info[level - 2][idx].rmap_pde; + return &linfo->rmap_pde; } /* @@ -887,19 +881,16 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, end = start + (memslot->npages << PAGE_SHIFT); if (hva >= start && hva < end) { gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; + gfn_t gfn = memslot->base_gfn + gfn_offset; ret = handler(kvm, &memslot->rmap[gfn_offset], data); for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) { - unsigned long idx; - int sh; - - sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j); - idx = ((memslot->base_gfn+gfn_offset) >> sh) - - (memslot->base_gfn >> sh); - ret |= handler(kvm, - &memslot->lpage_info[j][idx].rmap_pde, - data); + struct kvm_lpage_info *linfo; + + linfo = lpage_info_slot(gfn, memslot, + PT_DIRECTORY_LEVEL + j); + ret |= handler(kvm, &linfo->rmap_pde, data); } trace_kvm_age_page(hva, memslot, ret); retval |= ret; @@ -1161,7 +1152,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, } static int nonpaging_sync_page(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, bool clear_unsync) + struct kvm_mmu_page *sp) { return 1; } @@ -1291,7 +1282,7 @@ static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, if (clear_unsync) kvm_unlink_unsync_page(vcpu->kvm, sp); - if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) { + if (vcpu->arch.mmu.sync_page(vcpu, sp)) { kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); return 1; } @@ -1332,12 +1323,12 @@ static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn) continue; WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL); + kvm_unlink_unsync_page(vcpu->kvm, s); if ((s->role.cr4_pae != !!is_pae(vcpu)) || - (vcpu->arch.mmu.sync_page(vcpu, s, true))) { + (vcpu->arch.mmu.sync_page(vcpu, s))) { kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list); continue; } - kvm_unlink_unsync_page(vcpu->kvm, s); flush = true; } @@ -1963,9 +1954,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access, int user_fault, int write_fault, int dirty, int level, gfn_t gfn, pfn_t pfn, bool speculative, - bool can_unsync, bool reset_host_protection) + bool can_unsync, bool host_writable) { - u64 spte; + u64 spte, entry = *sptep; int ret = 0; /* @@ -1973,7 +1964,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, * whether the guest actually used the pte (in order to detect * demand paging). */ - spte = shadow_base_present_pte; + spte = PT_PRESENT_MASK; if (!speculative) spte |= shadow_accessed_mask; if (!dirty) @@ -1990,8 +1981,10 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn, kvm_is_mmio_pfn(pfn)); - if (reset_host_protection) + if (host_writable) spte |= SPTE_HOST_WRITEABLE; + else + pte_access &= ~ACC_WRITE_MASK; spte |= (u64)pfn << PAGE_SHIFT; @@ -2036,6 +2029,14 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, set_pte: update_spte(sptep, spte); + /* + * If we overwrite a writable spte with a read-only one we + * should flush remote TLBs. Otherwise rmap_write_protect + * will find a read-only spte, even though the writable spte + * might be cached on a CPU's TLB. + */ + if (is_writable_pte(entry) && !is_writable_pte(*sptep)) + kvm_flush_remote_tlbs(vcpu->kvm); done: return ret; } @@ -2045,7 +2046,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, int user_fault, int write_fault, int dirty, int *ptwrite, int level, gfn_t gfn, pfn_t pfn, bool speculative, - bool reset_host_protection) + bool host_writable) { int was_rmapped = 0; int rmap_count; @@ -2080,7 +2081,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault, dirty, level, gfn, pfn, speculative, true, - reset_host_protection)) { + host_writable)) { if (write_fault) *ptwrite = 1; kvm_mmu_flush_tlb(vcpu); @@ -2211,7 +2212,8 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) } static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, - int level, gfn_t gfn, pfn_t pfn) + int map_writable, int level, gfn_t gfn, pfn_t pfn, + bool prefault) { struct kvm_shadow_walk_iterator iterator; struct kvm_mmu_page *sp; @@ -2220,9 +2222,11 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) { if (iterator.level == level) { - mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL, + unsigned pte_access = ACC_ALL; + + mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access, 0, write, 1, &pt_write, - level, gfn, pfn, false, true); + level, gfn, pfn, prefault, map_writable); direct_pte_prefetch(vcpu, iterator.sptep); ++vcpu->stat.pf_fixed; break; @@ -2277,12 +2281,17 @@ static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn) return 1; } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, + gva_t gva, pfn_t *pfn, bool write, bool *writable); + +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn, + bool prefault) { int r; int level; pfn_t pfn; unsigned long mmu_seq; + bool map_writable; level = mapping_level(vcpu, gfn); @@ -2297,7 +2306,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, gfn); + + if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable)) + return 0; /* mmio */ if (is_error_pfn(pfn)) @@ -2307,7 +2318,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) if (mmu_notifier_retry(vcpu, mmu_seq)) goto out_unlock; kvm_mmu_free_some_pages(vcpu); - r = __direct_map(vcpu, v, write, level, gfn, pfn); + r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn, + prefault); spin_unlock(&vcpu->kvm->mmu_lock); @@ -2530,6 +2542,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu) hpa_t root = vcpu->arch.mmu.root_hpa; sp = page_header(root); mmu_sync_children(vcpu, sp); + trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); return; } for (i = 0; i < 4; ++i) { @@ -2552,23 +2565,24 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, struct x86_exception *exception) { - if (error) - *error = 0; + if (exception) + exception->error_code = 0; return vaddr; } static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, + struct x86_exception *exception) { - if (error) - *error = 0; + if (exception) + exception->error_code = 0; return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access); } static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) + u32 error_code, bool prefault) { gfn_t gfn; int r; @@ -2584,17 +2598,67 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, gfn = gva >> PAGE_SHIFT; return nonpaging_map(vcpu, gva & PAGE_MASK, - error_code & PFERR_WRITE_MASK, gfn); + error_code & PFERR_WRITE_MASK, gfn, prefault); +} + +static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn) +{ + struct kvm_arch_async_pf arch; + + arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; + arch.gfn = gfn; + arch.direct_map = vcpu->arch.mmu.direct_map; + arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu); + + return kvm_setup_async_pf(vcpu, gva, gfn, &arch); } -static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, - u32 error_code) +static bool can_do_async_pf(struct kvm_vcpu *vcpu) +{ + if (unlikely(!irqchip_in_kernel(vcpu->kvm) || + kvm_event_needs_reinjection(vcpu))) + return false; + + return kvm_x86_ops->interrupt_allowed(vcpu); +} + +static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, + gva_t gva, pfn_t *pfn, bool write, bool *writable) +{ + bool async; + + *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable); + + if (!async) + return false; /* *pfn has correct page already */ + + put_page(pfn_to_page(*pfn)); + + if (!prefault && can_do_async_pf(vcpu)) { + trace_kvm_try_async_get_page(gva, gfn); + if (kvm_find_async_pf_gfn(vcpu, gfn)) { + trace_kvm_async_pf_doublefault(gva, gfn); + kvm_make_request(KVM_REQ_APF_HALT, vcpu); + return true; + } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn)) + return true; + } + + *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable); + + return false; +} + +static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code, + bool prefault) { pfn_t pfn; int r; int level; gfn_t gfn = gpa >> PAGE_SHIFT; unsigned long mmu_seq; + int write = error_code & PFERR_WRITE_MASK; + bool map_writable; ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); @@ -2609,15 +2673,19 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, gfn); + + if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) + return 0; + + /* mmio */ if (is_error_pfn(pfn)) return kvm_handle_bad_page(vcpu->kvm, gfn, pfn); spin_lock(&vcpu->kvm->mmu_lock); if (mmu_notifier_retry(vcpu, mmu_seq)) goto out_unlock; kvm_mmu_free_some_pages(vcpu); - r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK, - level, gfn, pfn); + r = __direct_map(vcpu, gpa, write, map_writable, + level, gfn, pfn, prefault); spin_unlock(&vcpu->kvm->mmu_lock); return r; @@ -2659,18 +2727,19 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) static void paging_new_cr3(struct kvm_vcpu *vcpu) { - pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3); + pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu)); mmu_free_roots(vcpu); } static unsigned long get_cr3(struct kvm_vcpu *vcpu) { - return vcpu->arch.cr3; + return kvm_read_cr3(vcpu); } -static void inject_page_fault(struct kvm_vcpu *vcpu) +static void inject_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { - vcpu->arch.mmu.inject_page_fault(vcpu); + vcpu->arch.mmu.inject_page_fault(vcpu, fault); } static void paging_free(struct kvm_vcpu *vcpu) @@ -2816,6 +2885,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = vcpu->arch.walk_mmu; + context->base_role.word = 0; context->new_cr3 = nonpaging_new_cr3; context->page_fault = tdp_page_fault; context->free = nonpaging_free; @@ -3008,9 +3078,6 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, return; } - if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL)) - return; - ++vcpu->kvm->stat.mmu_pte_updated; if (!sp->role.cr4_pae) paging32_update_pte(vcpu, sp, spte, new); @@ -3264,12 +3331,13 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) } } -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code, + void *insn, int insn_len) { int r; enum emulation_result er; - r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false); if (r < 0) goto out; @@ -3282,7 +3350,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) if (r) goto out; - er = emulate_instruction(vcpu, cr2, error_code, 0); + er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len); switch (er) { case EMULATE_DONE: @@ -3377,11 +3445,14 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) if (!test_bit(slot, sp->slot_bitmap)) continue; + if (sp->role.level != PT_PAGE_TABLE_LEVEL) + continue; + pt = sp->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (is_writable_pte(pt[i])) - pt[i] &= ~PT_WRITABLE_MASK; + update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK); } kvm_flush_remote_tlbs(kvm); } @@ -3463,13 +3534,6 @@ static void mmu_destroy_caches(void) kmem_cache_destroy(mmu_page_header_cache); } -void kvm_mmu_module_exit(void) -{ - mmu_destroy_caches(); - percpu_counter_destroy(&kvm_total_used_mmu_pages); - unregister_shrinker(&mmu_shrinker); -} - int kvm_mmu_module_init(void) { pte_chain_cache = kmem_cache_create("kvm_pte_chain", @@ -3566,7 +3630,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu, static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu) { - (void)kvm_set_cr3(vcpu, vcpu->arch.cr3); + (void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu)); return 1; } @@ -3662,12 +3726,6 @@ int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4]) } EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy); -#ifdef CONFIG_KVM_MMU_AUDIT -#include "mmu_audit.c" -#else -static void mmu_audit_disable(void) { } -#endif - void kvm_mmu_destroy(struct kvm_vcpu *vcpu) { ASSERT(vcpu); @@ -3675,5 +3733,18 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu) destroy_kvm_mmu(vcpu); free_mmu_pages(vcpu); mmu_free_memory_caches(vcpu); +} + +#ifdef CONFIG_KVM_MMU_AUDIT +#include "mmu_audit.c" +#else +static void mmu_audit_disable(void) { } +#endif + +void kvm_mmu_module_exit(void) +{ + mmu_destroy_caches(); + percpu_counter_destroy(&kvm_total_used_mmu_pages); + unregister_shrinker(&mmu_shrinker); mmu_audit_disable(); } diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index ba2bcdd..5f6223b 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c @@ -19,11 +19,9 @@ #include <linux/ratelimit.h> -static int audit_point; - -#define audit_printk(fmt, args...) \ +#define audit_printk(kvm, fmt, args...) \ printk(KERN_ERR "audit: (%s) error: " \ - fmt, audit_point_name[audit_point], ##args) + fmt, audit_point_name[kvm->arch.audit_point], ##args) typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level); @@ -97,18 +95,21 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) if (sp->unsync) { if (level != PT_PAGE_TABLE_LEVEL) { - audit_printk("unsync sp: %p level = %d\n", sp, level); + audit_printk(vcpu->kvm, "unsync sp: %p " + "level = %d\n", sp, level); return; } if (*sptep == shadow_notrap_nonpresent_pte) { - audit_printk("notrap spte in unsync sp: %p\n", sp); + audit_printk(vcpu->kvm, "notrap spte in unsync " + "sp: %p\n", sp); return; } } if (sp->role.direct && *sptep == shadow_notrap_nonpresent_pte) { - audit_printk("notrap spte in direct sp: %p\n", sp); + audit_printk(vcpu->kvm, "notrap spte in direct sp: %p\n", + sp); return; } @@ -125,8 +126,9 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) hpa = pfn << PAGE_SHIFT; if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) - audit_printk("levels %d pfn %llx hpa %llx ent %llxn", - vcpu->arch.mmu.root_level, pfn, hpa, *sptep); + audit_printk(vcpu->kvm, "levels %d pfn %llx hpa %llx " + "ent %llxn", vcpu->arch.mmu.root_level, pfn, + hpa, *sptep); } static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) @@ -142,8 +144,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) if (!gfn_to_memslot(kvm, gfn)) { if (!printk_ratelimit()) return; - audit_printk("no memslot for gfn %llx\n", gfn); - audit_printk("index %ld of sp (gfn=%llx)\n", + audit_printk(kvm, "no memslot for gfn %llx\n", gfn); + audit_printk(kvm, "index %ld of sp (gfn=%llx)\n", (long int)(sptep - rev_sp->spt), rev_sp->gfn); dump_stack(); return; @@ -153,7 +155,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) if (!*rmapp) { if (!printk_ratelimit()) return; - audit_printk("no rmap for writable spte %llx\n", *sptep); + audit_printk(kvm, "no rmap for writable spte %llx\n", + *sptep); dump_stack(); } } @@ -168,8 +171,9 @@ static void audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level) { struct kvm_mmu_page *sp = page_header(__pa(sptep)); - if (audit_point == AUDIT_POST_SYNC && sp->unsync) - audit_printk("meet unsync sp(%p) after sync root.\n", sp); + if (vcpu->kvm->arch.audit_point == AUDIT_POST_SYNC && sp->unsync) + audit_printk(vcpu->kvm, "meet unsync sp(%p) after sync " + "root.\n", sp); } static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp) @@ -202,8 +206,9 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) spte = rmap_next(kvm, rmapp, NULL); while (spte) { if (is_writable_pte(*spte)) - audit_printk("shadow page has writable mappings: gfn " - "%llx role %x\n", sp->gfn, sp->role.word); + audit_printk(kvm, "shadow page has writable " + "mappings: gfn %llx role %x\n", + sp->gfn, sp->role.word); spte = rmap_next(kvm, rmapp, spte); } } @@ -238,7 +243,7 @@ static void kvm_mmu_audit(void *ignore, struct kvm_vcpu *vcpu, int point) if (!__ratelimit(&ratelimit_state)) return; - audit_point = point; + vcpu->kvm->arch.audit_point = point; audit_all_active_sps(vcpu->kvm); audit_vcpu_spte(vcpu); } diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index cd7a833..53210f1 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -72,7 +72,7 @@ struct guest_walker { unsigned pt_access; unsigned pte_access; gfn_t gfn; - u32 error_code; + struct x86_exception fault; }; static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl) @@ -266,21 +266,23 @@ walk: return 1; error: - walker->error_code = 0; + walker->fault.vector = PF_VECTOR; + walker->fault.error_code_valid = true; + walker->fault.error_code = 0; if (present) - walker->error_code |= PFERR_PRESENT_MASK; + walker->fault.error_code |= PFERR_PRESENT_MASK; - walker->error_code |= write_fault | user_fault; + walker->fault.error_code |= write_fault | user_fault; if (fetch_fault && mmu->nx) - walker->error_code |= PFERR_FETCH_MASK; + walker->fault.error_code |= PFERR_FETCH_MASK; if (rsvd_fault) - walker->error_code |= PFERR_RSVD_MASK; + walker->fault.error_code |= PFERR_RSVD_MASK; - vcpu->arch.fault.address = addr; - vcpu->arch.fault.error_code = walker->error_code; + walker->fault.address = addr; + walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu; - trace_kvm_mmu_walker_error(walker->error_code); + trace_kvm_mmu_walker_error(walker->fault.error_code); return 0; } @@ -299,25 +301,42 @@ static int FNAME(walk_addr_nested)(struct guest_walker *walker, addr, access); } +static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp, u64 *spte, + pt_element_t gpte) +{ + u64 nonpresent = shadow_trap_nonpresent_pte; + + if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)) + goto no_present; + + if (!is_present_gpte(gpte)) { + if (!sp->unsync) + nonpresent = shadow_notrap_nonpresent_pte; + goto no_present; + } + + if (!(gpte & PT_ACCESSED_MASK)) + goto no_present; + + return false; + +no_present: + drop_spte(vcpu->kvm, spte, nonpresent); + return true; +} + static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, const void *pte) { pt_element_t gpte; unsigned pte_access; pfn_t pfn; - u64 new_spte; gpte = *(const pt_element_t *)pte; - if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { - if (!is_present_gpte(gpte)) { - if (sp->unsync) - new_spte = shadow_trap_nonpresent_pte; - else - new_spte = shadow_notrap_nonpresent_pte; - __set_spte(spte, new_spte); - } + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) return; - } + pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte); pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn) @@ -329,7 +348,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, return; kvm_get_pfn(pfn); /* - * we call mmu_set_spte() with reset_host_protection = true beacuse that + * we call mmu_set_spte() with host_writable = true beacuse that * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1). */ mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0, @@ -364,7 +383,6 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, u64 *sptep) { struct kvm_mmu_page *sp; - struct kvm_mmu *mmu = &vcpu->arch.mmu; pt_element_t *gptep = gw->prefetch_ptes; u64 *spte; int i; @@ -395,14 +413,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, gpte = gptep[i]; - if (!is_present_gpte(gpte) || - is_rsvd_bits_set(mmu, gpte, PT_PAGE_TABLE_LEVEL)) { - if (!sp->unsync) - __set_spte(spte, shadow_notrap_nonpresent_pte); - continue; - } - - if (!(gpte & PT_ACCESSED_MASK)) + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) continue; pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); @@ -427,7 +438,8 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, struct guest_walker *gw, int user_fault, int write_fault, int hlevel, - int *ptwrite, pfn_t pfn) + int *ptwrite, pfn_t pfn, bool map_writable, + bool prefault) { unsigned access = gw->pt_access; struct kvm_mmu_page *sp = NULL; @@ -501,7 +513,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, mmu_set_spte(vcpu, it.sptep, access, gw->pte_access & access, user_fault, write_fault, dirty, ptwrite, it.level, - gw->gfn, pfn, false, true); + gw->gfn, pfn, prefault, map_writable); FNAME(pte_prefetch)(vcpu, gw, it.sptep); return it.sptep; @@ -527,8 +539,8 @@ out_gpte_changed: * Returns: 1 if we need to emulate the instruction, 0 otherwise, or * a negative value on error. */ -static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, - u32 error_code) +static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code, + bool prefault) { int write_fault = error_code & PFERR_WRITE_MASK; int user_fault = error_code & PFERR_USER_MASK; @@ -539,6 +551,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, pfn_t pfn; int level = PT_PAGE_TABLE_LEVEL; unsigned long mmu_seq; + bool map_writable; pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code); @@ -556,8 +569,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, */ if (!r) { pgprintk("%s: guest page fault\n", __func__); - inject_page_fault(vcpu); - vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ + if (!prefault) { + inject_page_fault(vcpu, &walker.fault); + /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; + } return 0; } @@ -568,7 +584,10 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, walker.gfn); + + if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault, + &map_writable)) + return 0; /* mmio */ if (is_error_pfn(pfn)) @@ -581,7 +600,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT); kvm_mmu_free_some_pages(vcpu); sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, - level, &write_pt, pfn); + level, &write_pt, pfn, map_writable, prefault); (void)sptep; pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__, sptep, *sptep, write_pt); @@ -661,7 +680,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) } static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access, - u32 *error) + struct x86_exception *exception) { struct guest_walker walker; gpa_t gpa = UNMAPPED_GVA; @@ -672,14 +691,15 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access, if (r) { gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; - } else if (error) - *error = walker.error_code; + } else if (exception) + *exception = walker.fault; return gpa; } static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, + struct x86_exception *exception) { struct guest_walker walker; gpa_t gpa = UNMAPPED_GVA; @@ -690,8 +710,8 @@ static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr, if (r) { gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; - } else if (error) - *error = walker.error_code; + } else if (exception) + *exception = walker.fault; return gpa; } @@ -730,12 +750,19 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, * Using the cached information from sp->gfns is safe because: * - The spte has a reference to the struct page, so the pfn for a given gfn * can't change unless all sptes pointing to it are nuked first. + * + * Note: + * We should flush all tlbs if spte is dropped even though guest is + * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page + * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't + * used by guest then tlbs are not flushed, so guest is allowed to access the + * freed pages. + * And we increase kvm->tlbs_dirty to delay tlbs flush in this case. */ -static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, - bool clear_unsync) +static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { int i, offset, nr_present; - bool reset_host_protection; + bool host_writable; gpa_t first_pte_gpa; offset = nr_present = 0; @@ -764,31 +791,27 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, return -EINVAL; gfn = gpte_to_gfn(gpte); - if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL) - || gfn != sp->gfns[i] || !is_present_gpte(gpte) - || !(gpte & PT_ACCESSED_MASK)) { - u64 nonpresent; - if (is_present_gpte(gpte) || !clear_unsync) - nonpresent = shadow_trap_nonpresent_pte; - else - nonpresent = shadow_notrap_nonpresent_pte; - drop_spte(vcpu->kvm, &sp->spt[i], nonpresent); + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) { + vcpu->kvm->tlbs_dirty++; + continue; + } + + if (gfn != sp->gfns[i]) { + drop_spte(vcpu->kvm, &sp->spt[i], + shadow_trap_nonpresent_pte); + vcpu->kvm->tlbs_dirty++; continue; } nr_present++; pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); - if (!(sp->spt[i] & SPTE_HOST_WRITEABLE)) { - pte_access &= ~ACC_WRITE_MASK; - reset_host_protection = 0; - } else { - reset_host_protection = 1; - } + host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE; + set_spte(vcpu, &sp->spt[i], pte_access, 0, 0, is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn, spte_to_pfn(sp->spt[i]), true, false, - reset_host_protection); + host_writable); } return !nr_present; diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index b81a9b7..25bd1bc 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -31,6 +31,7 @@ #include <asm/tlbflush.h> #include <asm/desc.h> +#include <asm/kvm_para.h> #include <asm/virtext.h> #include "trace.h" @@ -50,6 +51,10 @@ MODULE_LICENSE("GPL"); #define SVM_FEATURE_LBRV (1 << 1) #define SVM_FEATURE_SVML (1 << 2) #define SVM_FEATURE_NRIP (1 << 3) +#define SVM_FEATURE_TSC_RATE (1 << 4) +#define SVM_FEATURE_VMCB_CLEAN (1 << 5) +#define SVM_FEATURE_FLUSH_ASID (1 << 6) +#define SVM_FEATURE_DECODE_ASSIST (1 << 7) #define SVM_FEATURE_PAUSE_FILTER (1 << 10) #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ @@ -97,10 +102,8 @@ struct nested_state { unsigned long vmexit_rax; /* cache for intercepts of the guest */ - u16 intercept_cr_read; - u16 intercept_cr_write; - u16 intercept_dr_read; - u16 intercept_dr_write; + u32 intercept_cr; + u32 intercept_dr; u32 intercept_exceptions; u64 intercept; @@ -123,7 +126,12 @@ struct vcpu_svm { u64 next_rip; u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; - u64 host_gs_base; + struct { + u16 fs; + u16 gs; + u16 ldt; + u64 gs_base; + } host; u32 *msrpm; @@ -133,6 +141,7 @@ struct vcpu_svm { unsigned int3_injected; unsigned long int3_rip; + u32 apf_reason; }; #define MSR_INVALID 0xffffffffU @@ -180,14 +189,151 @@ static int nested_svm_vmexit(struct vcpu_svm *svm); static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); +enum { + VMCB_INTERCEPTS, /* Intercept vectors, TSC offset, + pause filter count */ + VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */ + VMCB_ASID, /* ASID */ + VMCB_INTR, /* int_ctl, int_vector */ + VMCB_NPT, /* npt_en, nCR3, gPAT */ + VMCB_CR, /* CR0, CR3, CR4, EFER */ + VMCB_DR, /* DR6, DR7 */ + VMCB_DT, /* GDT, IDT */ + VMCB_SEG, /* CS, DS, SS, ES, CPL */ + VMCB_CR2, /* CR2 only */ + VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */ + VMCB_DIRTY_MAX, +}; + +/* TPR and CR2 are always written before VMRUN */ +#define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2)) + +static inline void mark_all_dirty(struct vmcb *vmcb) +{ + vmcb->control.clean = 0; +} + +static inline void mark_all_clean(struct vmcb *vmcb) +{ + vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1) + & ~VMCB_ALWAYS_DIRTY_MASK; +} + +static inline void mark_dirty(struct vmcb *vmcb, int bit) +{ + vmcb->control.clean &= ~(1 << bit); +} + static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) { return container_of(vcpu, struct vcpu_svm, vcpu); } -static inline bool is_nested(struct vcpu_svm *svm) +static void recalc_intercepts(struct vcpu_svm *svm) +{ + struct vmcb_control_area *c, *h; + struct nested_state *g; + + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + + if (!is_guest_mode(&svm->vcpu)) + return; + + c = &svm->vmcb->control; + h = &svm->nested.hsave->control; + g = &svm->nested; + + c->intercept_cr = h->intercept_cr | g->intercept_cr; + c->intercept_dr = h->intercept_dr | g->intercept_dr; + c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions; + c->intercept = h->intercept | g->intercept; +} + +static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm) +{ + if (is_guest_mode(&svm->vcpu)) + return svm->nested.hsave; + else + return svm->vmcb; +} + +static inline void set_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + return vmcb->control.intercept_cr & (1U << bit); +} + +static inline void set_dr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_dr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline void set_exception_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit) { - return svm->nested.vmcb; + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline void set_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept |= (1ULL << bit); + + recalc_intercepts(svm); +} + +static inline void clr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept &= ~(1ULL << bit); + + recalc_intercepts(svm); } static inline void enable_gif(struct vcpu_svm *svm) @@ -264,11 +410,6 @@ static u32 svm_msrpm_offset(u32 msr) #define MAX_INST_SIZE 15 -static inline u32 svm_has(u32 feat) -{ - return svm_features & feat; -} - static inline void clgi(void) { asm volatile (__ex(SVM_CLGI)); @@ -284,16 +425,6 @@ static inline void invlpga(unsigned long addr, u32 asid) asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid)); } -static inline void force_new_asid(struct kvm_vcpu *vcpu) -{ - to_svm(vcpu)->asid_generation--; -} - -static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) -{ - force_new_asid(vcpu); -} - static int get_npt_level(void) { #ifdef CONFIG_X86_64 @@ -310,6 +441,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME; + mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); } static int is_external_interrupt(u32 info) @@ -347,7 +479,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) svm->next_rip = svm->vmcb->control.next_rip; if (!svm->next_rip) { - if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) != + if (emulate_instruction(vcpu, EMULTYPE_SKIP) != EMULATE_DONE) printk(KERN_DEBUG "%s: NOP\n", __func__); return; @@ -374,7 +506,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, nested_svm_check_exception(svm, nr, has_error_code, error_code)) return; - if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) { + if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) { unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu); /* @@ -670,7 +802,7 @@ static __init int svm_hardware_setup(void) svm_features = cpuid_edx(SVM_CPUID_FUNC); - if (!svm_has(SVM_FEATURE_NPT)) + if (!boot_cpu_has(X86_FEATURE_NPT)) npt_enabled = false; if (npt_enabled && !npt) { @@ -725,13 +857,15 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) struct vcpu_svm *svm = to_svm(vcpu); u64 g_tsc_offset = 0; - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { g_tsc_offset = svm->vmcb->control.tsc_offset - svm->nested.hsave->control.tsc_offset; svm->nested.hsave->control.tsc_offset = offset; } svm->vmcb->control.tsc_offset = offset + g_tsc_offset; + + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment) @@ -739,8 +873,9 @@ static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.tsc_offset += adjustment; - if (is_nested(svm)) + if (is_guest_mode(vcpu)) svm->nested.hsave->control.tsc_offset += adjustment; + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } static void init_vmcb(struct vcpu_svm *svm) @@ -749,62 +884,62 @@ static void init_vmcb(struct vcpu_svm *svm) struct vmcb_save_area *save = &svm->vmcb->save; svm->vcpu.fpu_active = 1; + svm->vcpu.arch.hflags = 0; - control->intercept_cr_read = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; - - control->intercept_cr_write = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK | - INTERCEPT_CR8_MASK; - - control->intercept_dr_read = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK | - INTERCEPT_DR4_MASK | - INTERCEPT_DR5_MASK | - INTERCEPT_DR6_MASK | - INTERCEPT_DR7_MASK; - - control->intercept_dr_write = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK | - INTERCEPT_DR4_MASK | - INTERCEPT_DR5_MASK | - INTERCEPT_DR6_MASK | - INTERCEPT_DR7_MASK; - - control->intercept_exceptions = (1 << PF_VECTOR) | - (1 << UD_VECTOR) | - (1 << MC_VECTOR); - - - control->intercept = (1ULL << INTERCEPT_INTR) | - (1ULL << INTERCEPT_NMI) | - (1ULL << INTERCEPT_SMI) | - (1ULL << INTERCEPT_SELECTIVE_CR0) | - (1ULL << INTERCEPT_CPUID) | - (1ULL << INTERCEPT_INVD) | - (1ULL << INTERCEPT_HLT) | - (1ULL << INTERCEPT_INVLPG) | - (1ULL << INTERCEPT_INVLPGA) | - (1ULL << INTERCEPT_IOIO_PROT) | - (1ULL << INTERCEPT_MSR_PROT) | - (1ULL << INTERCEPT_TASK_SWITCH) | - (1ULL << INTERCEPT_SHUTDOWN) | - (1ULL << INTERCEPT_VMRUN) | - (1ULL << INTERCEPT_VMMCALL) | - (1ULL << INTERCEPT_VMLOAD) | - (1ULL << INTERCEPT_VMSAVE) | - (1ULL << INTERCEPT_STGI) | - (1ULL << INTERCEPT_CLGI) | - (1ULL << INTERCEPT_SKINIT) | - (1ULL << INTERCEPT_WBINVD) | - (1ULL << INTERCEPT_MONITOR) | - (1ULL << INTERCEPT_MWAIT); + set_cr_intercept(svm, INTERCEPT_CR0_READ); + set_cr_intercept(svm, INTERCEPT_CR3_READ); + set_cr_intercept(svm, INTERCEPT_CR4_READ); + set_cr_intercept(svm, INTERCEPT_CR0_WRITE); + set_cr_intercept(svm, INTERCEPT_CR3_WRITE); + set_cr_intercept(svm, INTERCEPT_CR4_WRITE); + set_cr_intercept(svm, INTERCEPT_CR8_WRITE); + + set_dr_intercept(svm, INTERCEPT_DR0_READ); + set_dr_intercept(svm, INTERCEPT_DR1_READ); + set_dr_intercept(svm, INTERCEPT_DR2_READ); + set_dr_intercept(svm, INTERCEPT_DR3_READ); + set_dr_intercept(svm, INTERCEPT_DR4_READ); + set_dr_intercept(svm, INTERCEPT_DR5_READ); + set_dr_intercept(svm, INTERCEPT_DR6_READ); + set_dr_intercept(svm, INTERCEPT_DR7_READ); + + set_dr_intercept(svm, INTERCEPT_DR0_WRITE); + set_dr_intercept(svm, INTERCEPT_DR1_WRITE); + set_dr_intercept(svm, INTERCEPT_DR2_WRITE); + set_dr_intercept(svm, INTERCEPT_DR3_WRITE); + set_dr_intercept(svm, INTERCEPT_DR4_WRITE); + set_dr_intercept(svm, INTERCEPT_DR5_WRITE); + set_dr_intercept(svm, INTERCEPT_DR6_WRITE); + set_dr_intercept(svm, INTERCEPT_DR7_WRITE); + + set_exception_intercept(svm, PF_VECTOR); + set_exception_intercept(svm, UD_VECTOR); + set_exception_intercept(svm, MC_VECTOR); + + set_intercept(svm, INTERCEPT_INTR); + set_intercept(svm, INTERCEPT_NMI); + set_intercept(svm, INTERCEPT_SMI); + set_intercept(svm, INTERCEPT_SELECTIVE_CR0); + set_intercept(svm, INTERCEPT_CPUID); + set_intercept(svm, INTERCEPT_INVD); + set_intercept(svm, INTERCEPT_HLT); + set_intercept(svm, INTERCEPT_INVLPG); + set_intercept(svm, INTERCEPT_INVLPGA); + set_intercept(svm, INTERCEPT_IOIO_PROT); + set_intercept(svm, INTERCEPT_MSR_PROT); + set_intercept(svm, INTERCEPT_TASK_SWITCH); + set_intercept(svm, INTERCEPT_SHUTDOWN); + set_intercept(svm, INTERCEPT_VMRUN); + set_intercept(svm, INTERCEPT_VMMCALL); + set_intercept(svm, INTERCEPT_VMLOAD); + set_intercept(svm, INTERCEPT_VMSAVE); + set_intercept(svm, INTERCEPT_STGI); + set_intercept(svm, INTERCEPT_CLGI); + set_intercept(svm, INTERCEPT_SKINIT); + set_intercept(svm, INTERCEPT_WBINVD); + set_intercept(svm, INTERCEPT_MONITOR); + set_intercept(svm, INTERCEPT_MWAIT); + set_intercept(svm, INTERCEPT_XSETBV); control->iopm_base_pa = iopm_base; control->msrpm_base_pa = __pa(svm->msrpm); @@ -855,25 +990,27 @@ static void init_vmcb(struct vcpu_svm *svm) if (npt_enabled) { /* Setup VMCB for Nested Paging */ control->nested_ctl = 1; - control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) | - (1ULL << INTERCEPT_INVLPG)); - control->intercept_exceptions &= ~(1 << PF_VECTOR); - control->intercept_cr_read &= ~INTERCEPT_CR3_MASK; - control->intercept_cr_write &= ~INTERCEPT_CR3_MASK; + clr_intercept(svm, INTERCEPT_TASK_SWITCH); + clr_intercept(svm, INTERCEPT_INVLPG); + clr_exception_intercept(svm, PF_VECTOR); + clr_cr_intercept(svm, INTERCEPT_CR3_READ); + clr_cr_intercept(svm, INTERCEPT_CR3_WRITE); save->g_pat = 0x0007040600070406ULL; save->cr3 = 0; save->cr4 = 0; } - force_new_asid(&svm->vcpu); + svm->asid_generation = 0; svm->nested.vmcb = 0; svm->vcpu.arch.hflags = 0; - if (svm_has(SVM_FEATURE_PAUSE_FILTER)) { + if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { control->pause_filter_count = 3000; - control->intercept |= (1ULL << INTERCEPT_PAUSE); + set_intercept(svm, INTERCEPT_PAUSE); } + mark_all_dirty(svm->vmcb); + enable_gif(svm); } @@ -990,8 +1127,16 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if (unlikely(cpu != vcpu->cpu)) { svm->asid_generation = 0; + mark_all_dirty(svm->vmcb); } +#ifdef CONFIG_X86_64 + rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base); +#endif + savesegment(fs, svm->host.fs); + savesegment(gs, svm->host.gs); + svm->host.ldt = kvm_read_ldt(); + for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } @@ -1002,6 +1147,14 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) int i; ++vcpu->stat.host_state_reload; + kvm_load_ldt(svm->host.ldt); +#ifdef CONFIG_X86_64 + loadsegment(fs, svm->host.fs); + load_gs_index(svm->host.gs); + wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs); +#else + loadsegment(gs, svm->host.gs); +#endif for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } @@ -1021,7 +1174,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) switch (reg) { case VCPU_EXREG_PDPTR: BUG_ON(!npt_enabled); - load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); break; default: BUG(); @@ -1030,12 +1183,12 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) static void svm_set_vintr(struct vcpu_svm *svm) { - svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR; + set_intercept(svm, INTERCEPT_VINTR); } static void svm_clear_vintr(struct vcpu_svm *svm) { - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); + clr_intercept(svm, INTERCEPT_VINTR); } static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) @@ -1150,6 +1303,7 @@ static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) svm->vmcb->save.idtr.limit = dt->size; svm->vmcb->save.idtr.base = dt->address ; + mark_dirty(svm->vmcb, VMCB_DT); } static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) @@ -1166,19 +1320,23 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) svm->vmcb->save.gdtr.limit = dt->size; svm->vmcb->save.gdtr.base = dt->address ; + mark_dirty(svm->vmcb, VMCB_DT); } static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } +static void svm_decache_cr3(struct kvm_vcpu *vcpu) +{ +} + static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { } static void update_cr0_intercept(struct vcpu_svm *svm) { - struct vmcb *vmcb = svm->vmcb; ulong gcr0 = svm->vcpu.arch.cr0; u64 *hcr0 = &svm->vmcb->save.cr0; @@ -1188,27 +1346,14 @@ static void update_cr0_intercept(struct vcpu_svm *svm) *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK) | (gcr0 & SVM_CR0_SELECTIVE_MASK); + mark_dirty(svm->vmcb, VMCB_CR); if (gcr0 == *hcr0 && svm->vcpu.fpu_active) { - vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK; - vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK; - if (is_nested(svm)) { - struct vmcb *hsave = svm->nested.hsave; - - hsave->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK; - hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK; - vmcb->control.intercept_cr_read |= svm->nested.intercept_cr_read; - vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write; - } + clr_cr_intercept(svm, INTERCEPT_CR0_READ); + clr_cr_intercept(svm, INTERCEPT_CR0_WRITE); } else { - svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK; - svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK; - if (is_nested(svm)) { - struct vmcb *hsave = svm->nested.hsave; - - hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK; - hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK; - } + set_cr_intercept(svm, INTERCEPT_CR0_READ); + set_cr_intercept(svm, INTERCEPT_CR0_WRITE); } } @@ -1216,7 +1361,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { /* * We are here because we run in nested mode, the host kvm * intercepts cr0 writes but the l1 hypervisor does not. @@ -1268,6 +1413,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) */ cr0 &= ~(X86_CR0_CD | X86_CR0_NW); svm->vmcb->save.cr0 = cr0; + mark_dirty(svm->vmcb, VMCB_CR); update_cr0_intercept(svm); } @@ -1277,13 +1423,14 @@ static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4; if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) - force_new_asid(vcpu); + svm_flush_tlb(vcpu); vcpu->arch.cr4 = cr4; if (!npt_enabled) cr4 |= X86_CR4_PAE; cr4 |= host_cr4_mce; to_svm(vcpu)->vmcb->save.cr4 = cr4; + mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); } static void svm_set_segment(struct kvm_vcpu *vcpu, @@ -1312,26 +1459,25 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, = (svm->vmcb->save.cs.attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; + mark_dirty(svm->vmcb, VMCB_SEG); } static void update_db_intercept(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->control.intercept_exceptions &= - ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); + clr_exception_intercept(svm, DB_VECTOR); + clr_exception_intercept(svm, BP_VECTOR); if (svm->nmi_singlestep) - svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR); + set_exception_intercept(svm, DB_VECTOR); if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { if (vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) - svm->vmcb->control.intercept_exceptions |= - 1 << DB_VECTOR; + set_exception_intercept(svm, DB_VECTOR); if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) - svm->vmcb->control.intercept_exceptions |= - 1 << BP_VECTOR; + set_exception_intercept(svm, BP_VECTOR); } else vcpu->guest_debug = 0; } @@ -1345,21 +1491,9 @@ static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) else svm->vmcb->save.dr7 = vcpu->arch.dr7; - update_db_intercept(vcpu); -} - -static void load_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif -} + mark_dirty(svm->vmcb, VMCB_DR); -static void save_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif + update_db_intercept(vcpu); } static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) @@ -1372,6 +1506,8 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) svm->asid_generation = sd->asid_generation; svm->vmcb->control.asid = sd->next_asid++; + + mark_dirty(svm->vmcb, VMCB_ASID); } static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) @@ -1379,20 +1515,40 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.dr7 = value; + mark_dirty(svm->vmcb, VMCB_DR); } static int pf_interception(struct vcpu_svm *svm) { - u64 fault_address; + u64 fault_address = svm->vmcb->control.exit_info_2; u32 error_code; + int r = 1; - fault_address = svm->vmcb->control.exit_info_2; - error_code = svm->vmcb->control.exit_info_1; - - trace_kvm_page_fault(fault_address, error_code); - if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu)) - kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address); - return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); + switch (svm->apf_reason) { + default: + error_code = svm->vmcb->control.exit_info_1; + + trace_kvm_page_fault(fault_address, error_code); + if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu)) + kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address); + r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code, + svm->vmcb->control.insn_bytes, + svm->vmcb->control.insn_len); + break; + case KVM_PV_REASON_PAGE_NOT_PRESENT: + svm->apf_reason = 0; + local_irq_disable(); + kvm_async_pf_task_wait(fault_address); + local_irq_enable(); + break; + case KVM_PV_REASON_PAGE_READY: + svm->apf_reason = 0; + local_irq_disable(); + kvm_async_pf_task_wake(fault_address); + local_irq_enable(); + break; + } + return r; } static int db_interception(struct vcpu_svm *svm) @@ -1440,7 +1596,7 @@ static int ud_interception(struct vcpu_svm *svm) { int er; - er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD); + er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; @@ -1449,21 +1605,8 @@ static int ud_interception(struct vcpu_svm *svm) static void svm_fpu_activate(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - u32 excp; - - if (is_nested(svm)) { - u32 h_excp, n_excp; - - h_excp = svm->nested.hsave->control.intercept_exceptions; - n_excp = svm->nested.intercept_exceptions; - h_excp &= ~(1 << NM_VECTOR); - excp = h_excp | n_excp; - } else { - excp = svm->vmcb->control.intercept_exceptions; - excp &= ~(1 << NM_VECTOR); - } - svm->vmcb->control.intercept_exceptions = excp; + clr_exception_intercept(svm, NM_VECTOR); svm->vcpu.fpu_active = 1; update_cr0_intercept(svm); @@ -1570,7 +1713,7 @@ static int io_interception(struct vcpu_svm *svm) string = (io_info & SVM_IOIO_STR_MASK) != 0; in = (io_info & SVM_IOIO_TYPE_MASK) != 0; if (string || in) - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = io_info >> 16; size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; @@ -1624,17 +1767,19 @@ static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu, struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.nested_cr3 = root; - force_new_asid(vcpu); + mark_dirty(svm->vmcb, VMCB_NPT); + svm_flush_tlb(vcpu); } -static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu) +static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.exit_code = SVM_EXIT_NPF; svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = vcpu->arch.fault.error_code; - svm->vmcb->control.exit_info_2 = vcpu->arch.fault.address; + svm->vmcb->control.exit_info_1 = fault->error_code; + svm->vmcb->control.exit_info_2 = fault->address; nested_svm_vmexit(svm); } @@ -1680,7 +1825,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, { int vmexit; - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return 0; svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; @@ -1698,7 +1843,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, /* This function returns true if it is save to enable the irq window */ static inline bool nested_svm_intr(struct vcpu_svm *svm) { - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return true; if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) @@ -1737,7 +1882,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm) /* This function returns true if it is save to enable the nmi window */ static inline bool nested_svm_nmi(struct vcpu_svm *svm) { - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return true; if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI))) @@ -1836,8 +1981,8 @@ static int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_HOST; break; case SVM_EXIT_EXCP_BASE + PF_VECTOR: - /* When we're shadowing, trap PFs */ - if (!npt_enabled) + /* When we're shadowing, trap PFs, but not async PF */ + if (!npt_enabled && svm->apf_reason == 0) return NESTED_EXIT_HOST; break; case SVM_EXIT_EXCP_BASE + NM_VECTOR: @@ -1865,27 +2010,15 @@ static int nested_svm_intercept(struct vcpu_svm *svm) case SVM_EXIT_IOIO: vmexit = nested_svm_intercept_ioio(svm); break; - case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: { - u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0); - if (svm->nested.intercept_cr_read & cr_bits) + case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0); + if (svm->nested.intercept_cr & bit) vmexit = NESTED_EXIT_DONE; break; } - case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: { - u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0); - if (svm->nested.intercept_cr_write & cr_bits) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: { - u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0); - if (svm->nested.intercept_dr_read & dr_bits) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: { - u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0); - if (svm->nested.intercept_dr_write & dr_bits) + case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); + if (svm->nested.intercept_dr & bit) vmexit = NESTED_EXIT_DONE; break; } @@ -1893,6 +2026,10 @@ static int nested_svm_intercept(struct vcpu_svm *svm) u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); if (svm->nested.intercept_exceptions & excp_bits) vmexit = NESTED_EXIT_DONE; + /* async page fault always cause vmexit */ + else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) && + svm->apf_reason != 0) + vmexit = NESTED_EXIT_DONE; break; } case SVM_EXIT_ERR: { @@ -1926,10 +2063,8 @@ static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *fr struct vmcb_control_area *dst = &dst_vmcb->control; struct vmcb_control_area *from = &from_vmcb->control; - dst->intercept_cr_read = from->intercept_cr_read; - dst->intercept_cr_write = from->intercept_cr_write; - dst->intercept_dr_read = from->intercept_dr_read; - dst->intercept_dr_write = from->intercept_dr_write; + dst->intercept_cr = from->intercept_cr; + dst->intercept_dr = from->intercept_dr; dst->intercept_exceptions = from->intercept_exceptions; dst->intercept = from->intercept; dst->iopm_base_pa = from->iopm_base_pa; @@ -1970,7 +2105,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_vmcb) return 1; - /* Exit nested SVM mode */ + /* Exit Guest-Mode */ + leave_guest_mode(&svm->vcpu); svm->nested.vmcb = 0; /* Give the current vmcb to the guest */ @@ -1984,7 +2120,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.idtr = vmcb->save.idtr; nested_vmcb->save.efer = svm->vcpu.arch.efer; nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu); - nested_vmcb->save.cr3 = svm->vcpu.arch.cr3; + nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu); nested_vmcb->save.cr2 = vmcb->save.cr2; nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; nested_vmcb->save.rflags = vmcb->save.rflags; @@ -2061,6 +2197,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) svm->vmcb->save.cpl = 0; svm->vmcb->control.exit_int_info = 0; + mark_all_dirty(svm->vmcb); + nested_svm_unmap(page); nested_svm_uninit_mmu_context(&svm->vcpu); @@ -2148,8 +2286,8 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) nested_vmcb->control.event_inj, nested_vmcb->control.nested_ctl); - trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read, - nested_vmcb->control.intercept_cr_write, + trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff, + nested_vmcb->control.intercept_cr >> 16, nested_vmcb->control.intercept_exceptions, nested_vmcb->control.intercept); @@ -2177,7 +2315,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) if (npt_enabled) hsave->save.cr3 = vmcb->save.cr3; else - hsave->save.cr3 = svm->vcpu.arch.cr3; + hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); copy_vmcb_control_area(hsave, vmcb); @@ -2229,14 +2367,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL; /* cache intercepts */ - svm->nested.intercept_cr_read = nested_vmcb->control.intercept_cr_read; - svm->nested.intercept_cr_write = nested_vmcb->control.intercept_cr_write; - svm->nested.intercept_dr_read = nested_vmcb->control.intercept_dr_read; - svm->nested.intercept_dr_write = nested_vmcb->control.intercept_dr_write; + svm->nested.intercept_cr = nested_vmcb->control.intercept_cr; + svm->nested.intercept_dr = nested_vmcb->control.intercept_dr; svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; svm->nested.intercept = nested_vmcb->control.intercept; - force_new_asid(&svm->vcpu); + svm_flush_tlb(&svm->vcpu); svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; @@ -2245,29 +2381,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { /* We only want the cr8 intercept bits of the guest */ - svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK; - svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; + clr_cr_intercept(svm, INTERCEPT_CR8_READ); + clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); } /* We don't want to see VMMCALLs from a nested guest */ - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL); - - /* - * We don't want a nested guest to be more powerful than the guest, so - * all intercepts are ORed - */ - svm->vmcb->control.intercept_cr_read |= - nested_vmcb->control.intercept_cr_read; - svm->vmcb->control.intercept_cr_write |= - nested_vmcb->control.intercept_cr_write; - svm->vmcb->control.intercept_dr_read |= - nested_vmcb->control.intercept_dr_read; - svm->vmcb->control.intercept_dr_write |= - nested_vmcb->control.intercept_dr_write; - svm->vmcb->control.intercept_exceptions |= - nested_vmcb->control.intercept_exceptions; - - svm->vmcb->control.intercept |= nested_vmcb->control.intercept; + clr_intercept(svm, INTERCEPT_VMMCALL); svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl; svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; @@ -2278,11 +2397,21 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) nested_svm_unmap(page); - /* nested_vmcb is our indicator if nested SVM is activated */ + /* Enter Guest-Mode */ + enter_guest_mode(&svm->vcpu); + + /* + * Merge guest and host intercepts - must be called with vcpu in + * guest-mode to take affect here + */ + recalc_intercepts(svm); + svm->nested.vmcb = vmcb_gpa; enable_gif(svm); + mark_all_dirty(svm->vmcb); + return true; } @@ -2400,6 +2529,8 @@ static int clgi_interception(struct vcpu_svm *svm) svm_clear_vintr(svm); svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + mark_dirty(svm->vmcb, VMCB_INTR); + return 1; } @@ -2426,6 +2557,19 @@ static int skinit_interception(struct vcpu_svm *svm) return 1; } +static int xsetbv_interception(struct vcpu_svm *svm) +{ + u64 new_bv = kvm_read_edx_eax(&svm->vcpu); + u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX); + + if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) { + svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; + skip_emulated_instruction(&svm->vcpu); + } + + return 1; +} + static int invalid_op_interception(struct vcpu_svm *svm) { kvm_queue_exception(&svm->vcpu, UD_VECTOR); @@ -2507,19 +2651,92 @@ static int cpuid_interception(struct vcpu_svm *svm) static int iret_interception(struct vcpu_svm *svm) { ++svm->vcpu.stat.nmi_window_exits; - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET); + clr_intercept(svm, INTERCEPT_IRET); svm->vcpu.arch.hflags |= HF_IRET_MASK; return 1; } static int invlpg_interception(struct vcpu_svm *svm) { - return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE; + if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; + + kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1); + skip_emulated_instruction(&svm->vcpu); + return 1; } static int emulate_on_interception(struct vcpu_svm *svm) { - return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; +} + +#define CR_VALID (1ULL << 63) + +static int cr_interception(struct vcpu_svm *svm) +{ + int reg, cr; + unsigned long val; + int err; + + if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_on_interception(svm); + + if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) + return emulate_on_interception(svm); + + reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; + cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; + + err = 0; + if (cr >= 16) { /* mov to cr */ + cr -= 16; + val = kvm_register_read(&svm->vcpu, reg); + switch (cr) { + case 0: + err = kvm_set_cr0(&svm->vcpu, val); + break; + case 3: + err = kvm_set_cr3(&svm->vcpu, val); + break; + case 4: + err = kvm_set_cr4(&svm->vcpu, val); + break; + case 8: + err = kvm_set_cr8(&svm->vcpu, val); + break; + default: + WARN(1, "unhandled write to CR%d", cr); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + } else { /* mov from cr */ + switch (cr) { + case 0: + val = kvm_read_cr0(&svm->vcpu); + break; + case 2: + val = svm->vcpu.arch.cr2; + break; + case 3: + val = kvm_read_cr3(&svm->vcpu); + break; + case 4: + val = kvm_read_cr4(&svm->vcpu); + break; + case 8: + val = kvm_get_cr8(&svm->vcpu); + break; + default: + WARN(1, "unhandled read from CR%d", cr); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + kvm_register_write(&svm->vcpu, reg, val); + } + kvm_complete_insn_gp(&svm->vcpu, err); + + return 1; } static int cr0_write_interception(struct vcpu_svm *svm) @@ -2527,7 +2744,7 @@ static int cr0_write_interception(struct vcpu_svm *svm) struct kvm_vcpu *vcpu = &svm->vcpu; int r; - r = emulate_instruction(&svm->vcpu, 0, 0, 0); + r = cr_interception(svm); if (svm->nested.vmexit_rip) { kvm_register_write(vcpu, VCPU_REGS_RIP, svm->nested.vmexit_rip); @@ -2536,22 +2753,47 @@ static int cr0_write_interception(struct vcpu_svm *svm) svm->nested.vmexit_rip = 0; } - return r == EMULATE_DONE; + return r; +} + +static int dr_interception(struct vcpu_svm *svm) +{ + int reg, dr; + unsigned long val; + int err; + + if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_on_interception(svm); + + reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; + dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; + + if (dr >= 16) { /* mov to DRn */ + val = kvm_register_read(&svm->vcpu, reg); + kvm_set_dr(&svm->vcpu, dr - 16, val); + } else { + err = kvm_get_dr(&svm->vcpu, dr, &val); + if (!err) + kvm_register_write(&svm->vcpu, reg, val); + } + + return 1; } static int cr8_write_interception(struct vcpu_svm *svm) { struct kvm_run *kvm_run = svm->vcpu.run; + int r; u8 cr8_prev = kvm_get_cr8(&svm->vcpu); /* instruction emulation calls kvm_set_cr8() */ - emulate_instruction(&svm->vcpu, 0, 0, 0); + r = cr_interception(svm); if (irqchip_in_kernel(svm->vcpu.kvm)) { - svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; - return 1; + clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); + return r; } if (cr8_prev <= kvm_get_cr8(&svm->vcpu)) - return 1; + return r; kvm_run->exit_reason = KVM_EXIT_SET_TPR; return 0; } @@ -2562,14 +2804,9 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) switch (ecx) { case MSR_IA32_TSC: { - u64 tsc_offset; + struct vmcb *vmcb = get_host_vmcb(svm); - if (is_nested(svm)) - tsc_offset = svm->nested.hsave->control.tsc_offset; - else - tsc_offset = svm->vmcb->control.tsc_offset; - - *data = tsc_offset + native_read_tsc(); + *data = vmcb->control.tsc_offset + native_read_tsc(); break; } case MSR_STAR: @@ -2714,7 +2951,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) svm->vmcb->save.sysenter_esp = data; break; case MSR_IA32_DEBUGCTLMSR: - if (!svm_has(SVM_FEATURE_LBRV)) { + if (!boot_cpu_has(X86_FEATURE_LBRV)) { pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", __func__, data); break; @@ -2723,6 +2960,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) return 1; svm->vmcb->save.dbgctl = data; + mark_dirty(svm->vmcb, VMCB_LBR); if (data & (1ULL<<0)) svm_enable_lbrv(svm); else @@ -2775,6 +3013,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); svm_clear_vintr(svm); svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + mark_dirty(svm->vmcb, VMCB_INTR); /* * If the user space waits to inject interrupts, exit as soon as * possible @@ -2797,31 +3036,31 @@ static int pause_interception(struct vcpu_svm *svm) } static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { - [SVM_EXIT_READ_CR0] = emulate_on_interception, - [SVM_EXIT_READ_CR3] = emulate_on_interception, - [SVM_EXIT_READ_CR4] = emulate_on_interception, - [SVM_EXIT_READ_CR8] = emulate_on_interception, + [SVM_EXIT_READ_CR0] = cr_interception, + [SVM_EXIT_READ_CR3] = cr_interception, + [SVM_EXIT_READ_CR4] = cr_interception, + [SVM_EXIT_READ_CR8] = cr_interception, [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, [SVM_EXIT_WRITE_CR0] = cr0_write_interception, - [SVM_EXIT_WRITE_CR3] = emulate_on_interception, - [SVM_EXIT_WRITE_CR4] = emulate_on_interception, + [SVM_EXIT_WRITE_CR3] = cr_interception, + [SVM_EXIT_WRITE_CR4] = cr_interception, [SVM_EXIT_WRITE_CR8] = cr8_write_interception, - [SVM_EXIT_READ_DR0] = emulate_on_interception, - [SVM_EXIT_READ_DR1] = emulate_on_interception, - [SVM_EXIT_READ_DR2] = emulate_on_interception, - [SVM_EXIT_READ_DR3] = emulate_on_interception, - [SVM_EXIT_READ_DR4] = emulate_on_interception, - [SVM_EXIT_READ_DR5] = emulate_on_interception, - [SVM_EXIT_READ_DR6] = emulate_on_interception, - [SVM_EXIT_READ_DR7] = emulate_on_interception, - [SVM_EXIT_WRITE_DR0] = emulate_on_interception, - [SVM_EXIT_WRITE_DR1] = emulate_on_interception, - [SVM_EXIT_WRITE_DR2] = emulate_on_interception, - [SVM_EXIT_WRITE_DR3] = emulate_on_interception, - [SVM_EXIT_WRITE_DR4] = emulate_on_interception, - [SVM_EXIT_WRITE_DR5] = emulate_on_interception, - [SVM_EXIT_WRITE_DR6] = emulate_on_interception, - [SVM_EXIT_WRITE_DR7] = emulate_on_interception, + [SVM_EXIT_READ_DR0] = dr_interception, + [SVM_EXIT_READ_DR1] = dr_interception, + [SVM_EXIT_READ_DR2] = dr_interception, + [SVM_EXIT_READ_DR3] = dr_interception, + [SVM_EXIT_READ_DR4] = dr_interception, + [SVM_EXIT_READ_DR5] = dr_interception, + [SVM_EXIT_READ_DR6] = dr_interception, + [SVM_EXIT_READ_DR7] = dr_interception, + [SVM_EXIT_WRITE_DR0] = dr_interception, + [SVM_EXIT_WRITE_DR1] = dr_interception, + [SVM_EXIT_WRITE_DR2] = dr_interception, + [SVM_EXIT_WRITE_DR3] = dr_interception, + [SVM_EXIT_WRITE_DR4] = dr_interception, + [SVM_EXIT_WRITE_DR5] = dr_interception, + [SVM_EXIT_WRITE_DR6] = dr_interception, + [SVM_EXIT_WRITE_DR7] = dr_interception, [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, @@ -2854,6 +3093,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { [SVM_EXIT_WBINVD] = emulate_on_interception, [SVM_EXIT_MONITOR] = invalid_op_interception, [SVM_EXIT_MWAIT] = invalid_op_interception, + [SVM_EXIT_XSETBV] = xsetbv_interception, [SVM_EXIT_NPF] = pf_interception, }; @@ -2864,10 +3104,10 @@ void dump_vmcb(struct kvm_vcpu *vcpu) struct vmcb_save_area *save = &svm->vmcb->save; pr_err("VMCB Control Area:\n"); - pr_err("cr_read: %04x\n", control->intercept_cr_read); - pr_err("cr_write: %04x\n", control->intercept_cr_write); - pr_err("dr_read: %04x\n", control->intercept_dr_read); - pr_err("dr_write: %04x\n", control->intercept_dr_write); + pr_err("cr_read: %04x\n", control->intercept_cr & 0xffff); + pr_err("cr_write: %04x\n", control->intercept_cr >> 16); + pr_err("dr_read: %04x\n", control->intercept_dr & 0xffff); + pr_err("dr_write: %04x\n", control->intercept_dr >> 16); pr_err("exceptions: %08x\n", control->intercept_exceptions); pr_err("intercepts: %016llx\n", control->intercept); pr_err("pause filter count: %d\n", control->pause_filter_count); @@ -2950,15 +3190,23 @@ void dump_vmcb(struct kvm_vcpu *vcpu) } +static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) +{ + struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; + + *info1 = control->exit_info_1; + *info2 = control->exit_info_2; +} + static int handle_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; u32 exit_code = svm->vmcb->control.exit_code; - trace_kvm_exit(exit_code, vcpu); + trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); - if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK)) + if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE)) vcpu->arch.cr0 = svm->vmcb->save.cr0; if (npt_enabled) vcpu->arch.cr3 = svm->vmcb->save.cr3; @@ -2970,7 +3218,7 @@ static int handle_exit(struct kvm_vcpu *vcpu) return 1; } - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { int vmexit; trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code, @@ -3033,7 +3281,6 @@ static void pre_svm_run(struct vcpu_svm *svm) struct svm_cpu_data *sd = per_cpu(svm_data, cpu); - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; /* FIXME: handle wraparound of asid_generation */ if (svm->asid_generation != sd->asid_generation) new_asid(svm, sd); @@ -3045,7 +3292,7 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu) svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; vcpu->arch.hflags |= HF_NMI_MASK; - svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET); + set_intercept(svm, INTERCEPT_IRET); ++vcpu->stat.nmi_injections; } @@ -3058,6 +3305,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq) control->int_ctl &= ~V_INTR_PRIO_MASK; control->int_ctl |= V_IRQ_MASK | ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); + mark_dirty(svm->vmcb, VMCB_INTR); } static void svm_set_irq(struct kvm_vcpu *vcpu) @@ -3077,14 +3325,14 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; if (irr == -1) return; if (tpr >= irr) - svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK; + set_cr_intercept(svm, INTERCEPT_CR8_WRITE); } static int svm_nmi_allowed(struct kvm_vcpu *vcpu) @@ -3112,10 +3360,10 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) if (masked) { svm->vcpu.arch.hflags |= HF_NMI_MASK; - svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET); + set_intercept(svm, INTERCEPT_IRET); } else { svm->vcpu.arch.hflags &= ~HF_NMI_MASK; - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET); + clr_intercept(svm, INTERCEPT_IRET); } } @@ -3131,7 +3379,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) ret = !!(vmcb->save.rflags & X86_EFLAGS_IF); - if (is_nested(svm)) + if (is_guest_mode(vcpu)) return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK); return ret; @@ -3177,7 +3425,12 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) static void svm_flush_tlb(struct kvm_vcpu *vcpu) { - force_new_asid(vcpu); + struct vcpu_svm *svm = to_svm(vcpu); + + if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; + else + svm->asid_generation--; } static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) @@ -3188,10 +3441,10 @@ static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; - if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) { + if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) { int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; kvm_set_cr8(vcpu, cr8); } @@ -3202,7 +3455,7 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); u64 cr8; - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; cr8 = kvm_get_cr8(vcpu); @@ -3289,9 +3542,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) static void svm_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - u16 fs_selector; - u16 gs_selector; - u16 ldt_selector; svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; @@ -3308,10 +3558,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) sync_lapic_to_cr8(vcpu); - save_host_msrs(vcpu); - savesegment(fs, fs_selector); - savesegment(gs, gs_selector); - ldt_selector = kvm_read_ldt(); svm->vmcb->save.cr2 = vcpu->arch.cr2; clgi(); @@ -3389,19 +3635,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) #endif ); - vcpu->arch.cr2 = svm->vmcb->save.cr2; - vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; - vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; - vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; - - load_host_msrs(vcpu); - kvm_load_ldt(ldt_selector); - loadsegment(fs, fs_selector); #ifdef CONFIG_X86_64 - load_gs_index(gs_selector); - wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs); + wrmsrl(MSR_GS_BASE, svm->host.gs_base); #else - loadsegment(gs, gs_selector); + loadsegment(fs, svm->host.fs); #endif reload_tss(vcpu); @@ -3410,10 +3647,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) stgi(); + vcpu->arch.cr2 = svm->vmcb->save.cr2; + vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; + vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; + vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; + sync_cr8_to_lapic(vcpu); svm->next_rip = 0; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + + /* if exit due to PF check for async PF */ + if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) + svm->apf_reason = kvm_read_and_reset_pf_reason(); + if (npt_enabled) { vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR); @@ -3426,6 +3674,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + MC_VECTOR)) svm_handle_mce(svm); + + mark_all_clean(svm->vmcb); } #undef R @@ -3435,7 +3685,8 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.cr3 = root; - force_new_asid(vcpu); + mark_dirty(svm->vmcb, VMCB_CR); + svm_flush_tlb(vcpu); } static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root) @@ -3443,11 +3694,13 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.nested_cr3 = root; + mark_dirty(svm->vmcb, VMCB_NPT); /* Also sync guest cr3 here in case we live migrate */ - svm->vmcb->save.cr3 = vcpu->arch.cr3; + svm->vmcb->save.cr3 = kvm_read_cr3(vcpu); + mark_dirty(svm->vmcb, VMCB_CR); - force_new_asid(vcpu); + svm_flush_tlb(vcpu); } static int is_disabled(void) @@ -3494,10 +3747,6 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) { switch (func) { - case 0x00000001: - /* Mask out xsave bit as long as it is not supported by SVM */ - entry->ecx &= ~(bit(X86_FEATURE_XSAVE)); - break; case 0x80000001: if (nested) entry->ecx |= (1 << 2); /* Set SVM bit */ @@ -3511,7 +3760,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) additional features */ /* Support next_rip if host supports it */ - if (svm_has(SVM_FEATURE_NRIP)) + if (boot_cpu_has(X86_FEATURE_NRIPS)) entry->edx |= SVM_FEATURE_NRIP; /* Support NPT for the guest if enabled */ @@ -3571,6 +3820,7 @@ static const struct trace_print_flags svm_exit_reasons_str[] = { { SVM_EXIT_WBINVD, "wbinvd" }, { SVM_EXIT_MONITOR, "monitor" }, { SVM_EXIT_MWAIT, "mwait" }, + { SVM_EXIT_XSETBV, "xsetbv" }, { SVM_EXIT_NPF, "npf" }, { -1, NULL } }; @@ -3594,9 +3844,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR; - if (is_nested(svm)) - svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR; + set_exception_intercept(svm, NM_VECTOR); update_cr0_intercept(svm); } @@ -3627,6 +3875,7 @@ static struct kvm_x86_ops svm_x86_ops = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, + .decache_cr3 = svm_decache_cr3, .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr3 = svm_set_cr3, @@ -3667,7 +3916,9 @@ static struct kvm_x86_ops svm_x86_ops = { .get_tdp_level = get_npt_level, .get_mt_mask = svm_get_mt_mask, + .get_exit_info = svm_get_exit_info, .exit_reasons_str = svm_exit_reasons_str, + .get_lpage_level = svm_get_lpage_level, .cpuid_update = svm_cpuid_update, diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index a6544b8..1357d7c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -178,27 +178,36 @@ TRACE_EVENT(kvm_apic, #define trace_kvm_apic_read(reg, val) trace_kvm_apic(0, reg, val) #define trace_kvm_apic_write(reg, val) trace_kvm_apic(1, reg, val) +#define KVM_ISA_VMX 1 +#define KVM_ISA_SVM 2 + /* * Tracepoint for kvm guest exit: */ TRACE_EVENT(kvm_exit, - TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu), - TP_ARGS(exit_reason, vcpu), + TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu, u32 isa), + TP_ARGS(exit_reason, vcpu, isa), TP_STRUCT__entry( __field( unsigned int, exit_reason ) __field( unsigned long, guest_rip ) + __field( u32, isa ) + __field( u64, info1 ) + __field( u64, info2 ) ), TP_fast_assign( __entry->exit_reason = exit_reason; __entry->guest_rip = kvm_rip_read(vcpu); + __entry->isa = isa; + kvm_x86_ops->get_exit_info(vcpu, &__entry->info1, + &__entry->info2); ), - TP_printk("reason %s rip 0x%lx", + TP_printk("reason %s rip 0x%lx info %llx %llx", ftrace_print_symbols_seq(p, __entry->exit_reason, kvm_x86_ops->exit_reasons_str), - __entry->guest_rip) + __entry->guest_rip, __entry->info1, __entry->info2) ); /* diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 81fcbe9..bf89ec2 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -69,6 +69,9 @@ module_param(emulate_invalid_guest_state, bool, S_IRUGO); static int __read_mostly vmm_exclusive = 1; module_param(vmm_exclusive, bool, S_IRUGO); +static int __read_mostly yield_on_hlt = 1; +module_param(yield_on_hlt, bool, S_IRUGO); + #define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST \ (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD) #define KVM_GUEST_CR0_MASK \ @@ -177,6 +180,7 @@ static int init_rmode(struct kvm *kvm); static u64 construct_eptp(unsigned long root_hpa); static void kvm_cpu_vmxon(u64 addr); static void kvm_cpu_vmxoff(void); +static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); static DEFINE_PER_CPU(struct vmcs *, vmxarea); static DEFINE_PER_CPU(struct vmcs *, current_vmcs); @@ -188,6 +192,8 @@ static unsigned long *vmx_io_bitmap_b; static unsigned long *vmx_msr_bitmap_legacy; static unsigned long *vmx_msr_bitmap_longmode; +static bool cpu_has_load_ia32_efer; + static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); static DEFINE_SPINLOCK(vmx_vpid_lock); @@ -472,7 +478,7 @@ static void vmcs_clear(struct vmcs *vmcs) u8 error; asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr) : "cc", "memory"); if (error) printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n", @@ -485,7 +491,7 @@ static void vmcs_load(struct vmcs *vmcs) u8 error; asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr) : "cc", "memory"); if (error) printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n", @@ -565,10 +571,10 @@ static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa) static unsigned long vmcs_readl(unsigned long field) { - unsigned long value; + unsigned long value = 0; asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX) - : "=a"(value) : "d"(field) : "cc"); + : "+a"(value) : "d"(field) : "cc"); return value; } @@ -661,6 +667,12 @@ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) unsigned i; struct msr_autoload *m = &vmx->msr_autoload; + if (msr == MSR_EFER && cpu_has_load_ia32_efer) { + vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER); + vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER); + return; + } + for (i = 0; i < m->nr; ++i) if (m->guest[i].index == msr) break; @@ -680,6 +692,14 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, unsigned i; struct msr_autoload *m = &vmx->msr_autoload; + if (msr == MSR_EFER && cpu_has_load_ia32_efer) { + vmcs_write64(GUEST_IA32_EFER, guest_val); + vmcs_write64(HOST_IA32_EFER, host_val); + vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER); + vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER); + return; + } + for (i = 0; i < m->nr; ++i) if (m->guest[i].index == msr) break; @@ -1009,6 +1029,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) vmx_set_interrupt_shadow(vcpu, 0); } +static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* Ensure that we clear the HLT state in the VMCS. We don't need to + * explicitly skip the instruction because if the HLT state is set, then + * the instruction is already executing and RIP has already been + * advanced. */ + if (!yield_on_hlt && + vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) + vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); +} + static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, bool has_error_code, u32 error_code, bool reinject) @@ -1035,6 +1066,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, intr_info |= INTR_TYPE_HARD_EXCEPTION; vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); + vmx_clear_hlt(vcpu); } static bool vmx_rdtscp_supported(void) @@ -1305,8 +1337,11 @@ static __init int vmx_disabled_by_bios(void) && tboot_enabled()) return 1; if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX) - && !tboot_enabled()) + && !tboot_enabled()) { + printk(KERN_WARNING "kvm: disable TXT in the BIOS or " + " activate TXT before enabling KVM\n"); return 1; + } } return 0; @@ -1400,6 +1435,14 @@ static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, return 0; } +static __init bool allow_1_setting(u32 msr, u32 ctl) +{ + u32 vmx_msr_low, vmx_msr_high; + + rdmsr(msr, vmx_msr_low, vmx_msr_high); + return vmx_msr_high & ctl; +} + static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) { u32 vmx_msr_low, vmx_msr_high; @@ -1416,7 +1459,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) &_pin_based_exec_control) < 0) return -EIO; - min = CPU_BASED_HLT_EXITING | + min = #ifdef CONFIG_X86_64 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | @@ -1429,6 +1472,10 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) CPU_BASED_MWAIT_EXITING | CPU_BASED_MONITOR_EXITING | CPU_BASED_INVLPG_EXITING; + + if (yield_on_hlt) + min |= CPU_BASED_HLT_EXITING; + opt = CPU_BASED_TPR_SHADOW | CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; @@ -1510,6 +1557,12 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; + cpu_has_load_ia32_efer = + allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS, + VM_ENTRY_LOAD_IA32_EFER) + && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS, + VM_EXIT_LOAD_IA32_EFER); + return 0; } @@ -1683,9 +1736,13 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save) save->limit = vmcs_read32(sf->limit); save->ar = vmcs_read32(sf->ar_bytes); vmcs_write16(sf->selector, save->base >> 4); - vmcs_write32(sf->base, save->base & 0xfffff); + vmcs_write32(sf->base, save->base & 0xffff0); vmcs_write32(sf->limit, 0xffff); vmcs_write32(sf->ar_bytes, 0xf3); + if (save->base & 0xf) + printk_once(KERN_WARNING "kvm: segment base is not paragraph" + " aligned when entering protected mode (seg=%d)", + seg); } static void enter_rmode(struct kvm_vcpu *vcpu) @@ -1814,6 +1871,13 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits; } +static void vmx_decache_cr3(struct kvm_vcpu *vcpu) +{ + if (enable_ept && is_paging(vcpu)) + vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); +} + static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; @@ -1857,6 +1921,7 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0, unsigned long cr0, struct kvm_vcpu *vcpu) { + vmx_decache_cr3(vcpu); if (!(cr0 & X86_CR0_PG)) { /* From paging/starting to nonpaging */ vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, @@ -1937,7 +2002,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (enable_ept) { eptp = construct_eptp(cr3); vmcs_write64(EPT_POINTER, eptp); - guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 : + guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) : vcpu->kvm->arch.ept_identity_map_addr; ept_load_pdptrs(vcpu); } @@ -2725,7 +2790,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_IDTR_BASE, 0); vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); - vmcs_write32(GUEST_ACTIVITY_STATE, 0); + vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); @@ -2787,6 +2852,10 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu) return; } + if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { + enable_irq_window(vcpu); + return; + } cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); @@ -2814,6 +2883,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu) } else intr |= INTR_TYPE_EXT_INTR; vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); + vmx_clear_hlt(vcpu); } static void vmx_inject_nmi(struct kvm_vcpu *vcpu) @@ -2841,6 +2911,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu) } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); + vmx_clear_hlt(vcpu); } static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) @@ -2849,7 +2920,8 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) return 0; return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_NMI)); + (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI + | GUEST_INTR_STATE_NMI)); } static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) @@ -2910,7 +2982,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, * Cause the #SS fault with 0 error code in VM86 mode. */ if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) - if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE) + if (emulate_instruction(vcpu, 0) == EMULATE_DONE) return 1; /* * Forward all other exceptions that are valid in real mode. @@ -3007,7 +3079,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) } if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD); + er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(vcpu, UD_VECTOR); return 1; @@ -3026,7 +3098,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) if (kvm_event_needs_reinjection(vcpu)) kvm_mmu_unprotect_page_virt(vcpu, cr2); - return kvm_mmu_page_fault(vcpu, cr2, error_code); + return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0); } if (vmx->rmode.vm86_active && @@ -3098,7 +3170,7 @@ static int handle_io(struct kvm_vcpu *vcpu) ++vcpu->stat.io_exits; if (string || in) - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -3118,14 +3190,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[2] = 0xc1; } -static void complete_insn_gp(struct kvm_vcpu *vcpu, int err) -{ - if (err) - kvm_inject_gp(vcpu, 0); - else - skip_emulated_instruction(vcpu); -} - static int handle_cr(struct kvm_vcpu *vcpu) { unsigned long exit_qualification, val; @@ -3143,21 +3207,21 @@ static int handle_cr(struct kvm_vcpu *vcpu) switch (cr) { case 0: err = kvm_set_cr0(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 3: err = kvm_set_cr3(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 4: err = kvm_set_cr4(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 8: { u8 cr8_prev = kvm_get_cr8(vcpu); u8 cr8 = kvm_register_read(vcpu, reg); - kvm_set_cr8(vcpu, cr8); - skip_emulated_instruction(vcpu); + err = kvm_set_cr8(vcpu, cr8); + kvm_complete_insn_gp(vcpu, err); if (irqchip_in_kernel(vcpu->kvm)) return 1; if (cr8_prev <= cr8) @@ -3176,8 +3240,9 @@ static int handle_cr(struct kvm_vcpu *vcpu) case 1: /*mov from cr*/ switch (cr) { case 3: - kvm_register_write(vcpu, reg, vcpu->arch.cr3); - trace_kvm_cr_read(cr, vcpu->arch.cr3); + val = kvm_read_cr3(vcpu); + kvm_register_write(vcpu, reg, val); + trace_kvm_cr_read(cr, val); skip_emulated_instruction(vcpu); return 1; case 8: @@ -3349,6 +3414,11 @@ static int handle_vmx_insn(struct kvm_vcpu *vcpu) return 1; } +static int handle_invd(struct kvm_vcpu *vcpu) +{ + return emulate_instruction(vcpu, 0) == EMULATE_DONE; +} + static int handle_invlpg(struct kvm_vcpu *vcpu) { unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); @@ -3377,7 +3447,7 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; } static int handle_task_switch(struct kvm_vcpu *vcpu) @@ -3476,7 +3546,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); trace_kvm_page_fault(gpa, exit_qualification); - return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0); + return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0); } static u64 ept_rsvd_mask(u64 spte, int level) @@ -3592,7 +3662,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF)) return handle_interrupt_window(&vmx->vcpu); - err = emulate_instruction(vcpu, 0, 0, 0); + err = emulate_instruction(vcpu, 0); if (err == EMULATE_DO_MMIO) { ret = 0; @@ -3649,6 +3719,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_MSR_WRITE] = handle_wrmsr, [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, [EXIT_REASON_HLT] = handle_halt, + [EXIT_REASON_INVD] = handle_invd, [EXIT_REASON_INVLPG] = handle_invlpg, [EXIT_REASON_VMCALL] = handle_vmcall, [EXIT_REASON_VMCLEAR] = handle_vmx_insn, @@ -3676,6 +3747,12 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) +{ + *info1 = vmcs_readl(EXIT_QUALIFICATION); + *info2 = vmcs_read32(VM_EXIT_INTR_INFO); +} + /* * The guest has exited. See if we can fix it or if we need userspace * assistance. @@ -3686,17 +3763,12 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) u32 exit_reason = vmx->exit_reason; u32 vectoring_info = vmx->idt_vectoring_info; - trace_kvm_exit(exit_reason, vcpu); + trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX); /* If guest state is invalid, start emulating */ if (vmx->emulation_required && emulate_invalid_guest_state) return handle_invalid_guest_state(vcpu); - /* Access CR3 don't cause VMExit in paging mode, so we need - * to sync with guest real CR3. */ - if (enable_ept && is_paging(vcpu)) - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); - if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) { vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; vcpu->run->fail_entry.hardware_entry_failure_reason @@ -4013,7 +4085,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) ); vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) - | (1 << VCPU_EXREG_PDPTR)); + | (1 << VCPU_EXREG_PDPTR) + | (1 << VCPU_EXREG_CR3)); vcpu->arch.regs_dirty = 0; vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); @@ -4280,6 +4353,7 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, + .decache_cr3 = vmx_decache_cr3, .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr3 = vmx_set_cr3, @@ -4320,7 +4394,9 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_tdp_level = get_ept_level, .get_mt_mask = vmx_get_mt_mask, + .get_exit_info = vmx_get_exit_info, .exit_reasons_str = vmx_exit_reasons_str, + .get_lpage_level = vmx_get_lpage_level, .cpuid_update = vmx_cpuid_update, @@ -4396,8 +4472,6 @@ static int __init vmx_init(void) if (enable_ept) { bypass_guest_pf = 0; - kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK | - VMX_EPT_WRITABLE_MASK); kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull, VMX_EPT_EXECUTABLE_MASK); kvm_enable_tdp(); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 46a368c..bcc0efc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -43,6 +43,7 @@ #include <linux/slab.h> #include <linux/perf_event.h> #include <linux/uaccess.h> +#include <linux/hash.h> #include <trace/events/kvm.h> #define CREATE_TRACE_POINTS @@ -155,6 +156,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { u64 __read_mostly host_xcr0; +static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) +{ + int i; + for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++) + vcpu->arch.apf.gfns[i] = ~0; +} + static void kvm_on_user_return(struct user_return_notifier *urn) { unsigned slot; @@ -326,23 +334,28 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) } EXPORT_SYMBOL_GPL(kvm_requeue_exception); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu) +void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) { - unsigned error_code = vcpu->arch.fault.error_code; + if (err) + kvm_inject_gp(vcpu, 0); + else + kvm_x86_ops->skip_emulated_instruction(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) +{ ++vcpu->stat.pf_guest; - vcpu->arch.cr2 = vcpu->arch.fault.address; - kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); + vcpu->arch.cr2 = fault->address; + kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code); } -void kvm_propagate_fault(struct kvm_vcpu *vcpu) +void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { - if (mmu_is_nested(vcpu) && !vcpu->arch.fault.nested) - vcpu->arch.nested_mmu.inject_page_fault(vcpu); + if (mmu_is_nested(vcpu) && !fault->nested_page_fault) + vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); else - vcpu->arch.mmu.inject_page_fault(vcpu); - - vcpu->arch.fault.nested = false; + vcpu->arch.mmu.inject_page_fault(vcpu, fault); } void kvm_inject_nmi(struct kvm_vcpu *vcpu) @@ -460,8 +473,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) (unsigned long *)&vcpu->arch.regs_avail)) return true; - gfn = (vcpu->arch.cr3 & ~31u) >> PAGE_SHIFT; - offset = (vcpu->arch.cr3 & ~31u) & (PAGE_SIZE - 1); + gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT; + offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1); r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte), PFERR_USER_MASK | PFERR_WRITE_MASK); if (r < 0) @@ -506,12 +519,15 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } else #endif if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, - vcpu->arch.cr3)) + kvm_read_cr3(vcpu))) return 1; } kvm_x86_ops->set_cr0(vcpu, cr0); + if ((cr0 ^ old_cr0) & X86_CR0_PG) + kvm_clear_async_pf_completion_queue(vcpu); + if ((cr0 ^ old_cr0) & update_bits) kvm_mmu_reset_context(vcpu); return 0; @@ -595,7 +611,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return 1; } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) && ((cr4 ^ old_cr4) & pdptr_bits) - && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3)) + && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, + kvm_read_cr3(vcpu))) return 1; if (cr4 & X86_CR4_VMXE) @@ -615,7 +632,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4); int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { - if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { + if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { kvm_mmu_sync_roots(vcpu); kvm_mmu_flush_tlb(vcpu); return 0; @@ -650,12 +667,13 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) return 1; vcpu->arch.cr3 = cr3; + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); vcpu->arch.mmu.new_cr3(vcpu); return 0; } EXPORT_SYMBOL_GPL(kvm_set_cr3); -int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { if (cr8 & CR8_RESERVED_BITS) return 1; @@ -665,12 +683,6 @@ int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) vcpu->arch.cr8 = cr8; return 0; } - -void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (__kvm_set_cr8(vcpu, cr8)) - kvm_inject_gp(vcpu, 0); -} EXPORT_SYMBOL_GPL(kvm_set_cr8); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) @@ -775,12 +787,12 @@ EXPORT_SYMBOL_GPL(kvm_get_dr); * kvm-specific. Those are put in the beginning of the list. */ -#define KVM_SAVE_MSRS_BEGIN 7 +#define KVM_SAVE_MSRS_BEGIN 8 static u32 msrs_to_save[] = { MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, - HV_X64_MSR_APIC_ASSIST_PAGE, + HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, MSR_STAR, #ifdef CONFIG_X86_64 @@ -830,7 +842,6 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer) kvm_x86_ops->set_efer(vcpu, efer); vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled; - kvm_mmu_reset_context(vcpu); /* Update reserved bits */ if ((efer ^ old_efer) & EFER_NX) @@ -1418,6 +1429,30 @@ static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data) return 0; } +static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) +{ + gpa_t gpa = data & ~0x3f; + + /* Bits 2:5 are resrved, Should be zero */ + if (data & 0x3c) + return 1; + + vcpu->arch.apf.msr_val = data; + + if (!(data & KVM_ASYNC_PF_ENABLED)) { + kvm_clear_async_pf_completion_queue(vcpu); + kvm_async_pf_hash_reset(vcpu); + return 0; + } + + if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa)) + return 1; + + vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); + kvm_async_pf_wakeup_all(vcpu); + return 0; +} + int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) { switch (msr) { @@ -1499,6 +1534,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) } break; } + case MSR_KVM_ASYNC_PF_EN: + if (kvm_pv_enable_async_pf(vcpu, data)) + return 1; + break; case MSR_IA32_MCG_CTL: case MSR_IA32_MCG_STATUS: case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1: @@ -1775,6 +1814,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) case MSR_KVM_SYSTEM_TIME_NEW: data = vcpu->arch.time; break; + case MSR_KVM_ASYNC_PF_EN: + data = vcpu->arch.apf.msr_val; + break; case MSR_IA32_P5_MC_ADDR: case MSR_IA32_P5_MC_TYPE: case MSR_IA32_MCG_CAP: @@ -1904,6 +1946,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_NOP_IO_DELAY: case KVM_CAP_MP_STATE: case KVM_CAP_SYNC_MMU: + case KVM_CAP_USER_NMI: case KVM_CAP_REINJECT_CONTROL: case KVM_CAP_IRQ_INJECT_STATUS: case KVM_CAP_ASSIGN_DEV_IRQ: @@ -1922,6 +1965,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_DEBUGREGS: case KVM_CAP_X86_ROBUST_SINGLESTEP: case KVM_CAP_XSAVE: + case KVM_CAP_ASYNC_PF: r = 1; break; case KVM_CAP_COALESCED_MMIO: @@ -2185,6 +2229,11 @@ out: return r; } +static void cpuid_mask(u32 *word, int wordnum) +{ + *word &= boot_cpu_data.x86_capability[wordnum]; +} + static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, u32 index) { @@ -2259,7 +2308,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, break; case 1: entry->edx &= kvm_supported_word0_x86_features; + cpuid_mask(&entry->edx, 0); entry->ecx &= kvm_supported_word4_x86_features; + cpuid_mask(&entry->ecx, 4); /* we support x2apic emulation even if host does not support * it since we emulate x2apic in software */ entry->ecx |= F(X2APIC); @@ -2350,7 +2401,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, break; case 0x80000001: entry->edx &= kvm_supported_word1_x86_features; + cpuid_mask(&entry->edx, 1); entry->ecx &= kvm_supported_word6_x86_features; + cpuid_mask(&entry->ecx, 6); break; } @@ -3169,20 +3222,18 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_memslots *slots, *old_slots; unsigned long *dirty_bitmap; - r = -ENOMEM; - dirty_bitmap = vmalloc(n); - if (!dirty_bitmap) - goto out; + dirty_bitmap = memslot->dirty_bitmap_head; + if (memslot->dirty_bitmap == dirty_bitmap) + dirty_bitmap += n / sizeof(long); memset(dirty_bitmap, 0, n); r = -ENOMEM; slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); - if (!slots) { - vfree(dirty_bitmap); + if (!slots) goto out; - } memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); slots->memslots[log->slot].dirty_bitmap = dirty_bitmap; + slots->generation++; old_slots = kvm->memslots; rcu_assign_pointer(kvm->memslots, slots); @@ -3195,11 +3246,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, spin_unlock(&kvm->mmu_lock); r = -EFAULT; - if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) { - vfree(dirty_bitmap); + if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) goto out; - } - vfree(dirty_bitmap); } else { r = -EFAULT; if (clear_user(log->dirty_bitmap, n)) @@ -3266,8 +3314,10 @@ long kvm_arch_vm_ioctl(struct file *filp, if (vpic) { r = kvm_ioapic_init(kvm); if (r) { + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &vpic->dev); + mutex_unlock(&kvm->slots_lock); kfree(vpic); goto create_irqchip_unlock; } @@ -3278,10 +3328,12 @@ long kvm_arch_vm_ioctl(struct file *filp, smp_wmb(); r = kvm_setup_default_irq_routing(kvm); if (r) { + mutex_lock(&kvm->slots_lock); mutex_lock(&kvm->irq_lock); kvm_ioapic_destroy(kvm); kvm_destroy_pic(kvm); mutex_unlock(&kvm->irq_lock); + mutex_unlock(&kvm->slots_lock); } create_irqchip_unlock: mutex_unlock(&kvm->lock); @@ -3557,63 +3609,63 @@ static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) { gpa_t t_gpa; - u32 error; + struct x86_exception exception; BUG_ON(!mmu_is_nested(vcpu)); /* NPT walks are always user-walks */ access |= PFERR_USER_MASK; - t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &error); - if (t_gpa == UNMAPPED_GVA) - vcpu->arch.fault.nested = true; + t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception); return t_gpa; } -gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } - gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) + gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_FETCH_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } -gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_WRITE_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } /* uses this to access any guest's mapped memory without checking CPL */ -gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception); } static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu, u32 access, - u32 *error) + struct x86_exception *exception) { void *data = val; int r = X86EMUL_CONTINUE; while (bytes) { gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access, - error); + exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; - if (gpa == UNMAPPED_GVA) { - r = X86EMUL_PROPAGATE_FAULT; - goto out; - } + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; ret = kvm_read_guest(vcpu->kvm, gpa, data, toread); if (ret < 0) { r = X86EMUL_IO_NEEDED; @@ -3630,31 +3682,35 @@ out: /* used for instruction fetching */ static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, - access | PFERR_FETCH_MASK, error); + access | PFERR_FETCH_MASK, + exception); } static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, - error); + exception); } static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { - return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error); + return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception); } static int kvm_write_guest_virt_system(gva_t addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu, - u32 *error) + struct x86_exception *exception) { void *data = val; int r = X86EMUL_CONTINUE; @@ -3662,15 +3718,13 @@ static int kvm_write_guest_virt_system(gva_t addr, void *val, while (bytes) { gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, PFERR_WRITE_MASK, - error); + exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; - if (gpa == UNMAPPED_GVA) { - r = X86EMUL_PROPAGATE_FAULT; - goto out; - } + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite); if (ret < 0) { r = X86EMUL_IO_NEEDED; @@ -3688,7 +3742,7 @@ out: static int emulator_read_emulated(unsigned long addr, void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; @@ -3701,7 +3755,7 @@ static int emulator_read_emulated(unsigned long addr, return X86EMUL_CONTINUE; } - gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, error_code); + gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, exception); if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; @@ -3710,8 +3764,8 @@ static int emulator_read_emulated(unsigned long addr, if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) goto mmio; - if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL) - == X86EMUL_CONTINUE) + if (kvm_read_guest_virt(addr, val, bytes, vcpu, exception) + == X86EMUL_CONTINUE) return X86EMUL_CONTINUE; mmio: @@ -3735,7 +3789,7 @@ mmio: } int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, - const void *val, int bytes) + const void *val, int bytes) { int ret; @@ -3749,12 +3803,12 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, static int emulator_write_emulated_onepage(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; - gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error_code); + gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, exception); if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; @@ -3787,7 +3841,7 @@ mmio: int emulator_write_emulated(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { /* Crossing a page boundary? */ @@ -3795,7 +3849,7 @@ int emulator_write_emulated(unsigned long addr, int rc, now; now = -addr & ~PAGE_MASK; - rc = emulator_write_emulated_onepage(addr, val, now, error_code, + rc = emulator_write_emulated_onepage(addr, val, now, exception, vcpu); if (rc != X86EMUL_CONTINUE) return rc; @@ -3803,7 +3857,7 @@ int emulator_write_emulated(unsigned long addr, val += now; bytes -= now; } - return emulator_write_emulated_onepage(addr, val, bytes, error_code, + return emulator_write_emulated_onepage(addr, val, bytes, exception, vcpu); } @@ -3821,7 +3875,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, const void *old, const void *new, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; @@ -3879,7 +3933,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, emul_write: printk_once(KERN_WARNING "kvm: emulating exchange as write\n"); - return emulator_write_emulated(addr, new, bytes, error_code, vcpu); + return emulator_write_emulated(addr, new, bytes, exception, vcpu); } static int kernel_pio(struct kvm_vcpu *vcpu, void *pd) @@ -3904,7 +3958,7 @@ static int emulator_pio_in_emulated(int size, unsigned short port, void *val, if (vcpu->arch.pio.count) goto data_avail; - trace_kvm_pio(0, port, size, 1); + trace_kvm_pio(0, port, size, count); vcpu->arch.pio.port = port; vcpu->arch.pio.in = 1; @@ -3932,7 +3986,7 @@ static int emulator_pio_out_emulated(int size, unsigned short port, const void *val, unsigned int count, struct kvm_vcpu *vcpu) { - trace_kvm_pio(1, port, size, 1); + trace_kvm_pio(1, port, size, count); vcpu->arch.pio.port = port; vcpu->arch.pio.in = 0; @@ -3973,13 +4027,15 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) return X86EMUL_CONTINUE; if (kvm_x86_ops->has_wbinvd_exit()) { - preempt_disable(); + int cpu = get_cpu(); + + cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); smp_call_function_many(vcpu->arch.wbinvd_dirty_mask, wbinvd_ipi, NULL, 1); - preempt_enable(); + put_cpu(); cpumask_clear(vcpu->arch.wbinvd_dirty_mask); - } - wbinvd(); + } else + wbinvd(); return X86EMUL_CONTINUE; } EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); @@ -4019,7 +4075,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu) value = vcpu->arch.cr2; break; case 3: - value = vcpu->arch.cr3; + value = kvm_read_cr3(vcpu); break; case 4: value = kvm_read_cr4(vcpu); @@ -4053,7 +4109,7 @@ static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu) res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); break; case 8: - res = __kvm_set_cr8(vcpu, val & 0xfUL); + res = kvm_set_cr8(vcpu, val); break; default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); @@ -4206,12 +4262,13 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) static void inject_emulated_exception(struct kvm_vcpu *vcpu) { struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; - if (ctxt->exception == PF_VECTOR) - kvm_propagate_fault(vcpu); - else if (ctxt->error_code_valid) - kvm_queue_exception_e(vcpu, ctxt->exception, ctxt->error_code); + if (ctxt->exception.vector == PF_VECTOR) + kvm_propagate_fault(vcpu, &ctxt->exception); + else if (ctxt->exception.error_code_valid) + kvm_queue_exception_e(vcpu, ctxt->exception.vector, + ctxt->exception.error_code); else - kvm_queue_exception(vcpu, ctxt->exception); + kvm_queue_exception(vcpu, ctxt->exception.vector); } static void init_emulate_ctxt(struct kvm_vcpu *vcpu) @@ -4267,13 +4324,19 @@ EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); static int handle_emulation_failure(struct kvm_vcpu *vcpu) { + int r = EMULATE_DONE; + ++vcpu->stat.insn_emulation_fail; trace_kvm_emulate_insn_failed(vcpu); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (!is_guest_mode(vcpu)) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + r = EMULATE_FAIL; + } kvm_queue_exception(vcpu, UD_VECTOR); - return EMULATE_FAIL; + + return r; } static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) @@ -4302,10 +4365,11 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) return false; } -int emulate_instruction(struct kvm_vcpu *vcpu, - unsigned long cr2, - u16 error_code, - int emulation_type) +int x86_emulate_instruction(struct kvm_vcpu *vcpu, + unsigned long cr2, + int emulation_type, + void *insn, + int insn_len) { int r; struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode; @@ -4323,10 +4387,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu, if (!(emulation_type & EMULTYPE_NO_DECODE)) { init_emulate_ctxt(vcpu); vcpu->arch.emulate_ctxt.interruptibility = 0; - vcpu->arch.emulate_ctxt.exception = -1; + vcpu->arch.emulate_ctxt.have_exception = false; vcpu->arch.emulate_ctxt.perm_ok = false; - r = x86_decode_insn(&vcpu->arch.emulate_ctxt); + r = x86_decode_insn(&vcpu->arch.emulate_ctxt, insn, insn_len); if (r == X86EMUL_PROPAGATE_FAULT) goto done; @@ -4389,7 +4453,7 @@ restart: } done: - if (vcpu->arch.emulate_ctxt.exception >= 0) { + if (vcpu->arch.emulate_ctxt.have_exception) { inject_emulated_exception(vcpu); r = EMULATE_DONE; } else if (vcpu->arch.pio.count) { @@ -4413,7 +4477,7 @@ done: return r; } -EXPORT_SYMBOL_GPL(emulate_instruction); +EXPORT_SYMBOL_GPL(x86_emulate_instruction); int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) { @@ -4653,7 +4717,6 @@ int kvm_arch_init(void *opaque) kvm_x86_ops = ops; kvm_mmu_set_nonpresent_ptes(0ull, 0ull); - kvm_mmu_set_base_ptes(PT_PRESENT_MASK); kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, PT_DIRTY_MASK, PT64_NX_MASK, 0); @@ -5116,6 +5179,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->fpu_active = 0; kvm_x86_ops->fpu_deactivate(vcpu); } + if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { + /* Page is swapped out. Do synthetic halt */ + vcpu->arch.apf.halted = true; + r = 1; + goto out; + } } r = kvm_mmu_reload(vcpu); @@ -5244,7 +5313,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) r = 1; while (r > 0) { - if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && + !vcpu->arch.apf.halted) r = vcpu_enter_guest(vcpu); else { srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); @@ -5257,6 +5327,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; case KVM_MP_STATE_RUNNABLE: + vcpu->arch.apf.halted = false; break; case KVM_MP_STATE_SIPI_RECEIVED: default: @@ -5278,6 +5349,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->run->exit_reason = KVM_EXIT_INTR; ++vcpu->stat.request_irq_exits; } + + kvm_check_async_pf_completion(vcpu); + if (signal_pending(current)) { r = -EINTR; vcpu->run->exit_reason = KVM_EXIT_INTR; @@ -5302,6 +5376,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) int r; sigset_t sigsaved; + if (!tsk_used_math(current) && init_fpu(current)) + return -ENOMEM; + if (vcpu->sigset_active) sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); @@ -5313,8 +5390,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } /* re-sync apic's tpr */ - if (!irqchip_in_kernel(vcpu->kvm)) - kvm_set_cr8(vcpu, kvm_run->cr8); + if (!irqchip_in_kernel(vcpu->kvm)) { + if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { + r = -EINVAL; + goto out; + } + } if (vcpu->arch.pio.count || vcpu->mmio_needed) { if (vcpu->mmio_needed) { @@ -5323,7 +5404,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_needed = 0; } vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); - r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE); + r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); if (r != EMULATE_DONE) { r = 0; @@ -5436,7 +5517,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->cr0 = kvm_read_cr0(vcpu); sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = vcpu->arch.cr3; + sregs->cr3 = kvm_read_cr3(vcpu); sregs->cr4 = kvm_read_cr4(vcpu); sregs->cr8 = kvm_get_cr8(vcpu); sregs->efer = vcpu->arch.efer; @@ -5504,8 +5585,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->set_gdt(vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); kvm_set_cr8(vcpu, sregs->cr8); @@ -5522,7 +5604,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, if (sregs->cr4 & X86_CR4_OSXSAVE) update_cpuid(vcpu); if (!is_long_mode(vcpu) && is_pae(vcpu)) { - load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); mmu_reset_needed = 1; } @@ -5773,6 +5855,8 @@ free_vcpu: void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { + vcpu->arch.apf.msr_val = 0; + vcpu_load(vcpu); kvm_mmu_unload(vcpu); vcpu_put(vcpu); @@ -5792,6 +5876,11 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) vcpu->arch.dr7 = DR7_FIXED_1; kvm_make_request(KVM_REQ_EVENT, vcpu); + vcpu->arch.apf.msr_val = 0; + + kvm_clear_async_pf_completion_queue(vcpu); + kvm_async_pf_hash_reset(vcpu); + vcpu->arch.apf.halted = false; return kvm_x86_ops->vcpu_reset(vcpu); } @@ -5881,6 +5970,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) goto fail_free_mce_banks; + kvm_async_pf_hash_reset(vcpu); + return 0; fail_free_mce_banks: kfree(vcpu->arch.mce_banks); @@ -5906,13 +5997,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) free_page((unsigned long)vcpu->arch.pio_data); } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - - if (!kvm) - return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); @@ -5921,7 +6007,7 @@ struct kvm *kvm_arch_create_vm(void) spin_lock_init(&kvm->arch.tsc_write_lock); - return kvm; + return 0; } static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) @@ -5939,8 +6025,10 @@ static void kvm_free_vcpus(struct kvm *kvm) /* * Unpin any mmu pages first. */ - kvm_for_each_vcpu(i, vcpu, kvm) + kvm_for_each_vcpu(i, vcpu, kvm) { + kvm_clear_async_pf_completion_queue(vcpu); kvm_unload_vcpu_mmu(vcpu); + } kvm_for_each_vcpu(i, vcpu, kvm) kvm_arch_vcpu_free(vcpu); @@ -5964,13 +6052,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kfree(kvm->arch.vpic); kfree(kvm->arch.vioapic); kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); if (kvm->arch.apic_access_page) put_page(kvm->arch.apic_access_page); if (kvm->arch.ept_identity_pagetable) put_page(kvm->arch.ept_identity_pagetable); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -6051,7 +6136,9 @@ void kvm_arch_flush_shadow(struct kvm *kvm) int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { - return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE + return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && + !vcpu->arch.apf.halted) + || !list_empty_careful(&vcpu->async_pf.done) || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED || vcpu->arch.nmi_pending || (kvm_arch_interrupt_allowed(vcpu) && @@ -6110,6 +6197,147 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) } EXPORT_SYMBOL_GPL(kvm_set_rflags); +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) +{ + int r; + + if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) || + is_error_page(work->page)) + return; + + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + return; + + if (!vcpu->arch.mmu.direct_map && + work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu)) + return; + + vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true); +} + +static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) +{ + return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); +} + +static inline u32 kvm_async_pf_next_probe(u32 key) +{ + return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1); +} + +static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + u32 key = kvm_async_pf_hash_fn(gfn); + + while (vcpu->arch.apf.gfns[key] != ~0) + key = kvm_async_pf_next_probe(key); + + vcpu->arch.apf.gfns[key] = gfn; +} + +static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + int i; + u32 key = kvm_async_pf_hash_fn(gfn); + + for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) && + (vcpu->arch.apf.gfns[key] != gfn && + vcpu->arch.apf.gfns[key] != ~0); i++) + key = kvm_async_pf_next_probe(key); + + return key; +} + +bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; +} + +static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + u32 i, j, k; + + i = j = kvm_async_pf_gfn_slot(vcpu, gfn); + while (true) { + vcpu->arch.apf.gfns[i] = ~0; + do { + j = kvm_async_pf_next_probe(j); + if (vcpu->arch.apf.gfns[j] == ~0) + return; + k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); + /* + * k lies cyclically in ]i,j] + * | i.k.j | + * |....j i.k.| or |.k..j i...| + */ + } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); + vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; + i = j; + } +} + +static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) +{ + + return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, + sizeof(val)); +} + +void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + struct x86_exception fault; + + trace_kvm_async_pf_not_present(work->arch.token, work->gva); + kvm_add_async_pf_gfn(vcpu, work->arch.gfn); + + if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || + (vcpu->arch.apf.send_user_only && + kvm_x86_ops->get_cpl(vcpu) == 0)) + kvm_make_request(KVM_REQ_APF_HALT, vcpu); + else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) { + fault.vector = PF_VECTOR; + fault.error_code_valid = true; + fault.error_code = 0; + fault.nested_page_fault = false; + fault.address = work->arch.token; + kvm_inject_page_fault(vcpu, &fault); + } +} + +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + struct x86_exception fault; + + trace_kvm_async_pf_ready(work->arch.token, work->gva); + if (is_error_page(work->page)) + work->arch.token = ~0; /* broadcast wakeup */ + else + kvm_del_async_pf_gfn(vcpu, work->arch.gfn); + + if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) && + !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { + fault.vector = PF_VECTOR; + fault.error_code_valid = true; + fault.error_code = 0; + fault.nested_page_fault = false; + fault.address = work->arch.token; + kvm_inject_page_fault(vcpu, &fault); + } + vcpu->arch.apf.halted = false; +} + +bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) +{ + if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) + return true; + else + return !kvm_event_needs_reinjection(vcpu) && + kvm_x86_ops->interrupt_allowed(vcpu); +} + EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); diff --git a/arch/x86/mm/amdtopology_64.c b/arch/x86/mm/amdtopology_64.c index 08a0069..f21962c 100644 --- a/arch/x86/mm/amdtopology_64.c +++ b/arch/x86/mm/amdtopology_64.c @@ -27,6 +27,7 @@ #include <asm/amd_nb.h> static struct bootnode __initdata nodes[8]; +static unsigned char __initdata nodeids[8]; static nodemask_t __initdata nodes_parsed = NODE_MASK_NONE; static __init int find_northbridge(void) @@ -68,19 +69,6 @@ static __init void early_get_boot_cpu_id(void) #endif } -int __init amd_get_nodes(struct bootnode *physnodes) -{ - int i; - int ret = 0; - - for_each_node_mask(i, nodes_parsed) { - physnodes[ret].start = nodes[i].start; - physnodes[ret].end = nodes[i].end; - ret++; - } - return ret; -} - int __init amd_numa_init(unsigned long start_pfn, unsigned long end_pfn) { unsigned long start = PFN_PHYS(start_pfn); @@ -113,7 +101,7 @@ int __init amd_numa_init(unsigned long start_pfn, unsigned long end_pfn) base = read_pci_config(0, nb, 1, 0x40 + i*8); limit = read_pci_config(0, nb, 1, 0x44 + i*8); - nodeid = limit & 7; + nodeids[i] = nodeid = limit & 7; if ((base & 3) == 0) { if (i < numnodes) pr_info("Skipping disabled node %d\n", i); @@ -193,6 +181,76 @@ int __init amd_numa_init(unsigned long start_pfn, unsigned long end_pfn) return 0; } +#ifdef CONFIG_NUMA_EMU +static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = { + [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE +}; + +void __init amd_get_nodes(struct bootnode *physnodes) +{ + int i; + + for_each_node_mask(i, nodes_parsed) { + physnodes[i].start = nodes[i].start; + physnodes[i].end = nodes[i].end; + } +} + +static int __init find_node_by_addr(unsigned long addr) +{ + int ret = NUMA_NO_NODE; + int i; + + for (i = 0; i < 8; i++) + if (addr >= nodes[i].start && addr < nodes[i].end) { + ret = i; + break; + } + return ret; +} + +/* + * For NUMA emulation, fake proximity domain (_PXM) to node id mappings must be + * setup to represent the physical topology but reflect the emulated + * environment. For each emulated node, the real node which it appears on is + * found and a fake pxm to nid mapping is created which mirrors the actual + * locality. node_distance() then represents the correct distances between + * emulated nodes by using the fake acpi mappings to pxms. + */ +void __init amd_fake_nodes(const struct bootnode *nodes, int nr_nodes) +{ + unsigned int bits; + unsigned int cores; + unsigned int apicid_base = 0; + int i; + + bits = boot_cpu_data.x86_coreid_bits; + cores = 1 << bits; + early_get_boot_cpu_id(); + if (boot_cpu_physical_apicid > 0) + apicid_base = boot_cpu_physical_apicid; + + for (i = 0; i < nr_nodes; i++) { + int index; + int nid; + int j; + + nid = find_node_by_addr(nodes[i].start); + if (nid == NUMA_NO_NODE) + continue; + + index = nodeids[nid] << bits; + if (fake_apicid_to_node[index + apicid_base] == NUMA_NO_NODE) + for (j = apicid_base; j < cores + apicid_base; j++) + fake_apicid_to_node[index + j] = i; +#ifdef CONFIG_ACPI_NUMA + __acpi_map_pxm_to_node(nid, i); +#endif + } + memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node)); +} +#endif /* CONFIG_NUMA_EMU */ + int __init amd_scan_nodes(void) { unsigned int bits; diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index f89b5bb..c821074 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -45,6 +45,7 @@ #include <asm/bugs.h> #include <asm/tlb.h> #include <asm/tlbflush.h> +#include <asm/olpc_ofw.h> #include <asm/pgalloc.h> #include <asm/sections.h> #include <asm/paravirt.h> @@ -715,6 +716,7 @@ void __init paging_init(void) /* * NOTE: at this point the bootmem allocator is fully available. */ + olpc_dt_build_devicetree(); sparse_init(); zone_sizes_init(); } diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index 7762a51..1e72102 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c @@ -260,30 +260,30 @@ void __init numa_init_array(void) #ifdef CONFIG_NUMA_EMU /* Numa emulation */ static struct bootnode nodes[MAX_NUMNODES] __initdata; -static struct bootnode physnodes[MAX_NUMNODES] __initdata; +static struct bootnode physnodes[MAX_NUMNODES] __cpuinitdata; static char *cmdline __initdata; static int __init setup_physnodes(unsigned long start, unsigned long end, int acpi, int amd) { - int nr_nodes = 0; int ret = 0; int i; + memset(physnodes, 0, sizeof(physnodes)); #ifdef CONFIG_ACPI_NUMA if (acpi) - nr_nodes = acpi_get_nodes(physnodes); + acpi_get_nodes(physnodes, start, end); #endif #ifdef CONFIG_AMD_NUMA if (amd) - nr_nodes = amd_get_nodes(physnodes); + amd_get_nodes(physnodes); #endif /* * Basic sanity checking on the physical node map: there may be errors * if the SRAT or AMD code incorrectly reported the topology or the mem= * kernel parameter is used. */ - for (i = 0; i < nr_nodes; i++) { + for (i = 0; i < MAX_NUMNODES; i++) { if (physnodes[i].start == physnodes[i].end) continue; if (physnodes[i].start > end) { @@ -298,17 +298,6 @@ static int __init setup_physnodes(unsigned long start, unsigned long end, physnodes[i].start = start; if (physnodes[i].end > end) physnodes[i].end = end; - } - - /* - * Remove all nodes that have no memory or were truncated because of the - * limited address range. - */ - for (i = 0; i < nr_nodes; i++) { - if (physnodes[i].start == physnodes[i].end) - continue; - physnodes[ret].start = physnodes[i].start; - physnodes[ret].end = physnodes[i].end; ret++; } @@ -324,6 +313,24 @@ static int __init setup_physnodes(unsigned long start, unsigned long end, return ret; } +static void __init fake_physnodes(int acpi, int amd, int nr_nodes) +{ + int i; + + BUG_ON(acpi && amd); +#ifdef CONFIG_ACPI_NUMA + if (acpi) + acpi_fake_nodes(nodes, nr_nodes); +#endif +#ifdef CONFIG_AMD_NUMA + if (amd) + amd_fake_nodes(nodes, nr_nodes); +#endif + if (!acpi && !amd) + for (i = 0; i < nr_cpu_ids; i++) + numa_set_node(i, 0); +} + /* * Setups up nid to range from addr to addr + size. If the end * boundary is greater than max_addr, then max_addr is used instead. @@ -352,8 +359,7 @@ static int __init setup_node_range(int nid, u64 *addr, u64 size, u64 max_addr) * Sets up nr_nodes fake nodes interleaved over physical nodes ranging from addr * to max_addr. The return value is the number of nodes allocated. */ -static int __init split_nodes_interleave(u64 addr, u64 max_addr, - int nr_phys_nodes, int nr_nodes) +static int __init split_nodes_interleave(u64 addr, u64 max_addr, int nr_nodes) { nodemask_t physnode_mask = NODE_MASK_NONE; u64 size; @@ -384,7 +390,7 @@ static int __init split_nodes_interleave(u64 addr, u64 max_addr, return -1; } - for (i = 0; i < nr_phys_nodes; i++) + for (i = 0; i < MAX_NUMNODES; i++) if (physnodes[i].start != physnodes[i].end) node_set(i, physnode_mask); @@ -553,11 +559,9 @@ static int __init numa_emulation(unsigned long start_pfn, { u64 addr = start_pfn << PAGE_SHIFT; u64 max_addr = last_pfn << PAGE_SHIFT; - int num_phys_nodes; int num_nodes; int i; - num_phys_nodes = setup_physnodes(addr, max_addr, acpi, amd); /* * If the numa=fake command-line contains a 'M' or 'G', it represents * the fixed node size. Otherwise, if it is just a single number N, @@ -572,7 +576,7 @@ static int __init numa_emulation(unsigned long start_pfn, unsigned long n; n = simple_strtoul(cmdline, NULL, 0); - num_nodes = split_nodes_interleave(addr, max_addr, num_phys_nodes, n); + num_nodes = split_nodes_interleave(addr, max_addr, n); } if (num_nodes < 0) @@ -595,7 +599,8 @@ static int __init numa_emulation(unsigned long start_pfn, nodes[i].end >> PAGE_SHIFT); setup_node_bootmem(i, nodes[i].start, nodes[i].end); } - acpi_fake_nodes(nodes, num_nodes); + setup_physnodes(addr, max_addr, acpi, amd); + fake_physnodes(acpi, amd, num_nodes); numa_init_array(); return 0; } @@ -610,8 +615,12 @@ void __init initmem_init(unsigned long start_pfn, unsigned long last_pfn, nodes_clear(node_online_map); #ifdef CONFIG_NUMA_EMU + setup_physnodes(start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT, + acpi, amd); if (cmdline && !numa_emulation(start_pfn, last_pfn, acpi, amd)) return; + setup_physnodes(start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT, + acpi, amd); nodes_clear(node_possible_map); nodes_clear(node_online_map); #endif @@ -767,6 +776,7 @@ void __cpuinit numa_clear_node(int cpu) #ifndef CONFIG_DEBUG_PER_CPU_MAPS +#ifndef CONFIG_NUMA_EMU void __cpuinit numa_add_cpu(int cpu) { cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); @@ -776,34 +786,115 @@ void __cpuinit numa_remove_cpu(int cpu) { cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); } +#else +void __cpuinit numa_add_cpu(int cpu) +{ + unsigned long addr; + u16 apicid; + int physnid; + int nid = NUMA_NO_NODE; + + apicid = early_per_cpu(x86_cpu_to_apicid, cpu); + if (apicid != BAD_APICID) + nid = apicid_to_node[apicid]; + if (nid == NUMA_NO_NODE) + nid = early_cpu_to_node(cpu); + BUG_ON(nid == NUMA_NO_NODE || !node_online(nid)); + + /* + * Use the starting address of the emulated node to find which physical + * node it is allocated on. + */ + addr = node_start_pfn(nid) << PAGE_SHIFT; + for (physnid = 0; physnid < MAX_NUMNODES; physnid++) + if (addr >= physnodes[physnid].start && + addr < physnodes[physnid].end) + break; + + /* + * Map the cpu to each emulated node that is allocated on the physical + * node of the cpu's apic id. + */ + for_each_online_node(nid) { + addr = node_start_pfn(nid) << PAGE_SHIFT; + if (addr >= physnodes[physnid].start && + addr < physnodes[physnid].end) + cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); + } +} + +void __cpuinit numa_remove_cpu(int cpu) +{ + int i; + + for_each_online_node(i) + cpumask_clear_cpu(cpu, node_to_cpumask_map[i]); +} +#endif /* !CONFIG_NUMA_EMU */ #else /* CONFIG_DEBUG_PER_CPU_MAPS */ +static struct cpumask __cpuinit *debug_cpumask_set_cpu(int cpu, int enable) +{ + int node = early_cpu_to_node(cpu); + struct cpumask *mask; + char buf[64]; + + mask = node_to_cpumask_map[node]; + if (!mask) { + pr_err("node_to_cpumask_map[%i] NULL\n", node); + dump_stack(); + return NULL; + } + + cpulist_scnprintf(buf, sizeof(buf), mask); + printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n", + enable ? "numa_add_cpu" : "numa_remove_cpu", + cpu, node, buf); + return mask; +} /* * --------- debug versions of the numa functions --------- */ +#ifndef CONFIG_NUMA_EMU static void __cpuinit numa_set_cpumask(int cpu, int enable) { - int node = early_cpu_to_node(cpu); struct cpumask *mask; - char buf[64]; - mask = node_to_cpumask_map[node]; - if (mask == NULL) { - printk(KERN_ERR "node_to_cpumask_map[%i] NULL\n", node); - dump_stack(); + mask = debug_cpumask_set_cpu(cpu, enable); + if (!mask) return; - } if (enable) cpumask_set_cpu(cpu, mask); else cpumask_clear_cpu(cpu, mask); +} +#else +static void __cpuinit numa_set_cpumask(int cpu, int enable) +{ + int node = early_cpu_to_node(cpu); + struct cpumask *mask; + int i; - cpulist_scnprintf(buf, sizeof(buf), mask); - printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n", - enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf); + for_each_online_node(i) { + unsigned long addr; + + addr = node_start_pfn(i) << PAGE_SHIFT; + if (addr < physnodes[node].start || + addr >= physnodes[node].end) + continue; + mask = debug_cpumask_set_cpu(cpu, enable); + if (!mask) + return; + + if (enable) + cpumask_set_cpu(cpu, mask); + else + cpumask_clear_cpu(cpu, mask); + } } +#endif /* CONFIG_NUMA_EMU */ void __cpuinit numa_add_cpu(int cpu) { diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c index 171a0aa..603d285 100644 --- a/arch/x86/mm/srat_64.c +++ b/arch/x86/mm/srat_64.c @@ -349,18 +349,19 @@ static int __init nodes_cover_memory(const struct bootnode *nodes) void __init acpi_numa_arch_fixup(void) {} -int __init acpi_get_nodes(struct bootnode *physnodes) +#ifdef CONFIG_NUMA_EMU +void __init acpi_get_nodes(struct bootnode *physnodes, unsigned long start, + unsigned long end) { int i; - int ret = 0; for_each_node_mask(i, nodes_parsed) { - physnodes[ret].start = nodes[i].start; - physnodes[ret].end = nodes[i].end; - ret++; + cutoff_node(i, start, end); + physnodes[i].start = nodes[i].start; + physnodes[i].end = nodes[i].end; } - return ret; } +#endif /* CONFIG_NUMA_EMU */ /* Use the information discovered above to actually set up the nodes. */ int __init acpi_scan_nodes(unsigned long start, unsigned long end) @@ -505,8 +506,6 @@ void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes) { int i, j; - printk(KERN_INFO "Faking PXM affinity for fake nodes on real " - "topology.\n"); for (i = 0; i < num_nodes; i++) { int nid, pxm; @@ -526,6 +525,17 @@ void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes) fake_apicid_to_node[j] == NUMA_NO_NODE) fake_apicid_to_node[j] = i; } + + /* + * If there are apicid-to-node mappings for physical nodes that do not + * have a corresponding emulated node, it should default to a guaranteed + * value. + */ + for (i = 0; i < MAX_LOCAL_APIC; i++) + if (apicid_to_node[i] != NUMA_NO_NODE && + fake_apicid_to_node[i] == NUMA_NO_NODE) + fake_apicid_to_node[i] = 0; + for (i = 0; i < num_nodes; i++) __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i); memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node)); diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index f24a853..e2b7b0c 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -65,7 +65,6 @@ static int profile_exceptions_notify(struct notifier_block *self, switch (val) { case DIE_NMI: - case DIE_NMI_IPI: if (ctr_running) model->check_ctrs(args->regs, &__get_cpu_var(cpu_msrs)); else if (!nmi_enabled) @@ -361,7 +360,7 @@ static void nmi_cpu_setup(void *dummy) static struct notifier_block profile_exceptions_nb = { .notifier_call = profile_exceptions_notify, .next = NULL, - .priority = 2 + .priority = NMI_LOCAL_LOW_PRIOR, }; static void nmi_cpu_restore_registers(struct op_msrs *msrs) diff --git a/arch/x86/oprofile/nmi_timer_int.c b/arch/x86/oprofile/nmi_timer_int.c index 0636dd9..720bf5a 100644 --- a/arch/x86/oprofile/nmi_timer_int.c +++ b/arch/x86/oprofile/nmi_timer_int.c @@ -38,7 +38,7 @@ static int profile_timer_exceptions_notify(struct notifier_block *self, static struct notifier_block profile_timer_exceptions_nb = { .notifier_call = profile_timer_exceptions_notify, .next = NULL, - .priority = 0 + .priority = NMI_LOW_PRIOR, }; static int timer_start(void) diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index fc1e8fe..e27dffb 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -4,6 +4,7 @@ #include <linux/cpu.h> #include <linux/range.h> +#include <asm/amd_nb.h> #include <asm/pci_x86.h> #include <asm/pci-direct.h> @@ -378,6 +379,34 @@ static struct notifier_block __cpuinitdata amd_cpu_notifier = { .notifier_call = amd_cpu_notify, }; +static void __init pci_enable_pci_io_ecs(void) +{ +#ifdef CONFIG_AMD_NB + unsigned int i, n; + + for (n = i = 0; !n && amd_nb_bus_dev_ranges[i].dev_limit; ++i) { + u8 bus = amd_nb_bus_dev_ranges[i].bus; + u8 slot = amd_nb_bus_dev_ranges[i].dev_base; + u8 limit = amd_nb_bus_dev_ranges[i].dev_limit; + + for (; slot < limit; ++slot) { + u32 val = read_pci_config(bus, slot, 3, 0); + + if (!early_is_amd_nb(val)) + continue; + + val = read_pci_config(bus, slot, 3, 0x8c); + if (!(val & (ENABLE_CF8_EXT_CFG >> 32))) { + val |= ENABLE_CF8_EXT_CFG >> 32; + write_pci_config(bus, slot, 3, 0x8c, val); + } + ++n; + } + } + pr_info("Extended Config Space enabled on %u nodes\n", n); +#endif +} + static int __init pci_io_ecs_init(void) { int cpu; @@ -386,6 +415,10 @@ static int __init pci_io_ecs_init(void) if (boot_cpu_data.x86 < 0x10) return 0; + /* Try the PCI method first. */ + if (early_pci_allowed()) + pci_enable_pci_io_ecs(); + register_cpu_notifier(&amd_cpu_notifier); for_each_online_cpu(cpu) amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE, diff --git a/arch/x86/platform/mrst/early_printk_mrst.c b/arch/x86/platform/mrst/early_printk_mrst.c index 65df603..25bfdbb 100644 --- a/arch/x86/platform/mrst/early_printk_mrst.c +++ b/arch/x86/platform/mrst/early_printk_mrst.c @@ -103,7 +103,7 @@ struct dw_spi_reg { static unsigned long mrst_spi_paddr = MRST_REGBASE_SPI0; static u32 *pclk_spi0; -/* Always contains an accessable address, start with 0 */ +/* Always contains an accessible address, start with 0 */ static struct dw_spi_reg *pspi; static struct kmsg_dumper dw_dumper; diff --git a/arch/x86/platform/olpc/Makefile b/arch/x86/platform/olpc/Makefile index c31b8fc..e797428 100644 --- a/arch/x86/platform/olpc/Makefile +++ b/arch/x86/platform/olpc/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_OLPC) += olpc.o obj-$(CONFIG_OLPC_XO1) += olpc-xo1.o obj-$(CONFIG_OLPC_OPENFIRMWARE) += olpc_ofw.o +obj-$(CONFIG_OLPC_OPENFIRMWARE_DT) += olpc_dt.o diff --git a/arch/x86/platform/olpc/olpc_dt.c b/arch/x86/platform/olpc/olpc_dt.c new file mode 100644 index 0000000..dab8746 --- /dev/null +++ b/arch/x86/platform/olpc/olpc_dt.c @@ -0,0 +1,183 @@ +/* + * OLPC-specific OFW device tree support code. + * + * Paul Mackerras August 1996. + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner. + * {engebret|bergner}@us.ibm.com + * + * Adapted for sparc by David S. Miller davem@davemloft.net + * Adapted for x86/OLPC by Andres Salomon <dilinger@queued.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include <linux/kernel.h> +#include <linux/bootmem.h> +#include <linux/of.h> +#include <linux/of_pdt.h> +#include <asm/olpc_ofw.h> + +static phandle __init olpc_dt_getsibling(phandle node) +{ + const void *args[] = { (void *)node }; + void *res[] = { &node }; + + if ((s32)node == -1) + return 0; + + if (olpc_ofw("peer", args, res) || (s32)node == -1) + return 0; + + return node; +} + +static phandle __init olpc_dt_getchild(phandle node) +{ + const void *args[] = { (void *)node }; + void *res[] = { &node }; + + if ((s32)node == -1) + return 0; + + if (olpc_ofw("child", args, res) || (s32)node == -1) { + pr_err("PROM: %s: fetching child failed!\n", __func__); + return 0; + } + + return node; +} + +static int __init olpc_dt_getproplen(phandle node, const char *prop) +{ + const void *args[] = { (void *)node, prop }; + int len; + void *res[] = { &len }; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("getproplen", args, res)) { + pr_err("PROM: %s: getproplen failed!\n", __func__); + return -1; + } + + return len; +} + +static int __init olpc_dt_getproperty(phandle node, const char *prop, + char *buf, int bufsize) +{ + int plen; + + plen = olpc_dt_getproplen(node, prop); + if (plen > bufsize || plen < 1) { + return -1; + } else { + const void *args[] = { (void *)node, prop, buf, (void *)plen }; + void *res[] = { &plen }; + + if (olpc_ofw("getprop", args, res)) { + pr_err("PROM: %s: getprop failed!\n", __func__); + return -1; + } + } + + return plen; +} + +static int __init olpc_dt_nextprop(phandle node, char *prev, char *buf) +{ + const void *args[] = { (void *)node, prev, buf }; + int success; + void *res[] = { &success }; + + buf[0] = '\0'; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("nextprop", args, res) || success != 1) + return -1; + + return 0; +} + +static int __init olpc_dt_pkg2path(phandle node, char *buf, + const int buflen, int *len) +{ + const void *args[] = { (void *)node, buf, (void *)buflen }; + void *res[] = { len }; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("package-to-path", args, res) || *len < 1) + return -1; + + return 0; +} + +static unsigned int prom_early_allocated __initdata; + +void * __init prom_early_alloc(unsigned long size) +{ + static u8 *mem; + static size_t free_mem; + void *res; + + if (free_mem < size) { + const size_t chunk_size = max(PAGE_SIZE, size); + + /* + * To mimimize the number of allocations, grab at least + * PAGE_SIZE of memory (that's an arbitrary choice that's + * fast enough on the platforms we care about while minimizing + * wasted bootmem) and hand off chunks of it to callers. + */ + res = alloc_bootmem(chunk_size); + if (!res) + return NULL; + prom_early_allocated += chunk_size; + memset(res, 0, chunk_size); + free_mem = chunk_size; + mem = res; + } + + /* allocate from the local cache */ + free_mem -= size; + res = mem; + mem += size; + return res; +} + +static struct of_pdt_ops prom_olpc_ops __initdata = { + .nextprop = olpc_dt_nextprop, + .getproplen = olpc_dt_getproplen, + .getproperty = olpc_dt_getproperty, + .getchild = olpc_dt_getchild, + .getsibling = olpc_dt_getsibling, + .pkg2path = olpc_dt_pkg2path, +}; + +void __init olpc_dt_build_devicetree(void) +{ + phandle root; + + if (!olpc_ofw_is_installed()) + return; + + root = olpc_dt_getsibling(0); + if (!root) { + pr_err("PROM: unable to get root node from OFW!\n"); + return; + } + of_pdt_build_devicetree(root, &prom_olpc_ops); + + pr_info("PROM DT: Built device tree with %u bytes of memory.\n", + prom_early_allocated); +} diff --git a/arch/x86/platform/olpc/olpc_ofw.c b/arch/x86/platform/olpc/olpc_ofw.c index 7873204..e7604f6 100644 --- a/arch/x86/platform/olpc/olpc_ofw.c +++ b/arch/x86/platform/olpc/olpc_ofw.c @@ -110,3 +110,8 @@ void __init olpc_ofw_detect(void) (unsigned long)olpc_ofw_cif, (-start) >> 20); reserve_top_address(-start); } + +bool __init olpc_ofw_is_installed(void) +{ + return olpc_ofw_cif != NULL; +} |