diff options
Diffstat (limited to 'arch')
254 files changed, 7039 insertions, 1582 deletions
diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 0f1d849..c1f3e7c 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -506,7 +506,7 @@ set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds,RTC_SECONDS); CMOS_WRITE(real_minutes,RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c index 42ff90b..665ebf7 100644 --- a/arch/arm/common/it8152.c +++ b/arch/arm/common/it8152.c @@ -236,7 +236,7 @@ static struct resource it8152_mem = { /* * The following functions are needed for DMA bouncing. - * ITE8152 chip can addrees up to 64MByte, so all the devices + * ITE8152 chip can address up to 64MByte, so all the devices * connected to ITE8152 (PCI and USB) should have limited DMA window */ diff --git a/arch/arm/common/vic.c b/arch/arm/common/vic.c index ba65f6e..cb660bc 100644 --- a/arch/arm/common/vic.c +++ b/arch/arm/common/vic.c @@ -70,7 +70,7 @@ static inline struct vic_device *to_vic(struct sys_device *sys) * vic_init2 - common initialisation code * @base: Base of the VIC. * - * Common initialisation code for registeration + * Common initialisation code for registration * and resume. */ static void vic_init2(void __iomem *base) diff --git a/arch/arm/mach-at91/board-ecbat91.c b/arch/arm/mach-at91/board-ecbat91.c index 7b58c94..de2fd04 100644 --- a/arch/arm/mach-at91/board-ecbat91.c +++ b/arch/arm/mach-at91/board-ecbat91.c @@ -128,17 +128,17 @@ static struct spi_board_info __initdata ecb_at91spi_devices[] = { .platform_data = &my_flash0_platform, #endif }, - { /* User accessable spi - cs1 (250KHz) */ + { /* User accessible spi - cs1 (250KHz) */ .modalias = "spi-cs1", .chip_select = 1, .max_speed_hz = 250 * 1000, }, - { /* User accessable spi - cs2 (1MHz) */ + { /* User accessible spi - cs2 (1MHz) */ .modalias = "spi-cs2", .chip_select = 2, .max_speed_hz = 1 * 1000 * 1000, }, - { /* User accessable spi - cs3 (10MHz) */ + { /* User accessible spi - cs3 (10MHz) */ .modalias = "spi-cs3", .chip_select = 3, .max_speed_hz = 10 * 1000 * 1000, diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index dafbacc..ea53f4d 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -301,7 +301,7 @@ static void at91_pm_end(void) } -static struct platform_suspend_ops at91_pm_ops ={ +static const struct platform_suspend_ops at91_pm_ops = { .valid = at91_pm_valid_state, .begin = at91_pm_begin, .enter = at91_pm_enter, diff --git a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c index b3a61d8..96273ff 100644 --- a/arch/arm/mach-bcmring/csp/chipc/chipcHw.c +++ b/arch/arm/mach-bcmring/csp/chipc/chipcHw.c @@ -757,7 +757,7 @@ static int chipcHw_divide(int num, int denom) t = t << 1; } - /* Intialize the result */ + /* Initialize the result */ r = 0; do { diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c index 7b9bac2..6b9be2e 100644 --- a/arch/arm/mach-bcmring/csp/dmac/dmacHw.c +++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw.c @@ -893,7 +893,7 @@ int dmacHw_setDataDescriptor(dmacHw_CONFIG_t *pConfig, /* [ IN ] Configuration */ /****************************************************************************/ uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ - dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controler attribute of type dmacHw_CONTROLLER_ATTRIB_e */ + dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controller attribute of type dmacHw_CONTROLLER_ATTRIB_e */ ) { dmacHw_CBLK_t *pCblk = dmacHw_HANDLE_TO_CBLK(handle); diff --git a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c index ff7b436..77f84b4 100644 --- a/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c +++ b/arch/arm/mach-bcmring/csp/dmac/dmacHw_extra.c @@ -316,7 +316,7 @@ static void DisplayDescRing(void *pDescriptor, /* [ IN ] Descriptor buffer */ /** * @brief Check if DMA channel is the flow controller * -* @return 1 : If DMA is a flow controler +* @return 1 : If DMA is a flow controller * 0 : Peripheral is the flow controller * * @note diff --git a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c index 5c1c9a0..16225e4 100644 --- a/arch/arm/mach-bcmring/csp/tmr/tmrHw.c +++ b/arch/arm/mach-bcmring/csp/tmr/tmrHw.c @@ -558,7 +558,7 @@ static int tmrHw_divide(int num, int denom) t = t << 1; } - /* Intialize the result */ + /* Initialize the result */ r = 0; do { diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c index 77eb35c..8d1baf3 100644 --- a/arch/arm/mach-bcmring/dma.c +++ b/arch/arm/mach-bcmring/dma.c @@ -671,7 +671,7 @@ static int ConfigChannel(DMA_Handle_t handle) /****************************************************************************/ /** -* Intializes all of the data structures associated with the DMA. +* Initializes all of the data structures associated with the DMA. * @return * >= 0 - Initialization was successfull. * diff --git a/arch/arm/mach-bcmring/include/csp/dmacHw.h b/arch/arm/mach-bcmring/include/csp/dmacHw.h index 5d51013..6c8da2b 100644 --- a/arch/arm/mach-bcmring/include/csp/dmacHw.h +++ b/arch/arm/mach-bcmring/include/csp/dmacHw.h @@ -590,7 +590,7 @@ void dmacHw_printDebugInfo(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ /****************************************************************************/ uint32_t dmacHw_getDmaControllerAttribute(dmacHw_HANDLE_t handle, /* [ IN ] DMA Channel handle */ - dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controler attribute of type dmacHw_CONTROLLER_ATTRIB_e */ + dmacHw_CONTROLLER_ATTRIB_e attr /* [ IN ] DMA Controller attribute of type dmacHw_CONTROLLER_ATTRIB_e */ ); #endif /* _DMACHW_H */ diff --git a/arch/arm/mach-bcmring/include/csp/tmrHw.h b/arch/arm/mach-bcmring/include/csp/tmrHw.h index f1236d0..2cbb530 100644 --- a/arch/arm/mach-bcmring/include/csp/tmrHw.h +++ b/arch/arm/mach-bcmring/include/csp/tmrHw.h @@ -76,7 +76,7 @@ tmrHw_RATE_t tmrHw_setPeriodicTimerRate(tmrHw_ID_t timerId, /* [ IN ] Timer Id * certain time interval * * This function initializes a periodic timer to generate timer interrupt -* after every time interval in milisecond +* after every time interval in millisecond * * @return On success: Effective interval set in mili-second * On failure: 0 @@ -93,7 +93,7 @@ tmrHw_INTERVAL_t tmrHw_setPeriodicTimerInterval(tmrHw_ID_t timerId, /* [ IN ] T * after certain time interval * * This function initializes a periodic timer to generate a single ticks after -* certain time interval in milisecond +* certain time interval in millisecond * * @return On success: Effective interval set in mili-second * On failure: 0 diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h index cbf334d..d67e2f8 100644 --- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h +++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_priv.h @@ -28,7 +28,7 @@ /* Data type for DMA Link List Item */ typedef struct { - uint32_t sar; /* Source Adress Register. + uint32_t sar; /* Source Address Register. Address must be aligned to CTLx.SRC_TR_WIDTH. */ uint32_t dar; /* Destination Address Register. Address must be aligned to CTLx.DST_TR_WIDTH. */ diff --git a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h index 891cea8..f1ecf96 100644 --- a/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h +++ b/arch/arm/mach-bcmring/include/mach/csp/dmacHw_reg.h @@ -35,7 +35,7 @@ typedef struct { /* Data type representing DMA channel registers */ typedef struct { - dmacHw_REG64_t ChannelSar; /* Source Adress Register. 64 bits (upper 32 bits are reserved) + dmacHw_REG64_t ChannelSar; /* Source Address Register. 64 bits (upper 32 bits are reserved) Address must be aligned to CTLx.SRC_TR_WIDTH. */ dmacHw_REG64_t ChannelDar; /* Destination Address Register.64 bits (upper 32 bits are reserved) diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c index fab953b..1bd73a0 100644 --- a/arch/arm/mach-davinci/pm.c +++ b/arch/arm/mach-davinci/pm.c @@ -110,7 +110,7 @@ static int davinci_pm_enter(suspend_state_t state) return ret; } -static struct platform_suspend_ops davinci_pm_ops = { +static const struct platform_suspend_ops davinci_pm_ops = { .enter = davinci_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-gemini/include/mach/hardware.h b/arch/arm/mach-gemini/include/mach/hardware.h index 213a4fc..8c950e1 100644 --- a/arch/arm/mach-gemini/include/mach/hardware.h +++ b/arch/arm/mach-gemini/include/mach/hardware.h @@ -33,7 +33,7 @@ #define GEMINI_LPC_HOST_BASE 0x47000000 #define GEMINI_LPC_IO_BASE 0x47800000 #define GEMINI_INTERRUPT_BASE 0x48000000 -/* TODO: Different interrupt controlers when SMP +/* TODO: Different interrupt controllers when SMP * #define GEMINI_INTERRUPT0_BASE 0x48000000 * #define GEMINI_INTERRUPT1_BASE 0x49000000 */ diff --git a/arch/arm/mach-imx/mach-pcm038.c b/arch/arm/mach-imx/mach-pcm038.c index f667a26..5056148 100644 --- a/arch/arm/mach-imx/mach-pcm038.c +++ b/arch/arm/mach-imx/mach-pcm038.c @@ -254,10 +254,10 @@ static struct regulator_init_data cam_data = { static struct mc13783_regulator_init_data pcm038_regulators[] = { { - .id = MC13783_REGU_VCAM, + .id = MC13783_REG_VCAM, .init_data = &cam_data, }, { - .id = MC13783_REGU_VMMC1, + .id = MC13783_REG_VMMC1, .init_data = &sdhc1_data, }, }; diff --git a/arch/arm/mach-imx/pm-imx27.c b/arch/arm/mach-imx/pm-imx27.c index 6bf81ce..acf1769 100644 --- a/arch/arm/mach-imx/pm-imx27.c +++ b/arch/arm/mach-imx/pm-imx27.c @@ -32,7 +32,7 @@ static int mx27_suspend_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops mx27_suspend_ops = { +static const struct platform_suspend_ops mx27_suspend_ops = { .enter = mx27_suspend_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-kirkwood/openrd-setup.c b/arch/arm/mach-kirkwood/openrd-setup.c index c9d77fa..cfcca41 100644 --- a/arch/arm/mach-kirkwood/openrd-setup.c +++ b/arch/arm/mach-kirkwood/openrd-setup.c @@ -171,7 +171,7 @@ static void __init openrd_init(void) kirkwood_i2c_init(); - if (machine_is_openrd_client()) { + if (machine_is_openrd_client() || machine_is_openrd_ultimate()) { i2c_register_board_info(0, i2c_board_info, ARRAY_SIZE(i2c_board_info)); kirkwood_audio_init(); diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c index a6e2aed..e76d41b 100644 --- a/arch/arm/mach-lpc32xx/pm.c +++ b/arch/arm/mach-lpc32xx/pm.c @@ -123,7 +123,7 @@ static int lpc32xx_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops lpc32xx_pm_ops = { +static const struct platform_suspend_ops lpc32xx_pm_ops = { .valid = suspend_valid_only_mem, .enter = lpc32xx_pm_enter, }; diff --git a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S index 4dc99aa..1246715 100644 --- a/arch/arm/mach-msm/include/mach/entry-macro-qgic.S +++ b/arch/arm/mach-msm/include/mach/entry-macro-qgic.S @@ -26,7 +26,7 @@ * The interrupt numbering scheme is defined in the * interrupt controller spec. To wit: * - * Migrated the code from ARM MP port to be more consistant + * Migrated the code from ARM MP port to be more consistent * with interrupt processing , the following still holds true * however, all interrupts are treated the same regardless of * if they are local IPI or PPI diff --git a/arch/arm/mach-msm/io.c b/arch/arm/mach-msm/io.c index 800f327..1260007 100644 --- a/arch/arm/mach-msm/io.c +++ b/arch/arm/mach-msm/io.c @@ -154,7 +154,7 @@ __msm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) { if (mtype == MT_DEVICE) { /* The peripherals in the 88000000 - D0000000 range - * are only accessable by type MT_DEVICE_NONSHARED. + * are only accessible by type MT_DEVICE_NONSHARED. * Adjust mtype as necessary to make this "just work." */ if ((phys_addr >= 0x88000000) && (phys_addr < 0xD0000000)) diff --git a/arch/arm/mach-mx3/mach-mx31_3ds.c b/arch/arm/mach-mx3/mach-mx31_3ds.c index 4e516b4..899a969 100644 --- a/arch/arm/mach-mx3/mach-mx31_3ds.c +++ b/arch/arm/mach-mx3/mach-mx31_3ds.c @@ -140,10 +140,10 @@ static struct regulator_init_data gpo_init = { static struct mc13783_regulator_init_data mx31_3ds_regulators[] = { { - .id = MC13783_REGU_PWGT1SPI, /* Power Gate for ARM core. */ + .id = MC13783_REG_PWGT1SPI, /* Power Gate for ARM core. */ .init_data = &pwgtx_init, }, { - .id = MC13783_REGU_PWGT2SPI, /* Power Gate for L2 Cache. */ + .id = MC13783_REG_PWGT2SPI, /* Power Gate for L2 Cache. */ .init_data = &pwgtx_init, }, { diff --git a/arch/arm/mach-mx3/mach-mx31moboard.c b/arch/arm/mach-mx3/mach-mx31moboard.c index 203d21a..1aa8d65 100644 --- a/arch/arm/mach-mx3/mach-mx31moboard.c +++ b/arch/arm/mach-mx3/mach-mx31moboard.c @@ -216,11 +216,11 @@ static struct regulator_init_data cam_vreg_data = { static struct mc13783_regulator_init_data moboard_regulators[] = { { - .id = MC13783_REGU_VMMC1, + .id = MC13783_REG_VMMC1, .init_data = &sdhc_vreg_data, }, { - .id = MC13783_REGU_VCAM, + .id = MC13783_REG_VCAM, .init_data = &cam_vreg_data, }, }; diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c index 0cca23a..98ba978 100644 --- a/arch/arm/mach-omap1/pm.c +++ b/arch/arm/mach-omap1/pm.c @@ -647,7 +647,7 @@ static struct irqaction omap_wakeup_irq = { -static struct platform_suspend_ops omap_pm_ops ={ +static const struct platform_suspend_ops omap_pm_ops = { .prepare = omap_pm_prepare, .enter = omap_pm_enter, .finish = omap_pm_finish, diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile index cd7332f..1c0c2b0 100644 --- a/arch/arm/mach-omap2/Makefile +++ b/arch/arm/mach-omap2/Makefile @@ -187,16 +187,19 @@ obj-$(CONFIG_MACH_NOKIA_RX51) += board-rx51.o \ hsmmc.o obj-$(CONFIG_MACH_OMAP_ZOOM2) += board-zoom.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o \ board-zoom-debugboard.o obj-$(CONFIG_MACH_OMAP_ZOOM3) += board-zoom.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o \ board-zoom-debugboard.o obj-$(CONFIG_MACH_OMAP_3630SDP) += board-3630sdp.o \ board-zoom-peripherals.o \ + board-zoom-display.o \ board-flash.o \ hsmmc.o obj-$(CONFIG_MACH_CM_T35) += board-cm-t35.o \ diff --git a/arch/arm/mach-omap2/board-3430sdp.c b/arch/arm/mach-omap2/board-3430sdp.c index 3b39ef1..d4e41ef 100644 --- a/arch/arm/mach-omap2/board-3430sdp.c +++ b/arch/arm/mach-omap2/board-3430sdp.c @@ -38,6 +38,7 @@ #include <plat/dma.h> #include <plat/gpmc.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/gpmc-smc91x.h> @@ -270,13 +271,18 @@ static struct omap_dss_device sdp3430_lcd_device = { .platform_disable = sdp3430_panel_disable_lcd, }; +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = sdp3430_panel_enable_dvi, + .platform_disable = sdp3430_panel_disable_dvi, +}; + static struct omap_dss_device sdp3430_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = sdp3430_panel_enable_dvi, - .platform_disable = sdp3430_panel_disable_dvi, }; static struct omap_dss_device sdp3430_tv_device = { diff --git a/arch/arm/mach-omap2/board-3630sdp.c b/arch/arm/mach-omap2/board-3630sdp.c index 5d41dbe..6264564 100644 --- a/arch/arm/mach-omap2/board-3630sdp.c +++ b/arch/arm/mach-omap2/board-3630sdp.c @@ -207,6 +207,7 @@ static void __init omap_sdp_init(void) { omap3_mux_init(board_mux, OMAP_PACKAGE_CBP); zoom_peripherals_init(); + zoom_display_init(); board_smc91x_init(); board_flash_init(sdp_flash_partitions, chip_sel_sdp); enable_board_wakeup_source(); diff --git a/arch/arm/mach-omap2/board-am3517evm.c b/arch/arm/mach-omap2/board-am3517evm.c index bc15626..10d60b7 100644 --- a/arch/arm/mach-omap2/board-am3517evm.c +++ b/arch/arm/mach-omap2/board-am3517evm.c @@ -35,6 +35,7 @@ #include <plat/common.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include "mux.h" #include "control.h" @@ -303,13 +304,18 @@ static void am3517_evm_panel_disable_lcd(struct omap_dss_device *dssdev) lcd_enabled = 0; } +static struct panel_generic_dpi_data lcd_panel = { + .name = "sharp_lq", + .platform_enable = am3517_evm_panel_enable_lcd, + .platform_disable = am3517_evm_panel_disable_lcd, +}; + static struct omap_dss_device am3517_evm_lcd_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "lcd", - .driver_name = "sharp_lq_panel", + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 16, - .platform_enable = am3517_evm_panel_enable_lcd, - .platform_disable = am3517_evm_panel_disable_lcd, }; static int am3517_evm_panel_enable_tv(struct omap_dss_device *dssdev) @@ -346,13 +352,18 @@ static void am3517_evm_panel_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = am3517_evm_panel_enable_dvi, + .platform_disable = am3517_evm_panel_disable_dvi, +}; + static struct omap_dss_device am3517_evm_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = am3517_evm_panel_enable_dvi, - .platform_disable = am3517_evm_panel_disable_dvi, }; static struct omap_dss_device *am3517_evm_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c index 486a3de..dac1416 100644 --- a/arch/arm/mach-omap2/board-cm-t35.c +++ b/arch/arm/mach-omap2/board-cm-t35.c @@ -46,6 +46,7 @@ #include <plat/gpmc.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <mach/hardware.h> @@ -351,22 +352,32 @@ static void cm_t35_panel_disable_tv(struct omap_dss_device *dssdev) { } +static struct panel_generic_dpi_data lcd_panel = { + .name = "toppoly_tdo35s", + .platform_enable = cm_t35_panel_enable_lcd, + .platform_disable = cm_t35_panel_disable_lcd, +}; + static struct omap_dss_device cm_t35_lcd_device = { .name = "lcd", - .driver_name = "toppoly_tdo35s_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 18, - .platform_enable = cm_t35_panel_enable_lcd, - .platform_disable = cm_t35_panel_disable_lcd, +}; + +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = cm_t35_panel_enable_dvi, + .platform_disable = cm_t35_panel_disable_dvi, }; static struct omap_dss_device cm_t35_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = cm_t35_panel_enable_dvi, - .platform_disable = cm_t35_panel_disable_dvi, }; static struct omap_dss_device cm_t35_tv_device = { diff --git a/arch/arm/mach-omap2/board-devkit8000.c b/arch/arm/mach-omap2/board-devkit8000.c index 451e7ff..00bb1fc 100644 --- a/arch/arm/mach-omap2/board-devkit8000.c +++ b/arch/arm/mach-omap2/board-devkit8000.c @@ -46,6 +46,7 @@ #include <plat/nand.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <linux/input/matrix_keypad.h> @@ -149,23 +150,32 @@ static struct regulator_consumer_supply devkit8000_vmmc1_supply = static struct regulator_consumer_supply devkit8000_vio_supply = REGULATOR_SUPPLY("vcc", "spi2.0"); +static struct panel_generic_dpi_data lcd_panel = { + .name = "generic", + .platform_enable = devkit8000_panel_enable_lcd, + .platform_disable = devkit8000_panel_disable_lcd, +}; + static struct omap_dss_device devkit8000_lcd_device = { .name = "lcd", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 24, - .reset_gpio = -EINVAL, /* will be replaced */ - .platform_enable = devkit8000_panel_enable_lcd, - .platform_disable = devkit8000_panel_disable_lcd, }; + +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = devkit8000_panel_enable_dvi, + .platform_disable = devkit8000_panel_disable_dvi, +}; + static struct omap_dss_device devkit8000_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .reset_gpio = -EINVAL, /* will be replaced */ - .platform_enable = devkit8000_panel_enable_dvi, - .platform_disable = devkit8000_panel_disable_dvi, }; static struct omap_dss_device devkit8000_tv_device = { diff --git a/arch/arm/mach-omap2/board-igep0020.c b/arch/arm/mach-omap2/board-igep0020.c index 0afa301..ebaa230 100644 --- a/arch/arm/mach-omap2/board-igep0020.c +++ b/arch/arm/mach-omap2/board-igep0020.c @@ -31,6 +31,7 @@ #include <plat/gpmc.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/onenand.h> #include "mux.h" @@ -459,13 +460,18 @@ static void igep2_disable_dvi(struct omap_dss_device *dssdev) gpio_direction_output(IGEP2_GPIO_DVI_PUP, 0); } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = igep2_enable_dvi, + .platform_disable = igep2_disable_dvi, +}; + static struct omap_dss_device igep2_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = igep2_enable_dvi, - .platform_disable = igep2_disable_dvi, }; static struct omap_dss_device *igep2_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-omap3beagle.c b/arch/arm/mach-omap2/board-omap3beagle.c index 6c12760..a4fe8e1 100644 --- a/arch/arm/mach-omap2/board-omap3beagle.c +++ b/arch/arm/mach-omap2/board-omap3beagle.c @@ -41,6 +41,7 @@ #include <plat/board.h> #include <plat/common.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/gpmc.h> #include <plat/nand.h> #include <plat/usb.h> @@ -194,14 +195,19 @@ static void beagle_disable_dvi(struct omap_dss_device *dssdev) gpio_set_value(dssdev->reset_gpio, 0); } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = beagle_enable_dvi, + .platform_disable = beagle_disable_dvi, +}; + static struct omap_dss_device beagle_dvi_device = { .type = OMAP_DISPLAY_TYPE_DPI, .name = "dvi", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, .reset_gpio = 170, - .platform_enable = beagle_enable_dvi, - .platform_disable = beagle_disable_dvi, }; static struct omap_dss_device beagle_tv_device = { diff --git a/arch/arm/mach-omap2/board-omap3evm.c b/arch/arm/mach-omap2/board-omap3evm.c index 3de8d9b..323c380 100644 --- a/arch/arm/mach-omap2/board-omap3evm.c +++ b/arch/arm/mach-omap2/board-omap3evm.c @@ -43,6 +43,7 @@ #include <plat/common.h> #include <plat/mcspi.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include "mux.h" #include "sdram-micron-mt46h32m32lf-6.h" @@ -301,13 +302,18 @@ static void omap3_evm_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = omap3_evm_enable_dvi, + .platform_disable = omap3_evm_disable_dvi, +}; + static struct omap_dss_device omap3_evm_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = omap3_evm_enable_dvi, - .platform_disable = omap3_evm_disable_dvi, }; static struct omap_dss_device *omap3_evm_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-omap3stalker.c b/arch/arm/mach-omap2/board-omap3stalker.c index 9df9d93..2a2dad4 100644 --- a/arch/arm/mach-omap2/board-omap3stalker.c +++ b/arch/arm/mach-omap2/board-omap3stalker.c @@ -40,6 +40,7 @@ #include <plat/nand.h> #include <plat/usb.h> #include <plat/display.h> +#include <plat/panel-generic-dpi.h> #include <plat/mcspi.h> #include <linux/input/matrix_keypad.h> @@ -160,13 +161,18 @@ static void omap3_stalker_disable_lcd(struct omap_dss_device *dssdev) lcd_enabled = 0; } +static struct panel_generic_dpi_data lcd_panel = { + .name = "generic", + .platform_enable = omap3_stalker_enable_lcd, + .platform_disable = omap3_stalker_disable_lcd, +}; + static struct omap_dss_device omap3_stalker_lcd_device = { .name = "lcd", - .driver_name = "generic_panel", + .driver_name = "generic_dpi_panel", + .data = &lcd_panel, .phy.dpi.data_lines = 24, .type = OMAP_DISPLAY_TYPE_DPI, - .platform_enable = omap3_stalker_enable_lcd, - .platform_disable = omap3_stalker_disable_lcd, }; static int omap3_stalker_enable_tv(struct omap_dss_device *dssdev) @@ -208,13 +214,18 @@ static void omap3_stalker_disable_dvi(struct omap_dss_device *dssdev) dvi_enabled = 0; } +static struct panel_generic_dpi_data dvi_panel = { + .name = "generic", + .platform_enable = omap3_stalker_enable_dvi, + .platform_disable = omap3_stalker_disable_dvi, +}; + static struct omap_dss_device omap3_stalker_dvi_device = { .name = "dvi", - .driver_name = "generic_panel", .type = OMAP_DISPLAY_TYPE_DPI, + .driver_name = "generic_dpi_panel", + .data = &dvi_panel, .phy.dpi.data_lines = 24, - .platform_enable = omap3_stalker_enable_dvi, - .platform_disable = omap3_stalker_disable_dvi, }; static struct omap_dss_device *omap3_stalker_dss_devices[] = { diff --git a/arch/arm/mach-omap2/board-zoom-display.c b/arch/arm/mach-omap2/board-zoom-display.c new file mode 100644 index 0000000..6bcd436 --- /dev/null +++ b/arch/arm/mach-omap2/board-zoom-display.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2010 Texas Instruments Inc. + * + * Modified from mach-omap2/board-zoom-peripherals.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/gpio.h> +#include <linux/i2c/twl.h> +#include <linux/spi/spi.h> +#include <plat/mcspi.h> +#include <plat/display.h> + +#define LCD_PANEL_RESET_GPIO_PROD 96 +#define LCD_PANEL_RESET_GPIO_PILOT 55 +#define LCD_PANEL_QVGA_GPIO 56 + +static void zoom_lcd_panel_init(void) +{ + int ret; + unsigned char lcd_panel_reset_gpio; + + lcd_panel_reset_gpio = (omap_rev() > OMAP3430_REV_ES3_0) ? + LCD_PANEL_RESET_GPIO_PROD : + LCD_PANEL_RESET_GPIO_PILOT; + + ret = gpio_request(lcd_panel_reset_gpio, "lcd reset"); + if (ret) { + pr_err("Failed to get LCD reset GPIO (gpio%d).\n", + lcd_panel_reset_gpio); + return; + } + gpio_direction_output(lcd_panel_reset_gpio, 1); + + ret = gpio_request(LCD_PANEL_QVGA_GPIO, "lcd qvga"); + if (ret) { + pr_err("Failed to get LCD_PANEL_QVGA_GPIO (gpio%d).\n", + LCD_PANEL_QVGA_GPIO); + goto err0; + } + gpio_direction_output(LCD_PANEL_QVGA_GPIO, 1); + + return; +err0: + gpio_free(lcd_panel_reset_gpio); +} + +static int zoom_panel_enable_lcd(struct omap_dss_device *dssdev) +{ + return 0; +} + +static void zoom_panel_disable_lcd(struct omap_dss_device *dssdev) +{ +} + +/* + * PWMA/B register offsets (TWL4030_MODULE_PWMA) + */ +#define TWL_INTBR_PMBR1 0xD +#define TWL_INTBR_GPBR1 0xC +#define TWL_LED_PWMON 0x0 +#define TWL_LED_PWMOFF 0x1 + +static int zoom_set_bl_intensity(struct omap_dss_device *dssdev, int level) +{ + unsigned char c; + u8 mux_pwm, enb_pwm; + + if (level > 100) + return -1; + + twl_i2c_read_u8(TWL4030_MODULE_INTBR, &mux_pwm, TWL_INTBR_PMBR1); + twl_i2c_read_u8(TWL4030_MODULE_INTBR, &enb_pwm, TWL_INTBR_GPBR1); + + if (level == 0) { + /* disable pwm1 output and clock */ + enb_pwm = enb_pwm & 0xF5; + /* change pwm1 pin to gpio pin */ + mux_pwm = mux_pwm & 0xCF; + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + enb_pwm, TWL_INTBR_GPBR1); + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + mux_pwm, TWL_INTBR_PMBR1); + return 0; + } + + if (!((enb_pwm & 0xA) && (mux_pwm & 0x30))) { + /* change gpio pin to pwm1 pin */ + mux_pwm = mux_pwm | 0x30; + /* enable pwm1 output and clock*/ + enb_pwm = enb_pwm | 0x0A; + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + mux_pwm, TWL_INTBR_PMBR1); + twl_i2c_write_u8(TWL4030_MODULE_INTBR, + enb_pwm, TWL_INTBR_GPBR1); + } + + c = ((50 * (100 - level)) / 100) + 1; + twl_i2c_write_u8(TWL4030_MODULE_PWM1, 0x7F, TWL_LED_PWMOFF); + twl_i2c_write_u8(TWL4030_MODULE_PWM1, c, TWL_LED_PWMON); + + return 0; +} + +static struct omap_dss_device zoom_lcd_device = { + .name = "lcd", + .driver_name = "NEC_8048_panel", + .type = OMAP_DISPLAY_TYPE_DPI, + .phy.dpi.data_lines = 24, + .platform_enable = zoom_panel_enable_lcd, + .platform_disable = zoom_panel_disable_lcd, + .max_backlight_level = 100, + .set_backlight = zoom_set_bl_intensity, +}; + +static struct omap_dss_device *zoom_dss_devices[] = { + &zoom_lcd_device, +}; + +static struct omap_dss_board_info zoom_dss_data = { + .num_devices = ARRAY_SIZE(zoom_dss_devices), + .devices = zoom_dss_devices, + .default_device = &zoom_lcd_device, +}; + +static struct platform_device zoom_dss_device = { + .name = "omapdss", + .id = -1, + .dev = { + .platform_data = &zoom_dss_data, + }, +}; + +static struct omap2_mcspi_device_config dss_lcd_mcspi_config = { + .turbo_mode = 1, + .single_channel = 1, /* 0: slave, 1: master */ +}; + +static struct spi_board_info nec_8048_spi_board_info[] __initdata = { + [0] = { + .modalias = "nec_8048_spi", + .bus_num = 1, + .chip_select = 2, + .max_speed_hz = 375000, + .controller_data = &dss_lcd_mcspi_config, + }, +}; + +static struct platform_device *zoom_display_devices[] __initdata = { + &zoom_dss_device, +}; + +void __init zoom_display_init(void) +{ + platform_add_devices(zoom_display_devices, + ARRAY_SIZE(zoom_display_devices)); + spi_register_board_info(nec_8048_spi_board_info, + ARRAY_SIZE(nec_8048_spi_board_info)); + zoom_lcd_panel_init(); +} + diff --git a/arch/arm/mach-omap2/board-zoom-peripherals.c b/arch/arm/mach-omap2/board-zoom-peripherals.c index 3fbd0ed..14d95af 100644 --- a/arch/arm/mach-omap2/board-zoom-peripherals.c +++ b/arch/arm/mach-omap2/board-zoom-peripherals.c @@ -35,6 +35,8 @@ #define OMAP_ZOOM_WLAN_PMENA_GPIO (101) #define OMAP_ZOOM_WLAN_IRQ_GPIO (162) +#define LCD_PANEL_ENABLE_GPIO (7 + OMAP_MAX_GPIO_LINES) + /* Zoom2 has Qwerty keyboard*/ static uint32_t board_keymap[] = { KEY(0, 0, KEY_E), @@ -224,9 +226,43 @@ static struct omap2_hsmmc_info mmc[] = { {} /* Terminator */ }; +static struct regulator_consumer_supply zoom_vpll2_supply = + REGULATOR_SUPPLY("vdds_dsi", "omapdss"); + +static struct regulator_consumer_supply zoom_vdda_dac_supply = + REGULATOR_SUPPLY("vdda_dac", "omapdss"); + +static struct regulator_init_data zoom_vpll2 = { + .constraints = { + .min_uV = 1800000, + .max_uV = 1800000, + .valid_modes_mask = REGULATOR_MODE_NORMAL + | REGULATOR_MODE_STANDBY, + .valid_ops_mask = REGULATOR_CHANGE_MODE + | REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = 1, + .consumer_supplies = &zoom_vpll2_supply, +}; + +static struct regulator_init_data zoom_vdac = { + .constraints = { + .min_uV = 1800000, + .max_uV = 1800000, + .valid_modes_mask = REGULATOR_MODE_NORMAL + | REGULATOR_MODE_STANDBY, + .valid_ops_mask = REGULATOR_CHANGE_MODE + | REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = 1, + .consumer_supplies = &zoom_vdda_dac_supply, +}; + static int zoom_twl_gpio_setup(struct device *dev, unsigned gpio, unsigned ngpio) { + int ret; + /* gpio + 0 is "mmc0_cd" (input/IRQ) */ mmc[0].gpio_cd = gpio + 0; omap2_hsmmc_init(mmc); @@ -238,7 +274,15 @@ static int zoom_twl_gpio_setup(struct device *dev, zoom_vsim_supply.dev = mmc[0].dev; zoom_vmmc2_supply.dev = mmc[1].dev; - return 0; + ret = gpio_request(LCD_PANEL_ENABLE_GPIO, "lcd enable"); + if (ret) { + pr_err("Failed to get LCD_PANEL_ENABLE_GPIO (gpio%d).\n", + LCD_PANEL_ENABLE_GPIO); + return ret; + } + gpio_direction_output(LCD_PANEL_ENABLE_GPIO, 0); + + return ret; } /* EXTMUTE callback function */ @@ -301,7 +345,8 @@ static struct twl4030_platform_data zoom_twldata = { .vmmc1 = &zoom_vmmc1, .vmmc2 = &zoom_vmmc2, .vsim = &zoom_vsim, - + .vpll2 = &zoom_vpll2, + .vdac = &zoom_vdac, }; static struct i2c_board_info __initdata zoom_i2c_boardinfo[] = { diff --git a/arch/arm/mach-omap2/board-zoom.c b/arch/arm/mach-omap2/board-zoom.c index e041c53..e26754c 100644 --- a/arch/arm/mach-omap2/board-zoom.c +++ b/arch/arm/mach-omap2/board-zoom.c @@ -130,6 +130,7 @@ static void __init omap_zoom_init(void) ARRAY_SIZE(zoom_nand_partitions), ZOOM_NAND_CS); zoom_debugboard_init(); zoom_peripherals_init(); + zoom_display_init(); } MACHINE_START(OMAP_ZOOM2, "OMAP Zoom2 board") diff --git a/arch/arm/mach-omap2/cpuidle34xx.c b/arch/arm/mach-omap2/cpuidle34xx.c index f3e043f..11b89e9 100644 --- a/arch/arm/mach-omap2/cpuidle34xx.c +++ b/arch/arm/mach-omap2/cpuidle34xx.c @@ -252,7 +252,7 @@ static int omap3_enter_idle_bm(struct cpuidle_device *dev, * FIXME: we currently manage device-specific idle states * for PER and CORE in combination with CPU-specific * idle states. This is wrong, and device-specific - * idle managment needs to be separated out into + * idle management needs to be separated out into * its own code. */ diff --git a/arch/arm/mach-omap2/include/mach/board-zoom.h b/arch/arm/mach-omap2/include/mach/board-zoom.h index f93ca39..d20bd9c 100644 --- a/arch/arm/mach-omap2/include/mach/board-zoom.h +++ b/arch/arm/mach-omap2/include/mach/board-zoom.h @@ -1,9 +1,12 @@ /* * Defines for zoom boards */ +#include <plat/display.h> + #define ZOOM_NAND_CS 0 extern int __init zoom_debugboard_init(void); extern void __init zoom_peripherals_init(void); +extern void __init zoom_display_init(void); #define ZOOM2_HEADSET_EXTMUTE_GPIO 153 diff --git a/arch/arm/mach-omap2/pm24xx.c b/arch/arm/mach-omap2/pm24xx.c index dac2d1d..9e5dc8e 100644 --- a/arch/arm/mach-omap2/pm24xx.c +++ b/arch/arm/mach-omap2/pm24xx.c @@ -350,7 +350,7 @@ static void omap2_pm_end(void) enable_hlt(); } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap2_pm_begin, .enter = omap2_pm_enter, .end = omap2_pm_end, diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c index 5b323f2..8cbbead 100644 --- a/arch/arm/mach-omap2/pm34xx.c +++ b/arch/arm/mach-omap2/pm34xx.c @@ -605,7 +605,7 @@ static void omap3_pm_end(void) return; } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap3_pm_begin, .end = omap3_pm_end, .enter = omap3_pm_enter, diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c index e9f4862c..76cfff2 100644 --- a/arch/arm/mach-omap2/pm44xx.c +++ b/arch/arm/mach-omap2/pm44xx.c @@ -65,7 +65,7 @@ static void omap4_pm_end(void) return; } -static struct platform_suspend_ops omap_pm_ops = { +static const struct platform_suspend_ops omap_pm_ops = { .begin = omap4_pm_begin, .end = omap4_pm_end, .enter = omap4_pm_enter, diff --git a/arch/arm/mach-omap2/serial.c b/arch/arm/mach-omap2/serial.c index c645788..302da74 100644 --- a/arch/arm/mach-omap2/serial.c +++ b/arch/arm/mach-omap2/serial.c @@ -852,7 +852,7 @@ void __init omap_serial_init_port(struct omap_board_data *bdata) } /** - * omap_serial_init() - intialize all supported serial ports + * omap_serial_init() - initialize all supported serial ports * * Initializes all available UARTs as serial ports. Platforms * can call this function when they want to have default behaviour diff --git a/arch/arm/mach-pnx4008/pm.c b/arch/arm/mach-pnx4008/pm.c index ee3c29c..f3e60a0 100644 --- a/arch/arm/mach-pnx4008/pm.c +++ b/arch/arm/mach-pnx4008/pm.c @@ -119,7 +119,7 @@ static int pnx4008_pm_valid(suspend_state_t state) (state == PM_SUSPEND_MEM); } -static struct platform_suspend_ops pnx4008_pm_ops = { +static const struct platform_suspend_ops pnx4008_pm_ops = { .enter = pnx4008_pm_enter, .valid = pnx4008_pm_valid, }; diff --git a/arch/arm/mach-pxa/mxm8x10.c b/arch/arm/mach-pxa/mxm8x10.c index 462167a..cdf7f41 100644 --- a/arch/arm/mach-pxa/mxm8x10.c +++ b/arch/arm/mach-pxa/mxm8x10.c @@ -337,7 +337,7 @@ void __init mxm_8x10_mmc_init(void) } #endif -/* USB Open Host Controler Interface */ +/* USB Open Host Controller Interface */ static struct pxaohci_platform_data mxm_8x10_ohci_platform_data = { .port_mode = PMM_NPS_MODE, .flags = ENABLE_PORT_ALL diff --git a/arch/arm/mach-pxa/pm.c b/arch/arm/mach-pxa/pm.c index 166c15f..978e1b2 100644 --- a/arch/arm/mach-pxa/pm.c +++ b/arch/arm/mach-pxa/pm.c @@ -96,7 +96,7 @@ void pxa_pm_finish(void) pxa_cpu_pm_fns->finish(); } -static struct platform_suspend_ops pxa_pm_ops = { +static const struct platform_suspend_ops pxa_pm_ops = { .valid = pxa_pm_valid, .enter = pxa_pm_enter, .prepare = pxa_pm_prepare, diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c index e68d46d..785880f 100644 --- a/arch/arm/mach-pxa/sharpsl_pm.c +++ b/arch/arm/mach-pxa/sharpsl_pm.c @@ -869,7 +869,7 @@ static void sharpsl_apm_get_power_status(struct apm_power_info *info) } #ifdef CONFIG_PM -static struct platform_suspend_ops sharpsl_pm_ops = { +static const struct platform_suspend_ops sharpsl_pm_ops = { .prepare = pxa_pm_prepare, .finish = pxa_pm_finish, .enter = corgi_pxa_pm_enter, diff --git a/arch/arm/mach-s3c2410/mach-h1940.c b/arch/arm/mach-s3c2410/mach-h1940.c index d7ada8c..1a81fe1 100644 --- a/arch/arm/mach-s3c2410/mach-h1940.c +++ b/arch/arm/mach-s3c2410/mach-h1940.c @@ -387,7 +387,7 @@ static struct platform_device *h1940_devices[] __initdata = { &s3c_device_wdt, &s3c_device_i2c0, &s3c_device_iis, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c_device_usbgadget, &h1940_device_leds, &h1940_device_bluetooth, diff --git a/arch/arm/mach-s3c2440/mach-rx1950.c b/arch/arm/mach-s3c2440/mach-rx1950.c index e0622bb..eab6ae5 100644 --- a/arch/arm/mach-s3c2440/mach-rx1950.c +++ b/arch/arm/mach-s3c2440/mach-rx1950.c @@ -692,7 +692,7 @@ static struct platform_device *rx1950_devices[] __initdata = { &s3c_device_wdt, &s3c_device_i2c0, &s3c_device_iis, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c_device_usbgadget, &s3c_device_rtc, &s3c_device_nand, diff --git a/arch/arm/mach-s3c64xx/clock.c b/arch/arm/mach-s3c64xx/clock.c index 7e03f0a..1c98d2ff 100644 --- a/arch/arm/mach-s3c64xx/clock.c +++ b/arch/arm/mach-s3c64xx/clock.c @@ -695,7 +695,7 @@ static struct clksrc_clk clksrcs[] = { }, { .clk = { .name = "audio-bus", - .id = -1, /* There's only one IISv4 port */ + .id = 2, .ctrlbit = S3C6410_CLKCON_SCLK_AUDIO2, .enable = s3c64xx_sclk_ctrl, }, diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c index 76426a3..cad6702 100644 --- a/arch/arm/mach-s3c64xx/dev-audio.c +++ b/arch/arm/mach-s3c64xx/dev-audio.c @@ -22,7 +22,12 @@ #include <plat/audio.h> #include <plat/gpio-cfg.h> -static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) +static const char *rclksrc[] = { + [0] = "iis", + [1] = "audio-bus", +}; + +static int s3c64xx_i2s_cfg_gpio(struct platform_device *pdev) { unsigned int base; @@ -33,6 +38,12 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) case 1: base = S3C64XX_GPE(0); break; + case 2: + s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5)); + s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5)); + return 0; default: printk(KERN_DEBUG "Invalid I2S Controller number: %d\n", pdev->id); @@ -44,16 +55,6 @@ static int s3c64xx_i2sv3_cfg_gpio(struct platform_device *pdev) return 0; } -static int s3c64xx_i2sv4_cfg_gpio(struct platform_device *pdev) -{ - s3c_gpio_cfgpin(S3C64XX_GPC(4), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin(S3C64XX_GPC(5), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin(S3C64XX_GPC(7), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S3C64XX_GPH(6), 4, S3C_GPIO_SFN(5)); - - return 0; -} - static struct resource s3c64xx_iis0_resource[] = { [0] = { .start = S3C64XX_PA_IIS0, @@ -72,17 +73,22 @@ static struct resource s3c64xx_iis0_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2s0_pdata = { - .cfg_gpio = s3c64xx_i2sv3_cfg_gpio, +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .type = { + .i2s = { + .src_clk = rclksrc, + }, + }, }; struct platform_device s3c64xx_device_iis0 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 0, .num_resources = ARRAY_SIZE(s3c64xx_iis0_resource), .resource = s3c64xx_iis0_resource, .dev = { - .platform_data = &s3c_i2s0_pdata, + .platform_data = &i2sv3_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis0); @@ -105,17 +111,13 @@ static struct resource s3c64xx_iis1_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2s1_pdata = { - .cfg_gpio = s3c64xx_i2sv3_cfg_gpio, -}; - struct platform_device s3c64xx_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s3c64xx_iis1_resource), .resource = s3c64xx_iis1_resource, .dev = { - .platform_data = &s3c_i2s1_pdata, + .platform_data = &i2sv3_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis1); @@ -138,17 +140,23 @@ static struct resource s3c64xx_iisv4_resource[] = { }, }; -static struct s3c_audio_pdata s3c_i2sv4_pdata = { - .cfg_gpio = s3c64xx_i2sv4_cfg_gpio, +static struct s3c_audio_pdata i2sv4_pdata = { + .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN, + .src_clk = rclksrc, + }, + }, }; struct platform_device s3c64xx_device_iisv4 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 2, .num_resources = ARRAY_SIZE(s3c64xx_iisv4_resource), .resource = s3c64xx_iisv4_resource, .dev = { - .platform_data = &s3c_i2sv4_pdata, + .platform_data = &i2sv4_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iisv4); @@ -288,7 +296,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata; static u64 s3c64xx_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s3c64xx_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s3c64xx_ac97_resource), .resource = s3c64xx_ac97_resource, @@ -307,16 +315,3 @@ void __init s3c64xx_ac97_setup_gpio(int num) else s3c_ac97_pdata.cfg_gpio = s3c64xx_ac97_cfg_gpe; } - -static u64 s3c_device_audio_dmamask = 0xffffffffUL; - -struct platform_device s3c_device_pcm = { - .name = "s3c24xx-pcm-audio", - .id = -1, - .dev = { - .dma_mask = &s3c_device_audio_dmamask, - .coherent_dma_mask = 0xffffffffUL - } -}; -EXPORT_SYMBOL(s3c_device_pcm); - diff --git a/arch/arm/mach-s3c64xx/dma.c b/arch/arm/mach-s3c64xx/dma.c index e7d03ab..372ea68 100644 --- a/arch/arm/mach-s3c64xx/dma.c +++ b/arch/arm/mach-s3c64xx/dma.c @@ -740,7 +740,7 @@ static int __init s3c64xx_dma_init(void) /* Set all DMA configuration to be DMA, not SDMA */ writel(0xffffff, S3C_SYSREG(0x110)); - /* Register standard DMA controlers */ + /* Register standard DMA controllers */ s3c64xx_dma_init1(0, DMACH_UART0, IRQ_DMA0, 0x75000000); s3c64xx_dma_init1(8, DMACH_PCM1_TX, IRQ_DMA1, 0x75100000); diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c index 77488fa..e85192a 100644 --- a/arch/arm/mach-s3c64xx/mach-smdk6410.c +++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c @@ -283,7 +283,7 @@ static struct platform_device *smdk6410_devices[] __initdata = { &s3c_device_fb, &s3c_device_ohci, &s3c_device_usb_hsotg, - &s3c_device_pcm, + &samsung_asoc_dma, &s3c64xx_device_iisv4, &samsung_device_keypad, diff --git a/arch/arm/mach-s5p6442/dev-audio.c b/arch/arm/mach-s5p6442/dev-audio.c index 3462197..8719dc4 100644 --- a/arch/arm/mach-s5p6442/dev-audio.c +++ b/arch/arm/mach-s5p6442/dev-audio.c @@ -29,7 +29,7 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev) base = S5P6442_GPC1(0); break; - case -1: + case 0: base = S5P6442_GPC0(0); break; @@ -42,8 +42,19 @@ static int s5p6442_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static const char *rclksrc_v35[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + +static struct s3c_audio_pdata i2sv35_pdata = { .cfg_gpio = s5p6442_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_SEC_DAI | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc_v35, + }, + }, }; static struct resource s5p6442_iis0_resource[] = { @@ -62,15 +73,34 @@ static struct resource s5p6442_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5p6442_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p6442_iis0_resource), .resource = s5p6442_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv35_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "sclk_audio", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5p6442_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -93,12 +123,12 @@ static struct resource s5p6442_iis1_resource[] = { }; struct platform_device s5p6442_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5p6442_iis1_resource), .resource = s5p6442_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; diff --git a/arch/arm/mach-s5p64x0/clock-s5p6440.c b/arch/arm/mach-s5p64x0/clock-s5p6440.c index e4883dc..409c5fc 100644 --- a/arch/arm/mach-s5p64x0/clock-s5p6440.c +++ b/arch/arm/mach-s5p64x0/clock-s5p6440.c @@ -261,7 +261,7 @@ static struct clk init_clocks_disable[] = { .enable = s5p64x0_pclk_ctrl, .ctrlbit = (1 << 25), }, { - .name = "i2s_v40", + .name = "iis", .id = 0, .parent = &clk_pclk_low.clk, .enable = s5p64x0_pclk_ctrl, diff --git a/arch/arm/mach-s5p64x0/clock-s5p6450.c b/arch/arm/mach-s5p64x0/clock-s5p6450.c index 7dbf3c9..7fc6abd 100644 --- a/arch/arm/mach-s5p64x0/clock-s5p6450.c +++ b/arch/arm/mach-s5p64x0/clock-s5p6450.c @@ -256,7 +256,7 @@ static struct clk init_clocks_disable[] = { .ctrlbit = (1 << 22), }, { .name = "iis", - .id = -1, + .id = 0, .parent = &clk_pclk_low.clk, .enable = s5p64x0_pclk_ctrl, .ctrlbit = (1 << 26), diff --git a/arch/arm/mach-s5p64x0/dev-audio.c b/arch/arm/mach-s5p64x0/dev-audio.c index 396bacc..14f89e73 100644 --- a/arch/arm/mach-s5p64x0/dev-audio.c +++ b/arch/arm/mach-s5p64x0/dev-audio.c @@ -19,15 +19,19 @@ #include <mach/dma.h> #include <mach/irqs.h> -static int s5p6440_cfg_i2s(struct platform_device *pdev) +static const char *rclksrc[] = { + [0] = "iis", + [1] = "sclk_audio2", +}; + +static int s5p64x0_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { - case -1: + case 0: s3c_gpio_cfgpin_range(S5P6440_GPR(4), 5, S3C_GPIO_SFN(5)); s3c_gpio_cfgpin_range(S5P6440_GPR(13), 2, S3C_GPIO_SFN(5)); break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -36,31 +40,14 @@ static int s5p6440_cfg_i2s(struct platform_device *pdev) return 0; } -static int s5p6450_cfg_i2s(struct platform_device *pdev) -{ - /* configure GPIO for i2s port */ - switch (pdev->id) { - case -1: - s3c_gpio_cfgpin(S5P6450_GPB(4), S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S5P6450_GPR(4), 5, S3C_GPIO_SFN(5)); - s3c_gpio_cfgpin_range(S5P6450_GPR(13), 2, S3C_GPIO_SFN(5)); - - break; - - default: - printk(KERN_ERR "Invalid Device %d\n", pdev->id); - return -EINVAL; - } - - return 0; -} - -static struct s3c_audio_pdata s5p6440_i2s_pdata = { - .cfg_gpio = s5p6440_cfg_i2s, -}; - -static struct s3c_audio_pdata s5p6450_i2s_pdata = { - .cfg_gpio = s5p6450_cfg_i2s, +static struct s3c_audio_pdata s5p64x0_i2s_pdata = { + .cfg_gpio = s5p64x0_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN, + .src_clk = rclksrc, + }, + }, }; static struct resource s5p64x0_iis0_resource[] = { @@ -82,22 +69,22 @@ static struct resource s5p64x0_iis0_resource[] = { }; struct platform_device s5p6440_device_iis = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p64x0_iis0_resource), .resource = s5p64x0_iis0_resource, .dev = { - .platform_data = &s5p6440_i2s_pdata, + .platform_data = &s5p64x0_i2s_pdata, }, }; struct platform_device s5p6450_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5p64x0_iis0_resource), .resource = s5p64x0_iis0_resource, .dev = { - .platform_data = &s5p6450_i2s_pdata, + .platform_data = &s5p64x0_i2s_pdata, }, }; diff --git a/arch/arm/mach-s5pc100/dev-audio.c b/arch/arm/mach-s5pc100/dev-audio.c index 564e195..ab2d271 100644 --- a/arch/arm/mach-s5pc100/dev-audio.c +++ b/arch/arm/mach-s5pc100/dev-audio.c @@ -23,17 +23,14 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { + case 0: /* Dedicated pins */ + break; case 1: s3c_gpio_cfgpin_range(S5PC100_GPC(0), 5, S3C_GPIO_SFN(2)); break; - case 2: s3c_gpio_cfgpin_range(S5PC100_GPG3(0), 5, S3C_GPIO_SFN(4)); break; - - case -1: /* Dedicated pins */ - break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -42,8 +39,20 @@ static int s5pc100_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static const char *rclksrc_v5[] = { + [0] = "iis", + [1] = "i2sclkd2", +}; + +static struct s3c_audio_pdata i2sv5_pdata = { .cfg_gpio = s5pc100_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc_v5, + }, + }, }; static struct resource s5pc100_iis0_resource[] = { @@ -62,15 +71,34 @@ static struct resource s5pc100_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5pc100_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5pc100_iis0_resource), .resource = s5pc100_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "sclk_audio", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pc100_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -93,12 +121,12 @@ static struct resource s5pc100_iis1_resource[] = { }; struct platform_device s5pc100_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5pc100_iis1_resource), .resource = s5pc100_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -121,12 +149,12 @@ static struct resource s5pc100_iis2_resource[] = { }; struct platform_device s5pc100_device_iis2 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 2, .num_resources = ARRAY_SIZE(s5pc100_iis2_resource), .resource = s5pc100_iis2_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -253,7 +281,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = { static u64 s5pc100_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s5pc100_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s5pc100_ac97_resource), .resource = s5pc100_ac97_resource, diff --git a/arch/arm/mach-s5pc100/mach-smdkc100.c b/arch/arm/mach-s5pc100/mach-smdkc100.c index 18b405d..dd192a2 100644 --- a/arch/arm/mach-s5pc100/mach-smdkc100.c +++ b/arch/arm/mach-s5pc100/mach-smdkc100.c @@ -96,6 +96,7 @@ static struct s3c2410_uartcfg smdkc100_uartcfgs[] __initdata = { /* I2C0 */ static struct i2c_board_info i2c_devs0[] __initdata = { + {I2C_BOARD_INFO("wm8580", 0x1b),}, }; /* I2C1 */ @@ -190,6 +191,7 @@ static struct platform_device *smdkc100_devices[] __initdata = { &s3c_device_ts, &s3c_device_wdt, &smdkc100_lcd_powerdev, + &samsung_asoc_dma, &s5pc100_device_iis0, &samsung_device_keypad, &s5pc100_device_ac97, diff --git a/arch/arm/mach-s5pv210/clock.c b/arch/arm/mach-s5pv210/clock.c index 019c3a6..b774ff1 100644 --- a/arch/arm/mach-s5pv210/clock.c +++ b/arch/arm/mach-s5pv210/clock.c @@ -467,20 +467,20 @@ static struct clk init_clocks_disable[] = { .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1<<21), }, { - .name = "i2s_v50", + .name = "iis", .id = 0, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1<<4), }, { - .name = "i2s_v32", - .id = 0, + .name = "iis", + .id = 1, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1 << 5), }, { - .name = "i2s_v32", - .id = 1, + .name = "iis", + .id = 2, .parent = &clk_p, .enable = s5pv210_clk_ip3_ctrl, .ctrlbit = (1 << 6), diff --git a/arch/arm/mach-s5pv210/dev-audio.c b/arch/arm/mach-s5pv210/dev-audio.c index 1303fcb..8d58f19 100644 --- a/arch/arm/mach-s5pv210/dev-audio.c +++ b/arch/arm/mach-s5pv210/dev-audio.c @@ -19,22 +19,24 @@ #include <mach/dma.h> #include <mach/irqs.h> +static const char *rclksrc[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + static int s5pv210_cfg_i2s(struct platform_device *pdev) { /* configure GPIO for i2s port */ switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2)); + break; case 1: s3c_gpio_cfgpin_range(S5PV210_GPC0(0), 5, S3C_GPIO_SFN(2)); break; - case 2: s3c_gpio_cfgpin_range(S5PV210_GPC1(0), 5, S3C_GPIO_SFN(4)); break; - - case -1: - s3c_gpio_cfgpin_range(S5PV210_GPI(0), 7, S3C_GPIO_SFN(2)); - break; - default: printk(KERN_ERR "Invalid Device %d\n", pdev->id); return -EINVAL; @@ -43,8 +45,15 @@ static int s5pv210_cfg_i2s(struct platform_device *pdev) return 0; } -static struct s3c_audio_pdata s3c_i2s_pdata = { +static struct s3c_audio_pdata i2sv5_pdata = { .cfg_gpio = s5pv210_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc, + }, + }, }; static struct resource s5pv210_iis0_resource[] = { @@ -63,15 +72,34 @@ static struct resource s5pv210_iis0_resource[] = { .end = DMACH_I2S0_RX, .flags = IORESOURCE_DMA, }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, }; struct platform_device s5pv210_device_iis0 = { - .name = "s3c64xx-iis-v4", - .id = -1, + .name = "samsung-i2s", + .id = 0, .num_resources = ARRAY_SIZE(s5pv210_iis0_resource), .resource = s5pv210_iis0_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "iis", + [1] = "audio-bus", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pv210_cfg_i2s, + .type = { + .i2s = { + .src_clk = rclksrc_v3, + }, }, }; @@ -94,12 +122,12 @@ static struct resource s5pv210_iis1_resource[] = { }; struct platform_device s5pv210_device_iis1 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 1, .num_resources = ARRAY_SIZE(s5pv210_iis1_resource), .resource = s5pv210_iis1_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -122,12 +150,12 @@ static struct resource s5pv210_iis2_resource[] = { }; struct platform_device s5pv210_device_iis2 = { - .name = "s3c64xx-iis", + .name = "samsung-i2s", .id = 2, .num_resources = ARRAY_SIZE(s5pv210_iis2_resource), .resource = s5pv210_iis2_resource, .dev = { - .platform_data = &s3c_i2s_pdata, + .platform_data = &i2sv3_pdata, }, }; @@ -283,7 +311,7 @@ static struct s3c_audio_pdata s3c_ac97_pdata = { static u64 s5pv210_ac97_dmamask = DMA_BIT_MASK(32); struct platform_device s5pv210_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s5pv210_ac97_resource), .resource = s5pv210_ac97_resource, diff --git a/arch/arm/mach-s5pv310/Kconfig b/arch/arm/mach-s5pv310/Kconfig index 1150b36..d64efe0 100644 --- a/arch/arm/mach-s5pv310/Kconfig +++ b/arch/arm/mach-s5pv310/Kconfig @@ -11,6 +11,7 @@ if ARCH_S5PV310 config CPU_S5PV310 bool + select S3C_PL330_DMA help Enable S5PV310 CPU support diff --git a/arch/arm/mach-s5pv310/Makefile b/arch/arm/mach-s5pv310/Makefile index 84afc64..61e3cb6 100644 --- a/arch/arm/mach-s5pv310/Makefile +++ b/arch/arm/mach-s5pv310/Makefile @@ -13,7 +13,7 @@ obj- := # Core support for S5PV310 system obj-$(CONFIG_CPU_S5PV310) += cpu.o init.o clock.o irq-combiner.o -obj-$(CONFIG_CPU_S5PV310) += setup-i2c0.o time.o gpiolib.o irq-eint.o +obj-$(CONFIG_CPU_S5PV310) += setup-i2c0.o time.o gpiolib.o irq-eint.o dma.o obj-$(CONFIG_SMP) += platsmp.o headsmp.o obj-$(CONFIG_LOCAL_TIMERS) += localtimer.o @@ -27,6 +27,7 @@ obj-$(CONFIG_MACH_UNIVERSAL_C210) += mach-universal_c210.o # device support +obj-y += dev-audio.o obj-$(CONFIG_S5PV310_SETUP_I2C1) += setup-i2c1.o obj-$(CONFIG_S5PV310_SETUP_I2C2) += setup-i2c2.o obj-$(CONFIG_S5PV310_SETUP_I2C3) += setup-i2c3.o diff --git a/arch/arm/mach-s5pv310/dev-audio.c b/arch/arm/mach-s5pv310/dev-audio.c new file mode 100644 index 0000000..a196424 --- /dev/null +++ b/arch/arm/mach-s5pv310/dev-audio.c @@ -0,0 +1,364 @@ +/* linux/arch/arm/mach-s5pv310/dev-audio.c + * + * Copyright (c) 2010 Samsung Electronics Co. Ltd + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <linux/gpio.h> + +#include <plat/gpio-cfg.h> +#include <plat/audio.h> + +#include <mach/map.h> +#include <mach/dma.h> +#include <mach/irqs.h> + +static const char *rclksrc[] = { + [0] = "busclk", + [1] = "i2sclk", +}; + +static int s5pv310_cfg_i2s(struct platform_device *pdev) +{ + /* configure GPIO for i2s port */ + switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 7, S3C_GPIO_SFN(2)); + break; + case 1: + s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(2)); + break; + case 2: + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(4)); + break; + default: + printk(KERN_ERR "Invalid Device %d\n", pdev->id); + return -EINVAL; + } + + return 0; +} + +static struct s3c_audio_pdata i2sv5_pdata = { + .cfg_gpio = s5pv310_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_PRI_6CHAN | QUIRK_SEC_DAI + | QUIRK_NEED_RSTCLR, + .src_clk = rclksrc, + }, + }, +}; + +static struct resource s5pv310_i2s0_resource[] = { + [0] = { + .start = S5PV310_PA_I2S0, + .end = S5PV310_PA_I2S0 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S0_TX, + .end = DMACH_I2S0_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S0_RX, + .end = DMACH_I2S0_RX, + .flags = IORESOURCE_DMA, + }, + [3] = { + .start = DMACH_I2S0S_TX, + .end = DMACH_I2S0S_TX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s0 = { + .name = "samsung-i2s", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_i2s0_resource), + .resource = s5pv310_i2s0_resource, + .dev = { + .platform_data = &i2sv5_pdata, + }, +}; + +static const char *rclksrc_v3[] = { + [0] = "sclk_i2s", + [1] = "no_such_clock", +}; + +static struct s3c_audio_pdata i2sv3_pdata = { + .cfg_gpio = s5pv310_cfg_i2s, + .type = { + .i2s = { + .quirks = QUIRK_NO_MUXPSR, + .src_clk = rclksrc_v3, + }, + }, +}; + +static struct resource s5pv310_i2s1_resource[] = { + [0] = { + .start = S5PV310_PA_I2S1, + .end = S5PV310_PA_I2S1 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S1_TX, + .end = DMACH_I2S1_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S1_RX, + .end = DMACH_I2S1_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s1 = { + .name = "samsung-i2s", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_i2s1_resource), + .resource = s5pv310_i2s1_resource, + .dev = { + .platform_data = &i2sv3_pdata, + }, +}; + +static struct resource s5pv310_i2s2_resource[] = { + [0] = { + .start = S5PV310_PA_I2S2, + .end = S5PV310_PA_I2S2 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_I2S2_TX, + .end = DMACH_I2S2_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_I2S2_RX, + .end = DMACH_I2S2_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_i2s2 = { + .name = "samsung-i2s", + .id = 2, + .num_resources = ARRAY_SIZE(s5pv310_i2s2_resource), + .resource = s5pv310_i2s2_resource, + .dev = { + .platform_data = &i2sv3_pdata, + }, +}; + +/* PCM Controller platform_devices */ + +static int s5pv310_pcm_cfg_gpio(struct platform_device *pdev) +{ + switch (pdev->id) { + case 0: + s3c_gpio_cfgpin_range(S5PV310_GPZ(0), 5, S3C_GPIO_SFN(3)); + break; + case 1: + s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(3)); + break; + case 2: + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 5, S3C_GPIO_SFN(3)); + break; + default: + printk(KERN_DEBUG "Invalid PCM Controller number!"); + return -EINVAL; + } + + return 0; +} + +static struct s3c_audio_pdata s3c_pcm_pdata = { + .cfg_gpio = s5pv310_pcm_cfg_gpio, +}; + +static struct resource s5pv310_pcm0_resource[] = { + [0] = { + .start = S5PV310_PA_PCM0, + .end = S5PV310_PA_PCM0 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM0_TX, + .end = DMACH_PCM0_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM0_RX, + .end = DMACH_PCM0_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm0 = { + .name = "samsung-pcm", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_pcm0_resource), + .resource = s5pv310_pcm0_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +static struct resource s5pv310_pcm1_resource[] = { + [0] = { + .start = S5PV310_PA_PCM1, + .end = S5PV310_PA_PCM1 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM1_TX, + .end = DMACH_PCM1_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM1_RX, + .end = DMACH_PCM1_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm1 = { + .name = "samsung-pcm", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_pcm1_resource), + .resource = s5pv310_pcm1_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +static struct resource s5pv310_pcm2_resource[] = { + [0] = { + .start = S5PV310_PA_PCM2, + .end = S5PV310_PA_PCM2 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_PCM2_TX, + .end = DMACH_PCM2_TX, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_PCM2_RX, + .end = DMACH_PCM2_RX, + .flags = IORESOURCE_DMA, + }, +}; + +struct platform_device s5pv310_device_pcm2 = { + .name = "samsung-pcm", + .id = 2, + .num_resources = ARRAY_SIZE(s5pv310_pcm2_resource), + .resource = s5pv310_pcm2_resource, + .dev = { + .platform_data = &s3c_pcm_pdata, + }, +}; + +/* AC97 Controller platform devices */ + +static int s5pv310_ac97_cfg_gpio(struct platform_device *pdev) +{ + return s3c_gpio_cfgpin_range(S5PV310_GPC0(0), 5, S3C_GPIO_SFN(4)); +} + +static struct resource s5pv310_ac97_resource[] = { + [0] = { + .start = S5PV310_PA_AC97, + .end = S5PV310_PA_AC97 + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_AC97_PCMOUT, + .end = DMACH_AC97_PCMOUT, + .flags = IORESOURCE_DMA, + }, + [2] = { + .start = DMACH_AC97_PCMIN, + .end = DMACH_AC97_PCMIN, + .flags = IORESOURCE_DMA, + }, + [3] = { + .start = DMACH_AC97_MICIN, + .end = DMACH_AC97_MICIN, + .flags = IORESOURCE_DMA, + }, + [4] = { + .start = IRQ_AC97, + .end = IRQ_AC97, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_audio_pdata s3c_ac97_pdata = { + .cfg_gpio = s5pv310_ac97_cfg_gpio, +}; + +static u64 s5pv310_ac97_dmamask = DMA_BIT_MASK(32); + +struct platform_device s5pv310_device_ac97 = { + .name = "samsung-ac97", + .id = -1, + .num_resources = ARRAY_SIZE(s5pv310_ac97_resource), + .resource = s5pv310_ac97_resource, + .dev = { + .platform_data = &s3c_ac97_pdata, + .dma_mask = &s5pv310_ac97_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + }, +}; + +/* S/PDIF Controller platform_device */ + +static int s5pv310_spdif_cfg_gpio(struct platform_device *pdev) +{ + s3c_gpio_cfgpin_range(S5PV310_GPC1(0), 2, S3C_GPIO_SFN(3)); + + return 0; +} + +static struct resource s5pv310_spdif_resource[] = { + [0] = { + .start = S5PV310_PA_SPDIF, + .end = S5PV310_PA_SPDIF + 0x100 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = DMACH_SPDIF, + .end = DMACH_SPDIF, + .flags = IORESOURCE_DMA, + }, +}; + +static struct s3c_audio_pdata samsung_spdif_pdata = { + .cfg_gpio = s5pv310_spdif_cfg_gpio, +}; + +static u64 s5pv310_spdif_dmamask = DMA_BIT_MASK(32); + +struct platform_device s5pv310_device_spdif = { + .name = "samsung-spdif", + .id = -1, + .num_resources = ARRAY_SIZE(s5pv310_spdif_resource), + .resource = s5pv310_spdif_resource, + .dev = { + .platform_data = &samsung_spdif_pdata, + .dma_mask = &s5pv310_spdif_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + }, +}; diff --git a/arch/arm/mach-s5pv310/dma.c b/arch/arm/mach-s5pv310/dma.c new file mode 100644 index 0000000..20066c7 --- /dev/null +++ b/arch/arm/mach-s5pv310/dma.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2010 Samsung Electronics Co. Ltd. + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> + +#include <plat/devs.h> +#include <plat/irqs.h> + +#include <mach/map.h> +#include <mach/irqs.h> + +#include <plat/s3c-pl330-pdata.h> + +static u64 dma_dmamask = DMA_BIT_MASK(32); + +static struct resource s5pv310_pdma0_resource[] = { + [0] = { + .start = S5PV310_PA_PDMA0, + .end = S5PV310_PA_PDMA0 + SZ_4K, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = IRQ_PDMA0, + .end = IRQ_PDMA0, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_pl330_platdata s5pv310_pdma0_pdata = { + .peri = { + [0] = DMACH_PCM0_RX, + [1] = DMACH_PCM0_TX, + [2] = DMACH_PCM2_RX, + [3] = DMACH_PCM2_TX, + [4] = DMACH_MSM_REQ0, + [5] = DMACH_MSM_REQ2, + [6] = DMACH_SPI0_RX, + [7] = DMACH_SPI0_TX, + [8] = DMACH_SPI2_RX, + [9] = DMACH_SPI2_TX, + [10] = DMACH_I2S0S_TX, + [11] = DMACH_I2S0_RX, + [12] = DMACH_I2S0_TX, + [13] = DMACH_I2S2_RX, + [14] = DMACH_I2S2_TX, + [15] = DMACH_UART0_RX, + [16] = DMACH_UART0_TX, + [17] = DMACH_UART2_RX, + [18] = DMACH_UART2_TX, + [19] = DMACH_UART4_RX, + [20] = DMACH_UART4_TX, + [21] = DMACH_SLIMBUS0_RX, + [22] = DMACH_SLIMBUS0_TX, + [23] = DMACH_SLIMBUS2_RX, + [24] = DMACH_SLIMBUS2_TX, + [25] = DMACH_SLIMBUS4_RX, + [26] = DMACH_SLIMBUS4_TX, + [27] = DMACH_AC97_MICIN, + [28] = DMACH_AC97_PCMIN, + [29] = DMACH_AC97_PCMOUT, + [30] = DMACH_MAX, + [31] = DMACH_MAX, + }, +}; + +static struct platform_device s5pv310_device_pdma0 = { + .name = "s3c-pl330", + .id = 0, + .num_resources = ARRAY_SIZE(s5pv310_pdma0_resource), + .resource = s5pv310_pdma0_resource, + .dev = { + .dma_mask = &dma_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &s5pv310_pdma0_pdata, + }, +}; + +static struct resource s5pv310_pdma1_resource[] = { + [0] = { + .start = S5PV310_PA_PDMA1, + .end = S5PV310_PA_PDMA1 + SZ_4K, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = IRQ_PDMA1, + .end = IRQ_PDMA1, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct s3c_pl330_platdata s5pv310_pdma1_pdata = { + .peri = { + [0] = DMACH_PCM0_RX, + [1] = DMACH_PCM0_TX, + [2] = DMACH_PCM1_RX, + [3] = DMACH_PCM1_TX, + [4] = DMACH_MSM_REQ1, + [5] = DMACH_MSM_REQ3, + [6] = DMACH_SPI1_RX, + [7] = DMACH_SPI1_TX, + [8] = DMACH_I2S0S_TX, + [9] = DMACH_I2S0_RX, + [10] = DMACH_I2S0_TX, + [11] = DMACH_I2S1_RX, + [12] = DMACH_I2S1_TX, + [13] = DMACH_UART0_RX, + [14] = DMACH_UART0_TX, + [15] = DMACH_UART1_RX, + [16] = DMACH_UART1_TX, + [17] = DMACH_UART3_RX, + [18] = DMACH_UART3_TX, + [19] = DMACH_SLIMBUS1_RX, + [20] = DMACH_SLIMBUS1_TX, + [21] = DMACH_SLIMBUS3_RX, + [22] = DMACH_SLIMBUS3_TX, + [23] = DMACH_SLIMBUS5_RX, + [24] = DMACH_SLIMBUS5_TX, + [25] = DMACH_SLIMBUS0AUX_RX, + [26] = DMACH_SLIMBUS0AUX_TX, + [27] = DMACH_SPDIF, + [28] = DMACH_MAX, + [29] = DMACH_MAX, + [30] = DMACH_MAX, + [31] = DMACH_MAX, + }, +}; + +static struct platform_device s5pv310_device_pdma1 = { + .name = "s3c-pl330", + .id = 1, + .num_resources = ARRAY_SIZE(s5pv310_pdma1_resource), + .resource = s5pv310_pdma1_resource, + .dev = { + .dma_mask = &dma_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &s5pv310_pdma1_pdata, + }, +}; + +static struct platform_device *s5pv310_dmacs[] __initdata = { + &s5pv310_device_pdma0, + &s5pv310_device_pdma1, +}; + +static int __init s5pv310_dma_init(void) +{ + platform_add_devices(s5pv310_dmacs, ARRAY_SIZE(s5pv310_dmacs)); + + return 0; +} +arch_initcall(s5pv310_dma_init); diff --git a/arch/arm/mach-s5pv310/include/mach/dma.h b/arch/arm/mach-s5pv310/include/mach/dma.h new file mode 100644 index 0000000..81209eb --- /dev/null +++ b/arch/arm/mach-s5pv310/include/mach/dma.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2010 Samsung Electronics Co. Ltd. + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifndef __MACH_DMA_H +#define __MACH_DMA_H + +/* This platform uses the common S3C DMA API driver for PL330 */ +#include <plat/s3c-dma-pl330.h> + +#endif /* __MACH_DMA_H */ diff --git a/arch/arm/mach-s5pv310/include/mach/irqs.h b/arch/arm/mach-s5pv310/include/mach/irqs.h index 99e7dad..3c05c58 100644 --- a/arch/arm/mach-s5pv310/include/mach/irqs.h +++ b/arch/arm/mach-s5pv310/include/mach/irqs.h @@ -54,6 +54,9 @@ #define COMBINER_GROUP(x) ((x) * MAX_IRQ_IN_COMBINER + IRQ_SPI(64)) #define COMBINER_IRQ(x, y) (COMBINER_GROUP(x) + y) +#define IRQ_PDMA0 COMBINER_IRQ(21, 0) +#define IRQ_PDMA1 COMBINER_IRQ(21, 1) + #define IRQ_TIMER0_VIC COMBINER_IRQ(22, 0) #define IRQ_TIMER1_VIC COMBINER_IRQ(22, 1) #define IRQ_TIMER2_VIC COMBINER_IRQ(22, 2) diff --git a/arch/arm/mach-s5pv310/include/mach/map.h b/arch/arm/mach-s5pv310/include/mach/map.h index 7acf4e7..5399446 100644 --- a/arch/arm/mach-s5pv310/include/mach/map.h +++ b/arch/arm/mach-s5pv310/include/mach/map.h @@ -52,6 +52,11 @@ #define S5PV310_PA_GIC_DIST (0x10501000) #define S5PV310_PA_L2CC (0x10502000) +/* DMA */ +#define S5PV310_PA_MDMA 0x10810000 +#define S5PV310_PA_PDMA0 0x12680000 +#define S5PV310_PA_PDMA1 0x12690000 + #define S5PV310_PA_GPIO1 (0x11400000) #define S5PV310_PA_GPIO2 (0x11000000) #define S5PV310_PA_GPIO3 (0x03860000) @@ -60,6 +65,22 @@ #define S5PV310_PA_SROMC (0x12570000) +/* S/PDIF */ +#define S5PV310_PA_SPDIF 0xE1100000 + +/* I2S */ +#define S5PV310_PA_I2S0 0x03830000 +#define S5PV310_PA_I2S1 0xE3100000 +#define S5PV310_PA_I2S2 0xE2A00000 + +/* PCM */ +#define S5PV310_PA_PCM0 0x03840000 +#define S5PV310_PA_PCM1 0x13980000 +#define S5PV310_PA_PCM2 0x13990000 + +/* AC97 */ +#define S5PV310_PA_AC97 0x139A0000 + #define S5PV310_PA_UART (0x13800000) #define S5P_PA_UART(x) (S5PV310_PA_UART + ((x) * S3C_UART_OFFSET)) diff --git a/arch/arm/mach-sa1100/pm.c b/arch/arm/mach-sa1100/pm.c index c83fdc8..ab9fc44 100644 --- a/arch/arm/mach-sa1100/pm.c +++ b/arch/arm/mach-sa1100/pm.c @@ -120,7 +120,7 @@ unsigned long sleep_phys_sp(void *sp) return virt_to_phys(sp); } -static struct platform_suspend_ops sa11x0_pm_ops = { +static const struct platform_suspend_ops sa11x0_pm_ops = { .enter = sa11x0_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c index 712e433..3cf0951 100644 --- a/arch/arm/mach-shmobile/board-ap4evb.c +++ b/arch/arm/mach-shmobile/board-ap4evb.c @@ -709,6 +709,10 @@ static struct platform_device fsi_device = { }, }; +static struct platform_device fsi_ak4643_device = { + .name = "sh_fsi2_a_ak4643", +}; + static struct sh_mobile_lcdc_info sh_mobile_lcdc1_info = { .clock_source = LCDC_CLK_EXTERNAL, .ch[0] = { @@ -931,6 +935,7 @@ static struct platform_device *ap4evb_devices[] __initdata = { &sdhi1_device, &usb1_host_device, &fsi_device, + &fsi_ak4643_device, &sh_mmcif_device, &lcdc1_device, &lcdc_device, diff --git a/arch/arm/mach-shmobile/setup-sh7367.c b/arch/arm/mach-shmobile/setup-sh7367.c index 003008c..ce28141 100644 --- a/arch/arm/mach-shmobile/setup-sh7367.c +++ b/arch/arm/mach-shmobile/setup-sh7367.c @@ -35,6 +35,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00) }, @@ -52,6 +54,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20) }, @@ -69,6 +73,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40) }, @@ -86,6 +92,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60) }, @@ -103,6 +111,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20) }, @@ -120,6 +130,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40) }, @@ -137,6 +149,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60) }, diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c index 2e3e11e..ff0494f 100644 --- a/arch/arm/mach-shmobile/setup-sh7372.c +++ b/arch/arm/mach-shmobile/setup-sh7372.c @@ -38,6 +38,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c00), evt2irq(0x0c00), evt2irq(0x0c00), evt2irq(0x0c00) }, @@ -55,6 +57,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c20), evt2irq(0x0c20), evt2irq(0x0c20), evt2irq(0x0c20) }, @@ -72,6 +76,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c40), evt2irq(0x0c40), evt2irq(0x0c40), evt2irq(0x0c40) }, @@ -89,6 +95,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0c60), evt2irq(0x0c60), evt2irq(0x0c60), evt2irq(0x0c60) }, @@ -106,6 +114,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0d20), evt2irq(0x0d20), evt2irq(0x0d20), evt2irq(0x0d20) }, @@ -123,6 +133,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { evt2irq(0x0d40), evt2irq(0x0d40), evt2irq(0x0d40), evt2irq(0x0d40) }, @@ -140,6 +152,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFB, .irqs = { evt2irq(0x0d60), evt2irq(0x0d60), evt2irq(0x0d60), evt2irq(0x0d60) }, diff --git a/arch/arm/mach-shmobile/setup-sh7377.c b/arch/arm/mach-shmobile/setup-sh7377.c index 575dbd6..8099b0b 100644 --- a/arch/arm/mach-shmobile/setup-sh7377.c +++ b/arch/arm/mach-shmobile/setup-sh7377.c @@ -36,6 +36,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00), evt2irq(0xc00) }, @@ -53,6 +55,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20), evt2irq(0xc20) }, @@ -70,6 +74,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40), evt2irq(0xc40) }, @@ -87,6 +93,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60), evt2irq(0xc60) }, @@ -104,6 +112,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20), evt2irq(0xd20) }, @@ -121,6 +131,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40), evt2irq(0xd40) }, @@ -138,6 +150,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6cc0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80), intcs_evt2irq(0x1a80) }, @@ -155,6 +169,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60), evt2irq(0xd60) }, diff --git a/arch/arm/mach-shmobile/setup-sh73a0.c b/arch/arm/mach-shmobile/setup-sh73a0.c index f1eff8b..685c40a 100644 --- a/arch/arm/mach-shmobile/setup-sh73a0.c +++ b/arch/arm/mach-shmobile/setup-sh73a0.c @@ -36,6 +36,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xe6c40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(72), gic_spi(72), gic_spi(72), gic_spi(72) }, @@ -52,6 +54,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xe6c50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(73), gic_spi(73), gic_spi(73), gic_spi(73) }, @@ -68,6 +72,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xe6c60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(74), gic_spi(74), gic_spi(74), gic_spi(74) }, @@ -84,6 +90,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xe6c70000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(75), gic_spi(75), gic_spi(75), gic_spi(75) }, @@ -100,6 +108,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xe6c80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(78), gic_spi(78), gic_spi(78), gic_spi(78) }, @@ -116,6 +126,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xe6cb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(79), gic_spi(79), gic_spi(79), gic_spi(79) }, @@ -132,6 +144,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xe6cc0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(156), gic_spi(156), gic_spi(156), gic_spi(156) }, @@ -148,6 +162,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xe6cd0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFA, .irqs = { gic_spi(143), gic_spi(143), gic_spi(143), gic_spi(143) }, @@ -164,6 +180,8 @@ static struct platform_device scif7_device = { static struct plat_sci_port scif8_platform_data = { .mapbase = 0xe6c30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIFB, .irqs = { gic_spi(80), gic_spi(80), gic_spi(80), gic_spi(80) }, diff --git a/arch/arm/mach-spear3xx/spear300.c b/arch/arm/mach-spear3xx/spear300.c index 3560f8c..5aa2d54 100644 --- a/arch/arm/mach-spear3xx/spear300.c +++ b/arch/arm/mach-spear3xx/spear300.c @@ -371,7 +371,7 @@ struct pmx_driver pmx_driver = { }; /* Add spear300 specific devices here */ -/* arm gpio1 device registeration */ +/* arm gpio1 device registration */ static struct pl061_platform_data gpio1_plat_data = { .gpio_base = 8, .irq_base = SPEAR_GPIO1_INT_BASE, @@ -451,7 +451,7 @@ void __init spear300_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ shirq_ras1.regs.base = ioremap(SPEAR300_TELECOM_BASE, SPEAR300_TELECOM_REG_SIZE); if (shirq_ras1.regs.base) { diff --git a/arch/arm/mach-spear3xx/spear310.c b/arch/arm/mach-spear3xx/spear310.c index 96a1ab8..53b41b5 100644 --- a/arch/arm/mach-spear3xx/spear310.c +++ b/arch/arm/mach-spear3xx/spear310.c @@ -266,7 +266,7 @@ void __init spear310_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ base = ioremap(SPEAR310_SOC_CONFIG_BASE, SPEAR310_SOC_CONFIG_SIZE); if (base) { /* shirq 1 */ diff --git a/arch/arm/mach-spear3xx/spear320.c b/arch/arm/mach-spear3xx/spear320.c index 6a12195..88b4652 100644 --- a/arch/arm/mach-spear3xx/spear320.c +++ b/arch/arm/mach-spear3xx/spear320.c @@ -519,7 +519,7 @@ void __init spear320_init(void) /* call spear3xx family common init function */ spear3xx_init(); - /* shared irq registeration */ + /* shared irq registration */ base = ioremap(SPEAR320_SOC_CONFIG_BASE, SPEAR320_SOC_CONFIG_SIZE); if (base) { /* shirq 1 */ diff --git a/arch/arm/mach-spear3xx/spear3xx.c b/arch/arm/mach-spear3xx/spear3xx.c index e87313a..52f553c 100644 --- a/arch/arm/mach-spear3xx/spear3xx.c +++ b/arch/arm/mach-spear3xx/spear3xx.c @@ -22,7 +22,7 @@ #include <mach/spear.h> /* Add spear3xx machines common devices here */ -/* gpio device registeration */ +/* gpio device registration */ static struct pl061_platform_data gpio_plat_data = { .gpio_base = 0, .irq_base = SPEAR_GPIO_INT_BASE, @@ -41,7 +41,7 @@ struct amba_device gpio_device = { .irq = {IRQ_BASIC_GPIO, NO_IRQ}, }; -/* uart device registeration */ +/* uart device registration */ struct amba_device uart_device = { .dev = { .init_name = "uart", @@ -543,6 +543,6 @@ void spear_pmx_init(struct pmx_driver *pmx_driver, uint base, uint size) pmx_fail: if (ret) - printk(KERN_ERR "padmux: registeration failed. err no: %d\n", + printk(KERN_ERR "padmux: registration failed. err no: %d\n", ret); } diff --git a/arch/arm/mach-spear6xx/spear6xx.c b/arch/arm/mach-spear6xx/spear6xx.c index baf6bcc..f2fe14e 100644 --- a/arch/arm/mach-spear6xx/spear6xx.c +++ b/arch/arm/mach-spear6xx/spear6xx.c @@ -23,7 +23,7 @@ #include <mach/spear.h> /* Add spear6xx machines common devices here */ -/* uart device registeration */ +/* uart device registration */ struct amba_device uart_device[] = { { .dev = { @@ -50,7 +50,7 @@ struct amba_device uart_device[] = { } }; -/* gpio device registeration */ +/* gpio device registration */ static struct pl061_platform_data gpio_plat_data[] = { { .gpio_base = 0, diff --git a/arch/arm/mach-u300/Kconfig b/arch/arm/mach-u300/Kconfig index 801b21e..32a7b0f 100644 --- a/arch/arm/mach-u300/Kconfig +++ b/arch/arm/mach-u300/Kconfig @@ -64,7 +64,7 @@ config MACH_U300_DUAL_RAM bool "Dual RAM" help Select this if you want support for Dual RAM phones. - This is two RAM memorys on different EMIFs. + This is two RAM memories on different EMIFs. endchoice config U300_DEBUG diff --git a/arch/arm/mach-u300/include/mach/coh901318.h b/arch/arm/mach-u300/include/mach/coh901318.h index 193da2d..6193aaa 100644 --- a/arch/arm/mach-u300/include/mach/coh901318.h +++ b/arch/arm/mach-u300/include/mach/coh901318.h @@ -24,7 +24,7 @@ * @src_addr: transfer source address * @dst_addr: transfer destination address * @link_addr: physical address to next lli - * @virt_link_addr: virtual addres of next lli (only used by pool_free) + * @virt_link_addr: virtual address of next lli (only used by pool_free) * @phy_this: physical address of current lli (only used by pool_free) */ struct coh901318_lli { @@ -90,7 +90,7 @@ struct powersave { * struct coh901318_platform - platform arch structure * @chans_slave: specifying dma slave channels * @chans_memcpy: specifying dma memcpy channels - * @access_memory_state: requesting DMA memeory access (on / off) + * @access_memory_state: requesting DMA memory access (on / off) * @chan_conf: dma channel configurations * @max_channels: max number of dma chanenls */ diff --git a/arch/arm/mach-ux500/board-mop500-regulators.c b/arch/arm/mach-ux500/board-mop500-regulators.c index 1187f1f..533967c 100644 --- a/arch/arm/mach-ux500/board-mop500-regulators.c +++ b/arch/arm/mach-ux500/board-mop500-regulators.c @@ -3,99 +3,94 @@ * * License Terms: GNU General Public License v2 * - * Author: Sundar Iyer <sundar.iyer@stericsson.com> + * Authors: Sundar Iyer <sundar.iyer@stericsson.com> + * Bengt Jonsson <bengt.g.jonsson@stericsson.com> * * MOP500 board specific initialization for regulators */ #include <linux/kernel.h> #include <linux/regulator/machine.h> +#include <linux/regulator/ab8500.h> -/* supplies to the display/camera */ -static struct regulator_init_data ab8500_vaux1_regulator = { - .constraints = { - .name = "V-DISPLAY", - .min_uV = 2500000, - .max_uV = 2900000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, +/* AB8500 regulators */ +struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS] = { + /* supplies to the display/camera */ + [AB8500_LDO_AUX1] = { + .constraints = { + .name = "V-DISPLAY", + .min_uV = 2500000, + .max_uV = 2900000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supplies to the on-board eMMC */ -static struct regulator_init_data ab8500_vaux2_regulator = { - .constraints = { - .name = "V-eMMC1", - .min_uV = 1100000, - .max_uV = 3300000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, + /* supplies to the on-board eMMC */ + [AB8500_LDO_AUX2] = { + .constraints = { + .name = "V-eMMC1", + .min_uV = 1100000, + .max_uV = 3300000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for VAUX3, supplies to SDcard slots */ -static struct regulator_init_data ab8500_vaux3_regulator = { - .constraints = { - .name = "V-MMC-SD", - .min_uV = 1100000, - .max_uV = 3300000, - .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE| - REGULATOR_CHANGE_STATUS, + /* supply for VAUX3, supplies to SDcard slots */ + [AB8500_LDO_AUX3] = { + .constraints = { + .name = "V-MMC-SD", + .min_uV = 1100000, + .max_uV = 3300000, + .valid_ops_mask = REGULATOR_CHANGE_VOLTAGE | + REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for tvout, gpadc, TVOUT LDO */ -static struct regulator_init_data ab8500_vtvout_init = { - .constraints = { - .name = "V-TVOUT", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for tvout, gpadc, TVOUT LDO */ + [AB8500_LDO_TVOUT] = { + .constraints = { + .name = "V-TVOUT", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for ab8500-vaudio, VAUDIO LDO */ -static struct regulator_init_data ab8500_vaudio_init = { - .constraints = { - .name = "V-AUD", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for ab8500-vaudio, VAUDIO LDO */ + [AB8500_LDO_AUDIO] = { + .constraints = { + .name = "V-AUD", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-anamic1 VAMic1-LDO */ -static struct regulator_init_data ab8500_vamic1_init = { - .constraints = { - .name = "V-AMIC1", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-anamic1 VAMic1-LDO */ + [AB8500_LDO_ANAMIC1] = { + .constraints = { + .name = "V-AMIC1", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */ -static struct regulator_init_data ab8500_vamic2_init = { - .constraints = { - .name = "V-AMIC2", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-amic2, VAMIC2 LDO, reuse constants for AMIC1 */ + [AB8500_LDO_ANAMIC2] = { + .constraints = { + .name = "V-AMIC2", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-dmic, VDMIC LDO */ -static struct regulator_init_data ab8500_vdmic_init = { - .constraints = { - .name = "V-DMIC", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-dmic, VDMIC LDO */ + [AB8500_LDO_DMIC] = { + .constraints = { + .name = "V-DMIC", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for v-intcore12, VINTCORE12 LDO */ -static struct regulator_init_data ab8500_vintcore_init = { - .constraints = { - .name = "V-INTCORE", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for v-intcore12, VINTCORE12 LDO */ + [AB8500_LDO_INTCORE] = { + .constraints = { + .name = "V-INTCORE", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, -}; - -/* supply for U8500 CSI/DSI, VANA LDO */ -static struct regulator_init_data ab8500_vana_init = { - .constraints = { - .name = "V-CSI/DSI", - .valid_ops_mask = REGULATOR_CHANGE_STATUS, + /* supply for U8500 CSI/DSI, VANA LDO */ + [AB8500_LDO_ANA] = { + .constraints = { + .name = "V-CSI/DSI", + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, }, }; - diff --git a/arch/arm/mach-ux500/board-mop500-regulators.h b/arch/arm/mach-ux500/board-mop500-regulators.h new file mode 100644 index 0000000..2675fae --- /dev/null +++ b/arch/arm/mach-ux500/board-mop500-regulators.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) ST-Ericsson SA 2010 + * + * License Terms: GNU General Public License v2 + * + * Author: Bengt Jonsson <bengt.g.jonsson@stericsson.com> for ST-Ericsson + * + * MOP500 board specific initialization for regulators + */ + +#ifndef __BOARD_MOP500_REGULATORS_H +#define __BOARD_MOP500_REGULATORS_H + +#include <linux/regulator/machine.h> +#include <linux/regulator/ab8500.h> + +extern struct regulator_init_data ab8500_regulators[AB8500_NUM_REGULATORS]; + +#endif diff --git a/arch/arm/mach-ux500/board-mop500.c b/arch/arm/mach-ux500/board-mop500.c index a1c9ea1..a393f57 100644 --- a/arch/arm/mach-ux500/board-mop500.c +++ b/arch/arm/mach-ux500/board-mop500.c @@ -35,6 +35,7 @@ #include "devices-db8500.h" #include "pins-db8500.h" #include "board-mop500.h" +#include "board-mop500-regulators.h" static pin_cfg_t mop500_pins[] = { /* SSP0 */ @@ -80,6 +81,8 @@ static pin_cfg_t mop500_pins[] = { static struct ab8500_platform_data ab8500_platdata = { .irq_base = MOP500_AB8500_IRQ_BASE, + .regulator = ab8500_regulators, + .num_regulator = ARRAY_SIZE(ab8500_regulators), }; static struct resource ab8500_resources[] = { diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index c29f283..2b269c9 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -18,7 +18,6 @@ #include <asm/smp_plat.h> #include <asm/system.h> #include <asm/tlbflush.h> -#include <asm/smp_plat.h> #include "mm.h" diff --git a/arch/arm/plat-mxc/include/mach/irqs.h b/arch/arm/plat-mxc/include/mach/irqs.h index 58a49cc..ba65c92 100644 --- a/arch/arm/plat-mxc/include/mach/irqs.h +++ b/arch/arm/plat-mxc/include/mach/irqs.h @@ -70,7 +70,7 @@ extern int imx_irq_set_priority(unsigned char irq, unsigned char prio); /* all normal IRQs can be FIQs */ #define FIQ_START 0 -/* switch betwean IRQ and FIQ */ +/* switch between IRQ and FIQ */ extern int mxc_set_irq_fiq(unsigned int irq, unsigned int type); #endif /* __ASM_ARCH_MXC_IRQS_H__ */ diff --git a/arch/arm/plat-omap/include/plat/display.h b/arch/arm/plat-omap/include/plat/display.h index c915a66..537f4e4 100644 --- a/arch/arm/plat-omap/include/plat/display.h +++ b/arch/arm/plat-omap/include/plat/display.h @@ -42,6 +42,10 @@ #define DISPC_IRQ_SYNC_LOST (1 << 14) #define DISPC_IRQ_SYNC_LOST_DIGIT (1 << 15) #define DISPC_IRQ_WAKEUP (1 << 16) +#define DISPC_IRQ_SYNC_LOST2 (1 << 17) +#define DISPC_IRQ_VSYNC2 (1 << 18) +#define DISPC_IRQ_ACBIAS_COUNT_STAT2 (1 << 21) +#define DISPC_IRQ_FRAMEDONE2 (1 << 22) struct omap_dss_device; struct omap_overlay_manager; @@ -64,6 +68,7 @@ enum omap_plane { enum omap_channel { OMAP_DSS_CHANNEL_LCD = 0, OMAP_DSS_CHANNEL_DIGIT = 1, + OMAP_DSS_CHANNEL_LCD2 = 2, }; enum omap_color_mode { @@ -142,6 +147,7 @@ enum omap_dss_display_state { enum omap_dss_overlay_managers { OMAP_DSS_OVL_MGR_LCD, OMAP_DSS_OVL_MGR_TV, + OMAP_DSS_OVL_MGR_LCD2, }; enum omap_dss_rotation_type { @@ -268,6 +274,7 @@ struct omap_overlay_info { u16 out_width; /* if 0, out_width == width */ u16 out_height; /* if 0, out_height == height */ u8 global_alpha; + u8 pre_mult_alpha; }; struct omap_overlay { @@ -351,6 +358,8 @@ struct omap_dss_device { enum omap_display_type type; + enum omap_channel channel; + union { struct { u8 data_lines; diff --git a/arch/arm/plat-omap/include/plat/omap_hwmod.h b/arch/arm/plat-omap/include/plat/omap_hwmod.h index 6864a99..1eee85a 100644 --- a/arch/arm/plat-omap/include/plat/omap_hwmod.h +++ b/arch/arm/plat-omap/include/plat/omap_hwmod.h @@ -351,7 +351,7 @@ struct omap_hwmod_omap2_prcm { /** * struct omap_hwmod_omap4_prcm - OMAP4-specific PRCM data * @clkctrl_reg: PRCM address of the clock control register - * @rstctrl_reg: adress of the XXX_RSTCTRL register located in the PRM + * @rstctrl_reg: address of the XXX_RSTCTRL register located in the PRM * @submodule_wkdep_bit: bit shift of the WKDEP range */ struct omap_hwmod_omap4_prcm { diff --git a/arch/arm/plat-omap/include/plat/panel-generic-dpi.h b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h new file mode 100644 index 0000000..7906197 --- /dev/null +++ b/arch/arm/plat-omap/include/plat/panel-generic-dpi.h @@ -0,0 +1,37 @@ +/* + * Header for generic DPI panel driver + * + * Copyright (C) 2010 Canonical Ltd. + * Author: Bryan Wu <bryan.wu@canonical.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H +#define __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H + +#include "display.h" + +/** + * struct panel_generic_dpi_data - panel driver configuration data + * @name: panel name + * @platform_enable: platform specific panel enable function + * @platform_disable: platform specific panel disable function + */ +struct panel_generic_dpi_data { + const char *name; + int (*platform_enable)(struct omap_dss_device *dssdev); + void (*platform_disable)(struct omap_dss_device *dssdev); +}; + +#endif /* __ARCH_ARM_PLAT_OMAP_PANEL_GENERIC_DPI_H */ diff --git a/arch/arm/plat-s3c24xx/devs.c b/arch/arm/plat-s3c24xx/devs.c index 2f91057..8a42bc4 100644 --- a/arch/arm/plat-s3c24xx/devs.c +++ b/arch/arm/plat-s3c24xx/devs.c @@ -259,21 +259,6 @@ struct platform_device s3c_device_iis = { EXPORT_SYMBOL(s3c_device_iis); -/* ASoC PCM DMA */ - -static u64 s3c_device_audio_dmamask = 0xffffffffUL; - -struct platform_device s3c_device_pcm = { - .name = "s3c24xx-pcm-audio", - .id = -1, - .dev = { - .dma_mask = &s3c_device_audio_dmamask, - .coherent_dma_mask = 0xffffffffUL - } -}; - -EXPORT_SYMBOL(s3c_device_pcm); - /* RTC */ static struct resource s3c_rtc_resource[] = { @@ -496,8 +481,10 @@ static struct resource s3c_ac97_resource[] = { }, }; +static u64 s3c_device_audio_dmamask = 0xffffffffUL; + struct platform_device s3c_device_ac97 = { - .name = "s3c-ac97", + .name = "samsung-ac97", .id = -1, .num_resources = ARRAY_SIZE(s3c_ac97_resource), .resource = s3c_ac97_resource, diff --git a/arch/arm/plat-samsung/Makefile b/arch/arm/plat-samsung/Makefile index afcce47..19d8a16 100644 --- a/arch/arm/plat-samsung/Makefile +++ b/arch/arm/plat-samsung/Makefile @@ -17,6 +17,7 @@ obj-y += clock.o obj-y += pwm-clock.o obj-y += gpio.o obj-y += gpio-config.o +obj-y += dev-asocdma.o obj-$(CONFIG_SAMSUNG_GPIOLIB_4BIT) += gpiolib.o obj-$(CONFIG_SAMSUNG_CLKSRC) += clock-clksrc.o diff --git a/arch/arm/plat-samsung/dev-asocdma.c b/arch/arm/plat-samsung/dev-asocdma.c new file mode 100644 index 0000000..a068c4f --- /dev/null +++ b/arch/arm/plat-samsung/dev-asocdma.c @@ -0,0 +1,25 @@ +/* linux/arch/arm/plat-samsung/dev-asocdma.c + * + * Copyright (c) 2010 Samsung Electronics Co. Ltd + * Jaswinder Singh <jassi.brar@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <plat/devs.h> + +static u64 audio_dmamask = DMA_BIT_MASK(32); + +struct platform_device samsung_asoc_dma = { + .name = "samsung-audio", + .id = -1, + .dev = { + .dma_mask = &audio_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + } +}; +EXPORT_SYMBOL(samsung_asoc_dma); diff --git a/arch/arm/plat-samsung/include/plat/audio.h b/arch/arm/plat-samsung/include/plat/audio.h index 7712ff6..a0826ed 100644 --- a/arch/arm/plat-samsung/include/plat/audio.h +++ b/arch/arm/plat-samsung/include/plat/audio.h @@ -25,10 +25,34 @@ extern void s3c64xx_ac97_setup_gpio(int); #define S5PC100_SPDIF_GPG3 1 extern void s5pc100_spdif_setup_gpio(int); +struct samsung_i2s { +/* If the Primary DAI has 5.1 Channels */ +#define QUIRK_PRI_6CHAN (1 << 0) +/* If the I2S block has a Stereo Overlay Channel */ +#define QUIRK_SEC_DAI (1 << 1) +/* + * If the I2S block has no internal prescalar or MUX (I2SMOD[10] bit) + * The Machine driver must provide suitably set clock to the I2S block. + */ +#define QUIRK_NO_MUXPSR (1 << 2) +#define QUIRK_NEED_RSTCLR (1 << 3) + /* Quirks of the I2S controller */ + u32 quirks; + + /* + * Array of clock names that can be used to generate I2S signals. + * Also corresponds to clocks of I2SMOD[10] + */ + const char **src_clk; +}; + /** * struct s3c_audio_pdata - common platform data for audio device drivers * @cfg_gpio: Callback function to setup mux'ed pins in I2S/PCM/AC97 mode */ struct s3c_audio_pdata { int (*cfg_gpio)(struct platform_device *); + union { + struct samsung_i2s i2s; + } type; }; diff --git a/arch/arm/plat-samsung/include/plat/devs.h b/arch/arm/plat-samsung/include/plat/devs.h index 2d82a6c..e9e3b6e 100644 --- a/arch/arm/plat-samsung/include/plat/devs.h +++ b/arch/arm/plat-samsung/include/plat/devs.h @@ -32,7 +32,7 @@ extern struct platform_device s3c64xx_device_iisv4; extern struct platform_device s3c64xx_device_spi0; extern struct platform_device s3c64xx_device_spi1; -extern struct platform_device s3c_device_pcm; +extern struct platform_device samsung_asoc_dma; extern struct platform_device s3c64xx_device_pcm0; extern struct platform_device s3c64xx_device_pcm1; @@ -96,6 +96,15 @@ extern struct platform_device s5pv210_device_iis1; extern struct platform_device s5pv210_device_iis2; extern struct platform_device s5pv210_device_spdif; +extern struct platform_device s5pv310_device_ac97; +extern struct platform_device s5pv310_device_pcm0; +extern struct platform_device s5pv310_device_pcm1; +extern struct platform_device s5pv310_device_pcm2; +extern struct platform_device s5pv310_device_i2s0; +extern struct platform_device s5pv310_device_i2s1; +extern struct platform_device s5pv310_device_i2s2; +extern struct platform_device s5pv310_device_spdif; + extern struct platform_device s5p6442_device_pcm0; extern struct platform_device s5p6442_device_pcm1; extern struct platform_device s5p6442_device_iis0; diff --git a/arch/arm/plat-samsung/pm.c b/arch/arm/plat-samsung/pm.c index 27cfca5..5bf3f2f 100644 --- a/arch/arm/plat-samsung/pm.c +++ b/arch/arm/plat-samsung/pm.c @@ -355,7 +355,7 @@ static void s3c_pm_finish(void) s3c_pm_check_cleanup(); } -static struct platform_suspend_ops s3c_pm_ops = { +static const struct platform_suspend_ops s3c_pm_ops = { .enter = s3c_pm_enter, .prepare = s3c_pm_prepare, .finish = s3c_pm_finish, diff --git a/arch/avr32/mach-at32ap/pm.c b/arch/avr32/mach-at32ap/pm.c index f021edf..32d680e 100644 --- a/arch/avr32/mach-at32ap/pm.c +++ b/arch/avr32/mach-at32ap/pm.c @@ -176,7 +176,7 @@ out: return 0; } -static struct platform_suspend_ops avr32_pm_ops = { +static const struct platform_suspend_ops avr32_pm_ops = { .valid = avr32_pm_valid_state, .enter = avr32_pm_enter, }; diff --git a/arch/blackfin/mach-bf537/include/mach/defBF534.h b/arch/blackfin/mach-bf537/include/mach/defBF534.h index 32529a0..725bb35 100644 --- a/arch/blackfin/mach-bf537/include/mach/defBF534.h +++ b/arch/blackfin/mach-bf537/include/mach/defBF534.h @@ -1418,7 +1418,7 @@ #define SADD_LEN 0x0002 /* Slave Address Length */ #define STDVAL 0x0004 /* Slave Transmit Data Valid */ #define NAK 0x0008 /* NAK/ACK* Generated At Conclusion Of Transfer */ -#define GEN 0x0010 /* General Call Adrress Matching Enabled */ +#define GEN 0x0010 /* General Call Address Matching Enabled */ /* TWI_SLAVE_STAT Masks */ #define SDIR 0x0001 /* Slave Transfer Direction (Transmit/Receive*) */ diff --git a/arch/blackfin/mach-common/pm.c b/arch/blackfin/mach-common/pm.c index 42fa87e..3c648a0 100644 --- a/arch/blackfin/mach-common/pm.c +++ b/arch/blackfin/mach-common/pm.c @@ -223,7 +223,7 @@ static int bfin_pm_enter(suspend_state_t state) return 0; } -struct platform_suspend_ops bfin_pm_ops = { +static const struct platform_suspend_ops bfin_pm_ops = { .enter = bfin_pm_enter, .valid = bfin_pm_valid, }; diff --git a/arch/cris/arch-v32/lib/nand_init.S b/arch/cris/arch-v32/lib/nand_init.S index e705f5c..d671fed 100644 --- a/arch/cris/arch-v32/lib/nand_init.S +++ b/arch/cris/arch-v32/lib/nand_init.S @@ -139,7 +139,7 @@ copy_nand_to_ram: lsrq 8, $r4 move.b $r4, [$r1] ; Row address lsrq 8, $r4 - move.b $r4, [$r1] ; Row adddress + move.b $r4, [$r1] ; Row address moveq 20, $r4 2: bne 2b subq 1, $r4 diff --git a/arch/cris/include/asm/etraxgpio.h b/arch/cris/include/asm/etraxgpio.h index d474818..461c089 100644 --- a/arch/cris/include/asm/etraxgpio.h +++ b/arch/cris/include/asm/etraxgpio.h @@ -1,5 +1,5 @@ /* - * The following devices are accessable using this driver using + * The following devices are accessible using this driver using * GPIO_MAJOR (120) and a couple of minor numbers. * * For ETRAX 100LX (CONFIG_ETRAX_ARCH_V10): diff --git a/arch/cris/kernel/time.c b/arch/cris/kernel/time.c index b509643..4e73092 100644 --- a/arch/cris/kernel/time.c +++ b/arch/cris/kernel/time.c @@ -86,7 +86,7 @@ int set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds,RTC_SECONDS); CMOS_WRITE(real_minutes,RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/h8300/Kconfig.debug b/arch/h8300/Kconfig.debug index ee671c3..e8d1b23 100644 --- a/arch/h8300/Kconfig.debug +++ b/arch/h8300/Kconfig.debug @@ -48,7 +48,7 @@ config DEFAULT_CMDLINE builtin kernel commandline enabled. config KERNEL_COMMAND - string "Buildin commmand string" + string "Buildin command string" depends on DEFAULT_CMDLINE help builtin kernel commandline strings. diff --git a/arch/ia64/include/asm/kvm_host.h b/arch/ia64/include/asm/kvm_host.h index 2f229e5..2689ee5 100644 --- a/arch/ia64/include/asm/kvm_host.h +++ b/arch/ia64/include/asm/kvm_host.h @@ -590,6 +590,10 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu); int kvm_pal_emul(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); void kvm_sal_emul(struct kvm_vcpu *vcpu); +#define __KVM_HAVE_ARCH_VM_ALLOC 1 +struct kvm *kvm_arch_alloc_vm(void); +void kvm_arch_free_vm(struct kvm *kvm); + #endif /* __ASSEMBLY__*/ #endif diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index d92d5b5..ac76da0 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -617,11 +617,14 @@ pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, return get_unmapped_area(file, addr, len, pgoff, flags); } +/* forward declaration */ +static static const struct dentry_operations pfmfs_dentry_operations; static struct dentry * pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data) { - return mount_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC); + return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations, + PFMFS_MAGIC); } static struct file_system_type pfm_fs_type = { @@ -2232,7 +2235,6 @@ pfm_alloc_file(pfm_context_t *ctx) } path.mnt = mntget(pfmfs_mnt); - d_set_d_op(path.dentry, &pfmfs_dentry_operations); d_add(path.dentry, inode); file = alloc_file(&path, FMODE_READ, &pfm_file_ops); diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c index f56a631..70d224d 100644 --- a/arch/ia64/kvm/kvm-ia64.c +++ b/arch/ia64/kvm/kvm-ia64.c @@ -749,7 +749,7 @@ out: return r; } -static struct kvm *kvm_alloc_kvm(void) +struct kvm *kvm_arch_alloc_vm(void) { struct kvm *kvm; @@ -760,7 +760,7 @@ static struct kvm *kvm_alloc_kvm(void) vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE)); if (!vm_base) - return ERR_PTR(-ENOMEM); + return NULL; memset((void *)vm_base, 0, KVM_VM_DATA_SIZE); kvm = (struct kvm *)(vm_base + @@ -806,10 +806,12 @@ static void kvm_build_io_pmt(struct kvm *kvm) #define GUEST_PHYSICAL_RR4 0x2739 #define VMM_INIT_RR 0x1660 -static void kvm_init_vm(struct kvm *kvm) +int kvm_arch_init_vm(struct kvm *kvm) { BUG_ON(!kvm); + kvm->arch.is_sn2 = ia64_platform_is("sn2"); + kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0; kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4; kvm->arch.vmm_init_rr = VMM_INIT_RR; @@ -823,21 +825,8 @@ static void kvm_init_vm(struct kvm *kvm) /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); -} - -struct kvm *kvm_arch_create_vm(void) -{ - struct kvm *kvm = kvm_alloc_kvm(); - - if (IS_ERR(kvm)) - return ERR_PTR(-ENOMEM); - - kvm->arch.is_sn2 = ia64_platform_is("sn2"); - - kvm_init_vm(kvm); - - return kvm; + return 0; } static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, @@ -962,7 +951,9 @@ long kvm_arch_vm_ioctl(struct file *filp, goto out; r = kvm_setup_default_irq_routing(kvm); if (r) { + mutex_lock(&kvm->slots_lock); kvm_ioapic_destroy(kvm); + mutex_unlock(&kvm->slots_lock); goto out; } break; @@ -1357,7 +1348,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return -EINVAL; } -static void free_kvm(struct kvm *kvm) +void kvm_arch_free_vm(struct kvm *kvm) { unsigned long vm_base = kvm->arch.vm_base; @@ -1399,9 +1390,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm) #endif kfree(kvm->arch.vioapic); kvm_release_vm_pages(kvm); - kvm_free_physmem(kvm); - cleanup_srcu_struct(&kvm->srcu); - free_kvm(kvm); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) diff --git a/arch/ia64/kvm/mmio.c b/arch/ia64/kvm/mmio.c index fb8f9f5..f1e17d3 100644 --- a/arch/ia64/kvm/mmio.c +++ b/arch/ia64/kvm/mmio.c @@ -130,7 +130,7 @@ static void mmio_access(struct kvm_vcpu *vcpu, u64 src_pa, u64 *dest, local_irq_save(psr); - /*Intercept the acces for PIB range*/ + /*Intercept the access for PIB range*/ if (iot == GPFN_PIB) { if (!dir) lsapic_write(vcpu, src_pa, s, *dest); diff --git a/arch/m68k/ifpsp060/src/fpsp.S b/arch/m68k/ifpsp060/src/fpsp.S index 73613b5..26e85e2 100644 --- a/arch/m68k/ifpsp060/src/fpsp.S +++ b/arch/m68k/ifpsp060/src/fpsp.S @@ -3881,7 +3881,7 @@ _fpsp_fline: # FP Unimplemented Instruction stack frame and jump to that entry # point. # -# but, if the FPU is disabled, then we need to jump to the FPU diabled +# but, if the FPU is disabled, then we need to jump to the FPU disabled # entry point. movc %pcr,%d0 btst &0x1,%d0 diff --git a/arch/m68k/include/asm/m548xgpt.h b/arch/m68k/include/asm/m548xgpt.h index c8ef158..33b2eef 100644 --- a/arch/m68k/include/asm/m548xgpt.h +++ b/arch/m68k/include/asm/m548xgpt.h @@ -59,11 +59,13 @@ #define MCF_GPT_GMS_GPIO_INPUT (0x00000000) #define MCF_GPT_GMS_GPIO_OUTLO (0x00000020) #define MCF_GPT_GMS_GPIO_OUTHI (0x00000030) +#define MCF_GPT_GMS_GPIO_MASK (0x00000030) #define MCF_GPT_GMS_TMS_DISABLE (0x00000000) #define MCF_GPT_GMS_TMS_INCAPT (0x00000001) #define MCF_GPT_GMS_TMS_OUTCAPT (0x00000002) #define MCF_GPT_GMS_TMS_PWM (0x00000003) #define MCF_GPT_GMS_TMS_GPIO (0x00000004) +#define MCF_GPT_GMS_TMS_MASK (0x00000007) /* Bit definitions and macros for MCF_GPT_GCIR */ #define MCF_GPT_GCIR_CNT(x) (((x)&0x0000FFFF)<<0) diff --git a/arch/m68k/mac/psc.c b/arch/m68k/mac/psc.c index ba6ccab..a4c3eb6 100644 --- a/arch/m68k/mac/psc.c +++ b/arch/m68k/mac/psc.c @@ -88,7 +88,7 @@ void __init psc_init(void) /* * The PSC is always at the same spot, but using psc - * keeps things consisant with the psc_xxxx functions. + * keeps things consistent with the psc_xxxx functions. */ psc = (void *) PSC_BASE; diff --git a/arch/mips/alchemy/common/power.c b/arch/mips/alchemy/common/power.c index e5916a5..647e518 100644 --- a/arch/mips/alchemy/common/power.c +++ b/arch/mips/alchemy/common/power.c @@ -130,7 +130,7 @@ static void restore_core_regs(void) au_writel(sleep_usb[1], USBD_ENABLE); au_sync(); #else - /* enable accces to OTG memory */ + /* enable access to OTG memory */ au_writel(au_readl(USB_MSR_BASE + 4) | (1 << 6), USB_MSR_BASE + 4); au_sync(); diff --git a/arch/mips/alchemy/devboards/pm.c b/arch/mips/alchemy/devboards/pm.c index 4bbd313..acaf91b 100644 --- a/arch/mips/alchemy/devboards/pm.c +++ b/arch/mips/alchemy/devboards/pm.c @@ -110,7 +110,7 @@ static void db1x_pm_end(void) } -static struct platform_suspend_ops db1x_pm_ops = { +static const struct platform_suspend_ops db1x_pm_ops = { .valid = suspend_valid_only_mem, .begin = db1x_pm_begin, .enter = db1x_pm_enter, diff --git a/arch/mips/dec/time.c b/arch/mips/dec/time.c index 02f505f..ea57f39 100644 --- a/arch/mips/dec/time.c +++ b/arch/mips/dec/time.c @@ -104,7 +104,7 @@ int rtc_mips_set_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/mips/include/asm/mach-powertv/ioremap.h b/arch/mips/include/asm/mach-powertv/ioremap.h index 076f2ee..c86ef09 100644 --- a/arch/mips/include/asm/mach-powertv/ioremap.h +++ b/arch/mips/include/asm/mach-powertv/ioremap.h @@ -88,7 +88,7 @@ static inline dma_addr_t _dma_to_phys_offset_raw(dma_addr_t dma) } /* These are not portable and should not be used in drivers. Drivers should - * be using ioremap() and friends to map physical addreses to virtual + * be using ioremap() and friends to map physical addresses to virtual * addresses and dma_map*() and friends to map virtual addresses into DMA * addresses and back. */ diff --git a/arch/mips/include/asm/mc146818-time.h b/arch/mips/include/asm/mc146818-time.h index 199b457..4a08dbe 100644 --- a/arch/mips/include/asm/mc146818-time.h +++ b/arch/mips/include/asm/mc146818-time.h @@ -66,7 +66,7 @@ static inline int mc146818_set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/mips/jz4740/board-qi_lb60.c b/arch/mips/jz4740/board-qi_lb60.c index 5c0a357..2c0e107 100644 --- a/arch/mips/jz4740/board-qi_lb60.c +++ b/arch/mips/jz4740/board-qi_lb60.c @@ -65,7 +65,7 @@ static struct nand_ecclayout qi_lb60_ecclayout_1gb = { /* Early prototypes of the QI LB60 had only 1GB of NAND. * In order to support these devices aswell the partition and ecc layout is - * initalized depending on the NAND size */ + * initialized depending on the NAND size */ static struct mtd_partition qi_lb60_partitions_1gb[] = { { .name = "NAND BOOT partition", @@ -464,7 +464,7 @@ static int __init qi_lb60_board_setup(void) board_gpio_setup(); if (qi_lb60_init_platform_devices()) - panic("Failed to initalize platform devices\n"); + panic("Failed to initialize platform devices\n"); return 0; } diff --git a/arch/mips/jz4740/gpio.c b/arch/mips/jz4740/gpio.c index 38f60f3..88e6aed 100644 --- a/arch/mips/jz4740/gpio.c +++ b/arch/mips/jz4740/gpio.c @@ -546,7 +546,7 @@ static int __init jz4740_gpio_init(void) for (i = 0; i < ARRAY_SIZE(jz4740_gpio_chips); ++i) jz4740_gpio_chip_init(&jz4740_gpio_chips[i], i); - printk(KERN_INFO "JZ4740 GPIO initalized\n"); + printk(KERN_INFO "JZ4740 GPIO initialized\n"); return 0; } diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c index a999458..902d5b5 100644 --- a/arch/mips/jz4740/pm.c +++ b/arch/mips/jz4740/pm.c @@ -42,7 +42,7 @@ static int jz4740_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops jz4740_pm_ops = { +static const struct platform_suspend_ops jz4740_pm_ops = { .valid = suspend_valid_only_mem, .enter = jz4740_pm_enter, }; diff --git a/arch/mips/loongson/common/pm.c b/arch/mips/loongson/common/pm.c index 6c1fd90..f55e07a 100644 --- a/arch/mips/loongson/common/pm.c +++ b/arch/mips/loongson/common/pm.c @@ -147,7 +147,7 @@ static int loongson_pm_valid_state(suspend_state_t state) } } -static struct platform_suspend_ops loongson_pm_ops = { +static const struct platform_suspend_ops loongson_pm_ops = { .valid = loongson_pm_valid_state, .enter = loongson_pm_enter, }; diff --git a/arch/mips/mti-malta/malta-memory.c b/arch/mips/mti-malta/malta-memory.c index b27419c..a96d281 100644 --- a/arch/mips/mti-malta/malta-memory.c +++ b/arch/mips/mti-malta/malta-memory.c @@ -43,7 +43,7 @@ static struct prom_pmemblock mdesc[PROM_MAX_PMEMBLOCKS]; static char *mtypes[3] = { "Dont use memory", "YAMON PROM memory", - "Free memmory", + "Free memory", }; #endif diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c index 385f035..0583c463 100644 --- a/arch/mips/pci/pcie-octeon.c +++ b/arch/mips/pci/pcie-octeon.c @@ -900,7 +900,7 @@ static int cvmx_pcie_rc_initialize(int pcie_port) mem_access_subid.s.ror = 0; /* Disable Relaxed Ordering for Writes. */ mem_access_subid.s.row = 0; - /* PCIe Adddress Bits <63:34>. */ + /* PCIe Address Bits <63:34>. */ mem_access_subid.s.ba = 0; /* diff --git a/arch/mips/powertv/memory.c b/arch/mips/powertv/memory.c index 73880ad..fb3d296 100644 --- a/arch/mips/powertv/memory.c +++ b/arch/mips/powertv/memory.c @@ -57,7 +57,7 @@ unsigned long ptv_memsize; /* - * struct low_mem_reserved - Items in low memmory that are reserved + * struct low_mem_reserved - Items in low memory that are reserved * @start: Physical address of item * @size: Size, in bytes, of this item * @is_aliased: True if this is RAM aliased from another location. If false, diff --git a/arch/mips/txx9/generic/pci.c b/arch/mips/txx9/generic/pci.c index 9a0be81..96e69a0 100644 --- a/arch/mips/txx9/generic/pci.c +++ b/arch/mips/txx9/generic/pci.c @@ -107,7 +107,7 @@ int txx9_pci_mem_high __initdata; /* * allocate pci_controller and resources. - * mem_base, io_base: physical addresss. 0 for auto assignment. + * mem_base, io_base: physical address. 0 for auto assignment. * mem_size and io_size means max size on auto assignment. * pcic must be &txx9_primary_pcic or NULL. */ diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index 41ba385..8ed41cf 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -203,6 +203,7 @@ endmenu config SMP bool "Symmetric multi-processing support" default y + select USE_GENERIC_SMP_HELPERS depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050 ---help--- This enables support for systems with more than one CPU. If you have @@ -226,11 +227,6 @@ config NR_CPUS depends on SMP default "2" -config USE_GENERIC_SMP_HELPERS - bool - depends on SMP - default y - source "kernel/Kconfig.preempt" config MN10300_CURRENT_IN_E2 diff --git a/arch/mn10300/kernel/rtc.c b/arch/mn10300/kernel/rtc.c index e9e20f9..48d7058 100644 --- a/arch/mn10300/kernel/rtc.c +++ b/arch/mn10300/kernel/rtc.c @@ -89,7 +89,7 @@ static int set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/powerpc/include/asm/8xx_immap.h b/arch/powerpc/include/asm/8xx_immap.h index 4b0e152..6b6dc20 100644 --- a/arch/powerpc/include/asm/8xx_immap.h +++ b/arch/powerpc/include/asm/8xx_immap.h @@ -93,7 +93,7 @@ typedef struct mem_ctlr { } memctl8xx_t; /*----------------------------------------------------------------------- - * BR - Memory Controler: Base Register 16-9 + * BR - Memory Controller: Base Register 16-9 */ #define BR_BA_MSK 0xffff8000 /* Base Address Mask */ #define BR_AT_MSK 0x00007000 /* Address Type Mask */ @@ -110,7 +110,7 @@ typedef struct mem_ctlr { #define BR_V 0x00000001 /* Bank Valid */ /*----------------------------------------------------------------------- - * OR - Memory Controler: Option Register 16-11 + * OR - Memory Controller: Option Register 16-11 */ #define OR_AM_MSK 0xffff8000 /* Address Mask Mask */ #define OR_ATM_MSK 0x00007000 /* Address Type Mask Mask */ diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index e316847..badc983 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -1307,12 +1307,10 @@ struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id) int err = -ENOMEM; unsigned long p; - vcpu_book3s = vmalloc(sizeof(struct kvmppc_vcpu_book3s)); + vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); if (!vcpu_book3s) goto out; - memset(vcpu_book3s, 0, sizeof(struct kvmppc_vcpu_book3s)); - vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *) kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL); if (!vcpu_book3s->shadow_vcpu) diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 38f756f..9975846 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -145,18 +145,12 @@ void kvm_arch_check_processor_compat(void *rtn) *(int *)rtn = kvmppc_core_check_processor_compat(); } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm; - - kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - if (!kvm) - return ERR_PTR(-ENOMEM); - - return kvm; + return 0; } -static void kvmppc_free_vcpus(struct kvm *kvm) +void kvm_arch_destroy_vm(struct kvm *kvm) { unsigned int i; struct kvm_vcpu *vcpu; @@ -176,14 +170,6 @@ void kvm_arch_sync_events(struct kvm *kvm) { } -void kvm_arch_destroy_vm(struct kvm *kvm) -{ - kvmppc_free_vcpus(kvm); - kvm_free_physmem(kvm); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); -} - int kvm_dev_ioctl_check_extension(long ext) { int r; diff --git a/arch/powerpc/oprofile/op_model_cell.c b/arch/powerpc/oprofile/op_model_cell.c index 7fd90d0..c4d2b71 100644 --- a/arch/powerpc/oprofile/op_model_cell.c +++ b/arch/powerpc/oprofile/op_model_cell.c @@ -1469,7 +1469,7 @@ static int cell_global_start(struct op_counter_config *ctr) * The pm_interval register is setup to write the SPU PC value into the * trace buffer at the maximum rate possible. The trace buffer is configured * to store the PCs, wrapping when it is full. The performance counter is - * intialized to the max hardware count minus the number of events, N, between + * initialized to the max hardware count minus the number of events, N, between * samples. Once the N events have occured, a HW counter overflow occurs * causing the generation of a HW counter interrupt which also stops the * writing of the SPU PC values to the trace buffer. Hence the last PC diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c index 80234e5..eda0fc2 100644 --- a/arch/powerpc/platforms/52xx/lite5200_pm.c +++ b/arch/powerpc/platforms/52xx/lite5200_pm.c @@ -232,7 +232,7 @@ static void lite5200_pm_end(void) lite5200_pm_target_state = PM_SUSPEND_ON; } -static struct platform_suspend_ops lite5200_pm_ops = { +static const struct platform_suspend_ops lite5200_pm_ops = { .valid = lite5200_pm_valid, .begin = lite5200_pm_begin, .prepare = lite5200_pm_prepare, diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c index 568cef6..8310e8b 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c @@ -186,7 +186,7 @@ void mpc52xx_pm_finish(void) iounmap(mbar); } -static struct platform_suspend_ops mpc52xx_pm_ops = { +static const struct platform_suspend_ops mpc52xx_pm_ops = { .valid = mpc52xx_pm_valid, .prepare = mpc52xx_pm_prepare, .enter = mpc52xx_pm_enter, diff --git a/arch/powerpc/platforms/83xx/suspend-asm.S b/arch/powerpc/platforms/83xx/suspend-asm.S index 1930543..3d1ecd2 100644 --- a/arch/powerpc/platforms/83xx/suspend-asm.S +++ b/arch/powerpc/platforms/83xx/suspend-asm.S @@ -231,7 +231,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) ori r4, r4, 0x002a mtspr SPRN_DBAT0L, r4 lis r8, TMP_VIRT_IMMR@h - ori r4, r8, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r8, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT0U, r4 isync @@ -241,7 +241,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) ori r4, r4, 0x002a mtspr SPRN_DBAT1L, r4 lis r9, (TMP_VIRT_IMMR + 0x01000000)@h - ori r4, r9, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r9, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT1U, r4 isync @@ -253,7 +253,7 @@ _GLOBAL(mpc83xx_enter_deep_sleep) li r4, 0x0002 mtspr SPRN_DBAT2L, r4 lis r4, KERNELBASE@h - ori r4, r4, 0x001e /* 1 MByte accessable from Kernel Space only */ + ori r4, r4, 0x001e /* 1 MByte accessible from Kernel Space only */ mtspr SPRN_DBAT2U, r4 isync diff --git a/arch/powerpc/platforms/83xx/suspend.c b/arch/powerpc/platforms/83xx/suspend.c index 75ae77f..fd4f2f2 100644 --- a/arch/powerpc/platforms/83xx/suspend.c +++ b/arch/powerpc/platforms/83xx/suspend.c @@ -311,7 +311,7 @@ static int mpc83xx_is_pci_agent(void) return ret; } -static struct platform_suspend_ops mpc83xx_suspend_ops = { +static const struct platform_suspend_ops mpc83xx_suspend_ops = { .valid = mpc83xx_suspend_valid, .begin = mpc83xx_suspend_begin, .enter = mpc83xx_suspend_enter, diff --git a/arch/powerpc/platforms/85xx/mpc85xx_mds.c b/arch/powerpc/platforms/85xx/mpc85xx_mds.c index aa34cac..747d1ee 100644 --- a/arch/powerpc/platforms/85xx/mpc85xx_mds.c +++ b/arch/powerpc/platforms/85xx/mpc85xx_mds.c @@ -309,7 +309,7 @@ static void __init mpc85xx_mds_qe_init(void) /* P1021 has pins muxed for QE and other functions. To * enable QE UEC mode, we need to set bit QE0 for UCC1 * in Eth mode, QE0 and QE3 for UCC5 in Eth mode, QE9 - * and QE12 for QE MII management singals in PMUXCR + * and QE12 for QE MII management signals in PMUXCR * register. */ setbits32(pmuxcr, MPC85xx_PMUXCR_QE0 | diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c index b341018..6c4b583 100644 --- a/arch/powerpc/platforms/ps3/device-init.c +++ b/arch/powerpc/platforms/ps3/device-init.c @@ -566,10 +566,10 @@ static int ps3_setup_dynamic_device(const struct ps3_repository_device *repo) case PS3_DEV_TYPE_STOR_DISK: result = ps3_setup_storage_dev(repo, PS3_MATCH_ID_STOR_DISK); - /* Some devices are not accessable from the Other OS lpar. */ + /* Some devices are not accessible from the Other OS lpar. */ if (result == -ENODEV) { result = 0; - pr_debug("%s:%u: not accessable\n", __func__, + pr_debug("%s:%u: not accessible\n", __func__, __LINE__); } diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index 59d9712..92290ff 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -44,7 +44,7 @@ * @lock: * @ipi_debug_brk_mask: * - * The HV mantains per SMT thread mappings of HV outlet to HV plug on + * The HV maintains per SMT thread mappings of HV outlet to HV plug on * behalf of the guest. These mappings are implemented as 256 bit guest * supplied bitmaps indexed by plug number. The addresses of the bitmaps * are registered with the HV through lv1_configure_irq_state_bitmap(). diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c index e19ff02..f106662 100644 --- a/arch/powerpc/platforms/pseries/hvCall_inst.c +++ b/arch/powerpc/platforms/pseries/hvCall_inst.c @@ -55,7 +55,7 @@ static void hc_stop(struct seq_file *m, void *p) static int hc_show(struct seq_file *m, void *p) { unsigned long h_num = (unsigned long)p; - struct hcall_stats *hs = (struct hcall_stats *)m->private; + struct hcall_stats *hs = m->private; if (hs[h_num].num_calls) { if (cpu_has_feature(CPU_FTR_PURR)) diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c index ed72098..a8ca289 100644 --- a/arch/powerpc/platforms/pseries/suspend.c +++ b/arch/powerpc/platforms/pseries/suspend.c @@ -153,7 +153,7 @@ static struct sysdev_class suspend_sysdev_class = { .name = "power", }; -static struct platform_suspend_ops pseries_suspend_ops = { +static const struct platform_suspend_ops pseries_suspend_ops = { .valid = suspend_valid_only_mem, .begin = pseries_suspend_begin, .prepare_late = pseries_prepare_late, diff --git a/arch/powerpc/sysdev/fsl_pmc.c b/arch/powerpc/sysdev/fsl_pmc.c index 44de855..e9381bf 100644 --- a/arch/powerpc/sysdev/fsl_pmc.c +++ b/arch/powerpc/sysdev/fsl_pmc.c @@ -53,7 +53,7 @@ static int pmc_suspend_valid(suspend_state_t state) return 1; } -static struct platform_suspend_ops pmc_suspend_ops = { +static const struct platform_suspend_ops pmc_suspend_ops = { .valid = pmc_suspend_valid, .enter = pmc_suspend_enter, }; diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index 9725369..9f99bef 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -973,7 +973,6 @@ fsl_rio_dbell_handler(int irq, void *dev_instance) if (dsr & DOORBELL_DSR_QFI) { pr_info("RIO: doorbell queue full\n"); out_be32(&priv->msg_regs->dsr, DOORBELL_DSR_QFI); - goto out; } /* XXX Need to check/dispatch until queue empty */ diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 985d825..bade533 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -164,24 +164,18 @@ long kvm_arch_vm_ioctl(struct file *filp, return r; } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm; int rc; char debug_name[16]; rc = s390_enable_sie(); if (rc) - goto out_nokvm; - - rc = -ENOMEM; - kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - if (!kvm) - goto out_nokvm; + goto out_err; kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL); if (!kvm->arch.sca) - goto out_nosca; + goto out_err; sprintf(debug_name, "kvm-%u", current->pid); @@ -195,13 +189,11 @@ struct kvm *kvm_arch_create_vm(void) debug_register_view(kvm->arch.dbf, &debug_sprintf_view); VM_EVENT(kvm, 3, "%s", "vm created"); - return kvm; + return 0; out_nodbf: free_page((unsigned long)(kvm->arch.sca)); -out_nosca: - kfree(kvm); -out_nokvm: - return ERR_PTR(rc); +out_err: + return rc; } void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) @@ -240,11 +232,8 @@ void kvm_arch_sync_events(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); free_page((unsigned long)(kvm->arch.sca)); debug_unregister(kvm->arch.dbf); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); } /* Section: vcpu related */ diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig index 2018c7e..d893411 100644 --- a/arch/sh/boards/Kconfig +++ b/arch/sh/boards/Kconfig @@ -3,6 +3,9 @@ menu "Board support" config SOLUTION_ENGINE bool +config SH_ALPHA_BOARD + bool + config SH_SOLUTION_ENGINE bool "SolutionEngine" select SOLUTION_ENGINE @@ -320,6 +323,21 @@ config SH_SH2007 Compact Flash socket, two serial ports and PC-104 bus. More information at <http://sh2000.sh-linux.org>. +config SH_APSH4A3A + bool "AP-SH4A-3A" + select SH_ALPHA_BOARD + depends on CPU_SUBTYPE_SH7785 + help + Select AP-SH4A-3A if configuring for an ALPHAPROJECT AP-SH4A-3A. + +config SH_APSH4AD0A + bool "AP-SH4AD-0A" + select SH_ALPHA_BOARD + select SYS_SUPPORTS_PCI + depends on CPU_SUBTYPE_SH7786 + help + Select AP-SH4AD-0A if configuring for an ALPHAPROJECT AP-SH4AD-0A. + endmenu source "arch/sh/boards/mach-r2d/Kconfig" diff --git a/arch/sh/boards/Makefile b/arch/sh/boards/Makefile index be7d11d..975a0f6 100644 --- a/arch/sh/boards/Makefile +++ b/arch/sh/boards/Makefile @@ -13,3 +13,5 @@ obj-$(CONFIG_SH_ESPT) += board-espt.o obj-$(CONFIG_SH_POLARIS) += board-polaris.o obj-$(CONFIG_SH_TITAN) += board-titan.o obj-$(CONFIG_SH_SH7757LCR) += board-sh7757lcr.o +obj-$(CONFIG_SH_APSH4A3A) += board-apsh4a3a.o +obj-$(CONFIG_SH_APSH4AD0A) += board-apsh4ad0a.o diff --git a/arch/sh/boards/board-apsh4a3a.c b/arch/sh/boards/board-apsh4a3a.c new file mode 100644 index 0000000..8e2a270 --- /dev/null +++ b/arch/sh/boards/board-apsh4a3a.c @@ -0,0 +1,175 @@ +/* + * ALPHAPROJECT AP-SH4A-3A Support. + * + * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd. + * Copyright (C) 2008 Yoshihiro Shimoda + * Copyright (C) 2009 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/mtd/physmap.h> +#include <linux/smsc911x.h> +#include <linux/irq.h> +#include <linux/clk.h> +#include <asm/machvec.h> +#include <asm/sizes.h> +#include <asm/clock.h> + +static struct mtd_partition nor_flash_partitions[] = { + { + .name = "loader", + .offset = 0x00000000, + .size = 512 * 1024, + }, + { + .name = "bootenv", + .offset = MTDPART_OFS_APPEND, + .size = 512 * 1024, + }, + { + .name = "kernel", + .offset = MTDPART_OFS_APPEND, + .size = 4 * 1024 * 1024, + }, + { + .name = "data", + .offset = MTDPART_OFS_APPEND, + .size = MTDPART_SIZ_FULL, + }, +}; + +static struct physmap_flash_data nor_flash_data = { + .width = 4, + .parts = nor_flash_partitions, + .nr_parts = ARRAY_SIZE(nor_flash_partitions), +}; + +static struct resource nor_flash_resources[] = { + [0] = { + .start = 0x00000000, + .end = 0x01000000 - 1, + .flags = IORESOURCE_MEM, + } +}; + +static struct platform_device nor_flash_device = { + .name = "physmap-flash", + .dev = { + .platform_data = &nor_flash_data, + }, + .num_resources = ARRAY_SIZE(nor_flash_resources), + .resource = nor_flash_resources, +}; + +static struct resource smsc911x_resources[] = { + [0] = { + .name = "smsc911x-memory", + .start = 0xA4000000, + .end = 0xA4000000 + SZ_256 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .name = "smsc911x-irq", + .start = evt2irq(0x200), + .end = evt2irq(0x200), + .flags = IORESOURCE_IRQ, + }, +}; + +static struct smsc911x_platform_config smsc911x_config = { + .irq_polarity = SMSC911X_IRQ_POLARITY_ACTIVE_LOW, + .irq_type = SMSC911X_IRQ_TYPE_OPEN_DRAIN, + .flags = SMSC911X_USE_16BIT, + .phy_interface = PHY_INTERFACE_MODE_MII, +}; + +static struct platform_device smsc911x_device = { + .name = "smsc911x", + .id = -1, + .num_resources = ARRAY_SIZE(smsc911x_resources), + .resource = smsc911x_resources, + .dev = { + .platform_data = &smsc911x_config, + }, +}; + +static struct platform_device *apsh4a3a_devices[] __initdata = { + &nor_flash_device, + &smsc911x_device, +}; + +static int __init apsh4a3a_devices_setup(void) +{ + return platform_add_devices(apsh4a3a_devices, + ARRAY_SIZE(apsh4a3a_devices)); +} +device_initcall(apsh4a3a_devices_setup); + +static int apsh4a3a_clk_init(void) +{ + struct clk *clk; + int ret; + + clk = clk_get(NULL, "extal"); + if (!clk || IS_ERR(clk)) + return PTR_ERR(clk); + ret = clk_set_rate(clk, 33333000); + clk_put(clk); + + return ret; +} + +/* Initialize the board */ +static void __init apsh4a3a_setup(char **cmdline_p) +{ + printk(KERN_INFO "Alpha Project AP-SH4A-3A support:\n"); +} + +static void __init apsh4a3a_init_irq(void) +{ + plat_irq_setup_pins(IRQ_MODE_IRQ7654); +} + +/* Return the board specific boot mode pin configuration */ +static int apsh4a3a_mode_pins(void) +{ + int value = 0; + + /* These are the factory default settings of SW1 and SW2. + * If you change these dip switches then you will need to + * adjust the values below as well. + */ + value &= ~MODE_PIN0; /* Clock Mode 16 */ + value &= ~MODE_PIN1; + value &= ~MODE_PIN2; + value &= ~MODE_PIN3; + value |= MODE_PIN4; + value &= ~MODE_PIN5; /* 16-bit Area0 bus width */ + value |= MODE_PIN6; /* Area 0 SRAM interface */ + value |= MODE_PIN7; + value |= MODE_PIN8; /* Little Endian */ + value |= MODE_PIN9; /* Master Mode */ + value |= MODE_PIN10; /* Crystal resonator */ + value |= MODE_PIN11; /* Display Unit */ + value |= MODE_PIN12; + value &= ~MODE_PIN13; /* 29-bit address mode */ + value |= MODE_PIN14; /* No PLL step-up */ + + return value; +} + +/* + * The Machine Vector + */ +static struct sh_machine_vector mv_apsh4a3a __initmv = { + .mv_name = "AP-SH4A-3A", + .mv_setup = apsh4a3a_setup, + .mv_clk_init = apsh4a3a_clk_init, + .mv_init_irq = apsh4a3a_init_irq, + .mv_mode_pins = apsh4a3a_mode_pins, +}; diff --git a/arch/sh/boards/board-apsh4ad0a.c b/arch/sh/boards/board-apsh4ad0a.c new file mode 100644 index 0000000..e2bd218 --- /dev/null +++ b/arch/sh/boards/board-apsh4ad0a.c @@ -0,0 +1,125 @@ +/* + * ALPHAPROJECT AP-SH4AD-0A Support. + * + * Copyright (C) 2010 ALPHAPROJECT Co.,Ltd. + * Copyright (C) 2010 Matt Fleming + * Copyright (C) 2010 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/smsc911x.h> +#include <linux/irq.h> +#include <linux/clk.h> +#include <asm/machvec.h> +#include <asm/sizes.h> + +static struct resource smsc911x_resources[] = { + [0] = { + .name = "smsc911x-memory", + .start = 0xA4000000, + .end = 0xA4000000 + SZ_256 - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .name = "smsc911x-irq", + .start = evt2irq(0x200), + .end = evt2irq(0x200), + .flags = IORESOURCE_IRQ, + }, +}; + +static struct smsc911x_platform_config smsc911x_config = { + .irq_polarity = SMSC911X_IRQ_POLARITY_ACTIVE_LOW, + .irq_type = SMSC911X_IRQ_TYPE_OPEN_DRAIN, + .flags = SMSC911X_USE_16BIT, + .phy_interface = PHY_INTERFACE_MODE_MII, +}; + +static struct platform_device smsc911x_device = { + .name = "smsc911x", + .id = -1, + .num_resources = ARRAY_SIZE(smsc911x_resources), + .resource = smsc911x_resources, + .dev = { + .platform_data = &smsc911x_config, + }, +}; + +static struct platform_device *apsh4ad0a_devices[] __initdata = { + &smsc911x_device, +}; + +static int __init apsh4ad0a_devices_setup(void) +{ + return platform_add_devices(apsh4ad0a_devices, + ARRAY_SIZE(apsh4ad0a_devices)); +} +device_initcall(apsh4ad0a_devices_setup); + +static int apsh4ad0a_mode_pins(void) +{ + int value = 0; + + /* These are the factory default settings of SW1 and SW2. + * If you change these dip switches then you will need to + * adjust the values below as well. + */ + value |= MODE_PIN0; /* Clock Mode 3 */ + value |= MODE_PIN1; + value &= ~MODE_PIN2; + value &= ~MODE_PIN3; + value &= ~MODE_PIN4; /* 16-bit Area0 bus width */ + value |= MODE_PIN5; + value |= MODE_PIN6; + value |= MODE_PIN7; /* Normal mode */ + value |= MODE_PIN8; /* Little Endian */ + value |= MODE_PIN9; /* Crystal resonator */ + value &= ~MODE_PIN10; /* 29-bit address mode */ + value &= ~MODE_PIN11; /* PCI-E Root port */ + value &= ~MODE_PIN12; /* 4 lane + 1 lane */ + value |= MODE_PIN13; /* AUD Enable */ + value &= ~MODE_PIN14; /* Normal Operation */ + + return value; +} + +static int apsh4ad0a_clk_init(void) +{ + struct clk *clk; + int ret; + + clk = clk_get(NULL, "extal"); + if (!clk || IS_ERR(clk)) + return PTR_ERR(clk); + ret = clk_set_rate(clk, 33333000); + clk_put(clk); + + return ret; +} + +/* Initialize the board */ +static void __init apsh4ad0a_setup(char **cmdline_p) +{ + pr_info("Alpha Project AP-SH4AD-0A support:\n"); +} + +static void __init apsh4ad0a_init_irq(void) +{ + plat_irq_setup_pins(IRQ_MODE_IRQ3210); +} + +/* + * The Machine Vector + */ +static struct sh_machine_vector mv_apsh4ad0a __initmv = { + .mv_name = "AP-SH4AD-0A", + .mv_setup = apsh4ad0a_setup, + .mv_mode_pins = apsh4ad0a_mode_pins, + .mv_clk_init = apsh4ad0a_clk_init, + .mv_init_irq = apsh4ad0a_init_irq, +}; diff --git a/arch/sh/boards/board-edosk7705.c b/arch/sh/boards/board-edosk7705.c index 4cb3bb7..541d8a2 100644 --- a/arch/sh/boards/board-edosk7705.c +++ b/arch/sh/boards/board-edosk7705.c @@ -66,7 +66,7 @@ static int __init init_edosk7705_devices(void) return platform_add_devices(edosk7705_devices, ARRAY_SIZE(edosk7705_devices)); } -__initcall(init_edosk7705_devices); +device_initcall(init_edosk7705_devices); /* * The Machine Vector diff --git a/arch/sh/boards/board-edosk7760.c b/arch/sh/boards/board-edosk7760.c index 35dc099..f47ac82 100644 --- a/arch/sh/boards/board-edosk7760.c +++ b/arch/sh/boards/board-edosk7760.c @@ -182,7 +182,7 @@ static int __init init_edosk7760_devices(void) return platform_add_devices(edosk7760_devices, ARRAY_SIZE(edosk7760_devices)); } -__initcall(init_edosk7760_devices); +device_initcall(init_edosk7760_devices); /* * The Machine Vector diff --git a/arch/sh/boards/board-sh7785lcr.c b/arch/sh/boards/board-sh7785lcr.c index fe7e686..ee65ff0 100644 --- a/arch/sh/boards/board-sh7785lcr.c +++ b/arch/sh/boards/board-sh7785lcr.c @@ -284,7 +284,7 @@ static int __init sh7785lcr_devices_setup(void) return platform_add_devices(sh7785lcr_devices, ARRAY_SIZE(sh7785lcr_devices)); } -__initcall(sh7785lcr_devices_setup); +device_initcall(sh7785lcr_devices_setup); /* Initialize IRQ setting */ void __init init_sh7785lcr_IRQ(void) diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c index 07ea908..3e5fc3b 100644 --- a/arch/sh/boards/mach-ap325rxa/setup.c +++ b/arch/sh/boards/mach-ap325rxa/setup.c @@ -14,6 +14,8 @@ #include <linux/device.h> #include <linux/interrupt.h> #include <linux/platform_device.h> +#include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/mtd/sh_flctl.h> #include <linux/delay.h> @@ -430,11 +432,18 @@ static struct resource sdhi0_cn3_resources[] = { }, }; +static struct sh_mobile_sdhi_info sdhi0_cn3_data = { + .tmio_caps = MMC_CAP_SDIO_IRQ, +}; + static struct platform_device sdhi0_cn3_device = { .name = "sh_mobile_sdhi", .id = 0, /* "sdhi0" clock */ .num_resources = ARRAY_SIZE(sdhi0_cn3_resources), .resource = sdhi0_cn3_resources, + .dev = { + .platform_data = &sdhi0_cn3_data, + }, .archdata = { .hwblk_id = HWBLK_SDHI0, }, @@ -453,11 +462,18 @@ static struct resource sdhi1_cn7_resources[] = { }, }; +static struct sh_mobile_sdhi_info sdhi1_cn7_data = { + .tmio_caps = MMC_CAP_SDIO_IRQ, +}; + static struct platform_device sdhi1_cn7_device = { .name = "sh_mobile_sdhi", .id = 1, /* "sdhi1" clock */ .num_resources = ARRAY_SIZE(sdhi1_cn7_resources), .resource = sdhi1_cn7_resources, + .dev = { + .platform_data = &sdhi1_cn7_data, + }, .archdata = { .hwblk_id = HWBLK_SDHI1, }, diff --git a/arch/sh/boards/mach-cayman/setup.c b/arch/sh/boards/mach-cayman/setup.c index 7e8216a..e89e8e1 100644 --- a/arch/sh/boards/mach-cayman/setup.c +++ b/arch/sh/boards/mach-cayman/setup.c @@ -165,7 +165,7 @@ static int __init smsc_superio_setup(void) return 0; } -__initcall(smsc_superio_setup); +device_initcall(smsc_superio_setup); static void __iomem *cayman_ioport_map(unsigned long port, unsigned int len) { diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c index f48c492..33b6629 100644 --- a/arch/sh/boards/mach-ecovec24/setup.c +++ b/arch/sh/boards/mach-ecovec24/setup.c @@ -473,6 +473,7 @@ static struct sh_mobile_sdhi_info sdhi0_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, .set_pwr = sdhi0_set_pwr, + .tmio_caps = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD, }; static struct resource sdhi0_resources[] = { @@ -511,6 +512,7 @@ static void sdhi1_set_pwr(struct platform_device *pdev, int state) static struct sh_mobile_sdhi_info sdhi1_info = { .dma_slave_tx = SHDMA_SLAVE_SDHI1_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ | MMC_CAP_POWER_OFF_CARD, .set_pwr = sdhi1_set_pwr, }; diff --git a/arch/sh/boards/mach-hp6xx/pm.c b/arch/sh/boards/mach-hp6xx/pm.c index 4499a37..adc9b4b 100644 --- a/arch/sh/boards/mach-hp6xx/pm.c +++ b/arch/sh/boards/mach-hp6xx/pm.c @@ -143,7 +143,7 @@ static int hp6x0_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops hp6x0_pm_ops = { +static const struct platform_suspend_ops hp6x0_pm_ops = { .enter = hp6x0_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c index 9b60eaa..7504daa 100644 --- a/arch/sh/boards/mach-kfr2r09/setup.c +++ b/arch/sh/boards/mach-kfr2r09/setup.c @@ -11,6 +11,7 @@ #include <linux/platform_device.h> #include <linux/interrupt.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mfd/tmio.h> #include <linux/mtd/physmap.h> #include <linux/mtd/onenand.h> @@ -366,6 +367,7 @@ static struct sh_mobile_sdhi_info sh7724_sdhi0_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, .tmio_flags = TMIO_MMC_WRPROTECT_DISABLE, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device kfr2r09_sh_sdhi0_device = { diff --git a/arch/sh/boards/mach-landisk/irq.c b/arch/sh/boards/mach-landisk/irq.c index e79412a..c00ace3 100644 --- a/arch/sh/boards/mach-landisk/irq.c +++ b/arch/sh/boards/mach-landisk/irq.c @@ -1,9 +1,10 @@ /* - * arch/sh/boards/landisk/irq.c + * arch/sh/boards/mach-landisk/irq.c * * I-O DATA Device, Inc. LANDISK Support * * Copyright (C) 2005-2007 kogiidena + * Copyright (C) 2011 Nobuhiro Iwamatsu * * Copyright (C) 2001 Ian da Silva, Jeremy Siegel * Based largely on io_se.c. @@ -12,44 +13,54 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. */ + #include <linux/init.h> #include <linux/irq.h> #include <linux/interrupt.h> #include <linux/io.h> #include <mach-landisk/mach/iodata_landisk.h> -static void disable_landisk_irq(struct irq_data *data) -{ - unsigned char mask = 0xff ^ (0x01 << (data->irq - 5)); +enum { + UNUSED = 0, - __raw_writeb(__raw_readb(PA_IMASK) & mask, PA_IMASK); -} - -static void enable_landisk_irq(struct irq_data *data) -{ - unsigned char value = (0x01 << (data->irq - 5)); + PCI_INTA, /* PCI int A */ + PCI_INTB, /* PCI int B */ + PCI_INTC, /* PCI int C */ + PCI_INTD, /* PCI int D */ + ATA, /* ATA */ + FATA, /* CF */ + POWER, /* Power swtich */ + BUTTON, /* Button swtich */ +}; - __raw_writeb(__raw_readb(PA_IMASK) | value, PA_IMASK); -} +/* Vectors for LANDISK */ +static struct intc_vect vectors_landisk[] __initdata = { + INTC_IRQ(PCI_INTA, IRQ_PCIINTA), + INTC_IRQ(PCI_INTB, IRQ_PCIINTB), + INTC_IRQ(PCI_INTC, IRQ_PCIINTC), + INTC_IRQ(PCI_INTD, IRQ_PCIINTD), + INTC_IRQ(ATA, IRQ_ATA), + INTC_IRQ(FATA, IRQ_FATA), + INTC_IRQ(POWER, IRQ_POWER), + INTC_IRQ(BUTTON, IRQ_BUTTON), +}; -static struct irq_chip landisk_irq_chip __read_mostly = { - .name = "LANDISK", - .irq_mask = disable_landisk_irq, - .irq_unmask = enable_landisk_irq, +/* IRLMSK mask register layout for LANDISK */ +static struct intc_mask_reg mask_registers_landisk[] __initdata = { + { PA_IMASK, 0, 8, /* IRLMSK */ + { BUTTON, POWER, FATA, ATA, + PCI_INTD, PCI_INTC, PCI_INTB, PCI_INTA, + } + }, }; +static DECLARE_INTC_DESC(intc_desc_landisk, "landisk", vectors_landisk, NULL, + mask_registers_landisk, NULL, NULL); /* * Initialize IRQ setting */ void __init init_landisk_IRQ(void) { - int i; - - for (i = 5; i < 14; i++) { - disable_irq_nosync(i); - set_irq_chip_and_handler_name(i, &landisk_irq_chip, - handle_level_irq, "level"); - enable_landisk_irq(irq_get_irq_data(i)); - } + register_intc_controller(&intc_desc_landisk); __raw_writeb(0x00, PA_PWRINT_CLR); } diff --git a/arch/sh/boards/mach-landisk/setup.c b/arch/sh/boards/mach-landisk/setup.c index 50337acc..94186cf 100644 --- a/arch/sh/boards/mach-landisk/setup.c +++ b/arch/sh/boards/mach-landisk/setup.c @@ -21,8 +21,6 @@ #include <mach-landisk/mach/iodata_landisk.h> #include <asm/io.h> -void init_landisk_IRQ(void); - static void landisk_power_off(void) { __raw_writeb(0x01, PA_SHUTDOWN); @@ -83,7 +81,7 @@ static int __init landisk_devices_setup(void) ARRAY_SIZE(landisk_devices)); } -__initcall(landisk_devices_setup); +device_initcall(landisk_devices_setup); static void __init landisk_setup(char **cmdline_p) { diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index c8acfec..03a7ffe7 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -13,6 +13,7 @@ #include <linux/input.h> #include <linux/input/sh_keysc.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/mtd/nand.h> #include <linux/i2c.h> @@ -410,6 +411,7 @@ static struct resource sdhi_cn9_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi_cn9_device = { diff --git a/arch/sh/boards/mach-r2d/setup.c b/arch/sh/boards/mach-r2d/setup.c index b84df6a..4b98a52 100644 --- a/arch/sh/boards/mach-r2d/setup.c +++ b/arch/sh/boards/mach-r2d/setup.c @@ -258,7 +258,7 @@ static int __init rts7751r2d_devices_setup(void) return platform_add_devices(rts7751r2d_devices, ARRAY_SIZE(rts7751r2d_devices)); } -__initcall(rts7751r2d_devices_setup); +device_initcall(rts7751r2d_devices_setup); static void rts7751r2d_power_off(void) { diff --git a/arch/sh/boards/mach-sdk7786/setup.c b/arch/sh/boards/mach-sdk7786/setup.c index 75e4ddb..1521aa7 100644 --- a/arch/sh/boards/mach-sdk7786/setup.c +++ b/arch/sh/boards/mach-sdk7786/setup.c @@ -15,13 +15,13 @@ #include <linux/i2c.h> #include <linux/irq.h> #include <linux/clk.h> +#include <linux/clkdev.h> #include <mach/fpga.h> #include <mach/irq.h> #include <asm/machvec.h> #include <asm/heartbeat.h> #include <asm/sizes.h> #include <asm/clock.h> -#include <asm/clkdev.h> #include <asm/reboot.h> #include <asm/smp-ops.h> @@ -135,7 +135,7 @@ static int __init sdk7786_devices_setup(void) return sdk7786_i2c_setup(); } -__initcall(sdk7786_devices_setup); +device_initcall(sdk7786_devices_setup); static int sdk7786_mode_pins(void) { diff --git a/arch/sh/boards/mach-se/7206/setup.c b/arch/sh/boards/mach-se/7206/setup.c index 33039e0..8ab8330 100644 --- a/arch/sh/boards/mach-se/7206/setup.c +++ b/arch/sh/boards/mach-se/7206/setup.c @@ -77,7 +77,7 @@ static int __init se7206_devices_setup(void) { return platform_add_devices(se7206_devices, ARRAY_SIZE(se7206_devices)); } -__initcall(se7206_devices_setup); +device_initcall(se7206_devices_setup); static int se7206_mode_pins(void) { diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c index 527a0cd..5276793 100644 --- a/arch/sh/boards/mach-se/7724/setup.c +++ b/arch/sh/boards/mach-se/7724/setup.c @@ -15,6 +15,7 @@ #include <linux/interrupt.h> #include <linux/platform_device.h> #include <linux/mfd/sh_mobile_sdhi.h> +#include <linux/mmc/host.h> #include <linux/mtd/physmap.h> #include <linux/delay.h> #include <linux/smc91x.h> @@ -318,6 +319,10 @@ static struct platform_device fsi_device = { }, }; +static struct platform_device fsi_ak4642_device = { + .name = "sh_fsi_a_ak4642", +}; + /* KEYSC in SoC (Needs SW33-2 set to ON) */ static struct sh_keysc_info keysc_info = { .mode = SH_KEYSC_MODE_1, @@ -467,6 +472,7 @@ static struct resource sdhi0_cn7_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi0_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI0_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI0_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi0_cn7_device = { @@ -498,6 +504,7 @@ static struct resource sdhi1_cn8_resources[] = { static struct sh_mobile_sdhi_info sh7724_sdhi1_data = { .dma_slave_tx = SHDMA_SLAVE_SDHI1_TX, .dma_slave_rx = SHDMA_SLAVE_SDHI1_RX, + .tmio_caps = MMC_CAP_SDIO_IRQ, }; static struct platform_device sdhi1_cn8_device = { @@ -590,6 +597,7 @@ static struct platform_device *ms7724se_devices[] __initdata = { &sh7724_usb0_host_device, &sh7724_usb1_gadget_device, &fsi_device, + &fsi_ak4642_device, &sdhi0_cn7_device, &sdhi1_cn8_device, &irda_device, diff --git a/arch/sh/boards/mach-se/7751/setup.c b/arch/sh/boards/mach-se/7751/setup.c index 9fbc51b..4ed60c5 100644 --- a/arch/sh/boards/mach-se/7751/setup.c +++ b/arch/sh/boards/mach-se/7751/setup.c @@ -48,7 +48,7 @@ static int __init se7751_devices_setup(void) { return platform_add_devices(se7751_devices, ARRAY_SIZE(se7751_devices)); } -__initcall(se7751_devices_setup); +device_initcall(se7751_devices_setup); /* * The Machine Vector diff --git a/arch/sh/boards/mach-sh03/rtc.c b/arch/sh/boards/mach-sh03/rtc.c index 1b20099..f83ac79 100644 --- a/arch/sh/boards/mach-sh03/rtc.c +++ b/arch/sh/boards/mach-sh03/rtc.c @@ -108,7 +108,7 @@ static int set_rtc_mmss(unsigned long nowtime) __raw_writeb(real_minutes % 10, RTC_MIN1); __raw_writeb(real_minutes / 10, RTC_MIN10); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/sh/boards/mach-sh03/setup.c b/arch/sh/boards/mach-sh03/setup.c index af4a0c0..d4f79b2 100644 --- a/arch/sh/boards/mach-sh03/setup.c +++ b/arch/sh/boards/mach-sh03/setup.c @@ -96,7 +96,7 @@ static int __init sh03_devices_setup(void) return platform_add_devices(sh03_devices, ARRAY_SIZE(sh03_devices)); } -__initcall(sh03_devices_setup); +device_initcall(sh03_devices_setup); static struct sh_machine_vector mv_sh03 __initmv = { .mv_name = "Interface (CTP/PCI-SH03)", diff --git a/arch/sh/boot/romimage/mmcif-sh7724.c b/arch/sh/boot/romimage/mmcif-sh7724.c index 14863d7..c84e783 100644 --- a/arch/sh/boot/romimage/mmcif-sh7724.c +++ b/arch/sh/boot/romimage/mmcif-sh7724.c @@ -21,9 +21,6 @@ #define HIZCRC 0xa405015c #define DRVCRA 0xa405018a -enum { MMCIF_PROGRESS_ENTER, MMCIF_PROGRESS_INIT, - MMCIF_PROGRESS_LOAD, MMCIF_PROGRESS_DONE }; - /* SH7724 specific MMCIF loader * * loads the romImage from an MMC card starting from block 512 @@ -63,7 +60,9 @@ asmlinkage void mmcif_loader(unsigned char *buf, unsigned long no_bytes) mmcif_update_progress(MMCIF_PROGRESS_LOAD); /* load kernel via MMCIF interface */ - sh_mmcif_boot_slurp(MMCIF_BASE, buf, no_bytes); + sh_mmcif_boot_do_read(MMCIF_BASE, 512, + (no_bytes + SH_MMCIF_BBS - 1) / SH_MMCIF_BBS, + buf); /* disable clock to the MMCIF hardware block */ __raw_writel(__raw_readl(MSTPCR2) | 0x20000000, MSTPCR2); diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig new file mode 100644 index 0000000..6cb3279 --- /dev/null +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -0,0 +1,102 @@ +CONFIG_EXPERIMENTAL=y +CONFIG_SYSVIPC=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_SYSFS_DEPRECATED=y +CONFIG_SYSFS_DEPRECATED_V2=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +# CONFIG_BLK_DEV_BSG is not set +CONFIG_CPU_SUBTYPE_SH7785=y +CONFIG_MEMORY_START=0x0C000000 +CONFIG_FLATMEM_MANUAL=y +CONFIG_SH_STORE_QUEUES=y +CONFIG_SH_APSH4A3A=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_KEXEC=y +CONFIG_PREEMPT=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +# CONFIG_INET_LRO is not set +# CONFIG_IPV6 is not set +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +# CONFIG_FW_LOADER is not set +CONFIG_MTD=y +CONFIG_MTD_CONCAT=y +CONFIG_MTD_PARTITIONS=y +CONFIG_MTD_CHAR=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_PHYSMAP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_NETDEVICES=y +CONFIG_NET_ETHERNET=y +CONFIG_SMSC911X=y +# CONFIG_NETDEV_1000 is not set +# CONFIG_NETDEV_10000 is not set +# CONFIG_WLAN is not set +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO is not set +CONFIG_VT_HW_CONSOLE_BINDING=y +CONFIG_SERIAL_SH_SCI=y +CONFIG_SERIAL_SH_SCI_NR_UARTS=6 +CONFIG_SERIAL_SH_SCI_CONSOLE=y +CONFIG_HW_RANDOM=y +# CONFIG_HWMON is not set +CONFIG_FB=y +CONFIG_FB_SH7785FB=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +CONFIG_LOGO=y +# CONFIG_HID_SUPPORT is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_JFFS2_FS=y +CONFIG_CRAMFS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V4=y +CONFIG_CIFS=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_932=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +# CONFIG_ENABLE_WARN_DEPRECATED is not set +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_KERNEL=y +# CONFIG_DEBUG_PREEMPT is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO=y +# CONFIG_RCU_CPU_STALL_DETECTOR is not set +# CONFIG_FTRACE is not set +# CONFIG_CRYPTO_ANSI_CPRNG is not set +# CONFIG_CRYPTO_HW is not set diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig new file mode 100644 index 0000000..e71a531 --- /dev/null +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -0,0 +1,133 @@ +CONFIG_EXPERIMENTAL=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_RCU_TRACE=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUPS=y +CONFIG_CGROUP_NS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_RESOURCE_COUNTERS=y +CONFIG_CGROUP_MEM_RES_CTLR=y +CONFIG_BLK_CGROUP=y +CONFIG_NAMESPACES=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +# CONFIG_LBDAF is not set +# CONFIG_BLK_DEV_BSG is not set +CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_CPU_SUBTYPE_SH7786=y +CONFIG_MEMORY_SIZE=0x10000000 +CONFIG_HUGETLB_PAGE_SIZE_1MB=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_KSM=y +CONFIG_SH_STORE_QUEUES=y +CONFIG_SH_APSH4AD0A=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=m +CONFIG_CPU_FREQ_GOV_USERSPACE=m +CONFIG_CPU_FREQ_GOV_ONDEMAND=m +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m +CONFIG_SH_CPU_FREQ=y +CONFIG_KEXEC=y +CONFIG_SECCOMP=y +CONFIG_PREEMPT=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_PM=y +CONFIG_PM_DEBUG=y +CONFIG_PM_VERBOSE=y +CONFIG_PM_RUNTIME=y +CONFIG_CPU_IDLE=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_NET_KEY=y +CONFIG_INET=y +# CONFIG_INET_LRO is not set +# CONFIG_IPV6 is not set +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +# CONFIG_FW_LOADER is not set +CONFIG_MTD=y +CONFIG_MTD_CFI=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_SCSI_MULTI_LUN=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_NETDEVICES=y +CONFIG_MDIO_BITBANG=y +CONFIG_NET_ETHERNET=y +CONFIG_SMSC911X=y +# CONFIG_NETDEV_1000 is not set +# CONFIG_NETDEV_10000 is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO is not set +CONFIG_SERIAL_SH_SCI=y +CONFIG_SERIAL_SH_SCI_NR_UARTS=6 +CONFIG_SERIAL_SH_SCI_CONSOLE=y +# CONFIG_LEGACY_PTYS is not set +# CONFIG_HW_RANDOM is not set +# CONFIG_HWMON is not set +CONFIG_VIDEO_OUTPUT_CONTROL=y +CONFIG_FB=y +CONFIG_FB_SH7785FB=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_DEBUG=y +CONFIG_USB_MON=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_HUGETLBFS=y +CONFIG_JFFS2_FS=y +CONFIG_CRAMFS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V4=y +CONFIG_CIFS=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_932=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_SHIRQ=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_VM=y +# CONFIG_RCU_CPU_STALL_DETECTOR is not set +CONFIG_DWARF_UNWINDER=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig index 273f3fa..5f7f667 100644 --- a/arch/sh/configs/sh7757lcr_defconfig +++ b/arch/sh/configs/sh7757lcr_defconfig @@ -39,21 +39,15 @@ CONFIG_IPV6=y CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_RAM=y -# CONFIG_MISC_DEVICES is not set CONFIG_NETDEVICES=y -CONFIG_PHYLIB=y CONFIG_VITESSE_PHY=y -CONFIG_MDIO_BITBANG=y CONFIG_NET_ETHERNET=y -CONFIG_MII=y +CONFIG_SH_ETH=y # CONFIG_NETDEV_10000 is not set # CONFIG_WLAN is not set # CONFIG_KEYBOARD_ATKBD is not set # CONFIG_MOUSE_PS2 is not set # CONFIG_SERIO is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=2 CONFIG_SERIAL_SH_SCI=y CONFIG_SERIAL_SH_SCI_NR_UARTS=3 CONFIG_SERIAL_SH_SCI_CONSOLE=y @@ -63,7 +57,6 @@ CONFIG_SERIAL_SH_SCI_CONSOLE=y # CONFIG_USB_SUPPORT is not set CONFIG_EXT2_FS=y CONFIG_EXT3_FS=y -CONFIG_INOTIFY=y CONFIG_ISO9660_FS=y CONFIG_VFAT_FS=y CONFIG_PROC_KCORE=y @@ -76,10 +69,8 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y CONFIG_DEBUG_KERNEL=y -# CONFIG_DETECT_SOFTLOCKUP is not set # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_BUGVERBOSE is not set CONFIG_DEBUG_INFO=y -# CONFIG_RCU_CPU_STALL_DETECTOR is not set # CONFIG_FTRACE is not set # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/drivers/pci/fixups-landisk.c b/arch/sh/drivers/pci/fixups-landisk.c index bb1a6bb..95c6e2d 100644 --- a/arch/sh/drivers/pci/fixups-landisk.c +++ b/arch/sh/drivers/pci/fixups-landisk.c @@ -1,9 +1,10 @@ /* - * arch/sh/drivers/pci/ops-landisk.c + * arch/sh/drivers/pci/fixups-landisk.c * * PCI initialization for the I-O DATA Device, Inc. LANDISK board * * Copyright (C) 2006 kogiidena + * Copyright (C) 2010 Nobuhiro Iwamatsu * * May be copied or modified under the terms of the GNU General Public * License. See linux/COPYING for more information. @@ -15,6 +16,9 @@ #include <linux/pci.h> #include "pci-sh4.h" +#define PCIMCR_MRSET_OFF 0xBFFFFFFF +#define PCIMCR_RFSH_OFF 0xFFFFFFFB + int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin) { /* @@ -26,9 +30,29 @@ int pcibios_map_platform_irq(struct pci_dev *pdev, u8 slot, u8 pin) int irq = ((slot + pin - 1) & 0x3) + 5; if ((slot | (pin - 1)) > 0x3) { - printk("PCI: Bad IRQ mapping request for slot %d pin %c\n", + printk(KERN_WARNING "PCI: Bad IRQ mapping request for slot %d pin %c\n", slot, pin - 1 + 'A'); return -1; } return irq; } + +int pci_fixup_pcic(struct pci_channel *chan) +{ + unsigned long bcr1, mcr; + + bcr1 = __raw_readl(SH7751_BCR1); + bcr1 |= 0x40080000; /* Enable Bit 19 BREQEN, set PCIC to slave */ + pci_write_reg(chan, bcr1, SH4_PCIBCR1); + + mcr = __raw_readl(SH7751_MCR); + mcr = (mcr & PCIMCR_MRSET_OFF) & PCIMCR_RFSH_OFF; + pci_write_reg(chan, mcr, SH4_PCIMCR); + + pci_write_reg(chan, 0x0c000000, SH7751_PCICONF5); + pci_write_reg(chan, 0xd0000000, SH7751_PCICONF6); + pci_write_reg(chan, 0x0c000000, SH4_PCILAR0); + pci_write_reg(chan, 0x00000000, SH4_PCILAR1); + + return 0; +} diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index 89ab2c5..28c5aa5 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h @@ -11,11 +11,6 @@ * * While read{b,w,l,q} and write{b,w,l,q} contain memory barriers * automatically, there are also __raw versions, which do not. - * - * Historically, we have also had ctrl_in{b,w,l,q}/ctrl_out{b,w,l,q} for - * SuperH specific I/O (raw I/O to on-chip CPU peripherals). In practice - * these have the same semantics as the __raw variants, and as such, all - * new code should be using the __raw versions. */ #include <linux/errno.h> #include <asm/cache.h> @@ -231,52 +226,6 @@ __BUILD_IOPORT_STRING(q, u64) #endif -/* - * Legacy SuperH on-chip I/O functions - * - * These are all deprecated, all new (and especially cross-platform) code - * should be using the __raw_xxx() routines directly. - */ -static inline u8 __deprecated ctrl_inb(unsigned long addr) -{ - return __raw_readb(addr); -} - -static inline u16 __deprecated ctrl_inw(unsigned long addr) -{ - return __raw_readw(addr); -} - -static inline u32 __deprecated ctrl_inl(unsigned long addr) -{ - return __raw_readl(addr); -} - -static inline u64 __deprecated ctrl_inq(unsigned long addr) -{ - return __raw_readq(addr); -} - -static inline void __deprecated ctrl_outb(u8 v, unsigned long addr) -{ - __raw_writeb(v, addr); -} - -static inline void __deprecated ctrl_outw(u16 v, unsigned long addr) -{ - __raw_writew(v, addr); -} - -static inline void __deprecated ctrl_outl(u32 v, unsigned long addr) -{ - __raw_writel(v, addr); -} - -static inline void __deprecated ctrl_outq(u64 v, unsigned long addr) -{ - __raw_writeq(v, addr); -} - #define IO_SPACE_LIMIT 0xffffffff /* synco on SH-4A, otherwise a nop */ @@ -341,7 +290,15 @@ __ioremap_29bit(phys_addr_t offset, unsigned long size, pgprot_t prot) * mapping must be done by the PMB or by using page tables. */ if (likely(PXSEG(offset) < P3SEG && PXSEG(last_addr) < P3SEG)) { - if (unlikely(pgprot_val(prot) & _PAGE_CACHABLE)) + u64 flags = pgprot_val(prot); + + /* + * Anything using the legacy PTEA space attributes needs + * to be kicked down to page table mappings. + */ + if (unlikely(flags & _PAGE_PCC_MASK)) + return NULL; + if (unlikely(flags & _PAGE_CACHABLE)) return (void __iomem *)P1SEGADDR(offset); return (void __iomem *)P2SEGADDR(offset); diff --git a/arch/sh/include/asm/machvec.h b/arch/sh/include/asm/machvec.h index dd5d6e5..57c5c3d 100644 --- a/arch/sh/include/asm/machvec.h +++ b/arch/sh/include/asm/machvec.h @@ -31,6 +31,7 @@ struct sh_machine_vector { int (*mv_mode_pins)(void); void (*mv_mem_init)(void); + void (*mv_mem_reserve)(void); }; extern struct sh_machine_vector sh_mv; diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 43528ec..b799fe7 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -76,6 +76,10 @@ /* Wrapper for extended mode pgprot twiddling */ #define _PAGE_EXT(x) ((unsigned long long)(x) << 32) +#ifdef CONFIG_X2TLB +#define _PAGE_PCC_MASK 0x00000000 /* No legacy PTEA support */ +#else + /* software: moves to PTEA.TC (Timing Control) */ #define _PAGE_PCC_AREA5 0x00000000 /* use BSC registers for area5 */ #define _PAGE_PCC_AREA6 0x80000000 /* use BSC registers for area6 */ @@ -89,7 +93,8 @@ #define _PAGE_PCC_ATR8 0x60000000 /* Attribute Memory space, 8 bit bus */ #define _PAGE_PCC_ATR16 0x60000001 /* Attribute Memory space, 6 bit bus */ -#ifndef CONFIG_X2TLB +#define _PAGE_PCC_MASK 0xe0000001 + /* copy the ptea attributes */ static inline unsigned long copy_ptea_attributes(unsigned long x) { @@ -231,13 +236,7 @@ static inline unsigned long copy_ptea_attributes(unsigned long x) _PAGE_EXT_KERN_EXEC)) #define PAGE_KERNEL_PCC(slot, type) \ - __pgprot(_PAGE_PRESENT | _PAGE_DIRTY | \ - _PAGE_ACCESSED | _PAGE_FLAGS_HARD | \ - _PAGE_EXT(_PAGE_EXT_KERN_READ | \ - _PAGE_EXT_KERN_WRITE | \ - _PAGE_EXT_KERN_EXEC) \ - (slot ? _PAGE_PCC_AREA5 : _PAGE_PCC_AREA6) | \ - (type)) + __pgprot(0) #elif defined(CONFIG_MMU) /* SH-X TLB */ #define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_CACHABLE | \ diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h index c9e7cbc..9c7bdfc 100644 --- a/arch/sh/include/asm/processor.h +++ b/arch/sh/include/asm/processor.h @@ -35,7 +35,7 @@ enum cpu_type { CPU_SH7723, CPU_SH7724, CPU_SH7757, CPU_SHX3, /* SH4AL-DSP types */ - CPU_SH7343, CPU_SH7722, CPU_SH7366, + CPU_SH7343, CPU_SH7722, CPU_SH7366, CPU_SH7372, /* SH-5 types */ CPU_SH5_101, CPU_SH5_103, diff --git a/arch/sh/include/asm/processor_32.h b/arch/sh/include/asm/processor_32.h index e3c73cd..900f8d7 100644 --- a/arch/sh/include/asm/processor_32.h +++ b/arch/sh/include/asm/processor_32.h @@ -194,15 +194,17 @@ extern unsigned long get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[15]) #if defined(CONFIG_CPU_SH2A) || defined(CONFIG_CPU_SH4) + #define PREFETCH_STRIDE L1_CACHE_BYTES #define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW -static inline void prefetch(void *x) + +static inline void prefetch(const void *x) { __builtin_prefetch(x, 0, 3); } -static inline void prefetchw(void *x) +static inline void prefetchw(const void *x) { __builtin_prefetch(x, 1, 3); } diff --git a/arch/sh/include/mach-landisk/mach/iodata_landisk.h b/arch/sh/include/mach-landisk/mach/iodata_landisk.h index 6fb04ab..f432773 100644 --- a/arch/sh/include/mach-landisk/mach/iodata_landisk.h +++ b/arch/sh/include/mach-landisk/mach/iodata_landisk.h @@ -2,7 +2,7 @@ #define __ASM_SH_IODATA_LANDISK_H /* - * linux/include/asm-sh/landisk/iodata_landisk.h + * arch/sh/include/mach-landisk/mach/iodata_landisk.h * * Copyright (C) 2000 Atom Create Engineering Co., Ltd. * @@ -27,7 +27,7 @@ #define IRQ_PCIINTA 5 /* PCI INTA IRQ */ #define IRQ_PCIINTB 6 /* PCI INTB IRQ */ -#define IRQ_PCIINDC 7 /* PCI INTC IRQ */ +#define IRQ_PCIINTC 7 /* PCI INTC IRQ */ #define IRQ_PCIINTD 8 /* PCI INTD IRQ */ #define IRQ_ATA 9 /* ATA IRQ */ #define IRQ_FATA 10 /* FATA IRQ */ @@ -35,6 +35,8 @@ #define IRQ_BUTTON 12 /* USL-5P Button IRQ */ #define IRQ_FAULT 13 /* USL-5P Fault IRQ */ +void init_landisk_IRQ(void); + #define __IO_PREFIX landisk #include <asm/io_generic.h> diff --git a/arch/sh/kernel/cpu/proc.c b/arch/sh/kernel/cpu/proc.c index e80a936..f47be87 100644 --- a/arch/sh/kernel/cpu/proc.c +++ b/arch/sh/kernel/cpu/proc.c @@ -25,7 +25,7 @@ static const char *cpu_name[] = { [CPU_SH5_101] = "SH5-101", [CPU_SH5_103] = "SH5-103", [CPU_MXG] = "MX-G", [CPU_SH7723] = "SH7723", [CPU_SH7366] = "SH7366", [CPU_SH7724] = "SH7724", - [CPU_SH_NONE] = "Unknown" + [CPU_SH7372] = "SH7372", [CPU_SH_NONE] = "Unknown" }; const char *get_cpu_subtype(struct sh_cpuinfo *c) diff --git a/arch/sh/kernel/cpu/sh2/setup-sh7619.c b/arch/sh/kernel/cpu/sh2/setup-sh7619.c index c363851..0f8befc 100644 --- a/arch/sh/kernel/cpu/sh2/setup-sh7619.c +++ b/arch/sh/kernel/cpu/sh2/setup-sh7619.c @@ -62,6 +62,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7619", vectors, NULL, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xf8400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 88, 88, 88, 88 }, }; @@ -77,6 +79,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xf8410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 92, 92, 92, 92 }, }; @@ -92,6 +96,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xf8420000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 96, 96, 96, 96 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-mxg.c b/arch/sh/kernel/cpu/sh2a/setup-mxg.c index 6c96ea0..949bf2b 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-mxg.c +++ b/arch/sh/kernel/cpu/sh2a/setup-mxg.c @@ -201,6 +201,8 @@ static struct platform_device mtu2_2_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xff804000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 220, 220, 220, 220 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c index d08bf4c..9df558d 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7201.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7201.c @@ -180,6 +180,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7201", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 180, 180, 180, 180 } }; @@ -195,6 +197,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 184, 184, 184, 184 } }; @@ -210,6 +214,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 188, 188, 188, 188 } }; @@ -225,6 +231,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 192, 192, 192, 192 } }; @@ -240,6 +248,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xfffea000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 196, 196, 196, 196 } }; @@ -255,6 +265,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xfffea800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 200, 200, 200, 200 } }; @@ -270,6 +282,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xfffeb000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 204, 204, 204, 204 } }; @@ -285,6 +299,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xfffeb800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 208, 208, 208, 208 } }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c index 832f401..a43124e 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7203.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7203.c @@ -176,6 +176,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7203", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 192, 192, 192, 192 }, }; @@ -191,6 +193,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 196, 196, 196, 196 }, }; @@ -206,6 +210,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 200, 200, 200, 200 }, }; @@ -221,6 +227,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 204, 204, 204, 204 }, }; diff --git a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c index dc47b04..5d14f84 100644 --- a/arch/sh/kernel/cpu/sh2a/setup-sh7206.c +++ b/arch/sh/kernel/cpu/sh2a/setup-sh7206.c @@ -136,6 +136,8 @@ static DECLARE_INTC_DESC(intc_desc, "sh7206", vectors, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffe8000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 240, 240, 240, 240 }, }; @@ -151,6 +153,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfffe8800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 244, 244, 244, 244 }, }; @@ -166,6 +170,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfffe9000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 248, 248, 248, 248 }, }; @@ -181,6 +187,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfffe9800, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 252, 252, 252, 252 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7705.c b/arch/sh/kernel/cpu/sh3/setup-sh7705.c index baadd7f..cd2e702 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7705.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7705.c @@ -70,6 +70,9 @@ static DECLARE_INTC_DESC(intc_desc, "sh7705", vectors, NULL, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TIE | SCSCR_RIE | SCSCR_TE | + SCSCR_RE | SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 56, 56, 56 }, }; @@ -85,6 +88,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TIE | SCSCR_RIE | SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 52, 52, 52 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh770x.c b/arch/sh/kernel/cpu/sh3/setup-sh770x.c index 3cf8c8e..4551ad6 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh770x.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh770x.c @@ -109,6 +109,8 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfffffe80, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 23, 23, 23, 0 }, }; @@ -126,6 +128,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4000150, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 56, 56, 56, 56 }, }; @@ -143,6 +147,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xa4000140, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_IRDA, .irqs = { 52, 52, 52, 52 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7710.c b/arch/sh/kernel/cpu/sh3/setup-sh7710.c index b0c2fb4..78f6b01 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7710.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7710.c @@ -99,6 +99,9 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4400000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE | + SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 52, 52, 52 }, }; @@ -114,6 +117,9 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4410000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE | + SCSCR_CKE1 | SCSCR_CKE0, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 56, 56, 56, 56 }, }; diff --git a/arch/sh/kernel/cpu/sh3/setup-sh7720.c b/arch/sh/kernel/cpu/sh3/setup-sh7720.c index 24b1713..365b94a6 100644 --- a/arch/sh/kernel/cpu/sh3/setup-sh7720.c +++ b/arch/sh/kernel/cpu/sh3/setup-sh7720.c @@ -1,5 +1,5 @@ /* - * SH7720 Setup + * Setup code for SH7720, SH7721. * * Copyright (C) 2007 Markus Brunner, Mark Jonas * Copyright (C) 2009 Paul Mundt @@ -51,6 +51,8 @@ static struct platform_device rtc_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xa4430000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -66,6 +68,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xa4438000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_4, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; diff --git a/arch/sh/kernel/cpu/sh4/probe.c b/arch/sh/kernel/cpu/sh4/probe.c index b93458f..971cf0f 100644 --- a/arch/sh/kernel/cpu/sh4/probe.c +++ b/arch/sh/kernel/cpu/sh4/probe.c @@ -151,8 +151,14 @@ void __cpuinit cpu_probe(void) boot_cpu_data.flags |= CPU_HAS_L2_CACHE; break; case 0x10: + case 0x11: boot_cpu_data.type = CPU_SH7757; break; + case 0xd0: + case 0x40: /* yon-ten-go */ + boot_cpu_data.type = CPU_SH7372; + break; + } break; case 0x4000: /* 1st cut */ diff --git a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c index e916b18..5b28331 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh4-202.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh4-202.c @@ -18,6 +18,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7750.c b/arch/sh/kernel/cpu/sh4/setup-sh7750.c index 911d196..c2b0aaa 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh7750.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh7750.c @@ -14,6 +14,7 @@ #include <linux/io.h> #include <linux/sh_timer.h> #include <linux/serial_sci.h> +#include <asm/machtypes.h> static struct resource rtc_resources[] = { [0] = { @@ -35,33 +36,37 @@ static struct platform_device rtc_device = { .resource = rtc_resources, }; -static struct plat_sci_port scif0_platform_data = { +static struct plat_sci_port sci_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 23, 23, 23, 0 }, }; -static struct platform_device scif0_device = { +static struct platform_device sci_device = { .name = "sh-sci", .id = 0, .dev = { - .platform_data = &scif0_platform_data, + .platform_data = &sci_platform_data, }, }; -static struct plat_sci_port scif1_platform_data = { +static struct plat_sci_port scif_platform_data = { .mapbase = 0xffe80000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_TE | SCSCR_RE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; -static struct platform_device scif1_device = { +static struct platform_device scif_device = { .name = "sh-sci", .id = 1, .dev = { - .platform_data = &scif1_platform_data, + .platform_data = &scif_platform_data, }, }; @@ -210,8 +215,6 @@ static struct platform_device tmu4_device = { #endif static struct platform_device *sh7750_devices[] __initdata = { - &scif0_device, - &scif1_device, &rtc_device, &tmu0_device, &tmu1_device, @@ -226,14 +229,19 @@ static struct platform_device *sh7750_devices[] __initdata = { static int __init sh7750_devices_setup(void) { + if (mach_is_rts7751r2d()) { + platform_register_device(&scif_device); + } else { + platform_register_device(&sci_device); + platform_register_device(&scif_device); + } + return platform_add_devices(sh7750_devices, ARRAY_SIZE(sh7750_devices)); } arch_initcall(sh7750_devices_setup); static struct platform_device *sh7750_early_devices[] __initdata = { - &scif0_device, - &scif1_device, &tmu0_device, &tmu1_device, &tmu2_device, @@ -247,6 +255,14 @@ static struct platform_device *sh7750_early_devices[] __initdata = { void __init plat_early_device_setup(void) { + if (mach_is_rts7751r2d()) { + scif_platform_data.scscr |= SCSCR_CKE1; + early_platform_add_devices(&scif_device, 1); + } else { + early_platform_add_devices(&sci_device, 1); + early_platform_add_devices(&scif_device, 1); + } + early_platform_add_devices(sh7750_early_devices, ARRAY_SIZE(sh7750_early_devices)); } diff --git a/arch/sh/kernel/cpu/sh4/setup-sh7760.c b/arch/sh/kernel/cpu/sh4/setup-sh7760.c index 48ea8fe..78bbf23 100644 --- a/arch/sh/kernel/cpu/sh4/setup-sh7760.c +++ b/arch/sh/kernel/cpu/sh4/setup-sh7760.c @@ -129,6 +129,8 @@ static DECLARE_INTC_DESC(intc_desc_irq, "sh7760-irq", vectors_irq, groups, static struct plat_sci_port scif0_platform_data = { .mapbase = 0xfe600000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 53, 55, 54 }, }; @@ -145,6 +147,8 @@ static struct plat_sci_port scif1_platform_data = { .mapbase = 0xfe610000, .flags = UPF_BOOT_AUTOCONF, .type = PORT_SCIF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .irqs = { 72, 73, 75, 74 }, }; @@ -159,6 +163,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfe620000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 77, 79, 78 }, }; @@ -174,6 +180,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfe480000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCI, .irqs = { 80, 81, 82, 0 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c index 3681cafd..1b88483 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7343.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7343.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -34,6 +36,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -49,6 +53,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -64,6 +70,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffe30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 83, 83, 83, 83 }, }; @@ -360,6 +368,8 @@ void __init plat_early_device_setup(void) enum { UNUSED = 0, + ENABLED, + DISABLED, /* interrupt sources */ IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7, @@ -375,15 +385,13 @@ enum { I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI, I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI, SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI, - IRDA, - SDHI0, SDHI1, SDHI2, SDHI3, - CMT, TSIF, SIU, + IRDA, SDHI, CMT, TSIF, SIU, TMU0, TMU1, TMU2, JPU, LCDC, /* interrupt groups */ - DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, SDHI, USB, + DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C0, I2C1, SIM, USB, }; static struct intc_vect vectors[] __initdata = { @@ -412,8 +420,8 @@ static struct intc_vect vectors[] __initdata = { INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0), INTC_VECT(I2C0_ALI, 0xe00), INTC_VECT(I2C0_TACKI, 0xe20), INTC_VECT(I2C0_WAITI, 0xe40), INTC_VECT(I2C0_DTEI, 0xe60), - INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0), - INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0), + INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0), + INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0), INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20), INTC_VECT(SIU, 0xf80), INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420), @@ -431,7 +439,6 @@ static struct intc_group groups[] __initdata = { INTC_GROUP(I2C0, I2C0_ALI, I2C0_TACKI, I2C0_WAITI, I2C0_DTEI), INTC_GROUP(I2C1, I2C1_ALI, I2C1_TACKI, I2C1_WAITI, I2C1_DTEI), INTC_GROUP(SIM, SIM_TEI, SIM_TXI, SIM_RXI, SIM_ERI), - INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3), INTC_GROUP(USB, USBI0, USBI1), }; @@ -452,7 +459,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USBI1, USBI0 } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ @@ -488,9 +495,13 @@ static struct intc_mask_reg ack_registers[] __initdata = { { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } }, }; -static DECLARE_INTC_DESC_ACK(intc_desc, "sh7343", vectors, groups, - mask_registers, prio_registers, sense_registers, - ack_registers); +static struct intc_desc intc_desc __initdata = { + .name = "sh7343", + .force_enable = ENABLED, + .force_disable = DISABLED, + .hw = INTC_HW_DESC(vectors, groups, mask_registers, + prio_registers, sense_registers, ack_registers), +}; void __init plat_irq_setup(void) { diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c index 8dab9e1..82616af 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7366.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7366.c @@ -21,6 +21,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -319,6 +321,8 @@ void __init plat_early_device_setup(void) enum { UNUSED=0, + ENABLED, + DISABLED, /* interrupt sources */ IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7, @@ -332,14 +336,13 @@ enum { DENC, MSIOF, FLCTL_FLSTEI, FLCTL_FLENDI, FLCTL_FLTREQ0I, FLCTL_FLTREQ1I, I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI, - SDHI0, SDHI1, SDHI2, SDHI3, - CMT, TSIF, SIU, + SDHI, CMT, TSIF, SIU, TMU0, TMU1, TMU2, VEU2, LCDC, /* interrupt groups */ - DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C, SDHI, + DMAC0123, VIOVOU, MMC, DMAC45, FLCTL, I2C, }; static struct intc_vect vectors[] __initdata = { @@ -364,8 +367,8 @@ static struct intc_vect vectors[] __initdata = { INTC_VECT(FLCTL_FLTREQ0I, 0xdc0), INTC_VECT(FLCTL_FLTREQ1I, 0xde0), INTC_VECT(I2C_ALI, 0xe00), INTC_VECT(I2C_TACKI, 0xe20), INTC_VECT(I2C_WAITI, 0xe40), INTC_VECT(I2C_DTEI, 0xe60), - INTC_VECT(SDHI0, 0xe80), INTC_VECT(SDHI1, 0xea0), - INTC_VECT(SDHI2, 0xec0), INTC_VECT(SDHI3, 0xee0), + INTC_VECT(SDHI, 0xe80), INTC_VECT(SDHI, 0xea0), + INTC_VECT(SDHI, 0xec0), INTC_VECT(SDHI, 0xee0), INTC_VECT(CMT, 0xf00), INTC_VECT(TSIF, 0xf20), INTC_VECT(SIU, 0xf80), INTC_VECT(TMU0, 0x400), INTC_VECT(TMU1, 0x420), @@ -381,7 +384,6 @@ static struct intc_group groups[] __initdata = { INTC_GROUP(FLCTL, FLCTL_FLSTEI, FLCTL_FLENDI, FLCTL_FLTREQ0I, FLCTL_FLTREQ1I), INTC_GROUP(I2C, I2C_ALI, I2C_TACKI, I2C_WAITI, I2C_DTEI), - INTC_GROUP(SDHI, SDHI0, SDHI1, SDHI2, SDHI3), }; static struct intc_mask_reg mask_registers[] __initdata = { @@ -403,7 +405,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { SDHI3, SDHI2, SDHI1, SDHI0, 0, 0, 0, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, 0, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB, } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ @@ -441,9 +443,13 @@ static struct intc_mask_reg ack_registers[] __initdata = { { IRQ0, IRQ1, IRQ2, IRQ3, IRQ4, IRQ5, IRQ6, IRQ7 } }, }; -static DECLARE_INTC_DESC_ACK(intc_desc, "sh7366", vectors, groups, - mask_registers, prio_registers, sense_registers, - ack_registers); +static struct intc_desc intc_desc __initdata = { + .name = "sh7366", + .force_enable = ENABLED, + .force_disable = DISABLED, + .hw = INTC_HW_DESC(vectors, groups, mask_registers, + prio_registers, sense_registers, ack_registers), +}; void __init plat_irq_setup(void) { diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c index d551ed8..5813d80 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c @@ -181,6 +181,8 @@ struct platform_device dma_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -196,6 +198,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -211,6 +215,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -699,7 +705,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { DISABLED, DISABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } }, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, TWODG, SIU } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB_USBI1, USB_USBI0, } }, { 0xa40800a8, 0xa40800e8, 8, /* IMR10 / IMCR10 */ diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c index 0eadefd..0723822 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7723.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7723.c @@ -24,6 +24,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -39,6 +41,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -54,6 +58,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -69,6 +75,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xa4e30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 56, 56, 56, 56 }, }; @@ -84,6 +92,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xa4e40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 88, 88, 88, 88 }, }; @@ -99,6 +109,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xa4e50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 109, 109, 109, 109 }, }; @@ -719,7 +731,7 @@ static struct intc_group groups[] __initdata = { static struct intc_mask_reg mask_registers[] __initdata = { { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */ { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0, - 0, DISABLED, ENABLED, ENABLED } }, + 0, ENABLED, ENABLED, ENABLED } }, { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */ { VIO_VOUI, VIO_VEU2HI,VIO_BEUI,VIO_CEUI,DMAC0A_DEI3,DMAC0A_DEI2,DMAC0A_DEI1,DMAC0A_DEI0 } }, { 0xa4080088, 0xa40800c8, 8, /* IMR2 / IMCR2 */ @@ -736,7 +748,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C_DTEI, I2C_WAITI, I2C_TACKI, I2C_ALI, FLCTL_FLTREQ1I, FLCTL_FLTREQ0I, FLCTL_FLTENDI, FLCTL_FLSTEI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { 0, DISABLED, ENABLED, ENABLED, + { 0, ENABLED, ENABLED, ENABLED, 0, 0, SCIFA_SCIFA2, SIU_SIUI } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT_CMTI, 0, 0, USB_USI0,0 } }, diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c index 828c965..0333fe9 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7724.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7724.c @@ -257,6 +257,8 @@ static struct platform_device dma1_device = { static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 80, 80, 80, 80 }, }; @@ -272,6 +274,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 81, 81, 81, 81 }, }; @@ -287,6 +291,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe20000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 82, 82, 82, 82 }, }; @@ -302,6 +308,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xa4e30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 56, 56, 56, 56 }, }; @@ -317,6 +325,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xa4e40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 88, 88, 88, 88 }, }; @@ -332,6 +342,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xa4e50000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE, + .scbrr_algo_id = SCBRR_ALGO_3, .type = PORT_SCIFA, .irqs = { 109, 109, 109, 109 }, }; @@ -1144,7 +1156,7 @@ static struct intc_group groups[] __initdata = { static struct intc_mask_reg mask_registers[] __initdata = { { 0xa4080080, 0xa40800c0, 8, /* IMR0 / IMCR0 */ { 0, TMU1_TUNI2, TMU1_TUNI1, TMU1_TUNI0, - 0, DISABLED, ENABLED, ENABLED } }, + 0, ENABLED, ENABLED, ENABLED } }, { 0xa4080084, 0xa40800c4, 8, /* IMR1 / IMCR1 */ { VIO_VOU, VIO_VEU1, VIO_BEU0, VIO_CEU0, DMAC0A_DEI3, DMAC0A_DEI2, DMAC0A_DEI1, DMAC0A_DEI0 } }, @@ -1166,7 +1178,7 @@ static struct intc_mask_reg mask_registers[] __initdata = { { I2C0_DTEI, I2C0_WAITI, I2C0_TACKI, I2C0_ALI, I2C1_DTEI, I2C1_WAITI, I2C1_TACKI, I2C1_ALI } }, { 0xa40800a0, 0xa40800e0, 8, /* IMR8 / IMCR8 */ - { DISABLED, DISABLED, ENABLED, ENABLED, + { DISABLED, ENABLED, ENABLED, ENABLED, 0, 0, SCIFA5, FSI } }, { 0xa40800a4, 0xa40800e4, 8, /* IMR9 / IMCR9 */ { 0, 0, 0, CMT, 0, USB1, USB0, 0 } }, diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c index 749c638..9c1de26 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7757.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7757.c @@ -20,6 +20,8 @@ static struct plat_sci_port scif2_platform_data = { .mapbase = 0xfe4b0000, /* SCIF2 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -35,6 +37,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xfe4c0000, /* SCIF3 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -50,6 +54,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xfe4d0000, /* SCIF4 */ .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 104, 104, 104, 104 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c index 5b5f6b0..593eca6 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7763.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7763.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -34,6 +36,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe08000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -49,6 +53,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 104, 104, 104, 104 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c index 7270d7f..2c6aa22 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7770.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7770.c @@ -17,6 +17,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xff923000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 61, 61, 61, 61 }, }; @@ -32,6 +34,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xff924000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 62, 62, 62, 62 }, }; @@ -47,6 +51,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xff925000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 63, 63, 63, 63 }, }; @@ -62,6 +68,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xff926000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 64, 64, 64, 64 }, }; @@ -77,6 +85,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xff927000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 65, 65, 65, 65 }, }; @@ -92,6 +102,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xff928000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 66, 66, 66, 66 }, }; @@ -107,6 +119,8 @@ static struct platform_device scif5_device = { static struct plat_sci_port scif6_platform_data = { .mapbase = 0xff929000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 67, 67, 67, 67 }, }; @@ -122,6 +136,8 @@ static struct platform_device scif6_device = { static struct plat_sci_port scif7_platform_data = { .mapbase = 0xff92a000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 68, 68, 68, 68 }, }; @@ -137,6 +153,8 @@ static struct platform_device scif7_device = { static struct plat_sci_port scif8_platform_data = { .mapbase = 0xff92b000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 69, 69, 69, 69 }, }; @@ -152,6 +170,8 @@ static struct platform_device scif8_device = { static struct plat_sci_port scif9_platform_data = { .mapbase = 0xff92c000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_TOIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 70, 70, 70, 70 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c index 0f41486..08add7f 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7780.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7780.c @@ -20,6 +20,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffe00000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -35,6 +37,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffe10000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 76, 76, 76, 76 }, }; @@ -379,6 +383,7 @@ static int __init sh7780_devices_setup(void) ARRAY_SIZE(sh7780_devices)); } arch_initcall(sh7780_devices_setup); + static struct platform_device *sh7780_early_devices[] __initdata = { &scif0_device, &scif1_device, @@ -392,6 +397,13 @@ static struct platform_device *sh7780_early_devices[] __initdata = { void __init plat_early_device_setup(void) { + if (mach_is_sh2007()) { + scif0_platform_data.scscr &= ~SCSCR_CKE1; + scif0_platform_data.scbrr_algo_id = SCBRR_ALGO_2; + scif1_platform_data.scscr &= ~SCSCR_CKE1; + scif1_platform_data.scbrr_algo_id = SCBRR_ALGO_2; + } + early_platform_add_devices(sh7780_early_devices, ARRAY_SIZE(sh7780_early_devices)); } diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c index c9a572b..18d8fc1 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7785.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7785.c @@ -23,6 +23,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffea0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 40, 40, 40 }, }; @@ -38,6 +40,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffeb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 44, 44, 44, 44 }, }; @@ -53,6 +57,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffec0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 60, 60, 60, 60 }, }; @@ -68,6 +74,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffed0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 61, 61, 61, 61 }, }; @@ -83,6 +91,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xffee0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 62, 62, 62, 62 }, }; @@ -98,6 +108,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xffef0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 63, 63, 63, 63 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c index 0170dbd..1656b8c 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7786.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7786.c @@ -29,6 +29,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffea0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; @@ -47,6 +49,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffeb0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 44, 44, 44, 44 }, }; @@ -62,6 +66,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffec0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 50, 50, 50, 50 }, }; @@ -77,6 +83,8 @@ static struct platform_device scif2_device = { static struct plat_sci_port scif3_platform_data = { .mapbase = 0xffed0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 51, 51, 51, 51 }, }; @@ -92,6 +100,8 @@ static struct platform_device scif3_device = { static struct plat_sci_port scif4_platform_data = { .mapbase = 0xffee0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 52, 52, 52, 52 }, }; @@ -107,6 +117,8 @@ static struct platform_device scif4_device = { static struct plat_sci_port scif5_platform_data = { .mapbase = 0xffef0000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE | SCSCR_CKE1, + .scbrr_algo_id = SCBRR_ALGO_1, .type = PORT_SCIF, .irqs = { 53, 53, 53, 53 }, }; diff --git a/arch/sh/kernel/cpu/sh4a/setup-shx3.c b/arch/sh/kernel/cpu/sh4a/setup-shx3.c index 013f0b1..bb20880 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-shx3.c +++ b/arch/sh/kernel/cpu/sh4a/setup-shx3.c @@ -29,6 +29,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = 0xffc30000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 40, 41, 43, 42 }, }; @@ -44,6 +46,8 @@ static struct platform_device scif0_device = { static struct plat_sci_port scif1_platform_data = { .mapbase = 0xffc40000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 44, 45, 47, 46 }, }; @@ -59,6 +63,8 @@ static struct platform_device scif1_device = { static struct plat_sci_port scif2_platform_data = { .mapbase = 0xffc60000, .flags = UPF_BOOT_AUTOCONF, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 52, 53, 55, 54 }, }; diff --git a/arch/sh/kernel/cpu/sh5/setup-sh5.c b/arch/sh/kernel/cpu/sh5/setup-sh5.c index d910666..18419f1 100644 --- a/arch/sh/kernel/cpu/sh5/setup-sh5.c +++ b/arch/sh/kernel/cpu/sh5/setup-sh5.c @@ -19,6 +19,8 @@ static struct plat_sci_port scif0_platform_data = { .mapbase = PHYS_PERIPHERAL_BLOCK + 0x01030000, .flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP, + .scscr = SCSCR_RE | SCSCR_TE | SCSCR_REIE, + .scbrr_algo_id = SCBRR_ALGO_2, .type = PORT_SCIF, .irqs = { 39, 40, 42, 0 }, }; diff --git a/arch/sh/kernel/cpu/shmobile/pm.c b/arch/sh/kernel/cpu/shmobile/pm.c index e559687..a6f95ae 100644 --- a/arch/sh/kernel/cpu/shmobile/pm.c +++ b/arch/sh/kernel/cpu/shmobile/pm.c @@ -141,7 +141,7 @@ static int sh_pm_enter(suspend_state_t state) return 0; } -static struct platform_suspend_ops sh_pm_ops = { +static const struct platform_suspend_ops sh_pm_ops = { .enter = sh_pm_enter, .valid = suspend_valid_only_mem, }; diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c index eb4cc4e..d1bffbc 100644 --- a/arch/sh/mm/cache-sh5.c +++ b/arch/sh/mm/cache-sh5.c @@ -568,7 +568,7 @@ static void sh5_flush_dcache_page(void *page) } /* - * Flush the range [start,end] of kernel virtual adddress space from + * Flush the range [start,end] of kernel virtual address space from * the I-cache. The corresponding range must be purged from the * D-cache also because the SH-5 doesn't have cache snooping between * the caches. The addresses will be visible through the superpage diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 3385b28..0d3f912 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -2,7 +2,7 @@ * linux/arch/sh/mm/init.c * * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2002 - 2010 Paul Mundt + * Copyright (C) 2002 - 2011 Paul Mundt * * Based on linux/arch/i386/mm/init.c: * Copyright (C) 1995 Linus Torvalds @@ -325,11 +325,17 @@ void __init paging_init(void) int nid; memblock_init(); - sh_mv.mv_mem_init(); early_reserve_mem(); + /* + * Once the early reservations are out of the way, give the + * platforms a chance to kick out some memory. + */ + if (sh_mv.mv_mem_reserve) + sh_mv.mv_mem_reserve(); + memblock_enforce_memory_limit(memory_limit); memblock_analyze(); diff --git a/arch/sh/tools/mach-types b/arch/sh/tools/mach-types index 0e68465..6dd56c4 100644 --- a/arch/sh/tools/mach-types +++ b/arch/sh/tools/mach-types @@ -9,6 +9,7 @@ SE SH_SOLUTION_ENGINE HIGHLANDER SH_HIGHLANDER RTS7751R2D SH_RTS7751R2D RSK SH_RSK +ALPHA_BOARD SH_ALPHA_BOARD # # List of companion chips / MFDs. @@ -61,3 +62,5 @@ ESPT SH_ESPT POLARIS SH_POLARIS KFR2R09 SH_KFR2R09 ECOVEC SH_ECOVEC +APSH4A3A SH_APSH4A3A +APSH4AD0A SH_APSH4AD0A diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 42ad2ba..1e97709 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -622,7 +622,7 @@ static const char CHAFSR_PERR_msg[] = static const char CHAFSR_IERR_msg[] = "Internal processor error"; static const char CHAFSR_ISAP_msg[] = - "System request parity error on incoming addresss"; + "System request parity error on incoming address"; static const char CHAFSR_UCU_msg[] = "Uncorrectable E-cache ECC error for ifetch/data"; static const char CHAFSR_UCC_msg[] = diff --git a/arch/um/Kconfig.um b/arch/um/Kconfig.um index 50d6aa2..f8d1d0d 100644 --- a/arch/um/Kconfig.um +++ b/arch/um/Kconfig.um @@ -131,7 +131,7 @@ config NR_CPUS config HIGHMEM bool "Highmem support (EXPERIMENTAL)" - depends on !64BIT && EXPERIMENTAL + depends on !64BIT && BROKEN default n help This was used to allow UML to run with big amounts of memory. diff --git a/arch/um/drivers/line.c b/arch/um/drivers/line.c index 1664cce..050e4dd 100644 --- a/arch/um/drivers/line.c +++ b/arch/um/drivers/line.c @@ -821,12 +821,12 @@ void register_winch_irq(int fd, int tty_fd, int pid, struct tty_struct *tty, static void unregister_winch(struct tty_struct *tty) { - struct list_head *ele; + struct list_head *ele, *next; struct winch *winch; spin_lock(&winch_handler_lock); - list_for_each(ele, &winch_handlers) { + list_for_each_safe(ele, next, &winch_handlers) { winch = list_entry(ele, struct winch, list); if (winch->tty == tty) { free_winch(winch, 1); diff --git a/arch/um/drivers/mmapper_kern.c b/arch/um/drivers/mmapper_kern.c index 8501e7d..7e0619c 100644 --- a/arch/um/drivers/mmapper_kern.c +++ b/arch/um/drivers/mmapper_kern.c @@ -37,13 +37,7 @@ static ssize_t mmapper_write(struct file *file, const char __user *buf, if (*ppos > mmapper_size) return -EINVAL; - if (count > mmapper_size - *ppos) - count = mmapper_size - *ppos; - - if (copy_from_user(&v_buf[*ppos], buf, count)) - return -EFAULT; - - return count; + return simple_write_to_buffer(v_buf, mmapper_size, ppos, buf, count); } static long mmapper_ioctl(struct file *file, unsigned int cmd, unsigned long arg) @@ -137,3 +131,4 @@ module_exit(mmapper_exit); MODULE_AUTHOR("Greg Lonnon <glonnon@ridgerun.com>"); MODULE_DESCRIPTION("DSPLinux simulator mmapper driver"); +MODULE_LICENSE("GPL"); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b6fccb0..36ed2e2 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -51,6 +51,7 @@ config X86 select HAVE_KERNEL_GZIP select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_LZMA + select HAVE_KERNEL_XZ select HAVE_KERNEL_LZO select HAVE_HW_BREAKPOINT select HAVE_MIXED_BREAKPOINTS_REGS @@ -65,6 +66,7 @@ config X86 select HAVE_SPARSE_IRQ select GENERIC_IRQ_PROBE select GENERIC_PENDING_IRQ if SMP + select USE_GENERIC_SMP_HELPERS if SMP config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) @@ -203,10 +205,6 @@ config HAVE_INTEL_TXT def_bool y depends on EXPERIMENTAL && DMAR && ACPI -config USE_GENERIC_SMP_HELPERS - def_bool y - depends on SMP - config X86_32_SMP def_bool y depends on X86_32 && SMP @@ -2078,11 +2076,17 @@ config OLPC_OPENFIRMWARE bool "Support for OLPC's Open Firmware" depends on !X86_64 && !X86_PAE default n + select OF help This option adds support for the implementation of Open Firmware that is used on the OLPC XO-1 Children's Machine. If unsure, say N here. +config OLPC_OPENFIRMWARE_DT + bool + default y if OLPC_OPENFIRMWARE && PROC_DEVICETREE + select OF_PROMTREE + endif # X86_32 config AMD_NB diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 0c22955..09664ef 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -4,7 +4,7 @@ # create a compressed vmlinux image from the original vmlinux # -targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o +targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo head_$(BITS).o misc.o string.o cmdline.o early_serial_console.o piggy.o KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 KBUILD_CFLAGS += -fno-strict-aliasing -fPIC @@ -49,12 +49,15 @@ $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE $(call if_changed,bzip2) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE $(call if_changed,lzma) +$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE + $(call if_changed,xzkern) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE $(call if_changed,lzo) suffix-$(CONFIG_KERNEL_GZIP) := gz suffix-$(CONFIG_KERNEL_BZIP2) := bz2 suffix-$(CONFIG_KERNEL_LZMA) := lzma +suffix-$(CONFIG_KERNEL_XZ) := xz suffix-$(CONFIG_KERNEL_LZO) := lzo quiet_cmd_mkpiggy = MKPIGGY $@ diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 325c052..3a19d04 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -139,6 +139,10 @@ static int lines, cols; #include "../../../../lib/decompress_unlzma.c" #endif +#ifdef CONFIG_KERNEL_XZ +#include "../../../../lib/decompress_unxz.c" +#endif + #ifdef CONFIG_KERNEL_LZO #include "../../../../lib/decompress_unlzo.c" #endif diff --git a/arch/x86/boot/compressed/mkpiggy.c b/arch/x86/boot/compressed/mkpiggy.c index 5c22812..646aa78 100644 --- a/arch/x86/boot/compressed/mkpiggy.c +++ b/arch/x86/boot/compressed/mkpiggy.c @@ -74,7 +74,7 @@ int main(int argc, char *argv[]) offs = (olen > ilen) ? olen - ilen : 0; offs += olen >> 12; /* Add 8 bytes for each 32K block */ - offs += 32*1024 + 18; /* Add 32K + 18 bytes slack */ + offs += 64*1024 + 128; /* Add 64K + 128 bytes slack */ offs = (offs+4095) & ~4095; /* Round to a 4K boundary */ printf(".section \".rodata..compressed\",\"a\",@progbits\n"); diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index ff16756..8fe2a49 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -9,6 +9,20 @@ * Vinodh Gopal <vinodh.gopal@intel.com> * Kahraman Akdemir * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Erdinc Ozturk (erdinc.ozturk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Adrian Hoban <adrian.hoban@intel.com> + * James Guilford (james.guilford@intel.com) + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Tadeusz Struk (tadeusz.struk@intel.com) + * Wajdi Feghali (wajdi.k.feghali@intel.com) + * Copyright (c) 2010, Intel Corporation. + * + * Ported x86_64 version to x86: + * Author: Mathias Krause <minipli@googlemail.com> + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -18,8 +32,62 @@ #include <linux/linkage.h> #include <asm/inst.h> +#ifdef __x86_64__ +.data +POLY: .octa 0xC2000000000000000000000000000001 +TWOONE: .octa 0x00000001000000000000000000000001 + +# order of these constants should not change. +# more specifically, ALL_F should follow SHIFT_MASK, +# and ZERO should follow ALL_F + +SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F +MASK1: .octa 0x0000000000000000ffffffffffffffff +MASK2: .octa 0xffffffffffffffff0000000000000000 +SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 +ALL_F: .octa 0xffffffffffffffffffffffffffffffff +ZERO: .octa 0x00000000000000000000000000000000 +ONE: .octa 0x00000000000000000000000000000001 +F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0 +dec: .octa 0x1 +enc: .octa 0x2 + + .text + +#define STACK_OFFSET 8*3 +#define HashKey 16*0 // store HashKey <<1 mod poly here +#define HashKey_2 16*1 // store HashKey^2 <<1 mod poly here +#define HashKey_3 16*2 // store HashKey^3 <<1 mod poly here +#define HashKey_4 16*3 // store HashKey^4 <<1 mod poly here +#define HashKey_k 16*4 // store XOR of High 64 bits and Low 64 + // bits of HashKey <<1 mod poly here + //(for Karatsuba purposes) +#define HashKey_2_k 16*5 // store XOR of High 64 bits and Low 64 + // bits of HashKey^2 <<1 mod poly here + // (for Karatsuba purposes) +#define HashKey_3_k 16*6 // store XOR of High 64 bits and Low 64 + // bits of HashKey^3 <<1 mod poly here + // (for Karatsuba purposes) +#define HashKey_4_k 16*7 // store XOR of High 64 bits and Low 64 + // bits of HashKey^4 <<1 mod poly here + // (for Karatsuba purposes) +#define VARIABLE_OFFSET 16*8 + +#define arg1 rdi +#define arg2 rsi +#define arg3 rdx +#define arg4 rcx +#define arg5 r8 +#define arg6 r9 +#define arg7 STACK_OFFSET+8(%r14) +#define arg8 STACK_OFFSET+16(%r14) +#define arg9 STACK_OFFSET+24(%r14) +#define arg10 STACK_OFFSET+32(%r14) +#endif + + #define STATE1 %xmm0 #define STATE2 %xmm4 #define STATE3 %xmm5 @@ -32,12 +100,16 @@ #define IN IN1 #define KEY %xmm2 #define IV %xmm3 + #define BSWAP_MASK %xmm10 #define CTR %xmm11 #define INC %xmm12 +#ifdef __x86_64__ +#define AREG %rax #define KEYP %rdi #define OUTP %rsi +#define UKEYP OUTP #define INP %rdx #define LEN %rcx #define IVP %r8 @@ -46,6 +118,1588 @@ #define TKEYP T1 #define T2 %r11 #define TCTR_LOW T2 +#else +#define AREG %eax +#define KEYP %edi +#define OUTP AREG +#define UKEYP OUTP +#define INP %edx +#define LEN %esi +#define IVP %ebp +#define KLEN %ebx +#define T1 %ecx +#define TKEYP T1 +#endif + + +#ifdef __x86_64__ +/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +* +* +* Input: A and B (128-bits each, bit-reflected) +* Output: C = A*B*x mod poly, (i.e. >>1 ) +* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +* +*/ +.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5 + movdqa \GH, \TMP1 + pshufd $78, \GH, \TMP2 + pshufd $78, \HK, \TMP3 + pxor \GH, \TMP2 # TMP2 = a1+a0 + pxor \HK, \TMP3 # TMP3 = b1+b0 + PCLMULQDQ 0x11, \HK, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \HK, \GH # GH = a0*b0 + PCLMULQDQ 0x00, \TMP3, \TMP2 # TMP2 = (a0+a1)*(b1+b0) + pxor \GH, \TMP2 + pxor \TMP1, \TMP2 # TMP2 = (a0*b0)+(a1*b0) + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \GH + pxor \TMP2, \TMP1 # TMP2:GH holds the result of GH*HK + + # first phase of the reduction + + movdqa \GH, \TMP2 + movdqa \GH, \TMP3 + movdqa \GH, \TMP4 # copy GH into TMP2,TMP3 and TMP4 + # in in order to perform + # independent shifts + pslld $31, \TMP2 # packed right shift <<31 + pslld $30, \TMP3 # packed right shift <<30 + pslld $25, \TMP4 # packed right shift <<25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift TMP5 1 DW + pslldq $12, \TMP2 # left shift TMP2 3 DWs + pxor \TMP2, \GH + + # second phase of the reduction + + movdqa \GH,\TMP2 # copy GH into TMP2,TMP3 and TMP4 + # in in order to perform + # independent shifts + movdqa \GH,\TMP3 + movdqa \GH,\TMP4 + psrld $1,\TMP2 # packed left shift >>1 + psrld $2,\TMP3 # packed left shift >>2 + psrld $7,\TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \GH + pxor \TMP1, \GH # result is in TMP1 +.endm + +/* +* if a = number of total plaintext bytes +* b = floor(a/16) +* num_initial_blocks = b mod 4 +* encrypt the initial num_initial_blocks blocks and apply ghash on +* the ciphertext +* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers +* are clobbered +* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified +*/ + + +.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ +XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + mov arg7, %r10 # %r10 = AAD + mov arg8, %r12 # %r12 = aadLen + mov %r12, %r11 + pxor %xmm\i, %xmm\i +_get_AAD_loop\num_initial_blocks\operation: + movd (%r10), \TMP1 + pslldq $12, \TMP1 + psrldq $4, %xmm\i + pxor \TMP1, %xmm\i + add $4, %r10 + sub $4, %r12 + jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 + je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 +_get_AAD_loop2\num_initial_blocks\operation: + psrldq $4, %xmm\i + sub $4, %r12 + cmp %r11, %r12 + jne _get_AAD_loop2\num_initial_blocks\operation +_get_AAD_loop2_done\num_initial_blocks\operation: + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + + xor %r11, %r11 # initialise the data pointer offset as zero + + # start AES for num_initial_blocks blocks + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), \XMM0 # XMM0 = Y0 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM0 + +.if (\i == 5) || (\i == 6) || (\i == 7) +.irpc index, \i_seq + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + +.endr +.irpc index, \i_seq + pxor 16*0(%arg1), %xmm\index +.endr +.irpc index, \i_seq + movaps 0x10(%rdi), \TMP1 + AESENC \TMP1, %xmm\index # Round 1 +.endr +.irpc index, \i_seq + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x30(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x40(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x50(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x60(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x70(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x80(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x90(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0xa0(%arg1), \TMP1 + AESENCLAST \TMP1, %xmm\index # Round 10 +.endr +.irpc index, \i_seq + movdqu (%arg3 , %r11, 1), \TMP1 + pxor \TMP1, %xmm\index + movdqu %xmm\index, (%arg2 , %r11, 1) + # write back plaintext/ciphertext for num_initial_blocks + add $16, %r11 + + movdqa \TMP1, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index + + # prepare plaintext/ciphertext for GHASH computation +.endr +.endif + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks + +.if \i == 5 + pxor %xmm5, %xmm6 + GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 6 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 7 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.endif + cmp $64, %r13 + jl _initial_blocks_done\num_initial_blocks\operation + # no need for precomputed values +/* +* +* Precomputations for HashKey parallel with encryption of first 4 blocks. +* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i +*/ + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM1 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM2 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM3 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM4 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + + pxor 16*0(%arg1), \XMM1 + pxor 16*0(%arg1), \XMM2 + pxor 16*0(%arg1), \XMM3 + pxor 16*0(%arg1), \XMM4 + movdqa \TMP3, \TMP5 + pshufd $78, \TMP3, \TMP1 + pxor \TMP3, \TMP1 + movdqa \TMP1, HashKey_k(%rsp) + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^2<<1 (mod poly) + movdqa \TMP5, HashKey_2(%rsp) +# HashKey_2 = HashKey^2<<1 (mod poly) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_2_k(%rsp) +.irpc index, 1234 # do 4 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_3(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_3_k(%rsp) +.irpc index, 56789 # do next 5 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_4(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_4_k(%rsp) + movaps 0xa0(%arg1), \TMP2 + AESENCLAST \TMP2, \XMM1 + AESENCLAST \TMP2, \XMM2 + AESENCLAST \TMP2, \XMM3 + AESENCLAST \TMP2, \XMM4 + movdqu 16*0(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM1 + movdqu \XMM1, 16*0(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM1 + movdqu 16*1(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM2 + movdqu \XMM2, 16*1(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM2 + movdqu 16*2(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM3 + movdqu \XMM3, 16*2(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM3 + movdqu 16*3(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM4 + movdqu \XMM4, 16*3(%arg2 , %r11 , 1) + movdqa \TMP1, \XMM4 + add $64, %r11 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + pxor \XMMDst, \XMM1 +# combine GHASHed value with the corresponding ciphertext + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + +_initial_blocks_done\num_initial_blocks\operation: + +.endm + + +/* +* if a = number of total plaintext bytes +* b = floor(a/16) +* num_initial_blocks = b mod 4 +* encrypt the initial num_initial_blocks blocks and apply ghash on +* the ciphertext +* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers +* are clobbered +* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified +*/ + + +.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ +XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + mov arg7, %r10 # %r10 = AAD + mov arg8, %r12 # %r12 = aadLen + mov %r12, %r11 + pxor %xmm\i, %xmm\i +_get_AAD_loop\num_initial_blocks\operation: + movd (%r10), \TMP1 + pslldq $12, \TMP1 + psrldq $4, %xmm\i + pxor \TMP1, %xmm\i + add $4, %r10 + sub $4, %r12 + jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 + je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 +_get_AAD_loop2\num_initial_blocks\operation: + psrldq $4, %xmm\i + sub $4, %r12 + cmp %r11, %r12 + jne _get_AAD_loop2\num_initial_blocks\operation +_get_AAD_loop2_done\num_initial_blocks\operation: + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + + xor %r11, %r11 # initialise the data pointer offset as zero + + # start AES for num_initial_blocks blocks + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), \XMM0 # XMM0 = Y0 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM0 + +.if (\i == 5) || (\i == 6) || (\i == 7) +.irpc index, \i_seq + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, %xmm\index + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + +.endr +.irpc index, \i_seq + pxor 16*0(%arg1), %xmm\index +.endr +.irpc index, \i_seq + movaps 0x10(%rdi), \TMP1 + AESENC \TMP1, %xmm\index # Round 1 +.endr +.irpc index, \i_seq + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x30(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x40(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x50(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x60(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x70(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x80(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0x90(%arg1), \TMP1 + AESENC \TMP1, %xmm\index # Round 2 +.endr +.irpc index, \i_seq + movaps 0xa0(%arg1), \TMP1 + AESENCLAST \TMP1, %xmm\index # Round 10 +.endr +.irpc index, \i_seq + movdqu (%arg3 , %r11, 1), \TMP1 + pxor \TMP1, %xmm\index + movdqu %xmm\index, (%arg2 , %r11, 1) + # write back plaintext/ciphertext for num_initial_blocks + add $16, %r11 + + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, %xmm\index + + # prepare plaintext/ciphertext for GHASH computation +.endr +.endif + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks + +.if \i == 5 + pxor %xmm5, %xmm6 + GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 6 + pxor %xmm6, %xmm7 + GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.elseif \i == 7 + pxor %xmm7, %xmm8 + GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +.endif + cmp $64, %r13 + jl _initial_blocks_done\num_initial_blocks\operation + # no need for precomputed values +/* +* +* Precomputations for HashKey parallel with encryption of first 4 blocks. +* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i +*/ + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM1 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM2 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM3 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + + paddd ONE(%rip), \XMM0 # INCR Y0 + movdqa \XMM0, \XMM4 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + + pxor 16*0(%arg1), \XMM1 + pxor 16*0(%arg1), \XMM2 + pxor 16*0(%arg1), \XMM3 + pxor 16*0(%arg1), \XMM4 + movdqa \TMP3, \TMP5 + pshufd $78, \TMP3, \TMP1 + pxor \TMP3, \TMP1 + movdqa \TMP1, HashKey_k(%rsp) + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^2<<1 (mod poly) + movdqa \TMP5, HashKey_2(%rsp) +# HashKey_2 = HashKey^2<<1 (mod poly) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_2_k(%rsp) +.irpc index, 1234 # do 4 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_3(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_3_k(%rsp) +.irpc index, 56789 # do next 5 rounds + movaps 0x10*\index(%arg1), \TMP1 + AESENC \TMP1, \XMM1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 +.endr + GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7 +# TMP5 = HashKey^3<<1 (mod poly) + movdqa \TMP5, HashKey_4(%rsp) + pshufd $78, \TMP5, \TMP1 + pxor \TMP5, \TMP1 + movdqa \TMP1, HashKey_4_k(%rsp) + movaps 0xa0(%arg1), \TMP2 + AESENCLAST \TMP2, \XMM1 + AESENCLAST \TMP2, \XMM2 + AESENCLAST \TMP2, \XMM3 + AESENCLAST \TMP2, \XMM4 + movdqu 16*0(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM1 + movdqu 16*1(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM2 + movdqu 16*2(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM3 + movdqu 16*3(%arg3 , %r11 , 1), \TMP1 + pxor \TMP1, \XMM4 + movdqu \XMM1, 16*0(%arg2 , %r11 , 1) + movdqu \XMM2, 16*1(%arg2 , %r11 , 1) + movdqu \XMM3, 16*2(%arg2 , %r11 , 1) + movdqu \XMM4, 16*3(%arg2 , %r11 , 1) + + add $64, %r11 + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap + pxor \XMMDst, \XMM1 +# combine GHASHed value with the corresponding ciphertext + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap + movdqa SHUF_MASK(%rip), %xmm14 + PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap + +_initial_blocks_done\num_initial_blocks\operation: + +.endm + +/* +* encrypt 4 blocks at a time +* ghash the 4 previously encrypted ciphertext blocks +* arg1, %arg2, %arg3 are used as pointers only, not modified +* %r11 is the data offset value +*/ +.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \ +TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation + + movdqa \XMM1, \XMM5 + movdqa \XMM2, \XMM6 + movdqa \XMM3, \XMM7 + movdqa \XMM4, \XMM8 + + movdqa SHUF_MASK(%rip), %xmm15 + # multiply TMP5 * HashKey using karatsuba + + movdqa \XMM5, \TMP4 + pshufd $78, \XMM5, \TMP6 + pxor \XMM5, \TMP6 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 + movdqa \XMM0, \XMM1 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM2 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM3 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor (%arg1), \XMM1 + pxor (%arg1), \XMM2 + pxor (%arg1), \XMM3 + pxor (%arg1), \XMM4 + movdqa HashKey_4_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) + movaps 0x10(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 2 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movdqa \XMM6, \TMP1 + pshufd $78, \XMM6, \TMP2 + pxor \XMM6, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 + movaps 0x30(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 3 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 + movaps 0x40(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 4 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_3_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x50(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 5 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM6, \XMM5 + pxor \TMP2, \TMP6 + movdqa \XMM7, \TMP1 + pshufd $78, \XMM7, \TMP2 + pxor \XMM7, \TMP2 + movdqa HashKey_2(%rsp ), \TMP5 + + # Multiply TMP5 * HashKey using karatsuba + + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x60(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 6 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 + movaps 0x70(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 7 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_2_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x80(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 8 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM7, \XMM5 + pxor \TMP2, \TMP6 + + # Multiply XMM8 * HashKey + # XMM8 and TMP5 hold the values for the two operands + + movdqa \XMM8, \TMP1 + pshufd $78, \XMM8, \TMP2 + pxor \XMM8, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x90(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 9 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 + movaps 0xa0(%arg1), \TMP3 + AESENCLAST \TMP3, \XMM1 # Round 10 + AESENCLAST \TMP3, \XMM2 + AESENCLAST \TMP3, \XMM3 + AESENCLAST \TMP3, \XMM4 + movdqa HashKey_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqu (%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK + movdqu 16(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK + movdqu 32(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK + movdqu 48(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK + movdqu \XMM1, (%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM2, 16(%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM3, 32(%arg2,%r11,1) # Write to the ciphertext buffer + movdqu \XMM4, 48(%arg2,%r11,1) # Write to the ciphertext buffer + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor \TMP4, \TMP1 + pxor \XMM8, \XMM5 + pxor \TMP6, \TMP2 + pxor \TMP1, \TMP2 + pxor \XMM5, \TMP2 + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \XMM5 + pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 + + # first phase of reduction + + movdqa \XMM5, \TMP2 + movdqa \XMM5, \TMP3 + movdqa \XMM5, \TMP4 +# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently + pslld $31, \TMP2 # packed right shift << 31 + pslld $30, \TMP3 # packed right shift << 30 + pslld $25, \TMP4 # packed right shift << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift T5 1 DW + pslldq $12, \TMP2 # left shift T2 3 DWs + pxor \TMP2, \XMM5 + + # second phase of reduction + + movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 + movdqa \XMM5,\TMP3 + movdqa \XMM5,\TMP4 + psrld $1, \TMP2 # packed left shift >>1 + psrld $2, \TMP3 # packed left shift >>2 + psrld $7, \TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \XMM5 + pxor \TMP1, \XMM5 # result is in TMP1 + + pxor \XMM5, \XMM1 +.endm + +/* +* decrypt 4 blocks at a time +* ghash the 4 previously decrypted ciphertext blocks +* arg1, %arg2, %arg3 are used as pointers only, not modified +* %r11 is the data offset value +*/ +.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \ +TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation + + movdqa \XMM1, \XMM5 + movdqa \XMM2, \XMM6 + movdqa \XMM3, \XMM7 + movdqa \XMM4, \XMM8 + + movdqa SHUF_MASK(%rip), %xmm15 + # multiply TMP5 * HashKey using karatsuba + + movdqa \XMM5, \TMP4 + pshufd $78, \XMM5, \TMP6 + pxor \XMM5, \TMP6 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1 + movdqa \XMM0, \XMM1 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM2 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM3 + paddd ONE(%rip), \XMM0 # INCR CNT + movdqa \XMM0, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0 + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor (%arg1), \XMM1 + pxor (%arg1), \XMM2 + pxor (%arg1), \XMM3 + pxor (%arg1), \XMM4 + movdqa HashKey_4_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0) + movaps 0x10(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 1 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movaps 0x20(%arg1), \TMP1 + AESENC \TMP1, \XMM1 # Round 2 + AESENC \TMP1, \XMM2 + AESENC \TMP1, \XMM3 + AESENC \TMP1, \XMM4 + movdqa \XMM6, \TMP1 + pshufd $78, \XMM6, \TMP2 + pxor \XMM6, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1 + movaps 0x30(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 3 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0 + movaps 0x40(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 4 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_3_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x50(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 5 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM6, \XMM5 + pxor \TMP2, \TMP6 + movdqa \XMM7, \TMP1 + pshufd $78, \XMM7, \TMP2 + pxor \XMM7, \TMP2 + movdqa HashKey_2(%rsp ), \TMP5 + + # Multiply TMP5 * HashKey using karatsuba + + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x60(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 6 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0 + movaps 0x70(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 7 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + movdqa HashKey_2_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movaps 0x80(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 8 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + pxor \TMP1, \TMP4 +# accumulate the results in TMP4:XMM5, TMP6 holds the middle part + pxor \XMM7, \XMM5 + pxor \TMP2, \TMP6 + + # Multiply XMM8 * HashKey + # XMM8 and TMP5 hold the values for the two operands + + movdqa \XMM8, \TMP1 + pshufd $78, \XMM8, \TMP2 + pxor \XMM8, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + movaps 0x90(%arg1), \TMP3 + AESENC \TMP3, \XMM1 # Round 9 + AESENC \TMP3, \XMM2 + AESENC \TMP3, \XMM3 + AESENC \TMP3, \XMM4 + PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 + movaps 0xa0(%arg1), \TMP3 + AESENCLAST \TMP3, \XMM1 # Round 10 + AESENCLAST \TMP3, \XMM2 + AESENCLAST \TMP3, \XMM3 + AESENCLAST \TMP3, \XMM4 + movdqa HashKey_k(%rsp), \TMP5 + PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqu (%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK + movdqu \XMM1, (%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM1 + movdqu 16(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK + movdqu \XMM2, 16(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM2 + movdqu 32(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK + movdqu \XMM3, 32(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM3 + movdqu 48(%arg3,%r11,1), \TMP3 + pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK + movdqu \XMM4, 48(%arg2,%r11,1) # Write to plaintext buffer + movdqa \TMP3, \XMM4 + PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap + PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap + + pxor \TMP4, \TMP1 + pxor \XMM8, \XMM5 + pxor \TMP6, \TMP2 + pxor \TMP1, \TMP2 + pxor \XMM5, \TMP2 + movdqa \TMP2, \TMP3 + pslldq $8, \TMP3 # left shift TMP3 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP3, \XMM5 + pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5 + + # first phase of reduction + + movdqa \XMM5, \TMP2 + movdqa \XMM5, \TMP3 + movdqa \XMM5, \TMP4 +# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently + pslld $31, \TMP2 # packed right shift << 31 + pslld $30, \TMP3 # packed right shift << 30 + pslld $25, \TMP4 # packed right shift << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP5 + psrldq $4, \TMP5 # right shift T5 1 DW + pslldq $12, \TMP2 # left shift T2 3 DWs + pxor \TMP2, \XMM5 + + # second phase of reduction + + movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4 + movdqa \XMM5,\TMP3 + movdqa \XMM5,\TMP4 + psrld $1, \TMP2 # packed left shift >>1 + psrld $2, \TMP3 # packed left shift >>2 + psrld $7, \TMP4 # packed left shift >>7 + pxor \TMP3,\TMP2 # xor the shifted versions + pxor \TMP4,\TMP2 + pxor \TMP5, \TMP2 + pxor \TMP2, \XMM5 + pxor \TMP1, \XMM5 # result is in TMP1 + + pxor \XMM5, \XMM1 +.endm + +/* GHASH the last 4 ciphertext blocks. */ +.macro GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \ +TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst + + # Multiply TMP6 * HashKey (using Karatsuba) + + movdqa \XMM1, \TMP6 + pshufd $78, \XMM1, \TMP2 + pxor \XMM1, \TMP2 + movdqa HashKey_4(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP6 # TMP6 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM1 # XMM1 = a0*b0 + movdqa HashKey_4_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + movdqa \XMM1, \XMMDst + movdqa \TMP2, \XMM1 # result in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + + movdqa \XMM2, \TMP1 + pshufd $78, \XMM2, \TMP2 + pxor \XMM2, \TMP2 + movdqa HashKey_3(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM2 # XMM2 = a0*b0 + movdqa HashKey_3_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM2, \XMMDst + pxor \TMP2, \XMM1 +# results accumulated in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + + movdqa \XMM3, \TMP1 + pshufd $78, \XMM3, \TMP2 + pxor \XMM3, \TMP2 + movdqa HashKey_2(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM3 # XMM3 = a0*b0 + movdqa HashKey_2_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM3, \XMMDst + pxor \TMP2, \XMM1 # results accumulated in TMP6, XMMDst, XMM1 + + # Multiply TMP1 * HashKey (using Karatsuba) + movdqa \XMM4, \TMP1 + pshufd $78, \XMM4, \TMP2 + pxor \XMM4, \TMP2 + movdqa HashKey(%rsp), \TMP5 + PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1 + PCLMULQDQ 0x00, \TMP5, \XMM4 # XMM4 = a0*b0 + movdqa HashKey_k(%rsp), \TMP4 + PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0) + pxor \TMP1, \TMP6 + pxor \XMM4, \XMMDst + pxor \XMM1, \TMP2 + pxor \TMP6, \TMP2 + pxor \XMMDst, \TMP2 + # middle section of the temp results combined as in karatsuba algorithm + movdqa \TMP2, \TMP4 + pslldq $8, \TMP4 # left shift TMP4 2 DWs + psrldq $8, \TMP2 # right shift TMP2 2 DWs + pxor \TMP4, \XMMDst + pxor \TMP2, \TMP6 +# TMP6:XMMDst holds the result of the accumulated carry-less multiplications + # first phase of the reduction + movdqa \XMMDst, \TMP2 + movdqa \XMMDst, \TMP3 + movdqa \XMMDst, \TMP4 +# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently + pslld $31, \TMP2 # packed right shifting << 31 + pslld $30, \TMP3 # packed right shifting << 30 + pslld $25, \TMP4 # packed right shifting << 25 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + movdqa \TMP2, \TMP7 + psrldq $4, \TMP7 # right shift TMP7 1 DW + pslldq $12, \TMP2 # left shift TMP2 3 DWs + pxor \TMP2, \XMMDst + + # second phase of the reduction + movdqa \XMMDst, \TMP2 + # make 3 copies of XMMDst for doing 3 shift operations + movdqa \XMMDst, \TMP3 + movdqa \XMMDst, \TMP4 + psrld $1, \TMP2 # packed left shift >> 1 + psrld $2, \TMP3 # packed left shift >> 2 + psrld $7, \TMP4 # packed left shift >> 7 + pxor \TMP3, \TMP2 # xor the shifted versions + pxor \TMP4, \TMP2 + pxor \TMP7, \TMP2 + pxor \TMP2, \XMMDst + pxor \TMP6, \XMMDst # reduced result is in XMMDst +.endm + +/* Encryption of a single block done*/ +.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 + + pxor (%arg1), \XMM0 + movaps 16(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 32(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 48(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 64(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 80(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 96(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 112(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 128(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 144(%arg1), \TMP1 + AESENC \TMP1, \XMM0 + movaps 160(%arg1), \TMP1 + AESENCLAST \TMP1, \XMM0 +.endm + + +/***************************************************************************** +* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. +* u8 *out, // Plaintext output. Encrypt in-place is allowed. +* const u8 *in, // Ciphertext input +* u64 plaintext_len, // Length of data in bytes for decryption. +* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) +* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) +* // concatenated with 0x00000001. 16-byte aligned pointer. +* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. +* const u8 *aad, // Additional Authentication Data (AAD) +* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes +* u8 *auth_tag, // Authenticated Tag output. The driver will compare this to the +* // given authentication tag and only return the plaintext if they match. +* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 +* // (most likely), 12 or 8. +* +* Assumptions: +* +* keys: +* keys are pre-expanded and aligned to 16 bytes. we are using the first +* set of 11 keys in the data structure void *aes_ctx +* +* iv: +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Salt (From the SA) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Initialization Vector | +* | (This is the sequence number from IPSec header) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x1 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* +* +* AAD: +* AAD padded to 128 bits with 0 +* for example, assume AAD is a u32 vector +* +* if AAD is 8 bytes: +* AAD[3] = {A0, A1}; +* padded AAD in xmm register = {A1 A0 0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A1) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 32-bit Sequence Number (A0) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 32-bit Sequence Number +* +* if AAD is 12 bytes: +* AAD[3] = {A0, A1, A2}; +* padded AAD in xmm register = {A2 A1 A0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A2) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 64-bit Extended Sequence Number {A1,A0} | +* | | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 64-bit Extended Sequence Number +* +* aadLen: +* from the definition of the spec, aadLen can only be 8 or 12 bytes. +* The code supports 16 too but for other sizes, the code will fail. +* +* TLen: +* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +* For other sizes, the code will fail. +* +* poly = x^128 + x^127 + x^126 + x^121 + 1 +* +*****************************************************************************/ + +ENTRY(aesni_gcm_dec) + push %r12 + push %r13 + push %r14 + mov %rsp, %r14 +/* +* states of %xmm registers %xmm6:%xmm15 not saved +* all %xmm registers are clobbered +*/ + sub $VARIABLE_OFFSET, %rsp + and $~63, %rsp # align rsp to 64 bytes + mov %arg6, %r12 + movdqu (%r12), %xmm13 # %xmm13 = HashKey + movdqa SHUF_MASK(%rip), %xmm2 + PSHUFB_XMM %xmm2, %xmm13 + + +# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH) + + movdqa %xmm13, %xmm2 + psllq $1, %xmm13 + psrlq $63, %xmm2 + movdqa %xmm2, %xmm1 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + por %xmm2, %xmm13 + + # Reduction + + pshufd $0x24, %xmm1, %xmm2 + pcmpeqd TWOONE(%rip), %xmm2 + pand POLY(%rip), %xmm2 + pxor %xmm2, %xmm13 # %xmm13 holds the HashKey<<1 (mod poly) + + + # Decrypt first few blocks + + movdqa %xmm13, HashKey(%rsp) # store HashKey<<1 (mod poly) + mov %arg4, %r13 # save the number of bytes of plaintext/ciphertext + and $-16, %r13 # %r13 = %r13 - (%r13 mod 16) + mov %r13, %r12 + and $(3<<4), %r12 + jz _initial_num_blocks_is_0_decrypt + cmp $(2<<4), %r12 + jb _initial_num_blocks_is_1_decrypt + je _initial_num_blocks_is_2_decrypt +_initial_num_blocks_is_3_decrypt: + INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec + sub $48, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_2_decrypt: + INITIAL_BLOCKS_DEC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec + sub $32, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_1_decrypt: + INITIAL_BLOCKS_DEC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec + sub $16, %r13 + jmp _initial_blocks_decrypted +_initial_num_blocks_is_0_decrypt: + INITIAL_BLOCKS_DEC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec +_initial_blocks_decrypted: + cmp $0, %r13 + je _zero_cipher_left_decrypt + sub $64, %r13 + je _four_cipher_left_decrypt +_decrypt_by_4: + GHASH_4_ENCRYPT_4_PARALLEL_DEC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \ +%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec + add $64, %r11 + sub $64, %r13 + jne _decrypt_by_4 +_four_cipher_left_decrypt: + GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \ +%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8 +_zero_cipher_left_decrypt: + mov %arg4, %r13 + and $15, %r13 # %r13 = arg4 (mod 16) + je _multiple_of_16_bytes_decrypt + + # Handle the last <16 byte block seperately + + paddd ONE(%rip), %xmm0 # increment CNT to get Yn + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm0 + + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Yn) + sub $16, %r11 + add %r13, %r11 + movdqu (%arg3,%r11,1), %xmm1 # recieve the last <16 byte block + lea SHIFT_MASK+16(%rip), %r12 + sub %r13, %r12 +# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes +# (%r13 is the number of bytes in plaintext mod 16) + movdqu (%r12), %xmm2 # get the appropriate shuffle mask + PSHUFB_XMM %xmm2, %xmm1 # right shift 16-%r13 butes + + movdqa %xmm1, %xmm2 + pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn) + movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0 + pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0 + pand %xmm1, %xmm2 + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10 ,%xmm2 + + pxor %xmm2, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # GHASH computation for the last <16 byte block + sub %r13, %r11 + add $16, %r11 + + # output %r13 bytes + MOVQ_R64_XMM %xmm0, %rax + cmp $8, %r13 + jle _less_than_8_bytes_left_decrypt + mov %rax, (%arg2 , %r11, 1) + add $8, %r11 + psrldq $8, %xmm0 + MOVQ_R64_XMM %xmm0, %rax + sub $8, %r13 +_less_than_8_bytes_left_decrypt: + mov %al, (%arg2, %r11, 1) + add $1, %r11 + shr $8, %rax + sub $1, %r13 + jne _less_than_8_bytes_left_decrypt +_multiple_of_16_bytes_decrypt: + mov arg8, %r12 # %r13 = aadLen (number of bytes) + shl $3, %r12 # convert into number of bits + movd %r12d, %xmm15 # len(A) in %xmm15 + shl $3, %arg4 # len(C) in bits (*128) + MOVQ_R64_XMM %arg4, %xmm1 + pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000 + pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C) + pxor %xmm15, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # final GHASH computation + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm8 + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), %xmm0 # %xmm0 = Y0 + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0) + pxor %xmm8, %xmm0 +_return_T_decrypt: + mov arg9, %r10 # %r10 = authTag + mov arg10, %r11 # %r11 = auth_tag_len + cmp $16, %r11 + je _T_16_decrypt + cmp $12, %r11 + je _T_12_decrypt +_T_8_decrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + jmp _return_T_done_decrypt +_T_12_decrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + psrldq $8, %xmm0 + movd %xmm0, %eax + mov %eax, 8(%r10) + jmp _return_T_done_decrypt +_T_16_decrypt: + movdqu %xmm0, (%r10) +_return_T_done_decrypt: + mov %r14, %rsp + pop %r14 + pop %r13 + pop %r12 + ret + + +/***************************************************************************** +* void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. +* u8 *out, // Ciphertext output. Encrypt in-place is allowed. +* const u8 *in, // Plaintext input +* u64 plaintext_len, // Length of data in bytes for encryption. +* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association) +* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload) +* // concatenated with 0x00000001. 16-byte aligned pointer. +* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary. +* const u8 *aad, // Additional Authentication Data (AAD) +* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes +* u8 *auth_tag, // Authenticated Tag output. +* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely), +* // 12 or 8. +* +* Assumptions: +* +* keys: +* keys are pre-expanded and aligned to 16 bytes. we are using the +* first set of 11 keys in the data structure void *aes_ctx +* +* +* iv: +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Salt (From the SA) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | Initialization Vector | +* | (This is the sequence number from IPSec header) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x1 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* +* +* AAD: +* AAD padded to 128 bits with 0 +* for example, assume AAD is a u32 vector +* +* if AAD is 8 bytes: +* AAD[3] = {A0, A1}; +* padded AAD in xmm register = {A1 A0 0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A1) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 32-bit Sequence Number (A0) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 32-bit Sequence Number +* +* if AAD is 12 bytes: +* AAD[3] = {A0, A1, A2}; +* padded AAD in xmm register = {A2 A1 A0 0} +* +* 0 1 2 3 +* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | SPI (A2) | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 64-bit Extended Sequence Number {A1,A0} | +* | | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* | 0x0 | +* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +* +* AAD Format with 64-bit Extended Sequence Number +* +* aadLen: +* from the definition of the spec, aadLen can only be 8 or 12 bytes. +* The code supports 16 too but for other sizes, the code will fail. +* +* TLen: +* from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +* For other sizes, the code will fail. +* +* poly = x^128 + x^127 + x^126 + x^121 + 1 +***************************************************************************/ +ENTRY(aesni_gcm_enc) + push %r12 + push %r13 + push %r14 + mov %rsp, %r14 +# +# states of %xmm registers %xmm6:%xmm15 not saved +# all %xmm registers are clobbered +# + sub $VARIABLE_OFFSET, %rsp + and $~63, %rsp + mov %arg6, %r12 + movdqu (%r12), %xmm13 + movdqa SHUF_MASK(%rip), %xmm2 + PSHUFB_XMM %xmm2, %xmm13 + + +# precompute HashKey<<1 mod poly from the HashKey (required for GHASH) + + movdqa %xmm13, %xmm2 + psllq $1, %xmm13 + psrlq $63, %xmm2 + movdqa %xmm2, %xmm1 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + por %xmm2, %xmm13 + + # reduce HashKey<<1 + + pshufd $0x24, %xmm1, %xmm2 + pcmpeqd TWOONE(%rip), %xmm2 + pand POLY(%rip), %xmm2 + pxor %xmm2, %xmm13 + movdqa %xmm13, HashKey(%rsp) + mov %arg4, %r13 # %xmm13 holds HashKey<<1 (mod poly) + and $-16, %r13 + mov %r13, %r12 + + # Encrypt first few blocks + + and $(3<<4), %r12 + jz _initial_num_blocks_is_0_encrypt + cmp $(2<<4), %r12 + jb _initial_num_blocks_is_1_encrypt + je _initial_num_blocks_is_2_encrypt +_initial_num_blocks_is_3_encrypt: + INITIAL_BLOCKS_ENC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc + sub $48, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_2_encrypt: + INITIAL_BLOCKS_ENC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc + sub $32, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_1_encrypt: + INITIAL_BLOCKS_ENC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc + sub $16, %r13 + jmp _initial_blocks_encrypted +_initial_num_blocks_is_0_encrypt: + INITIAL_BLOCKS_ENC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \ +%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc +_initial_blocks_encrypted: + + # Main loop - Encrypt remaining blocks + + cmp $0, %r13 + je _zero_cipher_left_encrypt + sub $64, %r13 + je _four_cipher_left_encrypt +_encrypt_by_4_encrypt: + GHASH_4_ENCRYPT_4_PARALLEL_ENC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \ +%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc + add $64, %r11 + sub $64, %r13 + jne _encrypt_by_4_encrypt +_four_cipher_left_encrypt: + GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \ +%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8 +_zero_cipher_left_encrypt: + mov %arg4, %r13 + and $15, %r13 # %r13 = arg4 (mod 16) + je _multiple_of_16_bytes_encrypt + + # Handle the last <16 Byte block seperately + paddd ONE(%rip), %xmm0 # INCR CNT to get Yn + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm0 + + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) + sub $16, %r11 + add %r13, %r11 + movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte blocks + lea SHIFT_MASK+16(%rip), %r12 + sub %r13, %r12 + # adjust the shuffle mask pointer to be able to shift 16-r13 bytes + # (%r13 is the number of bytes in plaintext mod 16) + movdqu (%r12), %xmm2 # get the appropriate shuffle mask + PSHUFB_XMM %xmm2, %xmm1 # shift right 16-r13 byte + pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn) + movdqu ALL_F-SHIFT_MASK(%r12), %xmm1 + # get the appropriate mask to mask out top 16-r13 bytes of xmm0 + pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0 + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10,%xmm0 + + pxor %xmm0, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # GHASH computation for the last <16 byte block + sub %r13, %r11 + add $16, %r11 + PSHUFB_XMM %xmm10, %xmm1 + + # shuffle xmm0 back to output as ciphertext + + # Output %r13 bytes + MOVQ_R64_XMM %xmm0, %rax + cmp $8, %r13 + jle _less_than_8_bytes_left_encrypt + mov %rax, (%arg2 , %r11, 1) + add $8, %r11 + psrldq $8, %xmm0 + MOVQ_R64_XMM %xmm0, %rax + sub $8, %r13 +_less_than_8_bytes_left_encrypt: + mov %al, (%arg2, %r11, 1) + add $1, %r11 + shr $8, %rax + sub $1, %r13 + jne _less_than_8_bytes_left_encrypt +_multiple_of_16_bytes_encrypt: + mov arg8, %r12 # %r12 = addLen (number of bytes) + shl $3, %r12 + movd %r12d, %xmm15 # len(A) in %xmm15 + shl $3, %arg4 # len(C) in bits (*128) + MOVQ_R64_XMM %arg4, %xmm1 + pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000 + pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C) + pxor %xmm15, %xmm8 + GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 + # final GHASH computation + movdqa SHUF_MASK(%rip), %xmm10 + PSHUFB_XMM %xmm10, %xmm8 # perform a 16 byte swap + + mov %arg5, %rax # %rax = *Y0 + movdqu (%rax), %xmm0 # %xmm0 = Y0 + ENCRYPT_SINGLE_BLOCK %xmm0, %xmm15 # Encrypt(K, Y0) + pxor %xmm8, %xmm0 +_return_T_encrypt: + mov arg9, %r10 # %r10 = authTag + mov arg10, %r11 # %r11 = auth_tag_len + cmp $16, %r11 + je _T_16_encrypt + cmp $12, %r11 + je _T_12_encrypt +_T_8_encrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + jmp _return_T_done_encrypt +_T_12_encrypt: + MOVQ_R64_XMM %xmm0, %rax + mov %rax, (%r10) + psrldq $8, %xmm0 + movd %xmm0, %eax + mov %eax, 8(%r10) + jmp _return_T_done_encrypt +_T_16_encrypt: + movdqu %xmm0, (%r10) +_return_T_done_encrypt: + mov %r14, %rsp + pop %r14 + pop %r13 + pop %r12 + ret + +#endif + _key_expansion_128: _key_expansion_256a: @@ -55,10 +1709,11 @@ _key_expansion_256a: shufps $0b10001100, %xmm0, %xmm4 pxor %xmm4, %xmm0 pxor %xmm1, %xmm0 - movaps %xmm0, (%rcx) - add $0x10, %rcx + movaps %xmm0, (TKEYP) + add $0x10, TKEYP ret +.align 4 _key_expansion_192a: pshufd $0b01010101, %xmm1, %xmm1 shufps $0b00010000, %xmm0, %xmm4 @@ -76,12 +1731,13 @@ _key_expansion_192a: movaps %xmm0, %xmm1 shufps $0b01000100, %xmm0, %xmm6 - movaps %xmm6, (%rcx) + movaps %xmm6, (TKEYP) shufps $0b01001110, %xmm2, %xmm1 - movaps %xmm1, 16(%rcx) - add $0x20, %rcx + movaps %xmm1, 0x10(TKEYP) + add $0x20, TKEYP ret +.align 4 _key_expansion_192b: pshufd $0b01010101, %xmm1, %xmm1 shufps $0b00010000, %xmm0, %xmm4 @@ -96,10 +1752,11 @@ _key_expansion_192b: pxor %xmm3, %xmm2 pxor %xmm5, %xmm2 - movaps %xmm0, (%rcx) - add $0x10, %rcx + movaps %xmm0, (TKEYP) + add $0x10, TKEYP ret +.align 4 _key_expansion_256b: pshufd $0b10101010, %xmm1, %xmm1 shufps $0b00010000, %xmm2, %xmm4 @@ -107,8 +1764,8 @@ _key_expansion_256b: shufps $0b10001100, %xmm2, %xmm4 pxor %xmm4, %xmm2 pxor %xmm1, %xmm2 - movaps %xmm2, (%rcx) - add $0x10, %rcx + movaps %xmm2, (TKEYP) + add $0x10, TKEYP ret /* @@ -116,17 +1773,23 @@ _key_expansion_256b: * unsigned int key_len) */ ENTRY(aesni_set_key) - movups (%rsi), %xmm0 # user key (first 16 bytes) - movaps %xmm0, (%rdi) - lea 0x10(%rdi), %rcx # key addr - movl %edx, 480(%rdi) +#ifndef __x86_64__ + pushl KEYP + movl 8(%esp), KEYP # ctx + movl 12(%esp), UKEYP # in_key + movl 16(%esp), %edx # key_len +#endif + movups (UKEYP), %xmm0 # user key (first 16 bytes) + movaps %xmm0, (KEYP) + lea 0x10(KEYP), TKEYP # key addr + movl %edx, 480(KEYP) pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x cmp $24, %dl jb .Lenc_key128 je .Lenc_key192 - movups 0x10(%rsi), %xmm2 # other user key - movaps %xmm2, (%rcx) - add $0x10, %rcx + movups 0x10(UKEYP), %xmm2 # other user key + movaps %xmm2, (TKEYP) + add $0x10, TKEYP AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 call _key_expansion_256a AESKEYGENASSIST 0x1 %xmm0 %xmm1 @@ -155,7 +1818,7 @@ ENTRY(aesni_set_key) call _key_expansion_256a jmp .Ldec_key .Lenc_key192: - movq 0x10(%rsi), %xmm2 # other user key + movq 0x10(UKEYP), %xmm2 # other user key AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 call _key_expansion_192a AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2 @@ -195,33 +1858,47 @@ ENTRY(aesni_set_key) AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10 call _key_expansion_128 .Ldec_key: - sub $0x10, %rcx - movaps (%rdi), %xmm0 - movaps (%rcx), %xmm1 - movaps %xmm0, 240(%rcx) - movaps %xmm1, 240(%rdi) - add $0x10, %rdi - lea 240-16(%rcx), %rsi + sub $0x10, TKEYP + movaps (KEYP), %xmm0 + movaps (TKEYP), %xmm1 + movaps %xmm0, 240(TKEYP) + movaps %xmm1, 240(KEYP) + add $0x10, KEYP + lea 240-16(TKEYP), UKEYP .align 4 .Ldec_key_loop: - movaps (%rdi), %xmm0 + movaps (KEYP), %xmm0 AESIMC %xmm0 %xmm1 - movaps %xmm1, (%rsi) - add $0x10, %rdi - sub $0x10, %rsi - cmp %rcx, %rdi + movaps %xmm1, (UKEYP) + add $0x10, KEYP + sub $0x10, UKEYP + cmp TKEYP, KEYP jb .Ldec_key_loop - xor %rax, %rax + xor AREG, AREG +#ifndef __x86_64__ + popl KEYP +#endif ret /* * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) */ ENTRY(aesni_enc) +#ifndef __x86_64__ + pushl KEYP + pushl KLEN + movl 12(%esp), KEYP + movl 16(%esp), OUTP + movl 20(%esp), INP +#endif movl 480(KEYP), KLEN # key length movups (INP), STATE # input call _aesni_enc1 movups STATE, (OUTP) # output +#ifndef __x86_64__ + popl KLEN + popl KEYP +#endif ret /* @@ -236,6 +1913,7 @@ ENTRY(aesni_enc) * KEY * TKEYP (T1) */ +.align 4 _aesni_enc1: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -298,6 +1976,7 @@ _aesni_enc1: * KEY * TKEYP (T1) */ +.align 4 _aesni_enc4: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -391,11 +2070,22 @@ _aesni_enc4: * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) */ ENTRY(aesni_dec) +#ifndef __x86_64__ + pushl KEYP + pushl KLEN + movl 12(%esp), KEYP + movl 16(%esp), OUTP + movl 20(%esp), INP +#endif mov 480(KEYP), KLEN # key length add $240, KEYP movups (INP), STATE # input call _aesni_dec1 movups STATE, (OUTP) #output +#ifndef __x86_64__ + popl KLEN + popl KEYP +#endif ret /* @@ -410,6 +2100,7 @@ ENTRY(aesni_dec) * KEY * TKEYP (T1) */ +.align 4 _aesni_dec1: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -472,6 +2163,7 @@ _aesni_dec1: * KEY * TKEYP (T1) */ +.align 4 _aesni_dec4: movaps (KEYP), KEY # key mov KEYP, TKEYP @@ -566,6 +2258,15 @@ _aesni_dec4: * size_t len) */ ENTRY(aesni_ecb_enc) +#ifndef __x86_64__ + pushl LEN + pushl KEYP + pushl KLEN + movl 16(%esp), KEYP + movl 20(%esp), OUTP + movl 24(%esp), INP + movl 28(%esp), LEN +#endif test LEN, LEN # check length jz .Lecb_enc_ret mov 480(KEYP), KLEN @@ -602,6 +2303,11 @@ ENTRY(aesni_ecb_enc) cmp $16, LEN jge .Lecb_enc_loop1 .Lecb_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN +#endif ret /* @@ -609,6 +2315,15 @@ ENTRY(aesni_ecb_enc) * size_t len); */ ENTRY(aesni_ecb_dec) +#ifndef __x86_64__ + pushl LEN + pushl KEYP + pushl KLEN + movl 16(%esp), KEYP + movl 20(%esp), OUTP + movl 24(%esp), INP + movl 28(%esp), LEN +#endif test LEN, LEN jz .Lecb_dec_ret mov 480(KEYP), KLEN @@ -646,6 +2361,11 @@ ENTRY(aesni_ecb_dec) cmp $16, LEN jge .Lecb_dec_loop1 .Lecb_dec_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN +#endif ret /* @@ -653,6 +2373,17 @@ ENTRY(aesni_ecb_dec) * size_t len, u8 *iv) */ ENTRY(aesni_cbc_enc) +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl 20(%esp), KEYP + movl 24(%esp), OUTP + movl 28(%esp), INP + movl 32(%esp), LEN + movl 36(%esp), IVP +#endif cmp $16, LEN jb .Lcbc_enc_ret mov 480(KEYP), KLEN @@ -670,6 +2401,12 @@ ENTRY(aesni_cbc_enc) jge .Lcbc_enc_loop movups STATE, (IVP) .Lcbc_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif ret /* @@ -677,6 +2414,17 @@ ENTRY(aesni_cbc_enc) * size_t len, u8 *iv) */ ENTRY(aesni_cbc_dec) +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl 20(%esp), KEYP + movl 24(%esp), OUTP + movl 28(%esp), INP + movl 32(%esp), LEN + movl 36(%esp), IVP +#endif cmp $16, LEN jb .Lcbc_dec_just_ret mov 480(KEYP), KLEN @@ -690,16 +2438,30 @@ ENTRY(aesni_cbc_dec) movaps IN1, STATE1 movups 0x10(INP), IN2 movaps IN2, STATE2 +#ifdef __x86_64__ movups 0x20(INP), IN3 movaps IN3, STATE3 movups 0x30(INP), IN4 movaps IN4, STATE4 +#else + movups 0x20(INP), IN1 + movaps IN1, STATE3 + movups 0x30(INP), IN2 + movaps IN2, STATE4 +#endif call _aesni_dec4 pxor IV, STATE1 +#ifdef __x86_64__ pxor IN1, STATE2 pxor IN2, STATE3 pxor IN3, STATE4 movaps IN4, IV +#else + pxor (INP), STATE2 + pxor 0x10(INP), STATE3 + pxor IN1, STATE4 + movaps IN2, IV +#endif movups STATE1, (OUTP) movups STATE2, 0x10(OUTP) movups STATE3, 0x20(OUTP) @@ -727,8 +2489,15 @@ ENTRY(aesni_cbc_dec) .Lcbc_dec_ret: movups IV, (IVP) .Lcbc_dec_just_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif ret +#ifdef __x86_64__ .align 16 .Lbswap_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 @@ -744,6 +2513,7 @@ ENTRY(aesni_cbc_dec) * INC: == 1, in little endian * BSWAP_MASK == endian swapping mask */ +.align 4 _aesni_inc_init: movaps .Lbswap_mask, BSWAP_MASK movaps IV, CTR @@ -768,6 +2538,7 @@ _aesni_inc_init: * CTR: == output IV, in little endian * TCTR_LOW: == lower qword of CTR */ +.align 4 _aesni_inc: paddq INC, CTR add $1, TCTR_LOW @@ -839,3 +2610,4 @@ ENTRY(aesni_ctr_enc) movups IV, (IVP) .Lctr_enc_just_ret: ret +#endif diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 2cb3dcc..e1e60c7 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -5,6 +5,14 @@ * Copyright (C) 2008, Intel Corp. * Author: Huang Ying <ying.huang@intel.com> * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Adrian Hoban <adrian.hoban@intel.com> + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Tadeusz Struk (tadeusz.struk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Copyright (c) 2010, Intel Corporation. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -21,6 +29,10 @@ #include <crypto/ctr.h> #include <asm/i387.h> #include <asm/aes.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/aead.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) #define HAS_CTR @@ -42,8 +54,31 @@ struct async_aes_ctx { struct cryptd_ablkcipher *cryptd_tfm; }; -#define AESNI_ALIGN 16 +/* This data is stored at the end of the crypto_tfm struct. + * It's a type of per "session" data storage location. + * This needs to be 16 byte aligned. + */ +struct aesni_rfc4106_gcm_ctx { + u8 hash_subkey[16]; + struct crypto_aes_ctx aes_key_expanded; + u8 nonce[4]; + struct cryptd_aead *cryptd_tfm; +}; + +struct aesni_gcm_set_hash_subkey_result { + int err; + struct completion completion; +}; + +struct aesni_hash_subkey_req_data { + u8 iv[16]; + struct aesni_gcm_set_hash_subkey_result result; + struct scatterlist sg; +}; + +#define AESNI_ALIGN (16) #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) +#define RFC4106_HASH_SUBKEY_SIZE 16 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); @@ -59,9 +94,62 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +#ifdef CONFIG_X86_64 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +/* asmlinkage void aesni_gcm_enc() + * void *ctx, AES Key schedule. Starts on a 16 byte boundary. + * u8 *out, Ciphertext output. Encrypt in-place is allowed. + * const u8 *in, Plaintext input + * unsigned long plaintext_len, Length of data in bytes for encryption. + * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) + * concatenated with 8 byte Initialisation Vector (from IPSec ESP + * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. + * const u8 *aad, Additional Authentication Data (AAD) + * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this + * is going to be 8 or 12 bytes + * u8 *auth_tag, Authenticated Tag output. + * unsigned long auth_tag_len), Authenticated Tag Length in bytes. + * Valid values are 16 (most likely), 12 or 8. + */ +asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, + const u8 *in, unsigned long plaintext_len, u8 *iv, + u8 *hash_subkey, const u8 *aad, unsigned long aad_len, + u8 *auth_tag, unsigned long auth_tag_len); + +/* asmlinkage void aesni_gcm_dec() + * void *ctx, AES Key schedule. Starts on a 16 byte boundary. + * u8 *out, Plaintext output. Decrypt in-place is allowed. + * const u8 *in, Ciphertext input + * unsigned long ciphertext_len, Length of data in bytes for decryption. + * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) + * concatenated with 8 byte Initialisation Vector (from IPSec ESP + * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. + * const u8 *aad, Additional Authentication Data (AAD) + * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going + * to be 8 or 12 bytes + * u8 *auth_tag, Authenticated Tag output. + * unsigned long auth_tag_len) Authenticated Tag Length in bytes. + * Valid values are 16 (most likely), 12 or 8. + */ +asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, + const u8 *in, unsigned long ciphertext_len, u8 *iv, + u8 *hash_subkey, const u8 *aad, unsigned long aad_len, + u8 *auth_tag, unsigned long auth_tag_len); + +static inline struct +aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) +{ + return + (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *) + crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); +} +#endif + static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) { unsigned long addr = (unsigned long)raw_ctx; @@ -324,6 +412,7 @@ static struct crypto_alg blk_cbc_alg = { }, }; +#ifdef CONFIG_X86_64 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, struct blkcipher_walk *walk) { @@ -389,6 +478,7 @@ static struct crypto_alg blk_ctr_alg = { }, }, }; +#endif static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) @@ -536,6 +626,7 @@ static struct crypto_alg ablk_cbc_alg = { }, }; +#ifdef CONFIG_X86_64 static int ablk_ctr_init(struct crypto_tfm *tfm) { struct cryptd_ablkcipher *cryptd_tfm; @@ -612,6 +703,7 @@ static struct crypto_alg ablk_rfc3686_ctr_alg = { }, }; #endif +#endif #ifdef HAS_LRW static int ablk_lrw_init(struct crypto_tfm *tfm) @@ -730,6 +822,424 @@ static struct crypto_alg ablk_xts_alg = { }; #endif +#ifdef CONFIG_X86_64 +static int rfc4106_init(struct crypto_tfm *tfm) +{ + struct cryptd_aead *cryptd_tfm; + struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); + cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + ctx->cryptd_tfm = cryptd_tfm; + tfm->crt_aead.reqsize = sizeof(struct aead_request) + + crypto_aead_reqsize(&cryptd_tfm->base); + return 0; +} + +static void rfc4106_exit(struct crypto_tfm *tfm) +{ + struct aesni_rfc4106_gcm_ctx *ctx = + (struct aesni_rfc4106_gcm_ctx *) + PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); + if (!IS_ERR(ctx->cryptd_tfm)) + cryptd_free_aead(ctx->cryptd_tfm); + return; +} + +static void +rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) +{ + struct aesni_gcm_set_hash_subkey_result *result = req->data; + + if (err == -EINPROGRESS) + return; + result->err = err; + complete(&result->completion); +} + +static int +rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) +{ + struct crypto_ablkcipher *ctr_tfm; + struct ablkcipher_request *req; + int ret = -EINVAL; + struct aesni_hash_subkey_req_data *req_data; + + ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); + if (IS_ERR(ctr_tfm)) + return PTR_ERR(ctr_tfm); + + crypto_ablkcipher_clear_flags(ctr_tfm, ~0); + + ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); + if (ret) { + crypto_free_ablkcipher(ctr_tfm); + return ret; + } + + req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); + if (!req) { + crypto_free_ablkcipher(ctr_tfm); + return -EINVAL; + } + + req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); + if (!req_data) { + crypto_free_ablkcipher(ctr_tfm); + return -ENOMEM; + } + memset(req_data->iv, 0, sizeof(req_data->iv)); + + /* Clear the data in the hash sub key container to zero.*/ + /* We want to cipher all zeros to create the hash sub key. */ + memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); + + init_completion(&req_data->result.completion); + sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); + ablkcipher_request_set_tfm(req, ctr_tfm); + ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + rfc4106_set_hash_subkey_done, + &req_data->result); + + ablkcipher_request_set_crypt(req, &req_data->sg, + &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); + + ret = crypto_ablkcipher_encrypt(req); + if (ret == -EINPROGRESS || ret == -EBUSY) { + ret = wait_for_completion_interruptible + (&req_data->result.completion); + if (!ret) + ret = req_data->result.err; + } + ablkcipher_request_free(req); + kfree(req_data); + crypto_free_ablkcipher(ctr_tfm); + return ret; +} + +static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, + unsigned int key_len) +{ + int ret = 0; + struct crypto_tfm *tfm = crypto_aead_tfm(parent); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); + u8 *new_key_mem = NULL; + + if (key_len < 4) { + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + /*Account for 4 byte nonce at the end.*/ + key_len -= 4; + if (key_len != AES_KEYSIZE_128) { + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); + /*This must be on a 16 byte boundary!*/ + if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) + return -EINVAL; + + if ((unsigned long)key % AESNI_ALIGN) { + /*key is not aligned: use an auxuliar aligned pointer*/ + new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); + if (!new_key_mem) + return -ENOMEM; + + new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); + memcpy(new_key_mem, key, key_len); + key = new_key_mem; + } + + if (!irq_fpu_usable()) + ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), + key, key_len); + else { + kernel_fpu_begin(); + ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); + kernel_fpu_end(); + } + /*This must be on a 16 byte boundary!*/ + if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { + ret = -EINVAL; + goto exit; + } + ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); +exit: + kfree(new_key_mem); + return ret; +} + +/* This is the Integrity Check Value (aka the authentication tag length and can + * be 8, 12 or 16 bytes long. */ +static int rfc4106_set_authsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + crypto_aead_crt(parent)->authsize = authsize; + crypto_aead_crt(cryptd_child)->authsize = authsize; + return 0; +} + +static int rfc4106_encrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + return crypto_aead_encrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = cryptd_child->base.crt_aead.encrypt(req); + kernel_fpu_end(); + return ret; + } +} + +static int rfc4106_decrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + return crypto_aead_decrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = cryptd_child->base.crt_aead.decrypt(req); + kernel_fpu_end(); + return ret; + } +} + +static struct crypto_alg rfc4106_alg = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, + .cra_alignmask = 0, + .cra_type = &crypto_nivaead_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list), + .cra_init = rfc4106_init, + .cra_exit = rfc4106_exit, + .cra_u = { + .aead = { + .setkey = rfc4106_set_key, + .setauthsize = rfc4106_set_authsize, + .encrypt = rfc4106_encrypt, + .decrypt = rfc4106_decrypt, + .geniv = "seqiv", + .ivsize = 8, + .maxauthsize = 16, + }, + }, +}; + +static int __driver_rfc4106_encrypt(struct aead_request *req) +{ + u8 one_entry_in_sg = 0; + u8 *src, *dst, *assoc; + __be32 counter = cpu_to_be32(1); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 iv_tab[16+AESNI_ALIGN]; + u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); + struct scatter_walk src_sg_walk; + struct scatter_walk assoc_sg_walk; + struct scatter_walk dst_sg_walk; + unsigned int i; + + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length equal */ + /* to 8 or 12 bytes */ + if (unlikely(req->assoclen != 8 && req->assoclen != 12)) + return -EINVAL; + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { + one_entry_in_sg = 1; + scatterwalk_start(&src_sg_walk, req->src); + scatterwalk_start(&assoc_sg_walk, req->assoc); + src = scatterwalk_map(&src_sg_walk, 0); + assoc = scatterwalk_map(&assoc_sg_walk, 0); + dst = src; + if (unlikely(req->src != req->dst)) { + scatterwalk_start(&dst_sg_walk, req->dst); + dst = scatterwalk_map(&dst_sg_walk, 0); + } + + } else { + /* Allocate memory for src, dst, assoc */ + src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, + GFP_ATOMIC); + if (unlikely(!src)) + return -ENOMEM; + assoc = (src + req->cryptlen + auth_tag_len); + scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); + scatterwalk_map_and_copy(assoc, req->assoc, 0, + req->assoclen, 0); + dst = src; + } + + aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, + ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst + + ((unsigned long)req->cryptlen), auth_tag_len); + + /* The authTag (aka the Integrity Check Value) needs to be written + * back to the packet. */ + if (one_entry_in_sg) { + if (unlikely(req->src != req->dst)) { + scatterwalk_unmap(dst, 0); + scatterwalk_done(&dst_sg_walk, 0, 0); + } + scatterwalk_unmap(src, 0); + scatterwalk_unmap(assoc, 0); + scatterwalk_done(&src_sg_walk, 0, 0); + scatterwalk_done(&assoc_sg_walk, 0, 0); + } else { + scatterwalk_map_and_copy(dst, req->dst, 0, + req->cryptlen + auth_tag_len, 1); + kfree(src); + } + return 0; +} + +static int __driver_rfc4106_decrypt(struct aead_request *req) +{ + u8 one_entry_in_sg = 0; + u8 *src, *dst, *assoc; + unsigned long tempCipherLen = 0; + __be32 counter = cpu_to_be32(1); + int retval = 0; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 iv_and_authTag[32+AESNI_ALIGN]; + u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); + u8 *authTag = iv + 16; + struct scatter_walk src_sg_walk; + struct scatter_walk assoc_sg_walk; + struct scatter_walk dst_sg_walk; + unsigned int i; + + if (unlikely((req->cryptlen < auth_tag_len) || + (req->assoclen != 8 && req->assoclen != 12))) + return -EINVAL; + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length */ + /* equal to 8 or 12 bytes */ + + tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { + one_entry_in_sg = 1; + scatterwalk_start(&src_sg_walk, req->src); + scatterwalk_start(&assoc_sg_walk, req->assoc); + src = scatterwalk_map(&src_sg_walk, 0); + assoc = scatterwalk_map(&assoc_sg_walk, 0); + dst = src; + if (unlikely(req->src != req->dst)) { + scatterwalk_start(&dst_sg_walk, req->dst); + dst = scatterwalk_map(&dst_sg_walk, 0); + } + + } else { + /* Allocate memory for src, dst, assoc */ + src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); + if (!src) + return -ENOMEM; + assoc = (src + req->cryptlen + auth_tag_len); + scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); + scatterwalk_map_and_copy(assoc, req->assoc, 0, + req->assoclen, 0); + dst = src; + } + + aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, + ctx->hash_subkey, assoc, (unsigned long)req->assoclen, + authTag, auth_tag_len); + + /* Compare generated tag with passed in tag. */ + retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? + -EBADMSG : 0; + + if (one_entry_in_sg) { + if (unlikely(req->src != req->dst)) { + scatterwalk_unmap(dst, 0); + scatterwalk_done(&dst_sg_walk, 0, 0); + } + scatterwalk_unmap(src, 0); + scatterwalk_unmap(assoc, 0); + scatterwalk_done(&src_sg_walk, 0, 0); + scatterwalk_done(&assoc_sg_walk, 0, 0); + } else { + scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); + kfree(src); + } + return retval; +} + +static struct crypto_alg __rfc4106_alg = { + .cra_name = "__gcm-aes-aesni", + .cra_driver_name = "__driver-gcm-aes-aesni", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_AEAD, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, + .cra_alignmask = 0, + .cra_type = &crypto_aead_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list), + .cra_u = { + .aead = { + .encrypt = __driver_rfc4106_encrypt, + .decrypt = __driver_rfc4106_decrypt, + }, + }, +}; +#endif + static int __init aesni_init(void) { int err; @@ -738,6 +1248,7 @@ static int __init aesni_init(void) printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); return -ENODEV; } + if ((err = crypto_register_alg(&aesni_alg))) goto aes_err; if ((err = crypto_register_alg(&__aesni_alg))) @@ -746,18 +1257,24 @@ static int __init aesni_init(void) goto blk_ecb_err; if ((err = crypto_register_alg(&blk_cbc_alg))) goto blk_cbc_err; - if ((err = crypto_register_alg(&blk_ctr_alg))) - goto blk_ctr_err; if ((err = crypto_register_alg(&ablk_ecb_alg))) goto ablk_ecb_err; if ((err = crypto_register_alg(&ablk_cbc_alg))) goto ablk_cbc_err; +#ifdef CONFIG_X86_64 + if ((err = crypto_register_alg(&blk_ctr_alg))) + goto blk_ctr_err; if ((err = crypto_register_alg(&ablk_ctr_alg))) goto ablk_ctr_err; + if ((err = crypto_register_alg(&__rfc4106_alg))) + goto __aead_gcm_err; + if ((err = crypto_register_alg(&rfc4106_alg))) + goto aead_gcm_err; #ifdef HAS_CTR if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) goto ablk_rfc3686_ctr_err; #endif +#endif #ifdef HAS_LRW if ((err = crypto_register_alg(&ablk_lrw_alg))) goto ablk_lrw_err; @@ -770,7 +1287,6 @@ static int __init aesni_init(void) if ((err = crypto_register_alg(&ablk_xts_alg))) goto ablk_xts_err; #endif - return err; #ifdef HAS_XTS @@ -784,18 +1300,24 @@ ablk_pcbc_err: crypto_unregister_alg(&ablk_lrw_alg); ablk_lrw_err: #endif +#ifdef CONFIG_X86_64 #ifdef HAS_CTR crypto_unregister_alg(&ablk_rfc3686_ctr_alg); ablk_rfc3686_ctr_err: #endif + crypto_unregister_alg(&rfc4106_alg); +aead_gcm_err: + crypto_unregister_alg(&__rfc4106_alg); +__aead_gcm_err: crypto_unregister_alg(&ablk_ctr_alg); ablk_ctr_err: + crypto_unregister_alg(&blk_ctr_alg); +blk_ctr_err: +#endif crypto_unregister_alg(&ablk_cbc_alg); ablk_cbc_err: crypto_unregister_alg(&ablk_ecb_alg); ablk_ecb_err: - crypto_unregister_alg(&blk_ctr_alg); -blk_ctr_err: crypto_unregister_alg(&blk_cbc_alg); blk_cbc_err: crypto_unregister_alg(&blk_ecb_alg); @@ -818,13 +1340,17 @@ static void __exit aesni_exit(void) #ifdef HAS_LRW crypto_unregister_alg(&ablk_lrw_alg); #endif +#ifdef CONFIG_X86_64 #ifdef HAS_CTR crypto_unregister_alg(&ablk_rfc3686_ctr_alg); #endif + crypto_unregister_alg(&rfc4106_alg); + crypto_unregister_alg(&__rfc4106_alg); crypto_unregister_alg(&ablk_ctr_alg); + crypto_unregister_alg(&blk_ctr_alg); +#endif crypto_unregister_alg(&ablk_cbc_alg); crypto_unregister_alg(&ablk_ecb_alg); - crypto_unregister_alg(&blk_ctr_alg); crypto_unregister_alg(&blk_cbc_alg); crypto_unregister_alg(&blk_ecb_alg); crypto_unregister_alg(&__aesni_alg); diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h index 3b62ab5..5e1a2ee 100644 --- a/arch/x86/include/asm/boot.h +++ b/arch/x86/include/asm/boot.h @@ -32,11 +32,7 @@ #define BOOT_HEAP_SIZE 0x400000 #else /* !CONFIG_KERNEL_BZIP2 */ -#ifdef CONFIG_X86_64 -#define BOOT_HEAP_SIZE 0x7000 -#else -#define BOOT_HEAP_SIZE 0x4000 -#endif +#define BOOT_HEAP_SIZE 0x8000 #endif /* !CONFIG_KERNEL_BZIP2 */ diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index ba870bb..c704b38 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -10,6 +10,9 @@ #include <asm/apicdef.h> #include <asm/irq_vectors.h> +/* Even though we don't support this, supply it to appease OF */ +static inline void irq_dispose_mapping(unsigned int virq) { } + static inline int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h index b36c6b3..8e37deb 100644 --- a/arch/x86/include/asm/kvm_emulate.h +++ b/arch/x86/include/asm/kvm_emulate.h @@ -15,6 +15,14 @@ struct x86_emulate_ctxt; +struct x86_exception { + u8 vector; + bool error_code_valid; + u16 error_code; + bool nested_page_fault; + u64 address; /* cr2 or nested page fault gpa */ +}; + /* * x86_emulate_ops: * @@ -64,7 +72,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to read from memory. */ int (*read_std)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * write_std: Write bytes of standard (non-emulated/special) memory. @@ -74,7 +83,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to write to memory. */ int (*write_std)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * fetch: Read bytes of standard (non-emulated/special) memory. * Used for instruction fetch. @@ -83,7 +93,8 @@ struct x86_emulate_ops { * @bytes: [IN ] Number of bytes to read from memory. */ int (*fetch)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); + unsigned int bytes, struct kvm_vcpu *vcpu, + struct x86_exception *fault); /* * read_emulated: Read bytes from emulated/special memory area. @@ -94,7 +105,7 @@ struct x86_emulate_ops { int (*read_emulated)(unsigned long addr, void *val, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); /* @@ -107,7 +118,7 @@ struct x86_emulate_ops { int (*write_emulated)(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); /* @@ -122,7 +133,7 @@ struct x86_emulate_ops { const void *old, const void *new, unsigned int bytes, - unsigned int *error, + struct x86_exception *fault, struct kvm_vcpu *vcpu); int (*pio_in_emulated)(int size, unsigned short port, void *val, @@ -159,7 +170,10 @@ struct operand { }; union { unsigned long *reg; - unsigned long mem; + struct segmented_address { + ulong ea; + unsigned seg; + } mem; } addr; union { unsigned long val; @@ -226,9 +240,8 @@ struct x86_emulate_ctxt { bool perm_ok; /* do not check permissions if true */ - int exception; /* exception that happens during emulation or -1 */ - u32 error_code; /* error code for exception */ - bool error_code_valid; + bool have_exception; + struct x86_exception exception; /* decode cache */ struct decode_cache decode; @@ -252,7 +265,7 @@ struct x86_emulate_ctxt { #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 #endif -int x86_decode_insn(struct x86_emulate_ctxt *ctxt); +int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len); #define EMULATION_FAILED -1 #define EMULATION_OK 0 #define EMULATION_RESTART 1 diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f702f82..aa75f21 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -83,11 +83,14 @@ #define KVM_NR_FIXED_MTRR_REGION 88 #define KVM_NR_VAR_MTRR 8 +#define ASYNC_PF_PER_VCPU 64 + extern spinlock_t kvm_lock; extern struct list_head vm_list; struct kvm_vcpu; struct kvm; +struct kvm_async_pf; enum kvm_reg { VCPU_REGS_RAX = 0, @@ -114,6 +117,7 @@ enum kvm_reg { enum kvm_reg_ex { VCPU_EXREG_PDPTR = NR_VCPU_REGS, + VCPU_EXREG_CR3, }; enum { @@ -238,16 +242,18 @@ struct kvm_mmu { void (*new_cr3)(struct kvm_vcpu *vcpu); void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root); unsigned long (*get_cr3)(struct kvm_vcpu *vcpu); - int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*inject_page_fault)(struct kvm_vcpu *vcpu); + int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err, + bool prefault); + void (*inject_page_fault)(struct kvm_vcpu *vcpu, + struct x86_exception *fault); void (*free)(struct kvm_vcpu *vcpu); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva, u32 access, - u32 *error); + struct x86_exception *exception); gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access); void (*prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page); int (*sync_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, bool clear_unsync); + struct kvm_mmu_page *sp); void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva); hpa_t root_hpa; int root_level; @@ -315,16 +321,6 @@ struct kvm_vcpu_arch { */ struct kvm_mmu *walk_mmu; - /* - * This struct is filled with the necessary information to propagate a - * page fault into the guest - */ - struct { - u64 address; - unsigned error_code; - bool nested; - } fault; - /* only needed in kvm_pv_mmu_op() path, but it's hot so * put it here to avoid allocation */ struct kvm_pv_mmu_op_buffer mmu_op_buffer; @@ -412,6 +408,15 @@ struct kvm_vcpu_arch { u64 hv_vapic; cpumask_var_t wbinvd_dirty_mask; + + struct { + bool halted; + gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)]; + struct gfn_to_hva_cache data; + u64 msr_val; + u32 id; + bool send_user_only; + } apf; }; struct kvm_arch { @@ -456,6 +461,10 @@ struct kvm_arch { /* fields used by HYPER-V emulation */ u64 hv_guest_os_id; u64 hv_hypercall; + + #ifdef CONFIG_KVM_MMU_AUDIT + int audit_point; + #endif }; struct kvm_vm_stat { @@ -529,6 +538,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); + void (*decache_cr3)(struct kvm_vcpu *vcpu); void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); @@ -582,9 +592,17 @@ struct kvm_x86_ops { void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); + void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2); const struct trace_print_flags *exit_reasons_str; }; +struct kvm_arch_async_pf { + u32 token; + gfn_t gfn; + unsigned long cr3; + bool direct_map; +}; + extern struct kvm_x86_ops *kvm_x86_ops; int kvm_mmu_module_init(void); @@ -594,7 +612,6 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu); int kvm_mmu_create(struct kvm_vcpu *vcpu); int kvm_mmu_setup(struct kvm_vcpu *vcpu); void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); -void kvm_mmu_set_base_ptes(u64 base_pte); void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask); @@ -623,8 +640,15 @@ enum emulation_result { #define EMULTYPE_NO_DECODE (1 << 0) #define EMULTYPE_TRAP_UD (1 << 1) #define EMULTYPE_SKIP (1 << 2) -int emulate_instruction(struct kvm_vcpu *vcpu, - unsigned long cr2, u16 error_code, int emulation_type); +int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2, + int emulation_type, void *insn, int insn_len); + +static inline int emulate_instruction(struct kvm_vcpu *vcpu, + int emulation_type) +{ + return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); +} + void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); @@ -650,7 +674,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason, int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); -void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); @@ -668,11 +692,11 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gfn_t gfn, void *data, int offset, int len, u32 access); -void kvm_propagate_fault(struct kvm_vcpu *vcpu); +void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); int kvm_pic_set_irq(void *opaque, int irq, int level); @@ -690,16 +714,21 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu); -gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); -gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error); +gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); +gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception); int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); int kvm_fix_hypercall(struct kvm_vcpu *vcpu); -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code, + void *insn, int insn_len); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_enable_tdp(void); @@ -766,20 +795,25 @@ enum { #define HF_VINTR_MASK (1 << 2) #define HF_NMI_MASK (1 << 3) #define HF_IRET_MASK (1 << 4) +#define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ /* * Hardware virtualization extension instructions may fault if a * reboot turns off virtualization while processes are running. * Trap the fault and ignore the instruction if that happens. */ -asmlinkage void kvm_handle_fault_on_reboot(void); +asmlinkage void kvm_spurious_fault(void); +extern bool kvm_rebooting; #define __kvm_handle_fault_on_reboot(insn) \ "666: " insn "\n\t" \ + "668: \n\t" \ ".pushsection .fixup, \"ax\" \n" \ "667: \n\t" \ + "cmpb $0, kvm_rebooting \n\t" \ + "jne 668b \n\t" \ __ASM_SIZE(push) " $666b \n\t" \ - "jmp kvm_handle_fault_on_reboot \n\t" \ + "call kvm_spurious_fault \n\t" \ ".popsection \n\t" \ ".pushsection __ex_table, \"a\" \n\t" \ _ASM_PTR " 666b, 667b \n\t" \ @@ -799,4 +833,15 @@ void kvm_set_shared_msr(unsigned index, u64 val, u64 mask); bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); +void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); +bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); +extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); + +void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); + #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 7b562b6..a427bf7 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -20,6 +20,7 @@ * are available. The use of 0x11 and 0x12 is deprecated */ #define KVM_FEATURE_CLOCKSOURCE2 3 +#define KVM_FEATURE_ASYNC_PF 4 /* The last 8 bits are used to indicate how to interpret the flags field * in pvclock structure. If no bits are set, all flags are ignored. @@ -32,9 +33,13 @@ /* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */ #define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01 +#define MSR_KVM_ASYNC_PF_EN 0x4b564d02 #define KVM_MAX_MMU_OP_BATCH 32 +#define KVM_ASYNC_PF_ENABLED (1 << 0) +#define KVM_ASYNC_PF_SEND_ALWAYS (1 << 1) + /* Operations for KVM_HC_MMU_OP */ #define KVM_MMU_OP_WRITE_PTE 1 #define KVM_MMU_OP_FLUSH_TLB 2 @@ -61,10 +66,20 @@ struct kvm_mmu_op_release_pt { __u64 pt_phys; }; +#define KVM_PV_REASON_PAGE_NOT_PRESENT 1 +#define KVM_PV_REASON_PAGE_READY 2 + +struct kvm_vcpu_pv_apf_data { + __u32 reason; + __u8 pad[60]; + __u32 enabled; +}; + #ifdef __KERNEL__ #include <asm/processor.h> extern void kvmclock_init(void); +extern int kvm_register_clock(char *txt); /* This instruction is vmcall. On non-VT architectures, it will generate a @@ -160,8 +175,17 @@ static inline unsigned int kvm_arch_para_features(void) #ifdef CONFIG_KVM_GUEST void __init kvm_guest_init(void); +void kvm_async_pf_task_wait(u32 token); +void kvm_async_pf_task_wake(u32 token); +u32 kvm_read_and_reset_pf_reason(void); #else #define kvm_guest_init() do { } while (0) +#define kvm_async_pf_task_wait(T) do {} while(0) +#define kvm_async_pf_task_wake(T) do {} while(0) +static inline u32 kvm_read_and_reset_pf_reason(void) +{ + return 0; +} #endif #endif /* __KERNEL__ */ diff --git a/arch/x86/include/asm/olpc.h b/arch/x86/include/asm/olpc.h index 42a978c..f482010 100644 --- a/arch/x86/include/asm/olpc.h +++ b/arch/x86/include/asm/olpc.h @@ -107,10 +107,14 @@ extern int olpc_ec_mask_unset(uint8_t bits); /* GPIO assignments */ #define OLPC_GPIO_MIC_AC 1 -#define OLPC_GPIO_DCON_IRQ geode_gpio(7) +#define OLPC_GPIO_DCON_STAT0 5 +#define OLPC_GPIO_DCON_STAT1 6 +#define OLPC_GPIO_DCON_IRQ 7 #define OLPC_GPIO_THRM_ALRM geode_gpio(10) -#define OLPC_GPIO_SMB_CLK geode_gpio(14) -#define OLPC_GPIO_SMB_DATA geode_gpio(15) +#define OLPC_GPIO_DCON_LOAD 11 +#define OLPC_GPIO_DCON_BLANK 12 +#define OLPC_GPIO_SMB_CLK 14 +#define OLPC_GPIO_SMB_DATA 15 #define OLPC_GPIO_WORKAUX geode_gpio(24) #define OLPC_GPIO_LID geode_gpio(26) #define OLPC_GPIO_ECSCI geode_gpio(27) diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h index 2a84781..641988e 100644 --- a/arch/x86/include/asm/olpc_ofw.h +++ b/arch/x86/include/asm/olpc_ofw.h @@ -8,6 +8,8 @@ #ifdef CONFIG_OLPC_OPENFIRMWARE +extern bool olpc_ofw_is_installed(void); + /* run an OFW command by calling into the firmware */ #define olpc_ofw(name, args, res) \ __olpc_ofw((name), ARRAY_SIZE(args), args, ARRAY_SIZE(res), res) @@ -26,10 +28,17 @@ extern bool olpc_ofw_present(void); #else /* !CONFIG_OLPC_OPENFIRMWARE */ +static inline bool olpc_ofw_is_installed(void) { return false; } static inline void olpc_ofw_detect(void) { } static inline void setup_olpc_ofw_pgd(void) { } static inline bool olpc_ofw_present(void) { return false; } #endif /* !CONFIG_OLPC_OPENFIRMWARE */ +#ifdef CONFIG_OLPC_OPENFIRMWARE_DT +extern void olpc_dt_build_devicetree(void); +#else +static inline void olpc_dt_build_devicetree(void) { } +#endif /* CONFIG_OLPC_OPENFIRMWARE_DT */ + #endif /* _ASM_X86_OLPC_OFW_H */ diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index 271de94..b4389a4 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -92,7 +92,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, - unsigned long adddress) + unsigned long address) { ___pmd_free_tlb(tlb, pmd); } diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c6efecf..53fd1d5 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -901,7 +901,7 @@ extern unsigned long thread_saved_pc(struct task_struct *tsk); /* * The below -8 is to reserve 8 bytes on top of the ring0 stack. * This is necessary to guarantee that the entire "struct pt_regs" - * is accessable even if the CPU haven't stored the SS/ESP registers + * is accessible even if the CPU haven't stored the SS/ESP registers * on the stack (interrupt gate does not save these registers * when switching to the same priv ring). * Therefore beware: accessing the ss/esp fields of the diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h new file mode 100644 index 0000000..b4ec95f --- /dev/null +++ b/arch/x86/include/asm/prom.h @@ -0,0 +1 @@ +/* dummy prom.h; here to make linux/of.h's #includes happy */ diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 0e83105..f2b83bc 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -47,14 +47,13 @@ enum { INTERCEPT_MONITOR, INTERCEPT_MWAIT, INTERCEPT_MWAIT_COND, + INTERCEPT_XSETBV, }; struct __attribute__ ((__packed__)) vmcb_control_area { - u16 intercept_cr_read; - u16 intercept_cr_write; - u16 intercept_dr_read; - u16 intercept_dr_write; + u32 intercept_cr; + u32 intercept_dr; u32 intercept_exceptions; u64 intercept; u8 reserved_1[42]; @@ -81,14 +80,19 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u32 event_inj_err; u64 nested_cr3; u64 lbr_ctl; - u64 reserved_5; + u32 clean; + u32 reserved_5; u64 next_rip; - u8 reserved_6[816]; + u8 insn_len; + u8 insn_bytes[15]; + u8 reserved_6[800]; }; #define TLB_CONTROL_DO_NOTHING 0 #define TLB_CONTROL_FLUSH_ALL_ASID 1 +#define TLB_CONTROL_FLUSH_ASID 3 +#define TLB_CONTROL_FLUSH_ASID_LOCAL 7 #define V_TPR_MASK 0x0f @@ -204,19 +208,31 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK #define SVM_SELECTOR_CODE_MASK (1 << 3) -#define INTERCEPT_CR0_MASK 1 -#define INTERCEPT_CR3_MASK (1 << 3) -#define INTERCEPT_CR4_MASK (1 << 4) -#define INTERCEPT_CR8_MASK (1 << 8) - -#define INTERCEPT_DR0_MASK 1 -#define INTERCEPT_DR1_MASK (1 << 1) -#define INTERCEPT_DR2_MASK (1 << 2) -#define INTERCEPT_DR3_MASK (1 << 3) -#define INTERCEPT_DR4_MASK (1 << 4) -#define INTERCEPT_DR5_MASK (1 << 5) -#define INTERCEPT_DR6_MASK (1 << 6) -#define INTERCEPT_DR7_MASK (1 << 7) +#define INTERCEPT_CR0_READ 0 +#define INTERCEPT_CR3_READ 3 +#define INTERCEPT_CR4_READ 4 +#define INTERCEPT_CR8_READ 8 +#define INTERCEPT_CR0_WRITE (16 + 0) +#define INTERCEPT_CR3_WRITE (16 + 3) +#define INTERCEPT_CR4_WRITE (16 + 4) +#define INTERCEPT_CR8_WRITE (16 + 8) + +#define INTERCEPT_DR0_READ 0 +#define INTERCEPT_DR1_READ 1 +#define INTERCEPT_DR2_READ 2 +#define INTERCEPT_DR3_READ 3 +#define INTERCEPT_DR4_READ 4 +#define INTERCEPT_DR5_READ 5 +#define INTERCEPT_DR6_READ 6 +#define INTERCEPT_DR7_READ 7 +#define INTERCEPT_DR0_WRITE (16 + 0) +#define INTERCEPT_DR1_WRITE (16 + 1) +#define INTERCEPT_DR2_WRITE (16 + 2) +#define INTERCEPT_DR3_WRITE (16 + 3) +#define INTERCEPT_DR4_WRITE (16 + 4) +#define INTERCEPT_DR5_WRITE (16 + 5) +#define INTERCEPT_DR6_WRITE (16 + 6) +#define INTERCEPT_DR7_WRITE (16 + 7) #define SVM_EVTINJ_VEC_MASK 0xff @@ -246,6 +262,8 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXITINFOSHIFT_TS_REASON_JMP 38 #define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44 +#define SVM_EXITINFO_REG_MASK 0x0F + #define SVM_EXIT_READ_CR0 0x000 #define SVM_EXIT_READ_CR3 0x003 #define SVM_EXIT_READ_CR4 0x004 @@ -316,6 +334,7 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXIT_MONITOR 0x08a #define SVM_EXIT_MWAIT 0x08b #define SVM_EXIT_MWAIT_COND 0x08c +#define SVM_EXIT_XSETBV 0x08d #define SVM_EXIT_NPF 0x400 #define SVM_EXIT_ERR -1 diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index f66cda5..0310da6 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -30,6 +30,7 @@ asmlinkage void segment_not_present(void); asmlinkage void stack_segment(void); asmlinkage void general_protection(void); asmlinkage void page_fault(void); +asmlinkage void async_page_fault(void); asmlinkage void spurious_interrupt_bug(void); asmlinkage void coprocessor_error(void); asmlinkage void alignment_check(void); diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 9f0cbd9..84471b8 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -66,15 +66,23 @@ #define PIN_BASED_NMI_EXITING 0x00000008 #define PIN_BASED_VIRTUAL_NMIS 0x00000020 +#define VM_EXIT_SAVE_DEBUG_CONTROLS 0x00000002 #define VM_EXIT_HOST_ADDR_SPACE_SIZE 0x00000200 +#define VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL 0x00001000 #define VM_EXIT_ACK_INTR_ON_EXIT 0x00008000 #define VM_EXIT_SAVE_IA32_PAT 0x00040000 #define VM_EXIT_LOAD_IA32_PAT 0x00080000 +#define VM_EXIT_SAVE_IA32_EFER 0x00100000 +#define VM_EXIT_LOAD_IA32_EFER 0x00200000 +#define VM_EXIT_SAVE_VMX_PREEMPTION_TIMER 0x00400000 +#define VM_ENTRY_LOAD_DEBUG_CONTROLS 0x00000002 #define VM_ENTRY_IA32E_MODE 0x00000200 #define VM_ENTRY_SMM 0x00000400 #define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 +#define VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL 0x00002000 #define VM_ENTRY_LOAD_IA32_PAT 0x00004000 +#define VM_ENTRY_LOAD_IA32_EFER 0x00008000 /* VMCS Encodings */ enum vmcs_field { @@ -239,6 +247,7 @@ enum vmcs_field { #define EXIT_REASON_TASK_SWITCH 9 #define EXIT_REASON_CPUID 10 #define EXIT_REASON_HLT 12 +#define EXIT_REASON_INVD 13 #define EXIT_REASON_INVLPG 14 #define EXIT_REASON_RDPMC 15 #define EXIT_REASON_RDTSC 16 @@ -296,6 +305,12 @@ enum vmcs_field { #define GUEST_INTR_STATE_SMI 0x00000004 #define GUEST_INTR_STATE_NMI 0x00000008 +/* GUEST_ACTIVITY_STATE flags */ +#define GUEST_ACTIVITY_ACTIVE 0 +#define GUEST_ACTIVITY_HLT 1 +#define GUEST_ACTIVITY_SHUTDOWN 2 +#define GUEST_ACTIVITY_WAIT_SIPI 3 + /* * Exit Qualifications for MOV for Control Register Access */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index d2fdb08..57ca777 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1086,7 +1086,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, dma_dom->aperture_size += APERTURE_RANGE_SIZE; - /* Intialize the exclusion range if necessary */ + /* Initialize the exclusion range if necessary */ for_each_iommu(iommu) { if (iommu->exclusion_start && iommu->exclusion_start >= dma_dom->aperture[index]->offset @@ -1353,7 +1353,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) /* * Allocates a new protection domain usable for the dma_ops functions. - * It also intializes the page table and the address allocator data + * It also initializes the page table and the address allocator data * structures required for the dma_ops interface */ static struct dma_ops_domain *dma_ops_domain_alloc(void) diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 591e601..c8b4efa 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S @@ -1406,6 +1406,16 @@ ENTRY(general_protection) CFI_ENDPROC END(general_protection) +#ifdef CONFIG_KVM_GUEST +ENTRY(async_page_fault) + RING0_EC_FRAME + pushl $do_async_page_fault + CFI_ADJUST_CFA_OFFSET 4 + jmp error_code + CFI_ENDPROC +END(apf_page_fault) +#endif + /* * End of kprobes section */ diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index d3b895f..aed1ffb 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -1329,6 +1329,9 @@ errorentry xen_stack_segment do_stack_segment #endif errorentry general_protection do_general_protection errorentry page_fault do_page_fault +#ifdef CONFIG_KVM_GUEST +errorentry async_page_fault do_async_page_fault +#endif #ifdef CONFIG_X86_MCE paranoidzeroentry machine_check *machine_check_vector(%rip) #endif diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index 9f54b20..fc293dc 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -126,7 +126,7 @@ ENTRY(startup_32) movsl movl pa(boot_params) + NEW_CL_POINTER,%esi andl %esi,%esi - jz 1f # No comand line + jz 1f # No command line movl $pa(boot_command_line),%edi movl $(COMMAND_LINE_SIZE/4),%ecx rep diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index 58bb239..e60c38c 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -169,6 +169,7 @@ int init_fpu(struct task_struct *tsk) set_stopped_child_used_math(tsk); return 0; } +EXPORT_SYMBOL_GPL(init_fpu); /* * The xstateregs_active() routine is the same as the fpregs_active() routine, diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 3a43caa..52945da 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -4,6 +4,7 @@ #include <linux/cpu.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> +#include <linux/of.h> #include <linux/seq_file.h> #include <linux/smp.h> #include <linux/ftrace.h> @@ -275,6 +276,15 @@ void smp_x86_platform_ipi(struct pt_regs *regs) EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); +#ifdef CONFIG_OF +unsigned int irq_create_of_mapping(struct device_node *controller, + const u32 *intspec, unsigned int intsize) +{ + return intspec[0]; +} +EXPORT_SYMBOL_GPL(irq_create_of_mapping); +#endif + #ifdef CONFIG_HOTPLUG_CPU /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ void fixup_irqs(void) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 63b0ec8..8dc4466 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -27,16 +27,37 @@ #include <linux/mm.h> #include <linux/highmem.h> #include <linux/hardirq.h> +#include <linux/notifier.h> +#include <linux/reboot.h> +#include <linux/hash.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/kprobes.h> #include <asm/timer.h> +#include <asm/cpu.h> +#include <asm/traps.h> +#include <asm/desc.h> +#include <asm/tlbflush.h> #define MMU_QUEUE_SIZE 1024 +static int kvmapf = 1; + +static int parse_no_kvmapf(char *arg) +{ + kvmapf = 0; + return 0; +} + +early_param("no-kvmapf", parse_no_kvmapf); + struct kvm_para_state { u8 mmu_queue[MMU_QUEUE_SIZE]; int mmu_queue_len; }; static DEFINE_PER_CPU(struct kvm_para_state, para_state); +static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); static struct kvm_para_state *kvm_para_state(void) { @@ -50,6 +71,195 @@ static void kvm_io_delay(void) { } +#define KVM_TASK_SLEEP_HASHBITS 8 +#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS) + +struct kvm_task_sleep_node { + struct hlist_node link; + wait_queue_head_t wq; + u32 token; + int cpu; + bool halted; + struct mm_struct *mm; +}; + +static struct kvm_task_sleep_head { + spinlock_t lock; + struct hlist_head list; +} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE]; + +static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, + u32 token) +{ + struct hlist_node *p; + + hlist_for_each(p, &b->list) { + struct kvm_task_sleep_node *n = + hlist_entry(p, typeof(*n), link); + if (n->token == token) + return n; + } + + return NULL; +} + +void kvm_async_pf_task_wait(u32 token) +{ + u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); + struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; + struct kvm_task_sleep_node n, *e; + DEFINE_WAIT(wait); + int cpu, idle; + + cpu = get_cpu(); + idle = idle_cpu(cpu); + put_cpu(); + + spin_lock(&b->lock); + e = _find_apf_task(b, token); + if (e) { + /* dummy entry exist -> wake up was delivered ahead of PF */ + hlist_del(&e->link); + kfree(e); + spin_unlock(&b->lock); + return; + } + + n.token = token; + n.cpu = smp_processor_id(); + n.mm = current->active_mm; + n.halted = idle || preempt_count() > 1; + atomic_inc(&n.mm->mm_count); + init_waitqueue_head(&n.wq); + hlist_add_head(&n.link, &b->list); + spin_unlock(&b->lock); + + for (;;) { + if (!n.halted) + prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE); + if (hlist_unhashed(&n.link)) + break; + + if (!n.halted) { + local_irq_enable(); + schedule(); + local_irq_disable(); + } else { + /* + * We cannot reschedule. So halt. + */ + native_safe_halt(); + local_irq_disable(); + } + } + if (!n.halted) + finish_wait(&n.wq, &wait); + + return; +} +EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait); + +static void apf_task_wake_one(struct kvm_task_sleep_node *n) +{ + hlist_del_init(&n->link); + if (!n->mm) + return; + mmdrop(n->mm); + if (n->halted) + smp_send_reschedule(n->cpu); + else if (waitqueue_active(&n->wq)) + wake_up(&n->wq); +} + +static void apf_task_wake_all(void) +{ + int i; + + for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { + struct hlist_node *p, *next; + struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; + spin_lock(&b->lock); + hlist_for_each_safe(p, next, &b->list) { + struct kvm_task_sleep_node *n = + hlist_entry(p, typeof(*n), link); + if (n->cpu == smp_processor_id()) + apf_task_wake_one(n); + } + spin_unlock(&b->lock); + } +} + +void kvm_async_pf_task_wake(u32 token) +{ + u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); + struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; + struct kvm_task_sleep_node *n; + + if (token == ~0) { + apf_task_wake_all(); + return; + } + +again: + spin_lock(&b->lock); + n = _find_apf_task(b, token); + if (!n) { + /* + * async PF was not yet handled. + * Add dummy entry for the token. + */ + n = kmalloc(sizeof(*n), GFP_ATOMIC); + if (!n) { + /* + * Allocation failed! Busy wait while other cpu + * handles async PF. + */ + spin_unlock(&b->lock); + cpu_relax(); + goto again; + } + n->token = token; + n->cpu = smp_processor_id(); + n->mm = NULL; + init_waitqueue_head(&n->wq); + hlist_add_head(&n->link, &b->list); + } else + apf_task_wake_one(n); + spin_unlock(&b->lock); + return; +} +EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); + +u32 kvm_read_and_reset_pf_reason(void) +{ + u32 reason = 0; + + if (__get_cpu_var(apf_reason).enabled) { + reason = __get_cpu_var(apf_reason).reason; + __get_cpu_var(apf_reason).reason = 0; + } + + return reason; +} +EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); + +dotraplinkage void __kprobes +do_async_page_fault(struct pt_regs *regs, unsigned long error_code) +{ + switch (kvm_read_and_reset_pf_reason()) { + default: + do_page_fault(regs, error_code); + break; + case KVM_PV_REASON_PAGE_NOT_PRESENT: + /* page is swapped out by the host. */ + kvm_async_pf_task_wait((u32)read_cr2()); + break; + case KVM_PV_REASON_PAGE_READY: + kvm_async_pf_task_wake((u32)read_cr2()); + break; + } +} + static void kvm_mmu_op(void *buffer, unsigned len) { int r; @@ -231,10 +441,117 @@ static void __init paravirt_ops_setup(void) #endif } +void __cpuinit kvm_guest_cpu_init(void) +{ + if (!kvm_para_available()) + return; + + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) { + u64 pa = __pa(&__get_cpu_var(apf_reason)); + +#ifdef CONFIG_PREEMPT + pa |= KVM_ASYNC_PF_SEND_ALWAYS; +#endif + wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED); + __get_cpu_var(apf_reason).enabled = 1; + printk(KERN_INFO"KVM setup async PF for cpu %d\n", + smp_processor_id()); + } +} + +static void kvm_pv_disable_apf(void *unused) +{ + if (!__get_cpu_var(apf_reason).enabled) + return; + + wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); + __get_cpu_var(apf_reason).enabled = 0; + + printk(KERN_INFO"Unregister pv shared memory for cpu %d\n", + smp_processor_id()); +} + +static int kvm_pv_reboot_notify(struct notifier_block *nb, + unsigned long code, void *unused) +{ + if (code == SYS_RESTART) + on_each_cpu(kvm_pv_disable_apf, NULL, 1); + return NOTIFY_DONE; +} + +static struct notifier_block kvm_pv_reboot_nb = { + .notifier_call = kvm_pv_reboot_notify, +}; + +#ifdef CONFIG_SMP +static void __init kvm_smp_prepare_boot_cpu(void) +{ +#ifdef CONFIG_KVM_CLOCK + WARN_ON(kvm_register_clock("primary cpu clock")); +#endif + kvm_guest_cpu_init(); + native_smp_prepare_boot_cpu(); +} + +static void kvm_guest_cpu_online(void *dummy) +{ + kvm_guest_cpu_init(); +} + +static void kvm_guest_cpu_offline(void *dummy) +{ + kvm_pv_disable_apf(NULL); + apf_task_wake_all(); +} + +static int __cpuinit kvm_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + int cpu = (unsigned long)hcpu; + switch (action) { + case CPU_ONLINE: + case CPU_DOWN_FAILED: + case CPU_ONLINE_FROZEN: + smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0); + break; + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata kvm_cpu_notifier = { + .notifier_call = kvm_cpu_notify, +}; +#endif + +static void __init kvm_apf_trap_init(void) +{ + set_intr_gate(14, &async_page_fault); +} + void __init kvm_guest_init(void) { + int i; + if (!kvm_para_available()) return; paravirt_ops_setup(); + register_reboot_notifier(&kvm_pv_reboot_nb); + for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) + spin_lock_init(&async_pf_sleepers[i].lock); + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF)) + x86_init.irqs.trap_init = kvm_apf_trap_init; + +#ifdef CONFIG_SMP + smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; + register_cpu_notifier(&kvm_cpu_notifier); +#else + kvm_guest_cpu_init(); +#endif } diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index ca43ce3..f98d3ea 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -125,7 +125,7 @@ static struct clocksource kvm_clock = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static int kvm_register_clock(char *txt) +int kvm_register_clock(char *txt) { int cpu = smp_processor_id(); int low, high, ret; @@ -152,14 +152,6 @@ static void __cpuinit kvm_setup_secondary_clock(void) } #endif -#ifdef CONFIG_SMP -static void __init kvm_smp_prepare_boot_cpu(void) -{ - WARN_ON(kvm_register_clock("primary cpu clock")); - native_smp_prepare_boot_cpu(); -} -#endif - /* * After the clock is registered, the host will keep writing to the * registered memory location. If the guest happens to shutdown, this memory @@ -206,9 +198,6 @@ void __init kvmclock_init(void) x86_cpuinit.setup_percpu_clockev = kvm_setup_secondary_clock; #endif -#ifdef CONFIG_SMP - smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; -#endif machine_ops.shutdown = kvm_shutdown; #ifdef CONFIG_KEXEC machine_ops.crash_shutdown = kvm_crash_shutdown; diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 1cfbbfc..6f39cab 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -76,7 +76,7 @@ int mach_set_rtc_mmss(unsigned long nowtime) CMOS_WRITE(real_seconds, RTC_SECONDS); CMOS_WRITE(real_minutes, RTC_MINUTES); } else { - printk(KERN_WARNING + printk_once(KERN_NOTICE "set_rtc_mmss: can't update from %d to %d\n", cmos_minutes, real_minutes); retval = -1; diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index ddc131f..50f6364 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -28,6 +28,7 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_EVENTFD select KVM_APIC_ARCHITECTURE + select KVM_ASYNC_PF select USER_RETURN_NOTIFIER select KVM_MMIO ---help--- diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 31a7035..f15501f43 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,5 +1,5 @@ -EXTRA_CFLAGS += -Ivirt/kvm -Iarch/x86/kvm +ccflags-y += -Ivirt/kvm -Iarch/x86/kvm CFLAGS_x86.o := -I. CFLAGS_svm.o := -I. @@ -9,6 +9,7 @@ kvm-y += $(addprefix ../../../virt/kvm/, kvm_main.o ioapic.o \ coalesced_mmio.o irq_comm.o eventfd.o \ assigned-dev.o) kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o) +kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o) kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ i8254.o timer.o diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 38b6e8d..caf9667 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -20,16 +20,8 @@ * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 */ -#ifndef __KERNEL__ -#include <stdio.h> -#include <stdint.h> -#include <public/xen.h> -#define DPRINTF(_f, _a ...) printf(_f , ## _a) -#else #include <linux/kvm_host.h> #include "kvm_cache_regs.h" -#define DPRINTF(x...) do {} while (0) -#endif #include <linux/module.h> #include <asm/kvm_emulate.h> @@ -418,9 +410,9 @@ address_mask(struct decode_cache *c, unsigned long reg) } static inline unsigned long -register_address(struct decode_cache *c, unsigned long base, unsigned long reg) +register_address(struct decode_cache *c, unsigned long reg) { - return base + address_mask(c, reg); + return address_mask(c, reg); } static inline void @@ -452,60 +444,55 @@ static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, return ops->get_cached_segment_base(seg, ctxt->vcpu); } -static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - struct decode_cache *c) +static unsigned seg_override(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + struct decode_cache *c) { if (!c->has_seg_override) return 0; - return seg_base(ctxt, ops, c->seg_override); + return c->seg_override; } -static unsigned long es_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) +static ulong linear(struct x86_emulate_ctxt *ctxt, + struct segmented_address addr) { - return seg_base(ctxt, ops, VCPU_SREG_ES); -} - -static unsigned long ss_base(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - return seg_base(ctxt, ops, VCPU_SREG_SS); -} + struct decode_cache *c = &ctxt->decode; + ulong la; -static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, - u32 error, bool valid) -{ - ctxt->exception = vec; - ctxt->error_code = error; - ctxt->error_code_valid = valid; + la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea; + if (c->ad_bytes != 8) + la &= (u32)-1; + return la; } -static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err) +static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, + u32 error, bool valid) { - emulate_exception(ctxt, GP_VECTOR, err, true); + ctxt->exception.vector = vec; + ctxt->exception.error_code = error; + ctxt->exception.error_code_valid = valid; + return X86EMUL_PROPAGATE_FAULT; } -static void emulate_pf(struct x86_emulate_ctxt *ctxt) +static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err) { - emulate_exception(ctxt, PF_VECTOR, 0, true); + return emulate_exception(ctxt, GP_VECTOR, err, true); } -static void emulate_ud(struct x86_emulate_ctxt *ctxt) +static int emulate_ud(struct x86_emulate_ctxt *ctxt) { - emulate_exception(ctxt, UD_VECTOR, 0, false); + return emulate_exception(ctxt, UD_VECTOR, 0, false); } -static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err) +static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err) { - emulate_exception(ctxt, TS_VECTOR, err, true); + return emulate_exception(ctxt, TS_VECTOR, err, true); } static int emulate_de(struct x86_emulate_ctxt *ctxt) { - emulate_exception(ctxt, DE_VECTOR, 0, false); - return X86EMUL_PROPAGATE_FAULT; + return emulate_exception(ctxt, DE_VECTOR, 0, false); } static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, @@ -520,7 +507,7 @@ static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, cur_size = fc->end - fc->start; size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip)); rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size, - size, ctxt->vcpu, NULL); + size, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; fc->end += size; @@ -564,7 +551,7 @@ static void *decode_register(u8 modrm_reg, unsigned long *regs, static int read_descriptor(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - ulong addr, + struct segmented_address addr, u16 *size, unsigned long *address, int op_bytes) { int rc; @@ -572,10 +559,13 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt, if (op_bytes == 2) op_bytes = 3; *address = 0; - rc = ops->read_std(addr, (unsigned long *)size, 2, ctxt->vcpu, NULL); + rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2, + ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; - rc = ops->read_std(addr + 2, address, op_bytes, ctxt->vcpu, NULL); + addr.ea += 2; + rc = ops->read_std(linear(ctxt, addr), address, op_bytes, + ctxt->vcpu, &ctxt->exception); return rc; } @@ -768,7 +758,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, break; } } - op->addr.mem = modrm_ea; + op->addr.mem.ea = modrm_ea; done: return rc; } @@ -783,13 +773,13 @@ static int decode_abs(struct x86_emulate_ctxt *ctxt, op->type = OP_MEM; switch (c->ad_bytes) { case 2: - op->addr.mem = insn_fetch(u16, 2, c->eip); + op->addr.mem.ea = insn_fetch(u16, 2, c->eip); break; case 4: - op->addr.mem = insn_fetch(u32, 4, c->eip); + op->addr.mem.ea = insn_fetch(u32, 4, c->eip); break; case 8: - op->addr.mem = insn_fetch(u64, 8, c->eip); + op->addr.mem.ea = insn_fetch(u64, 8, c->eip); break; } done: @@ -808,7 +798,7 @@ static void fetch_bit_operand(struct decode_cache *c) else if (c->src.bytes == 4) sv = (s32)c->src.val & (s32)mask; - c->dst.addr.mem += (sv >> 3); + c->dst.addr.mem.ea += (sv >> 3); } /* only subword offset */ @@ -821,7 +811,6 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, { int rc; struct read_cache *mc = &ctxt->decode.mem_read; - u32 err; while (size) { int n = min(size, 8u); @@ -829,10 +818,8 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt, if (mc->pos < mc->end) goto read_cached; - rc = ops->read_emulated(addr, mc->data + mc->end, n, &err, - ctxt->vcpu); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + rc = ops->read_emulated(addr, mc->data + mc->end, n, + &ctxt->exception, ctxt->vcpu); if (rc != X86EMUL_CONTINUE) return rc; mc->end += n; @@ -907,19 +894,15 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt, struct desc_ptr dt; u16 index = selector >> 3; int ret; - u32 err; ulong addr; get_descriptor_table_ptr(ctxt, ops, selector, &dt); - if (dt.size < index * 8 + 7) { - emulate_gp(ctxt, selector & 0xfffc); - return X86EMUL_PROPAGATE_FAULT; - } + if (dt.size < index * 8 + 7) + return emulate_gp(ctxt, selector & 0xfffc); addr = dt.address + index * 8; - ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, + &ctxt->exception); return ret; } @@ -931,21 +914,17 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt, { struct desc_ptr dt; u16 index = selector >> 3; - u32 err; ulong addr; int ret; get_descriptor_table_ptr(ctxt, ops, selector, &dt); - if (dt.size < index * 8 + 7) { - emulate_gp(ctxt, selector & 0xfffc); - return X86EMUL_PROPAGATE_FAULT; - } + if (dt.size < index * 8 + 7) + return emulate_gp(ctxt, selector & 0xfffc); addr = dt.address + index * 8; - ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); + ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, + &ctxt->exception); return ret; } @@ -1092,7 +1071,6 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, { int rc; struct decode_cache *c = &ctxt->decode; - u32 err; switch (c->dst.type) { case OP_REG: @@ -1101,21 +1079,19 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, case OP_MEM: if (c->lock_prefix) rc = ops->cmpxchg_emulated( - c->dst.addr.mem, + linear(ctxt, c->dst.addr.mem), &c->dst.orig_val, &c->dst.val, c->dst.bytes, - &err, + &ctxt->exception, ctxt->vcpu); else rc = ops->write_emulated( - c->dst.addr.mem, + linear(ctxt, c->dst.addr.mem), &c->dst.val, c->dst.bytes, - &err, + &ctxt->exception, ctxt->vcpu); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); if (rc != X86EMUL_CONTINUE) return rc; break; @@ -1137,8 +1113,8 @@ static inline void emulate_push(struct x86_emulate_ctxt *ctxt, c->dst.bytes = c->op_bytes; c->dst.val = c->src.val; register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes); - c->dst.addr.mem = register_address(c, ss_base(ctxt, ops), - c->regs[VCPU_REGS_RSP]); + c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]); + c->dst.addr.mem.seg = VCPU_SREG_SS; } static int emulate_pop(struct x86_emulate_ctxt *ctxt, @@ -1147,10 +1123,11 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt, { struct decode_cache *c = &ctxt->decode; int rc; + struct segmented_address addr; - rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops), - c->regs[VCPU_REGS_RSP]), - dest, len); + addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]); + addr.seg = VCPU_SREG_SS; + rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len); if (rc != X86EMUL_CONTINUE) return rc; @@ -1184,10 +1161,8 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt, change_mask |= EFLG_IF; break; case X86EMUL_MODE_VM86: - if (iopl < 3) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (iopl < 3) + return emulate_gp(ctxt, 0); change_mask |= EFLG_IF; break; default: /* real mode */ @@ -1198,9 +1173,6 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt, *(unsigned long *)dest = (ctxt->eflags & ~change_mask) | (val & change_mask); - if (rc == X86EMUL_PROPAGATE_FAULT) - emulate_pf(ctxt); - return rc; } @@ -1287,7 +1259,6 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, gva_t cs_addr; gva_t eip_addr; u16 cs, eip; - u32 err; /* TODO: Add limit checks */ c->src.val = ctxt->eflags; @@ -1317,11 +1288,11 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, eip_addr = dt.address + (irq << 2); cs_addr = dt.address + (irq << 2) + 2; - rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &err); + rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; - rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &err); + rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception); if (rc != X86EMUL_CONTINUE) return rc; @@ -1370,10 +1341,8 @@ static int emulate_iret_real(struct x86_emulate_ctxt *ctxt, if (rc != X86EMUL_CONTINUE) return rc; - if (temp_eip & ~0xffff) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (temp_eip & ~0xffff) + return emulate_gp(ctxt, 0); rc = emulate_pop(ctxt, ops, &cs, c->op_bytes); @@ -1624,10 +1593,8 @@ emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* syscall is not available in real mode */ if (ctxt->mode == X86EMUL_MODE_REAL || - ctxt->mode == X86EMUL_MODE_VM86) { - emulate_ud(ctxt); - return X86EMUL_PROPAGATE_FAULT; - } + ctxt->mode == X86EMUL_MODE_VM86) + return emulate_ud(ctxt); setup_syscalls_segments(ctxt, ops, &cs, &ss); @@ -1678,34 +1645,26 @@ emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) u16 cs_sel, ss_sel; /* inject #GP if in real mode */ - if (ctxt->mode == X86EMUL_MODE_REAL) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (ctxt->mode == X86EMUL_MODE_REAL) + return emulate_gp(ctxt, 0); /* XXX sysenter/sysexit have not been tested in 64bit mode. * Therefore, we inject an #UD. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) { - emulate_ud(ctxt); - return X86EMUL_PROPAGATE_FAULT; - } + if (ctxt->mode == X86EMUL_MODE_PROT64) + return emulate_ud(ctxt); setup_syscalls_segments(ctxt, ops, &cs, &ss); ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data); switch (ctxt->mode) { case X86EMUL_MODE_PROT32: - if ((msr_data & 0xfffc) == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if ((msr_data & 0xfffc) == 0x0) + return emulate_gp(ctxt, 0); break; case X86EMUL_MODE_PROT64: - if (msr_data == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (msr_data == 0x0) + return emulate_gp(ctxt, 0); break; } @@ -1745,10 +1704,8 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* inject #GP if in real mode or Virtual 8086 mode */ if (ctxt->mode == X86EMUL_MODE_REAL || - ctxt->mode == X86EMUL_MODE_VM86) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + ctxt->mode == X86EMUL_MODE_VM86) + return emulate_gp(ctxt, 0); setup_syscalls_segments(ctxt, ops, &cs, &ss); @@ -1763,18 +1720,14 @@ emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (usermode) { case X86EMUL_MODE_PROT32: cs_sel = (u16)(msr_data + 16); - if ((msr_data & 0xfffc) == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if ((msr_data & 0xfffc) == 0x0) + return emulate_gp(ctxt, 0); ss_sel = (u16)(msr_data + 24); break; case X86EMUL_MODE_PROT64: cs_sel = (u16)(msr_data + 32); - if (msr_data == 0x0) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (msr_data == 0x0) + return emulate_gp(ctxt, 0); ss_sel = cs_sel + 8; cs.d = 0; cs.l = 1; @@ -1934,33 +1887,27 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt, { struct tss_segment_16 tss_seg; int ret; - u32 err, new_tss_base = get_desc_base(new_desc); + u32 new_tss_base = get_desc_base(new_desc); ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } save_state_to_tss16(ctxt, ops, &tss_seg); ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } if (old_tss_sel != 0xffff) { tss_seg.prev_task_link = old_tss_sel; @@ -1968,12 +1915,10 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt, ret = ops->write_std(new_tss_base, &tss_seg.prev_task_link, sizeof tss_seg.prev_task_link, - ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + ctxt->vcpu, &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } } return load_state_from_tss16(ctxt, ops, &tss_seg); @@ -2013,10 +1958,8 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt, struct decode_cache *c = &ctxt->decode; int ret; - if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) + return emulate_gp(ctxt, 0); c->eip = tss->eip; ctxt->eflags = tss->eflags | 2; c->regs[VCPU_REGS_RAX] = tss->eax; @@ -2076,33 +2019,27 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt, { struct tss_segment_32 tss_seg; int ret; - u32 err, new_tss_base = get_desc_base(new_desc); + u32 new_tss_base = get_desc_base(new_desc); ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } save_state_to_tss32(ctxt, ops, &tss_seg); ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu, - &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } if (old_tss_sel != 0xffff) { tss_seg.prev_task_link = old_tss_sel; @@ -2110,12 +2047,10 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt, ret = ops->write_std(new_tss_base, &tss_seg.prev_task_link, sizeof tss_seg.prev_task_link, - ctxt->vcpu, &err); - if (ret == X86EMUL_PROPAGATE_FAULT) { + ctxt->vcpu, &ctxt->exception); + if (ret != X86EMUL_CONTINUE) /* FIXME: need to provide precise fault address */ - emulate_pf(ctxt); return ret; - } } return load_state_from_tss32(ctxt, ops, &tss_seg); @@ -2146,10 +2081,8 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt, if (reason != TASK_SWITCH_IRET) { if ((tss_selector & 3) > next_tss_desc.dpl || - ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) + return emulate_gp(ctxt, 0); } desc_limit = desc_limit_scaled(&next_tss_desc); @@ -2231,14 +2164,15 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt, return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; } -static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base, +static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg, int reg, struct operand *op) { struct decode_cache *c = &ctxt->decode; int df = (ctxt->eflags & EFLG_DF) ? -1 : 1; register_address_increment(c, &c->regs[reg], df * op->bytes); - op->addr.mem = register_address(c, base, c->regs[reg]); + op->addr.mem.ea = register_address(c, c->regs[reg]); + op->addr.mem.seg = seg; } static int em_push(struct x86_emulate_ctxt *ctxt) @@ -2369,10 +2303,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt) struct decode_cache *c = &ctxt->decode; u64 tsc = 0; - if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) { - emulate_gp(ctxt, 0); - return X86EMUL_PROPAGATE_FAULT; - } + if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) + return emulate_gp(ctxt, 0); ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc); c->regs[VCPU_REGS_RAX] = (u32)tsc; c->regs[VCPU_REGS_RDX] = tsc >> 32; @@ -2647,7 +2579,7 @@ static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op, op->type = OP_IMM; op->bytes = size; - op->addr.mem = c->eip; + op->addr.mem.ea = c->eip; /* NB. Immediates are sign-extended as necessary. */ switch (op->bytes) { case 1: @@ -2678,7 +2610,7 @@ done: } int -x86_decode_insn(struct x86_emulate_ctxt *ctxt) +x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len) { struct x86_emulate_ops *ops = ctxt->ops; struct decode_cache *c = &ctxt->decode; @@ -2689,7 +2621,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt) struct operand memop = { .type = OP_NONE }; c->eip = ctxt->eip; - c->fetch.start = c->fetch.end = c->eip; + c->fetch.start = c->eip; + c->fetch.end = c->fetch.start + insn_len; + if (insn_len > 0) + memcpy(c->fetch.data, insn, insn_len); ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS); switch (mode) { @@ -2803,10 +2738,8 @@ done_prefixes: c->execute = opcode.u.execute; /* Unrecognised? */ - if (c->d == 0 || (c->d & Undefined)) { - DPRINTF("Cannot emulate %02x\n", c->b); + if (c->d == 0 || (c->d & Undefined)) return -1; - } if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) c->op_bytes = 8; @@ -2831,14 +2764,13 @@ done_prefixes: if (!c->has_seg_override) set_seg_override(c, VCPU_SREG_DS); - if (memop.type == OP_MEM && !(!c->twobyte && c->b == 0x8d)) - memop.addr.mem += seg_override_base(ctxt, ops, c); + memop.addr.mem.seg = seg_override(ctxt, ops, c); if (memop.type == OP_MEM && c->ad_bytes != 8) - memop.addr.mem = (u32)memop.addr.mem; + memop.addr.mem.ea = (u32)memop.addr.mem.ea; if (memop.type == OP_MEM && c->rip_relative) - memop.addr.mem += c->eip; + memop.addr.mem.ea += c->eip; /* * Decode and fetch the source operand: register, memory @@ -2890,14 +2822,14 @@ done_prefixes: case SrcSI: c->src.type = OP_MEM; c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->src.addr.mem = - register_address(c, seg_override_base(ctxt, ops, c), - c->regs[VCPU_REGS_RSI]); + c->src.addr.mem.ea = + register_address(c, c->regs[VCPU_REGS_RSI]); + c->src.addr.mem.seg = seg_override(ctxt, ops, c), c->src.val = 0; break; case SrcImmFAddr: c->src.type = OP_IMM; - c->src.addr.mem = c->eip; + c->src.addr.mem.ea = c->eip; c->src.bytes = c->op_bytes + 2; insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip); break; @@ -2944,7 +2876,7 @@ done_prefixes: break; case DstImmUByte: c->dst.type = OP_IMM; - c->dst.addr.mem = c->eip; + c->dst.addr.mem.ea = c->eip; c->dst.bytes = 1; c->dst.val = insn_fetch(u8, 1, c->eip); break; @@ -2969,9 +2901,9 @@ done_prefixes: case DstDI: c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.addr.mem = - register_address(c, es_base(ctxt, ops), - c->regs[VCPU_REGS_RDI]); + c->dst.addr.mem.ea = + register_address(c, c->regs[VCPU_REGS_RDI]); + c->dst.addr.mem.seg = VCPU_SREG_ES; c->dst.val = 0; break; case ImplicitOps: @@ -3020,24 +2952,24 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) ctxt->decode.mem_read.pos = 0; if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } /* LOCK prefix is allowed only with some instructions */ if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } /* Privileged instruction can be executed only in CPL=0 */ if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } @@ -3050,7 +2982,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) } if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) { - rc = read_emulated(ctxt, ops, c->src.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem), c->src.valptr, c->src.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3058,7 +2990,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) } if (c->src2.type == OP_MEM) { - rc = read_emulated(ctxt, ops, c->src2.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem), &c->src2.val, c->src2.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3070,7 +3002,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt) if ((c->dst.type == OP_MEM) && !(c->d & Mov)) { /* optimisation - avoid slow emulated read if Mov */ - rc = read_emulated(ctxt, ops, c->dst.addr.mem, + rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem), &c->dst.val, c->dst.bytes); if (rc != X86EMUL_CONTINUE) goto done; @@ -3215,13 +3147,13 @@ special_insn: break; case 0x8c: /* mov r/m, sreg */ if (c->modrm_reg > VCPU_SREG_GS) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu); break; case 0x8d: /* lea r16/r32, m */ - c->dst.val = c->src.addr.mem; + c->dst.val = c->src.addr.mem.ea; break; case 0x8e: { /* mov seg, r/m16 */ uint16_t sel; @@ -3230,7 +3162,7 @@ special_insn: if (c->modrm_reg == VCPU_SREG_CS || c->modrm_reg > VCPU_SREG_GS) { - emulate_ud(ctxt); + rc = emulate_ud(ctxt); goto done; } @@ -3268,7 +3200,6 @@ special_insn: break; case 0xa6 ... 0xa7: /* cmps */ c->dst.type = OP_NONE; /* Disable writeback. */ - DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.addr.mem, c->dst.addr.mem); goto cmp; case 0xa8 ... 0xa9: /* test ax, imm */ goto test; @@ -3363,7 +3294,7 @@ special_insn: do_io_in: c->dst.bytes = min(c->dst.bytes, 4u); if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val, @@ -3377,7 +3308,7 @@ special_insn: c->src.bytes = min(c->src.bytes, 4u); if (!emulator_io_permited(ctxt, ops, c->dst.val, c->src.bytes)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } ops->pio_out_emulated(c->src.bytes, c->dst.val, @@ -3402,14 +3333,14 @@ special_insn: break; case 0xfa: /* cli */ if (emulator_bad_iopl(ctxt, ops)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } else ctxt->eflags &= ~X86_EFLAGS_IF; break; case 0xfb: /* sti */ if (emulator_bad_iopl(ctxt, ops)) { - emulate_gp(ctxt, 0); + rc = emulate_gp(ctxt, 0); goto done; } else { ctxt->interruptibility = KVM_X86_SHADOW_INT_STI; @@ -3449,11 +3380,11 @@ writeback: c->dst.type = saved_dst_type; if ((c->d & SrcMask) == SrcSI) - string_addr_inc(ctxt, seg_override_base(ctxt, ops, c), + string_addr_inc(ctxt, seg_override(ctxt, ops, c), VCPU_REGS_RSI, &c->src); if ((c->d & DstMask) == DstDI) - string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI, + string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI, &c->dst); if (c->rep_prefix && (c->d & String)) { @@ -3482,6 +3413,8 @@ writeback: ctxt->eip = c->eip; done: + if (rc == X86EMUL_PROPAGATE_FAULT) + ctxt->have_exception = true; return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; twobyte_insn: @@ -3544,9 +3477,11 @@ twobyte_insn: break; case 5: /* not defined */ emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; case 7: /* invlpg*/ - emulate_invlpg(ctxt->vcpu, c->src.addr.mem); + emulate_invlpg(ctxt->vcpu, + linear(ctxt, c->src.addr.mem)); /* Disable writeback. */ c->dst.type = OP_NONE; break; @@ -3573,6 +3508,7 @@ twobyte_insn: case 5 ... 7: case 9 ... 15: emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu); @@ -3581,6 +3517,7 @@ twobyte_insn: if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) && (c->modrm_reg == 4 || c->modrm_reg == 5)) { emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu); @@ -3588,6 +3525,7 @@ twobyte_insn: case 0x22: /* mov reg, cr */ if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } c->dst.type = OP_NONE; @@ -3596,6 +3534,7 @@ twobyte_insn: if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) && (c->modrm_reg == 4 || c->modrm_reg == 5)) { emulate_ud(ctxt); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } @@ -3604,6 +3543,7 @@ twobyte_insn: ~0ULL : ~0U), ctxt->vcpu) < 0) { /* #UD condition is already handled by the code above */ emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } @@ -3615,6 +3555,7 @@ twobyte_insn: | ((u64)c->regs[VCPU_REGS_RDX] << 32); if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } rc = X86EMUL_CONTINUE; @@ -3623,6 +3564,7 @@ twobyte_insn: /* rdmsr */ if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) { emulate_gp(ctxt, 0); + rc = X86EMUL_PROPAGATE_FAULT; goto done; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; @@ -3785,6 +3727,5 @@ twobyte_insn: goto writeback; cannot_emulate: - DPRINTF("Cannot emulate %02x\n", c->b); return -1; } diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 975bb45..3377d53 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -73,6 +73,13 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) return vcpu->arch.cr4 & mask; } +static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu) +{ + if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) + kvm_x86_ops->decache_cr3(vcpu); + return vcpu->arch.cr3; +} + static inline ulong kvm_read_cr4(struct kvm_vcpu *vcpu) { return kvm_read_cr4_bits(vcpu, ~0UL); @@ -84,4 +91,19 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & -1u) << 32); } +static inline void enter_guest_mode(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hflags |= HF_GUEST_MASK; +} + +static inline void leave_guest_mode(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hflags &= ~HF_GUEST_MASK; +} + +static inline bool is_guest_mode(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.hflags & HF_GUEST_MASK; +} + #endif diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 413f897..93cf9d0 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -277,7 +277,8 @@ static void apic_update_ppr(struct kvm_lapic *apic) if (old_ppr != ppr) { apic_set_reg(apic, APIC_PROCPRI, ppr); - kvm_make_request(KVM_REQ_EVENT, apic->vcpu); + if (ppr < old_ppr) + kvm_make_request(KVM_REQ_EVENT, apic->vcpu); } } diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index fbb04ae..9cafbb4 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -18,9 +18,11 @@ * */ +#include "irq.h" #include "mmu.h" #include "x86.h" #include "kvm_cache_regs.h" +#include "x86.h" #include <linux/kvm_host.h> #include <linux/types.h> @@ -194,7 +196,6 @@ static struct percpu_counter kvm_total_used_mmu_pages; static u64 __read_mostly shadow_trap_nonpresent_pte; static u64 __read_mostly shadow_notrap_nonpresent_pte; -static u64 __read_mostly shadow_base_present_pte; static u64 __read_mostly shadow_nx_mask; static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ static u64 __read_mostly shadow_user_mask; @@ -213,12 +214,6 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) } EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); -void kvm_mmu_set_base_ptes(u64 base_pte) -{ - shadow_base_present_pte = base_pte; -} -EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes); - void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask) { @@ -482,46 +477,46 @@ static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) } /* - * Return the pointer to the largepage write count for a given - * gfn, handling slots that are not large page aligned. + * Return the pointer to the large page information for a given gfn, + * handling slots that are not large page aligned. */ -static int *slot_largepage_idx(gfn_t gfn, - struct kvm_memory_slot *slot, - int level) +static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, + struct kvm_memory_slot *slot, + int level) { unsigned long idx; idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); - return &slot->lpage_info[level - 2][idx].write_count; + return &slot->lpage_info[level - 2][idx]; } static void account_shadowed(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; - int *write_count; + struct kvm_lpage_info *linfo; int i; slot = gfn_to_memslot(kvm, gfn); for (i = PT_DIRECTORY_LEVEL; i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) { - write_count = slot_largepage_idx(gfn, slot, i); - *write_count += 1; + linfo = lpage_info_slot(gfn, slot, i); + linfo->write_count += 1; } } static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; - int *write_count; + struct kvm_lpage_info *linfo; int i; slot = gfn_to_memslot(kvm, gfn); for (i = PT_DIRECTORY_LEVEL; i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) { - write_count = slot_largepage_idx(gfn, slot, i); - *write_count -= 1; - WARN_ON(*write_count < 0); + linfo = lpage_info_slot(gfn, slot, i); + linfo->write_count -= 1; + WARN_ON(linfo->write_count < 0); } } @@ -530,12 +525,12 @@ static int has_wrprotected_page(struct kvm *kvm, int level) { struct kvm_memory_slot *slot; - int *largepage_idx; + struct kvm_lpage_info *linfo; slot = gfn_to_memslot(kvm, gfn); if (slot) { - largepage_idx = slot_largepage_idx(gfn, slot, level); - return *largepage_idx; + linfo = lpage_info_slot(gfn, slot, level); + return linfo->write_count; } return 1; @@ -590,16 +585,15 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn) static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level) { struct kvm_memory_slot *slot; - unsigned long idx; + struct kvm_lpage_info *linfo; slot = gfn_to_memslot(kvm, gfn); if (likely(level == PT_PAGE_TABLE_LEVEL)) return &slot->rmap[gfn - slot->base_gfn]; - idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - - (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); + linfo = lpage_info_slot(gfn, slot, level); - return &slot->lpage_info[level - 2][idx].rmap_pde; + return &linfo->rmap_pde; } /* @@ -887,19 +881,16 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, end = start + (memslot->npages << PAGE_SHIFT); if (hva >= start && hva < end) { gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; + gfn_t gfn = memslot->base_gfn + gfn_offset; ret = handler(kvm, &memslot->rmap[gfn_offset], data); for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) { - unsigned long idx; - int sh; - - sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j); - idx = ((memslot->base_gfn+gfn_offset) >> sh) - - (memslot->base_gfn >> sh); - ret |= handler(kvm, - &memslot->lpage_info[j][idx].rmap_pde, - data); + struct kvm_lpage_info *linfo; + + linfo = lpage_info_slot(gfn, memslot, + PT_DIRECTORY_LEVEL + j); + ret |= handler(kvm, &linfo->rmap_pde, data); } trace_kvm_age_page(hva, memslot, ret); retval |= ret; @@ -1161,7 +1152,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, } static int nonpaging_sync_page(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, bool clear_unsync) + struct kvm_mmu_page *sp) { return 1; } @@ -1291,7 +1282,7 @@ static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, if (clear_unsync) kvm_unlink_unsync_page(vcpu->kvm, sp); - if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) { + if (vcpu->arch.mmu.sync_page(vcpu, sp)) { kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); return 1; } @@ -1332,12 +1323,12 @@ static void kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn) continue; WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL); + kvm_unlink_unsync_page(vcpu->kvm, s); if ((s->role.cr4_pae != !!is_pae(vcpu)) || - (vcpu->arch.mmu.sync_page(vcpu, s, true))) { + (vcpu->arch.mmu.sync_page(vcpu, s))) { kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list); continue; } - kvm_unlink_unsync_page(vcpu->kvm, s); flush = true; } @@ -1963,9 +1954,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, unsigned pte_access, int user_fault, int write_fault, int dirty, int level, gfn_t gfn, pfn_t pfn, bool speculative, - bool can_unsync, bool reset_host_protection) + bool can_unsync, bool host_writable) { - u64 spte; + u64 spte, entry = *sptep; int ret = 0; /* @@ -1973,7 +1964,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, * whether the guest actually used the pte (in order to detect * demand paging). */ - spte = shadow_base_present_pte; + spte = PT_PRESENT_MASK; if (!speculative) spte |= shadow_accessed_mask; if (!dirty) @@ -1990,8 +1981,10 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn, kvm_is_mmio_pfn(pfn)); - if (reset_host_protection) + if (host_writable) spte |= SPTE_HOST_WRITEABLE; + else + pte_access &= ~ACC_WRITE_MASK; spte |= (u64)pfn << PAGE_SHIFT; @@ -2036,6 +2029,14 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, set_pte: update_spte(sptep, spte); + /* + * If we overwrite a writable spte with a read-only one we + * should flush remote TLBs. Otherwise rmap_write_protect + * will find a read-only spte, even though the writable spte + * might be cached on a CPU's TLB. + */ + if (is_writable_pte(entry) && !is_writable_pte(*sptep)) + kvm_flush_remote_tlbs(vcpu->kvm); done: return ret; } @@ -2045,7 +2046,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, int user_fault, int write_fault, int dirty, int *ptwrite, int level, gfn_t gfn, pfn_t pfn, bool speculative, - bool reset_host_protection) + bool host_writable) { int was_rmapped = 0; int rmap_count; @@ -2080,7 +2081,7 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault, dirty, level, gfn, pfn, speculative, true, - reset_host_protection)) { + host_writable)) { if (write_fault) *ptwrite = 1; kvm_mmu_flush_tlb(vcpu); @@ -2211,7 +2212,8 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) } static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, - int level, gfn_t gfn, pfn_t pfn) + int map_writable, int level, gfn_t gfn, pfn_t pfn, + bool prefault) { struct kvm_shadow_walk_iterator iterator; struct kvm_mmu_page *sp; @@ -2220,9 +2222,11 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) { if (iterator.level == level) { - mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL, + unsigned pte_access = ACC_ALL; + + mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, pte_access, 0, write, 1, &pt_write, - level, gfn, pfn, false, true); + level, gfn, pfn, prefault, map_writable); direct_pte_prefetch(vcpu, iterator.sptep); ++vcpu->stat.pf_fixed; break; @@ -2277,12 +2281,17 @@ static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn) return 1; } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, + gva_t gva, pfn_t *pfn, bool write, bool *writable); + +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn, + bool prefault) { int r; int level; pfn_t pfn; unsigned long mmu_seq; + bool map_writable; level = mapping_level(vcpu, gfn); @@ -2297,7 +2306,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, gfn); + + if (try_async_pf(vcpu, prefault, gfn, v, &pfn, write, &map_writable)) + return 0; /* mmio */ if (is_error_pfn(pfn)) @@ -2307,7 +2318,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) if (mmu_notifier_retry(vcpu, mmu_seq)) goto out_unlock; kvm_mmu_free_some_pages(vcpu); - r = __direct_map(vcpu, v, write, level, gfn, pfn); + r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn, + prefault); spin_unlock(&vcpu->kvm->mmu_lock); @@ -2530,6 +2542,7 @@ static void mmu_sync_roots(struct kvm_vcpu *vcpu) hpa_t root = vcpu->arch.mmu.root_hpa; sp = page_header(root); mmu_sync_children(vcpu, sp); + trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); return; } for (i = 0; i < 4; ++i) { @@ -2552,23 +2565,24 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, struct x86_exception *exception) { - if (error) - *error = 0; + if (exception) + exception->error_code = 0; return vaddr; } static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, + struct x86_exception *exception) { - if (error) - *error = 0; + if (exception) + exception->error_code = 0; return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access); } static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) + u32 error_code, bool prefault) { gfn_t gfn; int r; @@ -2584,17 +2598,67 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, gfn = gva >> PAGE_SHIFT; return nonpaging_map(vcpu, gva & PAGE_MASK, - error_code & PFERR_WRITE_MASK, gfn); + error_code & PFERR_WRITE_MASK, gfn, prefault); +} + +static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn) +{ + struct kvm_arch_async_pf arch; + + arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id; + arch.gfn = gfn; + arch.direct_map = vcpu->arch.mmu.direct_map; + arch.cr3 = vcpu->arch.mmu.get_cr3(vcpu); + + return kvm_setup_async_pf(vcpu, gva, gfn, &arch); } -static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, - u32 error_code) +static bool can_do_async_pf(struct kvm_vcpu *vcpu) +{ + if (unlikely(!irqchip_in_kernel(vcpu->kvm) || + kvm_event_needs_reinjection(vcpu))) + return false; + + return kvm_x86_ops->interrupt_allowed(vcpu); +} + +static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, + gva_t gva, pfn_t *pfn, bool write, bool *writable) +{ + bool async; + + *pfn = gfn_to_pfn_async(vcpu->kvm, gfn, &async, write, writable); + + if (!async) + return false; /* *pfn has correct page already */ + + put_page(pfn_to_page(*pfn)); + + if (!prefault && can_do_async_pf(vcpu)) { + trace_kvm_try_async_get_page(gva, gfn); + if (kvm_find_async_pf_gfn(vcpu, gfn)) { + trace_kvm_async_pf_doublefault(gva, gfn); + kvm_make_request(KVM_REQ_APF_HALT, vcpu); + return true; + } else if (kvm_arch_setup_async_pf(vcpu, gva, gfn)) + return true; + } + + *pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write, writable); + + return false; +} + +static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code, + bool prefault) { pfn_t pfn; int r; int level; gfn_t gfn = gpa >> PAGE_SHIFT; unsigned long mmu_seq; + int write = error_code & PFERR_WRITE_MASK; + bool map_writable; ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); @@ -2609,15 +2673,19 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, gfn); + + if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) + return 0; + + /* mmio */ if (is_error_pfn(pfn)) return kvm_handle_bad_page(vcpu->kvm, gfn, pfn); spin_lock(&vcpu->kvm->mmu_lock); if (mmu_notifier_retry(vcpu, mmu_seq)) goto out_unlock; kvm_mmu_free_some_pages(vcpu); - r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK, - level, gfn, pfn); + r = __direct_map(vcpu, gpa, write, map_writable, + level, gfn, pfn, prefault); spin_unlock(&vcpu->kvm->mmu_lock); return r; @@ -2659,18 +2727,19 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) static void paging_new_cr3(struct kvm_vcpu *vcpu) { - pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3); + pgprintk("%s: cr3 %lx\n", __func__, kvm_read_cr3(vcpu)); mmu_free_roots(vcpu); } static unsigned long get_cr3(struct kvm_vcpu *vcpu) { - return vcpu->arch.cr3; + return kvm_read_cr3(vcpu); } -static void inject_page_fault(struct kvm_vcpu *vcpu) +static void inject_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { - vcpu->arch.mmu.inject_page_fault(vcpu); + vcpu->arch.mmu.inject_page_fault(vcpu, fault); } static void paging_free(struct kvm_vcpu *vcpu) @@ -2816,6 +2885,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = vcpu->arch.walk_mmu; + context->base_role.word = 0; context->new_cr3 = nonpaging_new_cr3; context->page_fault = tdp_page_fault; context->free = nonpaging_free; @@ -3008,9 +3078,6 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, return; } - if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL)) - return; - ++vcpu->kvm->stat.mmu_pte_updated; if (!sp->role.cr4_pae) paging32_update_pte(vcpu, sp, spte, new); @@ -3264,12 +3331,13 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) } } -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code, + void *insn, int insn_len) { int r; enum emulation_result er; - r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false); if (r < 0) goto out; @@ -3282,7 +3350,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) if (r) goto out; - er = emulate_instruction(vcpu, cr2, error_code, 0); + er = x86_emulate_instruction(vcpu, cr2, 0, insn, insn_len); switch (er) { case EMULATE_DONE: @@ -3377,11 +3445,14 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) if (!test_bit(slot, sp->slot_bitmap)) continue; + if (sp->role.level != PT_PAGE_TABLE_LEVEL) + continue; + pt = sp->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (is_writable_pte(pt[i])) - pt[i] &= ~PT_WRITABLE_MASK; + update_spte(&pt[i], pt[i] & ~PT_WRITABLE_MASK); } kvm_flush_remote_tlbs(kvm); } @@ -3463,13 +3534,6 @@ static void mmu_destroy_caches(void) kmem_cache_destroy(mmu_page_header_cache); } -void kvm_mmu_module_exit(void) -{ - mmu_destroy_caches(); - percpu_counter_destroy(&kvm_total_used_mmu_pages); - unregister_shrinker(&mmu_shrinker); -} - int kvm_mmu_module_init(void) { pte_chain_cache = kmem_cache_create("kvm_pte_chain", @@ -3566,7 +3630,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu, static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu) { - (void)kvm_set_cr3(vcpu, vcpu->arch.cr3); + (void)kvm_set_cr3(vcpu, kvm_read_cr3(vcpu)); return 1; } @@ -3662,12 +3726,6 @@ int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4]) } EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy); -#ifdef CONFIG_KVM_MMU_AUDIT -#include "mmu_audit.c" -#else -static void mmu_audit_disable(void) { } -#endif - void kvm_mmu_destroy(struct kvm_vcpu *vcpu) { ASSERT(vcpu); @@ -3675,5 +3733,18 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu) destroy_kvm_mmu(vcpu); free_mmu_pages(vcpu); mmu_free_memory_caches(vcpu); +} + +#ifdef CONFIG_KVM_MMU_AUDIT +#include "mmu_audit.c" +#else +static void mmu_audit_disable(void) { } +#endif + +void kvm_mmu_module_exit(void) +{ + mmu_destroy_caches(); + percpu_counter_destroy(&kvm_total_used_mmu_pages); + unregister_shrinker(&mmu_shrinker); mmu_audit_disable(); } diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index ba2bcdd..5f6223b 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c @@ -19,11 +19,9 @@ #include <linux/ratelimit.h> -static int audit_point; - -#define audit_printk(fmt, args...) \ +#define audit_printk(kvm, fmt, args...) \ printk(KERN_ERR "audit: (%s) error: " \ - fmt, audit_point_name[audit_point], ##args) + fmt, audit_point_name[kvm->arch.audit_point], ##args) typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level); @@ -97,18 +95,21 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) if (sp->unsync) { if (level != PT_PAGE_TABLE_LEVEL) { - audit_printk("unsync sp: %p level = %d\n", sp, level); + audit_printk(vcpu->kvm, "unsync sp: %p " + "level = %d\n", sp, level); return; } if (*sptep == shadow_notrap_nonpresent_pte) { - audit_printk("notrap spte in unsync sp: %p\n", sp); + audit_printk(vcpu->kvm, "notrap spte in unsync " + "sp: %p\n", sp); return; } } if (sp->role.direct && *sptep == shadow_notrap_nonpresent_pte) { - audit_printk("notrap spte in direct sp: %p\n", sp); + audit_printk(vcpu->kvm, "notrap spte in direct sp: %p\n", + sp); return; } @@ -125,8 +126,9 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) hpa = pfn << PAGE_SHIFT; if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) - audit_printk("levels %d pfn %llx hpa %llx ent %llxn", - vcpu->arch.mmu.root_level, pfn, hpa, *sptep); + audit_printk(vcpu->kvm, "levels %d pfn %llx hpa %llx " + "ent %llxn", vcpu->arch.mmu.root_level, pfn, + hpa, *sptep); } static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) @@ -142,8 +144,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) if (!gfn_to_memslot(kvm, gfn)) { if (!printk_ratelimit()) return; - audit_printk("no memslot for gfn %llx\n", gfn); - audit_printk("index %ld of sp (gfn=%llx)\n", + audit_printk(kvm, "no memslot for gfn %llx\n", gfn); + audit_printk(kvm, "index %ld of sp (gfn=%llx)\n", (long int)(sptep - rev_sp->spt), rev_sp->gfn); dump_stack(); return; @@ -153,7 +155,8 @@ static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) if (!*rmapp) { if (!printk_ratelimit()) return; - audit_printk("no rmap for writable spte %llx\n", *sptep); + audit_printk(kvm, "no rmap for writable spte %llx\n", + *sptep); dump_stack(); } } @@ -168,8 +171,9 @@ static void audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level) { struct kvm_mmu_page *sp = page_header(__pa(sptep)); - if (audit_point == AUDIT_POST_SYNC && sp->unsync) - audit_printk("meet unsync sp(%p) after sync root.\n", sp); + if (vcpu->kvm->arch.audit_point == AUDIT_POST_SYNC && sp->unsync) + audit_printk(vcpu->kvm, "meet unsync sp(%p) after sync " + "root.\n", sp); } static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp) @@ -202,8 +206,9 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) spte = rmap_next(kvm, rmapp, NULL); while (spte) { if (is_writable_pte(*spte)) - audit_printk("shadow page has writable mappings: gfn " - "%llx role %x\n", sp->gfn, sp->role.word); + audit_printk(kvm, "shadow page has writable " + "mappings: gfn %llx role %x\n", + sp->gfn, sp->role.word); spte = rmap_next(kvm, rmapp, spte); } } @@ -238,7 +243,7 @@ static void kvm_mmu_audit(void *ignore, struct kvm_vcpu *vcpu, int point) if (!__ratelimit(&ratelimit_state)) return; - audit_point = point; + vcpu->kvm->arch.audit_point = point; audit_all_active_sps(vcpu->kvm); audit_vcpu_spte(vcpu); } diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index cd7a833..53210f1 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -72,7 +72,7 @@ struct guest_walker { unsigned pt_access; unsigned pte_access; gfn_t gfn; - u32 error_code; + struct x86_exception fault; }; static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl) @@ -266,21 +266,23 @@ walk: return 1; error: - walker->error_code = 0; + walker->fault.vector = PF_VECTOR; + walker->fault.error_code_valid = true; + walker->fault.error_code = 0; if (present) - walker->error_code |= PFERR_PRESENT_MASK; + walker->fault.error_code |= PFERR_PRESENT_MASK; - walker->error_code |= write_fault | user_fault; + walker->fault.error_code |= write_fault | user_fault; if (fetch_fault && mmu->nx) - walker->error_code |= PFERR_FETCH_MASK; + walker->fault.error_code |= PFERR_FETCH_MASK; if (rsvd_fault) - walker->error_code |= PFERR_RSVD_MASK; + walker->fault.error_code |= PFERR_RSVD_MASK; - vcpu->arch.fault.address = addr; - vcpu->arch.fault.error_code = walker->error_code; + walker->fault.address = addr; + walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu; - trace_kvm_mmu_walker_error(walker->error_code); + trace_kvm_mmu_walker_error(walker->fault.error_code); return 0; } @@ -299,25 +301,42 @@ static int FNAME(walk_addr_nested)(struct guest_walker *walker, addr, access); } +static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp, u64 *spte, + pt_element_t gpte) +{ + u64 nonpresent = shadow_trap_nonpresent_pte; + + if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)) + goto no_present; + + if (!is_present_gpte(gpte)) { + if (!sp->unsync) + nonpresent = shadow_notrap_nonpresent_pte; + goto no_present; + } + + if (!(gpte & PT_ACCESSED_MASK)) + goto no_present; + + return false; + +no_present: + drop_spte(vcpu->kvm, spte, nonpresent); + return true; +} + static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, const void *pte) { pt_element_t gpte; unsigned pte_access; pfn_t pfn; - u64 new_spte; gpte = *(const pt_element_t *)pte; - if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { - if (!is_present_gpte(gpte)) { - if (sp->unsync) - new_spte = shadow_trap_nonpresent_pte; - else - new_spte = shadow_notrap_nonpresent_pte; - __set_spte(spte, new_spte); - } + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) return; - } + pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte); pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn) @@ -329,7 +348,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, return; kvm_get_pfn(pfn); /* - * we call mmu_set_spte() with reset_host_protection = true beacuse that + * we call mmu_set_spte() with host_writable = true beacuse that * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1). */ mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0, @@ -364,7 +383,6 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, u64 *sptep) { struct kvm_mmu_page *sp; - struct kvm_mmu *mmu = &vcpu->arch.mmu; pt_element_t *gptep = gw->prefetch_ptes; u64 *spte; int i; @@ -395,14 +413,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, gpte = gptep[i]; - if (!is_present_gpte(gpte) || - is_rsvd_bits_set(mmu, gpte, PT_PAGE_TABLE_LEVEL)) { - if (!sp->unsync) - __set_spte(spte, shadow_notrap_nonpresent_pte); - continue; - } - - if (!(gpte & PT_ACCESSED_MASK)) + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) continue; pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); @@ -427,7 +438,8 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, struct guest_walker *gw, int user_fault, int write_fault, int hlevel, - int *ptwrite, pfn_t pfn) + int *ptwrite, pfn_t pfn, bool map_writable, + bool prefault) { unsigned access = gw->pt_access; struct kvm_mmu_page *sp = NULL; @@ -501,7 +513,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, mmu_set_spte(vcpu, it.sptep, access, gw->pte_access & access, user_fault, write_fault, dirty, ptwrite, it.level, - gw->gfn, pfn, false, true); + gw->gfn, pfn, prefault, map_writable); FNAME(pte_prefetch)(vcpu, gw, it.sptep); return it.sptep; @@ -527,8 +539,8 @@ out_gpte_changed: * Returns: 1 if we need to emulate the instruction, 0 otherwise, or * a negative value on error. */ -static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, - u32 error_code) +static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code, + bool prefault) { int write_fault = error_code & PFERR_WRITE_MASK; int user_fault = error_code & PFERR_USER_MASK; @@ -539,6 +551,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, pfn_t pfn; int level = PT_PAGE_TABLE_LEVEL; unsigned long mmu_seq; + bool map_writable; pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code); @@ -556,8 +569,11 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, */ if (!r) { pgprintk("%s: guest page fault\n", __func__); - inject_page_fault(vcpu); - vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ + if (!prefault) { + inject_page_fault(vcpu, &walker.fault); + /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; + } return 0; } @@ -568,7 +584,10 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - pfn = gfn_to_pfn(vcpu->kvm, walker.gfn); + + if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault, + &map_writable)) + return 0; /* mmio */ if (is_error_pfn(pfn)) @@ -581,7 +600,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT); kvm_mmu_free_some_pages(vcpu); sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, - level, &write_pt, pfn); + level, &write_pt, pfn, map_writable, prefault); (void)sptep; pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__, sptep, *sptep, write_pt); @@ -661,7 +680,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) } static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access, - u32 *error) + struct x86_exception *exception) { struct guest_walker walker; gpa_t gpa = UNMAPPED_GVA; @@ -672,14 +691,15 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access, if (r) { gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; - } else if (error) - *error = walker.error_code; + } else if (exception) + *exception = walker.fault; return gpa; } static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr, - u32 access, u32 *error) + u32 access, + struct x86_exception *exception) { struct guest_walker walker; gpa_t gpa = UNMAPPED_GVA; @@ -690,8 +710,8 @@ static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr, if (r) { gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; - } else if (error) - *error = walker.error_code; + } else if (exception) + *exception = walker.fault; return gpa; } @@ -730,12 +750,19 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, * Using the cached information from sp->gfns is safe because: * - The spte has a reference to the struct page, so the pfn for a given gfn * can't change unless all sptes pointing to it are nuked first. + * + * Note: + * We should flush all tlbs if spte is dropped even though guest is + * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page + * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't + * used by guest then tlbs are not flushed, so guest is allowed to access the + * freed pages. + * And we increase kvm->tlbs_dirty to delay tlbs flush in this case. */ -static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, - bool clear_unsync) +static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { int i, offset, nr_present; - bool reset_host_protection; + bool host_writable; gpa_t first_pte_gpa; offset = nr_present = 0; @@ -764,31 +791,27 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, return -EINVAL; gfn = gpte_to_gfn(gpte); - if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL) - || gfn != sp->gfns[i] || !is_present_gpte(gpte) - || !(gpte & PT_ACCESSED_MASK)) { - u64 nonpresent; - if (is_present_gpte(gpte) || !clear_unsync) - nonpresent = shadow_trap_nonpresent_pte; - else - nonpresent = shadow_notrap_nonpresent_pte; - drop_spte(vcpu->kvm, &sp->spt[i], nonpresent); + if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) { + vcpu->kvm->tlbs_dirty++; + continue; + } + + if (gfn != sp->gfns[i]) { + drop_spte(vcpu->kvm, &sp->spt[i], + shadow_trap_nonpresent_pte); + vcpu->kvm->tlbs_dirty++; continue; } nr_present++; pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte); - if (!(sp->spt[i] & SPTE_HOST_WRITEABLE)) { - pte_access &= ~ACC_WRITE_MASK; - reset_host_protection = 0; - } else { - reset_host_protection = 1; - } + host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE; + set_spte(vcpu, &sp->spt[i], pte_access, 0, 0, is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn, spte_to_pfn(sp->spt[i]), true, false, - reset_host_protection); + host_writable); } return !nr_present; diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index b81a9b7..25bd1bc 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -31,6 +31,7 @@ #include <asm/tlbflush.h> #include <asm/desc.h> +#include <asm/kvm_para.h> #include <asm/virtext.h> #include "trace.h" @@ -50,6 +51,10 @@ MODULE_LICENSE("GPL"); #define SVM_FEATURE_LBRV (1 << 1) #define SVM_FEATURE_SVML (1 << 2) #define SVM_FEATURE_NRIP (1 << 3) +#define SVM_FEATURE_TSC_RATE (1 << 4) +#define SVM_FEATURE_VMCB_CLEAN (1 << 5) +#define SVM_FEATURE_FLUSH_ASID (1 << 6) +#define SVM_FEATURE_DECODE_ASSIST (1 << 7) #define SVM_FEATURE_PAUSE_FILTER (1 << 10) #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ @@ -97,10 +102,8 @@ struct nested_state { unsigned long vmexit_rax; /* cache for intercepts of the guest */ - u16 intercept_cr_read; - u16 intercept_cr_write; - u16 intercept_dr_read; - u16 intercept_dr_write; + u32 intercept_cr; + u32 intercept_dr; u32 intercept_exceptions; u64 intercept; @@ -123,7 +126,12 @@ struct vcpu_svm { u64 next_rip; u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; - u64 host_gs_base; + struct { + u16 fs; + u16 gs; + u16 ldt; + u64 gs_base; + } host; u32 *msrpm; @@ -133,6 +141,7 @@ struct vcpu_svm { unsigned int3_injected; unsigned long int3_rip; + u32 apf_reason; }; #define MSR_INVALID 0xffffffffU @@ -180,14 +189,151 @@ static int nested_svm_vmexit(struct vcpu_svm *svm); static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); +enum { + VMCB_INTERCEPTS, /* Intercept vectors, TSC offset, + pause filter count */ + VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */ + VMCB_ASID, /* ASID */ + VMCB_INTR, /* int_ctl, int_vector */ + VMCB_NPT, /* npt_en, nCR3, gPAT */ + VMCB_CR, /* CR0, CR3, CR4, EFER */ + VMCB_DR, /* DR6, DR7 */ + VMCB_DT, /* GDT, IDT */ + VMCB_SEG, /* CS, DS, SS, ES, CPL */ + VMCB_CR2, /* CR2 only */ + VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */ + VMCB_DIRTY_MAX, +}; + +/* TPR and CR2 are always written before VMRUN */ +#define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2)) + +static inline void mark_all_dirty(struct vmcb *vmcb) +{ + vmcb->control.clean = 0; +} + +static inline void mark_all_clean(struct vmcb *vmcb) +{ + vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1) + & ~VMCB_ALWAYS_DIRTY_MASK; +} + +static inline void mark_dirty(struct vmcb *vmcb, int bit) +{ + vmcb->control.clean &= ~(1 << bit); +} + static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) { return container_of(vcpu, struct vcpu_svm, vcpu); } -static inline bool is_nested(struct vcpu_svm *svm) +static void recalc_intercepts(struct vcpu_svm *svm) +{ + struct vmcb_control_area *c, *h; + struct nested_state *g; + + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + + if (!is_guest_mode(&svm->vcpu)) + return; + + c = &svm->vmcb->control; + h = &svm->nested.hsave->control; + g = &svm->nested; + + c->intercept_cr = h->intercept_cr | g->intercept_cr; + c->intercept_dr = h->intercept_dr | g->intercept_dr; + c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions; + c->intercept = h->intercept | g->intercept; +} + +static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm) +{ + if (is_guest_mode(&svm->vcpu)) + return svm->nested.hsave; + else + return svm->vmcb; +} + +static inline void set_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_cr &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + return vmcb->control.intercept_cr & (1U << bit); +} + +static inline void set_dr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_dr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_dr &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline void set_exception_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions |= (1U << bit); + + recalc_intercepts(svm); +} + +static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit) { - return svm->nested.vmcb; + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept_exceptions &= ~(1U << bit); + + recalc_intercepts(svm); +} + +static inline void set_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept |= (1ULL << bit); + + recalc_intercepts(svm); +} + +static inline void clr_intercept(struct vcpu_svm *svm, int bit) +{ + struct vmcb *vmcb = get_host_vmcb(svm); + + vmcb->control.intercept &= ~(1ULL << bit); + + recalc_intercepts(svm); } static inline void enable_gif(struct vcpu_svm *svm) @@ -264,11 +410,6 @@ static u32 svm_msrpm_offset(u32 msr) #define MAX_INST_SIZE 15 -static inline u32 svm_has(u32 feat) -{ - return svm_features & feat; -} - static inline void clgi(void) { asm volatile (__ex(SVM_CLGI)); @@ -284,16 +425,6 @@ static inline void invlpga(unsigned long addr, u32 asid) asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid)); } -static inline void force_new_asid(struct kvm_vcpu *vcpu) -{ - to_svm(vcpu)->asid_generation--; -} - -static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) -{ - force_new_asid(vcpu); -} - static int get_npt_level(void) { #ifdef CONFIG_X86_64 @@ -310,6 +441,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME; + mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); } static int is_external_interrupt(u32 info) @@ -347,7 +479,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) svm->next_rip = svm->vmcb->control.next_rip; if (!svm->next_rip) { - if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) != + if (emulate_instruction(vcpu, EMULTYPE_SKIP) != EMULATE_DONE) printk(KERN_DEBUG "%s: NOP\n", __func__); return; @@ -374,7 +506,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, nested_svm_check_exception(svm, nr, has_error_code, error_code)) return; - if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) { + if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) { unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu); /* @@ -670,7 +802,7 @@ static __init int svm_hardware_setup(void) svm_features = cpuid_edx(SVM_CPUID_FUNC); - if (!svm_has(SVM_FEATURE_NPT)) + if (!boot_cpu_has(X86_FEATURE_NPT)) npt_enabled = false; if (npt_enabled && !npt) { @@ -725,13 +857,15 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) struct vcpu_svm *svm = to_svm(vcpu); u64 g_tsc_offset = 0; - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { g_tsc_offset = svm->vmcb->control.tsc_offset - svm->nested.hsave->control.tsc_offset; svm->nested.hsave->control.tsc_offset = offset; } svm->vmcb->control.tsc_offset = offset + g_tsc_offset; + + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment) @@ -739,8 +873,9 @@ static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.tsc_offset += adjustment; - if (is_nested(svm)) + if (is_guest_mode(vcpu)) svm->nested.hsave->control.tsc_offset += adjustment; + mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } static void init_vmcb(struct vcpu_svm *svm) @@ -749,62 +884,62 @@ static void init_vmcb(struct vcpu_svm *svm) struct vmcb_save_area *save = &svm->vmcb->save; svm->vcpu.fpu_active = 1; + svm->vcpu.arch.hflags = 0; - control->intercept_cr_read = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; - - control->intercept_cr_write = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK | - INTERCEPT_CR8_MASK; - - control->intercept_dr_read = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK | - INTERCEPT_DR4_MASK | - INTERCEPT_DR5_MASK | - INTERCEPT_DR6_MASK | - INTERCEPT_DR7_MASK; - - control->intercept_dr_write = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK | - INTERCEPT_DR4_MASK | - INTERCEPT_DR5_MASK | - INTERCEPT_DR6_MASK | - INTERCEPT_DR7_MASK; - - control->intercept_exceptions = (1 << PF_VECTOR) | - (1 << UD_VECTOR) | - (1 << MC_VECTOR); - - - control->intercept = (1ULL << INTERCEPT_INTR) | - (1ULL << INTERCEPT_NMI) | - (1ULL << INTERCEPT_SMI) | - (1ULL << INTERCEPT_SELECTIVE_CR0) | - (1ULL << INTERCEPT_CPUID) | - (1ULL << INTERCEPT_INVD) | - (1ULL << INTERCEPT_HLT) | - (1ULL << INTERCEPT_INVLPG) | - (1ULL << INTERCEPT_INVLPGA) | - (1ULL << INTERCEPT_IOIO_PROT) | - (1ULL << INTERCEPT_MSR_PROT) | - (1ULL << INTERCEPT_TASK_SWITCH) | - (1ULL << INTERCEPT_SHUTDOWN) | - (1ULL << INTERCEPT_VMRUN) | - (1ULL << INTERCEPT_VMMCALL) | - (1ULL << INTERCEPT_VMLOAD) | - (1ULL << INTERCEPT_VMSAVE) | - (1ULL << INTERCEPT_STGI) | - (1ULL << INTERCEPT_CLGI) | - (1ULL << INTERCEPT_SKINIT) | - (1ULL << INTERCEPT_WBINVD) | - (1ULL << INTERCEPT_MONITOR) | - (1ULL << INTERCEPT_MWAIT); + set_cr_intercept(svm, INTERCEPT_CR0_READ); + set_cr_intercept(svm, INTERCEPT_CR3_READ); + set_cr_intercept(svm, INTERCEPT_CR4_READ); + set_cr_intercept(svm, INTERCEPT_CR0_WRITE); + set_cr_intercept(svm, INTERCEPT_CR3_WRITE); + set_cr_intercept(svm, INTERCEPT_CR4_WRITE); + set_cr_intercept(svm, INTERCEPT_CR8_WRITE); + + set_dr_intercept(svm, INTERCEPT_DR0_READ); + set_dr_intercept(svm, INTERCEPT_DR1_READ); + set_dr_intercept(svm, INTERCEPT_DR2_READ); + set_dr_intercept(svm, INTERCEPT_DR3_READ); + set_dr_intercept(svm, INTERCEPT_DR4_READ); + set_dr_intercept(svm, INTERCEPT_DR5_READ); + set_dr_intercept(svm, INTERCEPT_DR6_READ); + set_dr_intercept(svm, INTERCEPT_DR7_READ); + + set_dr_intercept(svm, INTERCEPT_DR0_WRITE); + set_dr_intercept(svm, INTERCEPT_DR1_WRITE); + set_dr_intercept(svm, INTERCEPT_DR2_WRITE); + set_dr_intercept(svm, INTERCEPT_DR3_WRITE); + set_dr_intercept(svm, INTERCEPT_DR4_WRITE); + set_dr_intercept(svm, INTERCEPT_DR5_WRITE); + set_dr_intercept(svm, INTERCEPT_DR6_WRITE); + set_dr_intercept(svm, INTERCEPT_DR7_WRITE); + + set_exception_intercept(svm, PF_VECTOR); + set_exception_intercept(svm, UD_VECTOR); + set_exception_intercept(svm, MC_VECTOR); + + set_intercept(svm, INTERCEPT_INTR); + set_intercept(svm, INTERCEPT_NMI); + set_intercept(svm, INTERCEPT_SMI); + set_intercept(svm, INTERCEPT_SELECTIVE_CR0); + set_intercept(svm, INTERCEPT_CPUID); + set_intercept(svm, INTERCEPT_INVD); + set_intercept(svm, INTERCEPT_HLT); + set_intercept(svm, INTERCEPT_INVLPG); + set_intercept(svm, INTERCEPT_INVLPGA); + set_intercept(svm, INTERCEPT_IOIO_PROT); + set_intercept(svm, INTERCEPT_MSR_PROT); + set_intercept(svm, INTERCEPT_TASK_SWITCH); + set_intercept(svm, INTERCEPT_SHUTDOWN); + set_intercept(svm, INTERCEPT_VMRUN); + set_intercept(svm, INTERCEPT_VMMCALL); + set_intercept(svm, INTERCEPT_VMLOAD); + set_intercept(svm, INTERCEPT_VMSAVE); + set_intercept(svm, INTERCEPT_STGI); + set_intercept(svm, INTERCEPT_CLGI); + set_intercept(svm, INTERCEPT_SKINIT); + set_intercept(svm, INTERCEPT_WBINVD); + set_intercept(svm, INTERCEPT_MONITOR); + set_intercept(svm, INTERCEPT_MWAIT); + set_intercept(svm, INTERCEPT_XSETBV); control->iopm_base_pa = iopm_base; control->msrpm_base_pa = __pa(svm->msrpm); @@ -855,25 +990,27 @@ static void init_vmcb(struct vcpu_svm *svm) if (npt_enabled) { /* Setup VMCB for Nested Paging */ control->nested_ctl = 1; - control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) | - (1ULL << INTERCEPT_INVLPG)); - control->intercept_exceptions &= ~(1 << PF_VECTOR); - control->intercept_cr_read &= ~INTERCEPT_CR3_MASK; - control->intercept_cr_write &= ~INTERCEPT_CR3_MASK; + clr_intercept(svm, INTERCEPT_TASK_SWITCH); + clr_intercept(svm, INTERCEPT_INVLPG); + clr_exception_intercept(svm, PF_VECTOR); + clr_cr_intercept(svm, INTERCEPT_CR3_READ); + clr_cr_intercept(svm, INTERCEPT_CR3_WRITE); save->g_pat = 0x0007040600070406ULL; save->cr3 = 0; save->cr4 = 0; } - force_new_asid(&svm->vcpu); + svm->asid_generation = 0; svm->nested.vmcb = 0; svm->vcpu.arch.hflags = 0; - if (svm_has(SVM_FEATURE_PAUSE_FILTER)) { + if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { control->pause_filter_count = 3000; - control->intercept |= (1ULL << INTERCEPT_PAUSE); + set_intercept(svm, INTERCEPT_PAUSE); } + mark_all_dirty(svm->vmcb); + enable_gif(svm); } @@ -990,8 +1127,16 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if (unlikely(cpu != vcpu->cpu)) { svm->asid_generation = 0; + mark_all_dirty(svm->vmcb); } +#ifdef CONFIG_X86_64 + rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base); +#endif + savesegment(fs, svm->host.fs); + savesegment(gs, svm->host.gs); + svm->host.ldt = kvm_read_ldt(); + for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } @@ -1002,6 +1147,14 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) int i; ++vcpu->stat.host_state_reload; + kvm_load_ldt(svm->host.ldt); +#ifdef CONFIG_X86_64 + loadsegment(fs, svm->host.fs); + load_gs_index(svm->host.gs); + wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs); +#else + loadsegment(gs, svm->host.gs); +#endif for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } @@ -1021,7 +1174,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) switch (reg) { case VCPU_EXREG_PDPTR: BUG_ON(!npt_enabled); - load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); break; default: BUG(); @@ -1030,12 +1183,12 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) static void svm_set_vintr(struct vcpu_svm *svm) { - svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR; + set_intercept(svm, INTERCEPT_VINTR); } static void svm_clear_vintr(struct vcpu_svm *svm) { - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); + clr_intercept(svm, INTERCEPT_VINTR); } static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) @@ -1150,6 +1303,7 @@ static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) svm->vmcb->save.idtr.limit = dt->size; svm->vmcb->save.idtr.base = dt->address ; + mark_dirty(svm->vmcb, VMCB_DT); } static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) @@ -1166,19 +1320,23 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) svm->vmcb->save.gdtr.limit = dt->size; svm->vmcb->save.gdtr.base = dt->address ; + mark_dirty(svm->vmcb, VMCB_DT); } static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } +static void svm_decache_cr3(struct kvm_vcpu *vcpu) +{ +} + static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { } static void update_cr0_intercept(struct vcpu_svm *svm) { - struct vmcb *vmcb = svm->vmcb; ulong gcr0 = svm->vcpu.arch.cr0; u64 *hcr0 = &svm->vmcb->save.cr0; @@ -1188,27 +1346,14 @@ static void update_cr0_intercept(struct vcpu_svm *svm) *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK) | (gcr0 & SVM_CR0_SELECTIVE_MASK); + mark_dirty(svm->vmcb, VMCB_CR); if (gcr0 == *hcr0 && svm->vcpu.fpu_active) { - vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK; - vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK; - if (is_nested(svm)) { - struct vmcb *hsave = svm->nested.hsave; - - hsave->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK; - hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK; - vmcb->control.intercept_cr_read |= svm->nested.intercept_cr_read; - vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write; - } + clr_cr_intercept(svm, INTERCEPT_CR0_READ); + clr_cr_intercept(svm, INTERCEPT_CR0_WRITE); } else { - svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK; - svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK; - if (is_nested(svm)) { - struct vmcb *hsave = svm->nested.hsave; - - hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK; - hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK; - } + set_cr_intercept(svm, INTERCEPT_CR0_READ); + set_cr_intercept(svm, INTERCEPT_CR0_WRITE); } } @@ -1216,7 +1361,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { /* * We are here because we run in nested mode, the host kvm * intercepts cr0 writes but the l1 hypervisor does not. @@ -1268,6 +1413,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) */ cr0 &= ~(X86_CR0_CD | X86_CR0_NW); svm->vmcb->save.cr0 = cr0; + mark_dirty(svm->vmcb, VMCB_CR); update_cr0_intercept(svm); } @@ -1277,13 +1423,14 @@ static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4; if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) - force_new_asid(vcpu); + svm_flush_tlb(vcpu); vcpu->arch.cr4 = cr4; if (!npt_enabled) cr4 |= X86_CR4_PAE; cr4 |= host_cr4_mce; to_svm(vcpu)->vmcb->save.cr4 = cr4; + mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); } static void svm_set_segment(struct kvm_vcpu *vcpu, @@ -1312,26 +1459,25 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, = (svm->vmcb->save.cs.attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; + mark_dirty(svm->vmcb, VMCB_SEG); } static void update_db_intercept(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->control.intercept_exceptions &= - ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); + clr_exception_intercept(svm, DB_VECTOR); + clr_exception_intercept(svm, BP_VECTOR); if (svm->nmi_singlestep) - svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR); + set_exception_intercept(svm, DB_VECTOR); if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { if (vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) - svm->vmcb->control.intercept_exceptions |= - 1 << DB_VECTOR; + set_exception_intercept(svm, DB_VECTOR); if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) - svm->vmcb->control.intercept_exceptions |= - 1 << BP_VECTOR; + set_exception_intercept(svm, BP_VECTOR); } else vcpu->guest_debug = 0; } @@ -1345,21 +1491,9 @@ static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) else svm->vmcb->save.dr7 = vcpu->arch.dr7; - update_db_intercept(vcpu); -} - -static void load_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif -} + mark_dirty(svm->vmcb, VMCB_DR); -static void save_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif + update_db_intercept(vcpu); } static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) @@ -1372,6 +1506,8 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) svm->asid_generation = sd->asid_generation; svm->vmcb->control.asid = sd->next_asid++; + + mark_dirty(svm->vmcb, VMCB_ASID); } static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) @@ -1379,20 +1515,40 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.dr7 = value; + mark_dirty(svm->vmcb, VMCB_DR); } static int pf_interception(struct vcpu_svm *svm) { - u64 fault_address; + u64 fault_address = svm->vmcb->control.exit_info_2; u32 error_code; + int r = 1; - fault_address = svm->vmcb->control.exit_info_2; - error_code = svm->vmcb->control.exit_info_1; - - trace_kvm_page_fault(fault_address, error_code); - if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu)) - kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address); - return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); + switch (svm->apf_reason) { + default: + error_code = svm->vmcb->control.exit_info_1; + + trace_kvm_page_fault(fault_address, error_code); + if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu)) + kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address); + r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code, + svm->vmcb->control.insn_bytes, + svm->vmcb->control.insn_len); + break; + case KVM_PV_REASON_PAGE_NOT_PRESENT: + svm->apf_reason = 0; + local_irq_disable(); + kvm_async_pf_task_wait(fault_address); + local_irq_enable(); + break; + case KVM_PV_REASON_PAGE_READY: + svm->apf_reason = 0; + local_irq_disable(); + kvm_async_pf_task_wake(fault_address); + local_irq_enable(); + break; + } + return r; } static int db_interception(struct vcpu_svm *svm) @@ -1440,7 +1596,7 @@ static int ud_interception(struct vcpu_svm *svm) { int er; - er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD); + er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; @@ -1449,21 +1605,8 @@ static int ud_interception(struct vcpu_svm *svm) static void svm_fpu_activate(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - u32 excp; - - if (is_nested(svm)) { - u32 h_excp, n_excp; - - h_excp = svm->nested.hsave->control.intercept_exceptions; - n_excp = svm->nested.intercept_exceptions; - h_excp &= ~(1 << NM_VECTOR); - excp = h_excp | n_excp; - } else { - excp = svm->vmcb->control.intercept_exceptions; - excp &= ~(1 << NM_VECTOR); - } - svm->vmcb->control.intercept_exceptions = excp; + clr_exception_intercept(svm, NM_VECTOR); svm->vcpu.fpu_active = 1; update_cr0_intercept(svm); @@ -1570,7 +1713,7 @@ static int io_interception(struct vcpu_svm *svm) string = (io_info & SVM_IOIO_STR_MASK) != 0; in = (io_info & SVM_IOIO_TYPE_MASK) != 0; if (string || in) - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = io_info >> 16; size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; @@ -1624,17 +1767,19 @@ static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu, struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.nested_cr3 = root; - force_new_asid(vcpu); + mark_dirty(svm->vmcb, VMCB_NPT); + svm_flush_tlb(vcpu); } -static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu) +static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.exit_code = SVM_EXIT_NPF; svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = vcpu->arch.fault.error_code; - svm->vmcb->control.exit_info_2 = vcpu->arch.fault.address; + svm->vmcb->control.exit_info_1 = fault->error_code; + svm->vmcb->control.exit_info_2 = fault->address; nested_svm_vmexit(svm); } @@ -1680,7 +1825,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, { int vmexit; - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return 0; svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; @@ -1698,7 +1843,7 @@ static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, /* This function returns true if it is save to enable the irq window */ static inline bool nested_svm_intr(struct vcpu_svm *svm) { - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return true; if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) @@ -1737,7 +1882,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm) /* This function returns true if it is save to enable the nmi window */ static inline bool nested_svm_nmi(struct vcpu_svm *svm) { - if (!is_nested(svm)) + if (!is_guest_mode(&svm->vcpu)) return true; if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI))) @@ -1836,8 +1981,8 @@ static int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_HOST; break; case SVM_EXIT_EXCP_BASE + PF_VECTOR: - /* When we're shadowing, trap PFs */ - if (!npt_enabled) + /* When we're shadowing, trap PFs, but not async PF */ + if (!npt_enabled && svm->apf_reason == 0) return NESTED_EXIT_HOST; break; case SVM_EXIT_EXCP_BASE + NM_VECTOR: @@ -1865,27 +2010,15 @@ static int nested_svm_intercept(struct vcpu_svm *svm) case SVM_EXIT_IOIO: vmexit = nested_svm_intercept_ioio(svm); break; - case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: { - u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0); - if (svm->nested.intercept_cr_read & cr_bits) + case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0); + if (svm->nested.intercept_cr & bit) vmexit = NESTED_EXIT_DONE; break; } - case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: { - u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0); - if (svm->nested.intercept_cr_write & cr_bits) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: { - u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0); - if (svm->nested.intercept_dr_read & dr_bits) - vmexit = NESTED_EXIT_DONE; - break; - } - case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: { - u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0); - if (svm->nested.intercept_dr_write & dr_bits) + case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { + u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); + if (svm->nested.intercept_dr & bit) vmexit = NESTED_EXIT_DONE; break; } @@ -1893,6 +2026,10 @@ static int nested_svm_intercept(struct vcpu_svm *svm) u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); if (svm->nested.intercept_exceptions & excp_bits) vmexit = NESTED_EXIT_DONE; + /* async page fault always cause vmexit */ + else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) && + svm->apf_reason != 0) + vmexit = NESTED_EXIT_DONE; break; } case SVM_EXIT_ERR: { @@ -1926,10 +2063,8 @@ static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *fr struct vmcb_control_area *dst = &dst_vmcb->control; struct vmcb_control_area *from = &from_vmcb->control; - dst->intercept_cr_read = from->intercept_cr_read; - dst->intercept_cr_write = from->intercept_cr_write; - dst->intercept_dr_read = from->intercept_dr_read; - dst->intercept_dr_write = from->intercept_dr_write; + dst->intercept_cr = from->intercept_cr; + dst->intercept_dr = from->intercept_dr; dst->intercept_exceptions = from->intercept_exceptions; dst->intercept = from->intercept; dst->iopm_base_pa = from->iopm_base_pa; @@ -1970,7 +2105,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_vmcb) return 1; - /* Exit nested SVM mode */ + /* Exit Guest-Mode */ + leave_guest_mode(&svm->vcpu); svm->nested.vmcb = 0; /* Give the current vmcb to the guest */ @@ -1984,7 +2120,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.idtr = vmcb->save.idtr; nested_vmcb->save.efer = svm->vcpu.arch.efer; nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu); - nested_vmcb->save.cr3 = svm->vcpu.arch.cr3; + nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu); nested_vmcb->save.cr2 = vmcb->save.cr2; nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; nested_vmcb->save.rflags = vmcb->save.rflags; @@ -2061,6 +2197,8 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) svm->vmcb->save.cpl = 0; svm->vmcb->control.exit_int_info = 0; + mark_all_dirty(svm->vmcb); + nested_svm_unmap(page); nested_svm_uninit_mmu_context(&svm->vcpu); @@ -2148,8 +2286,8 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) nested_vmcb->control.event_inj, nested_vmcb->control.nested_ctl); - trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read, - nested_vmcb->control.intercept_cr_write, + trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff, + nested_vmcb->control.intercept_cr >> 16, nested_vmcb->control.intercept_exceptions, nested_vmcb->control.intercept); @@ -2177,7 +2315,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) if (npt_enabled) hsave->save.cr3 = vmcb->save.cr3; else - hsave->save.cr3 = svm->vcpu.arch.cr3; + hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); copy_vmcb_control_area(hsave, vmcb); @@ -2229,14 +2367,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL; /* cache intercepts */ - svm->nested.intercept_cr_read = nested_vmcb->control.intercept_cr_read; - svm->nested.intercept_cr_write = nested_vmcb->control.intercept_cr_write; - svm->nested.intercept_dr_read = nested_vmcb->control.intercept_dr_read; - svm->nested.intercept_dr_write = nested_vmcb->control.intercept_dr_write; + svm->nested.intercept_cr = nested_vmcb->control.intercept_cr; + svm->nested.intercept_dr = nested_vmcb->control.intercept_dr; svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; svm->nested.intercept = nested_vmcb->control.intercept; - force_new_asid(&svm->vcpu); + svm_flush_tlb(&svm->vcpu); svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; @@ -2245,29 +2381,12 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { /* We only want the cr8 intercept bits of the guest */ - svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK; - svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; + clr_cr_intercept(svm, INTERCEPT_CR8_READ); + clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); } /* We don't want to see VMMCALLs from a nested guest */ - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL); - - /* - * We don't want a nested guest to be more powerful than the guest, so - * all intercepts are ORed - */ - svm->vmcb->control.intercept_cr_read |= - nested_vmcb->control.intercept_cr_read; - svm->vmcb->control.intercept_cr_write |= - nested_vmcb->control.intercept_cr_write; - svm->vmcb->control.intercept_dr_read |= - nested_vmcb->control.intercept_dr_read; - svm->vmcb->control.intercept_dr_write |= - nested_vmcb->control.intercept_dr_write; - svm->vmcb->control.intercept_exceptions |= - nested_vmcb->control.intercept_exceptions; - - svm->vmcb->control.intercept |= nested_vmcb->control.intercept; + clr_intercept(svm, INTERCEPT_VMMCALL); svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl; svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; @@ -2278,11 +2397,21 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) nested_svm_unmap(page); - /* nested_vmcb is our indicator if nested SVM is activated */ + /* Enter Guest-Mode */ + enter_guest_mode(&svm->vcpu); + + /* + * Merge guest and host intercepts - must be called with vcpu in + * guest-mode to take affect here + */ + recalc_intercepts(svm); + svm->nested.vmcb = vmcb_gpa; enable_gif(svm); + mark_all_dirty(svm->vmcb); + return true; } @@ -2400,6 +2529,8 @@ static int clgi_interception(struct vcpu_svm *svm) svm_clear_vintr(svm); svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + mark_dirty(svm->vmcb, VMCB_INTR); + return 1; } @@ -2426,6 +2557,19 @@ static int skinit_interception(struct vcpu_svm *svm) return 1; } +static int xsetbv_interception(struct vcpu_svm *svm) +{ + u64 new_bv = kvm_read_edx_eax(&svm->vcpu); + u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX); + + if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) { + svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; + skip_emulated_instruction(&svm->vcpu); + } + + return 1; +} + static int invalid_op_interception(struct vcpu_svm *svm) { kvm_queue_exception(&svm->vcpu, UD_VECTOR); @@ -2507,19 +2651,92 @@ static int cpuid_interception(struct vcpu_svm *svm) static int iret_interception(struct vcpu_svm *svm) { ++svm->vcpu.stat.nmi_window_exits; - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET); + clr_intercept(svm, INTERCEPT_IRET); svm->vcpu.arch.hflags |= HF_IRET_MASK; return 1; } static int invlpg_interception(struct vcpu_svm *svm) { - return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE; + if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; + + kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1); + skip_emulated_instruction(&svm->vcpu); + return 1; } static int emulate_on_interception(struct vcpu_svm *svm) { - return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; +} + +#define CR_VALID (1ULL << 63) + +static int cr_interception(struct vcpu_svm *svm) +{ + int reg, cr; + unsigned long val; + int err; + + if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_on_interception(svm); + + if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) + return emulate_on_interception(svm); + + reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; + cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; + + err = 0; + if (cr >= 16) { /* mov to cr */ + cr -= 16; + val = kvm_register_read(&svm->vcpu, reg); + switch (cr) { + case 0: + err = kvm_set_cr0(&svm->vcpu, val); + break; + case 3: + err = kvm_set_cr3(&svm->vcpu, val); + break; + case 4: + err = kvm_set_cr4(&svm->vcpu, val); + break; + case 8: + err = kvm_set_cr8(&svm->vcpu, val); + break; + default: + WARN(1, "unhandled write to CR%d", cr); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + } else { /* mov from cr */ + switch (cr) { + case 0: + val = kvm_read_cr0(&svm->vcpu); + break; + case 2: + val = svm->vcpu.arch.cr2; + break; + case 3: + val = kvm_read_cr3(&svm->vcpu); + break; + case 4: + val = kvm_read_cr4(&svm->vcpu); + break; + case 8: + val = kvm_get_cr8(&svm->vcpu); + break; + default: + WARN(1, "unhandled read from CR%d", cr); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + kvm_register_write(&svm->vcpu, reg, val); + } + kvm_complete_insn_gp(&svm->vcpu, err); + + return 1; } static int cr0_write_interception(struct vcpu_svm *svm) @@ -2527,7 +2744,7 @@ static int cr0_write_interception(struct vcpu_svm *svm) struct kvm_vcpu *vcpu = &svm->vcpu; int r; - r = emulate_instruction(&svm->vcpu, 0, 0, 0); + r = cr_interception(svm); if (svm->nested.vmexit_rip) { kvm_register_write(vcpu, VCPU_REGS_RIP, svm->nested.vmexit_rip); @@ -2536,22 +2753,47 @@ static int cr0_write_interception(struct vcpu_svm *svm) svm->nested.vmexit_rip = 0; } - return r == EMULATE_DONE; + return r; +} + +static int dr_interception(struct vcpu_svm *svm) +{ + int reg, dr; + unsigned long val; + int err; + + if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) + return emulate_on_interception(svm); + + reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; + dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; + + if (dr >= 16) { /* mov to DRn */ + val = kvm_register_read(&svm->vcpu, reg); + kvm_set_dr(&svm->vcpu, dr - 16, val); + } else { + err = kvm_get_dr(&svm->vcpu, dr, &val); + if (!err) + kvm_register_write(&svm->vcpu, reg, val); + } + + return 1; } static int cr8_write_interception(struct vcpu_svm *svm) { struct kvm_run *kvm_run = svm->vcpu.run; + int r; u8 cr8_prev = kvm_get_cr8(&svm->vcpu); /* instruction emulation calls kvm_set_cr8() */ - emulate_instruction(&svm->vcpu, 0, 0, 0); + r = cr_interception(svm); if (irqchip_in_kernel(svm->vcpu.kvm)) { - svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK; - return 1; + clr_cr_intercept(svm, INTERCEPT_CR8_WRITE); + return r; } if (cr8_prev <= kvm_get_cr8(&svm->vcpu)) - return 1; + return r; kvm_run->exit_reason = KVM_EXIT_SET_TPR; return 0; } @@ -2562,14 +2804,9 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) switch (ecx) { case MSR_IA32_TSC: { - u64 tsc_offset; + struct vmcb *vmcb = get_host_vmcb(svm); - if (is_nested(svm)) - tsc_offset = svm->nested.hsave->control.tsc_offset; - else - tsc_offset = svm->vmcb->control.tsc_offset; - - *data = tsc_offset + native_read_tsc(); + *data = vmcb->control.tsc_offset + native_read_tsc(); break; } case MSR_STAR: @@ -2714,7 +2951,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) svm->vmcb->save.sysenter_esp = data; break; case MSR_IA32_DEBUGCTLMSR: - if (!svm_has(SVM_FEATURE_LBRV)) { + if (!boot_cpu_has(X86_FEATURE_LBRV)) { pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", __func__, data); break; @@ -2723,6 +2960,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) return 1; svm->vmcb->save.dbgctl = data; + mark_dirty(svm->vmcb, VMCB_LBR); if (data & (1ULL<<0)) svm_enable_lbrv(svm); else @@ -2775,6 +3013,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); svm_clear_vintr(svm); svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + mark_dirty(svm->vmcb, VMCB_INTR); /* * If the user space waits to inject interrupts, exit as soon as * possible @@ -2797,31 +3036,31 @@ static int pause_interception(struct vcpu_svm *svm) } static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { - [SVM_EXIT_READ_CR0] = emulate_on_interception, - [SVM_EXIT_READ_CR3] = emulate_on_interception, - [SVM_EXIT_READ_CR4] = emulate_on_interception, - [SVM_EXIT_READ_CR8] = emulate_on_interception, + [SVM_EXIT_READ_CR0] = cr_interception, + [SVM_EXIT_READ_CR3] = cr_interception, + [SVM_EXIT_READ_CR4] = cr_interception, + [SVM_EXIT_READ_CR8] = cr_interception, [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, [SVM_EXIT_WRITE_CR0] = cr0_write_interception, - [SVM_EXIT_WRITE_CR3] = emulate_on_interception, - [SVM_EXIT_WRITE_CR4] = emulate_on_interception, + [SVM_EXIT_WRITE_CR3] = cr_interception, + [SVM_EXIT_WRITE_CR4] = cr_interception, [SVM_EXIT_WRITE_CR8] = cr8_write_interception, - [SVM_EXIT_READ_DR0] = emulate_on_interception, - [SVM_EXIT_READ_DR1] = emulate_on_interception, - [SVM_EXIT_READ_DR2] = emulate_on_interception, - [SVM_EXIT_READ_DR3] = emulate_on_interception, - [SVM_EXIT_READ_DR4] = emulate_on_interception, - [SVM_EXIT_READ_DR5] = emulate_on_interception, - [SVM_EXIT_READ_DR6] = emulate_on_interception, - [SVM_EXIT_READ_DR7] = emulate_on_interception, - [SVM_EXIT_WRITE_DR0] = emulate_on_interception, - [SVM_EXIT_WRITE_DR1] = emulate_on_interception, - [SVM_EXIT_WRITE_DR2] = emulate_on_interception, - [SVM_EXIT_WRITE_DR3] = emulate_on_interception, - [SVM_EXIT_WRITE_DR4] = emulate_on_interception, - [SVM_EXIT_WRITE_DR5] = emulate_on_interception, - [SVM_EXIT_WRITE_DR6] = emulate_on_interception, - [SVM_EXIT_WRITE_DR7] = emulate_on_interception, + [SVM_EXIT_READ_DR0] = dr_interception, + [SVM_EXIT_READ_DR1] = dr_interception, + [SVM_EXIT_READ_DR2] = dr_interception, + [SVM_EXIT_READ_DR3] = dr_interception, + [SVM_EXIT_READ_DR4] = dr_interception, + [SVM_EXIT_READ_DR5] = dr_interception, + [SVM_EXIT_READ_DR6] = dr_interception, + [SVM_EXIT_READ_DR7] = dr_interception, + [SVM_EXIT_WRITE_DR0] = dr_interception, + [SVM_EXIT_WRITE_DR1] = dr_interception, + [SVM_EXIT_WRITE_DR2] = dr_interception, + [SVM_EXIT_WRITE_DR3] = dr_interception, + [SVM_EXIT_WRITE_DR4] = dr_interception, + [SVM_EXIT_WRITE_DR5] = dr_interception, + [SVM_EXIT_WRITE_DR6] = dr_interception, + [SVM_EXIT_WRITE_DR7] = dr_interception, [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, @@ -2854,6 +3093,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { [SVM_EXIT_WBINVD] = emulate_on_interception, [SVM_EXIT_MONITOR] = invalid_op_interception, [SVM_EXIT_MWAIT] = invalid_op_interception, + [SVM_EXIT_XSETBV] = xsetbv_interception, [SVM_EXIT_NPF] = pf_interception, }; @@ -2864,10 +3104,10 @@ void dump_vmcb(struct kvm_vcpu *vcpu) struct vmcb_save_area *save = &svm->vmcb->save; pr_err("VMCB Control Area:\n"); - pr_err("cr_read: %04x\n", control->intercept_cr_read); - pr_err("cr_write: %04x\n", control->intercept_cr_write); - pr_err("dr_read: %04x\n", control->intercept_dr_read); - pr_err("dr_write: %04x\n", control->intercept_dr_write); + pr_err("cr_read: %04x\n", control->intercept_cr & 0xffff); + pr_err("cr_write: %04x\n", control->intercept_cr >> 16); + pr_err("dr_read: %04x\n", control->intercept_dr & 0xffff); + pr_err("dr_write: %04x\n", control->intercept_dr >> 16); pr_err("exceptions: %08x\n", control->intercept_exceptions); pr_err("intercepts: %016llx\n", control->intercept); pr_err("pause filter count: %d\n", control->pause_filter_count); @@ -2950,15 +3190,23 @@ void dump_vmcb(struct kvm_vcpu *vcpu) } +static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) +{ + struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; + + *info1 = control->exit_info_1; + *info2 = control->exit_info_2; +} + static int handle_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; u32 exit_code = svm->vmcb->control.exit_code; - trace_kvm_exit(exit_code, vcpu); + trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); - if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK)) + if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE)) vcpu->arch.cr0 = svm->vmcb->save.cr0; if (npt_enabled) vcpu->arch.cr3 = svm->vmcb->save.cr3; @@ -2970,7 +3218,7 @@ static int handle_exit(struct kvm_vcpu *vcpu) return 1; } - if (is_nested(svm)) { + if (is_guest_mode(vcpu)) { int vmexit; trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code, @@ -3033,7 +3281,6 @@ static void pre_svm_run(struct vcpu_svm *svm) struct svm_cpu_data *sd = per_cpu(svm_data, cpu); - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; /* FIXME: handle wraparound of asid_generation */ if (svm->asid_generation != sd->asid_generation) new_asid(svm, sd); @@ -3045,7 +3292,7 @@ static void svm_inject_nmi(struct kvm_vcpu *vcpu) svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; vcpu->arch.hflags |= HF_NMI_MASK; - svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET); + set_intercept(svm, INTERCEPT_IRET); ++vcpu->stat.nmi_injections; } @@ -3058,6 +3305,7 @@ static inline void svm_inject_irq(struct vcpu_svm *svm, int irq) control->int_ctl &= ~V_INTR_PRIO_MASK; control->int_ctl |= V_IRQ_MASK | ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); + mark_dirty(svm->vmcb, VMCB_INTR); } static void svm_set_irq(struct kvm_vcpu *vcpu) @@ -3077,14 +3325,14 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; if (irr == -1) return; if (tpr >= irr) - svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK; + set_cr_intercept(svm, INTERCEPT_CR8_WRITE); } static int svm_nmi_allowed(struct kvm_vcpu *vcpu) @@ -3112,10 +3360,10 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) if (masked) { svm->vcpu.arch.hflags |= HF_NMI_MASK; - svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET); + set_intercept(svm, INTERCEPT_IRET); } else { svm->vcpu.arch.hflags &= ~HF_NMI_MASK; - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET); + clr_intercept(svm, INTERCEPT_IRET); } } @@ -3131,7 +3379,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) ret = !!(vmcb->save.rflags & X86_EFLAGS_IF); - if (is_nested(svm)) + if (is_guest_mode(vcpu)) return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK); return ret; @@ -3177,7 +3425,12 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) static void svm_flush_tlb(struct kvm_vcpu *vcpu) { - force_new_asid(vcpu); + struct vcpu_svm *svm = to_svm(vcpu); + + if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; + else + svm->asid_generation--; } static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) @@ -3188,10 +3441,10 @@ static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; - if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) { + if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) { int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; kvm_set_cr8(vcpu, cr8); } @@ -3202,7 +3455,7 @@ static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); u64 cr8; - if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK)) + if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK)) return; cr8 = kvm_get_cr8(vcpu); @@ -3289,9 +3542,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) static void svm_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - u16 fs_selector; - u16 gs_selector; - u16 ldt_selector; svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; @@ -3308,10 +3558,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) sync_lapic_to_cr8(vcpu); - save_host_msrs(vcpu); - savesegment(fs, fs_selector); - savesegment(gs, gs_selector); - ldt_selector = kvm_read_ldt(); svm->vmcb->save.cr2 = vcpu->arch.cr2; clgi(); @@ -3389,19 +3635,10 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) #endif ); - vcpu->arch.cr2 = svm->vmcb->save.cr2; - vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; - vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; - vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; - - load_host_msrs(vcpu); - kvm_load_ldt(ldt_selector); - loadsegment(fs, fs_selector); #ifdef CONFIG_X86_64 - load_gs_index(gs_selector); - wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs); + wrmsrl(MSR_GS_BASE, svm->host.gs_base); #else - loadsegment(gs, gs_selector); + loadsegment(fs, svm->host.fs); #endif reload_tss(vcpu); @@ -3410,10 +3647,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) stgi(); + vcpu->arch.cr2 = svm->vmcb->save.cr2; + vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; + vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; + vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; + sync_cr8_to_lapic(vcpu); svm->next_rip = 0; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + + /* if exit due to PF check for async PF */ + if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) + svm->apf_reason = kvm_read_and_reset_pf_reason(); + if (npt_enabled) { vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR); @@ -3426,6 +3674,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + MC_VECTOR)) svm_handle_mce(svm); + + mark_all_clean(svm->vmcb); } #undef R @@ -3435,7 +3685,8 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.cr3 = root; - force_new_asid(vcpu); + mark_dirty(svm->vmcb, VMCB_CR); + svm_flush_tlb(vcpu); } static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root) @@ -3443,11 +3694,13 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root) struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.nested_cr3 = root; + mark_dirty(svm->vmcb, VMCB_NPT); /* Also sync guest cr3 here in case we live migrate */ - svm->vmcb->save.cr3 = vcpu->arch.cr3; + svm->vmcb->save.cr3 = kvm_read_cr3(vcpu); + mark_dirty(svm->vmcb, VMCB_CR); - force_new_asid(vcpu); + svm_flush_tlb(vcpu); } static int is_disabled(void) @@ -3494,10 +3747,6 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) { switch (func) { - case 0x00000001: - /* Mask out xsave bit as long as it is not supported by SVM */ - entry->ecx &= ~(bit(X86_FEATURE_XSAVE)); - break; case 0x80000001: if (nested) entry->ecx |= (1 << 2); /* Set SVM bit */ @@ -3511,7 +3760,7 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) additional features */ /* Support next_rip if host supports it */ - if (svm_has(SVM_FEATURE_NRIP)) + if (boot_cpu_has(X86_FEATURE_NRIPS)) entry->edx |= SVM_FEATURE_NRIP; /* Support NPT for the guest if enabled */ @@ -3571,6 +3820,7 @@ static const struct trace_print_flags svm_exit_reasons_str[] = { { SVM_EXIT_WBINVD, "wbinvd" }, { SVM_EXIT_MONITOR, "monitor" }, { SVM_EXIT_MWAIT, "mwait" }, + { SVM_EXIT_XSETBV, "xsetbv" }, { SVM_EXIT_NPF, "npf" }, { -1, NULL } }; @@ -3594,9 +3844,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR; - if (is_nested(svm)) - svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR; + set_exception_intercept(svm, NM_VECTOR); update_cr0_intercept(svm); } @@ -3627,6 +3875,7 @@ static struct kvm_x86_ops svm_x86_ops = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, + .decache_cr3 = svm_decache_cr3, .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr3 = svm_set_cr3, @@ -3667,7 +3916,9 @@ static struct kvm_x86_ops svm_x86_ops = { .get_tdp_level = get_npt_level, .get_mt_mask = svm_get_mt_mask, + .get_exit_info = svm_get_exit_info, .exit_reasons_str = svm_exit_reasons_str, + .get_lpage_level = svm_get_lpage_level, .cpuid_update = svm_cpuid_update, diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index a6544b8..1357d7c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -178,27 +178,36 @@ TRACE_EVENT(kvm_apic, #define trace_kvm_apic_read(reg, val) trace_kvm_apic(0, reg, val) #define trace_kvm_apic_write(reg, val) trace_kvm_apic(1, reg, val) +#define KVM_ISA_VMX 1 +#define KVM_ISA_SVM 2 + /* * Tracepoint for kvm guest exit: */ TRACE_EVENT(kvm_exit, - TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu), - TP_ARGS(exit_reason, vcpu), + TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu, u32 isa), + TP_ARGS(exit_reason, vcpu, isa), TP_STRUCT__entry( __field( unsigned int, exit_reason ) __field( unsigned long, guest_rip ) + __field( u32, isa ) + __field( u64, info1 ) + __field( u64, info2 ) ), TP_fast_assign( __entry->exit_reason = exit_reason; __entry->guest_rip = kvm_rip_read(vcpu); + __entry->isa = isa; + kvm_x86_ops->get_exit_info(vcpu, &__entry->info1, + &__entry->info2); ), - TP_printk("reason %s rip 0x%lx", + TP_printk("reason %s rip 0x%lx info %llx %llx", ftrace_print_symbols_seq(p, __entry->exit_reason, kvm_x86_ops->exit_reasons_str), - __entry->guest_rip) + __entry->guest_rip, __entry->info1, __entry->info2) ); /* diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 81fcbe9..bf89ec2 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -69,6 +69,9 @@ module_param(emulate_invalid_guest_state, bool, S_IRUGO); static int __read_mostly vmm_exclusive = 1; module_param(vmm_exclusive, bool, S_IRUGO); +static int __read_mostly yield_on_hlt = 1; +module_param(yield_on_hlt, bool, S_IRUGO); + #define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST \ (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD) #define KVM_GUEST_CR0_MASK \ @@ -177,6 +180,7 @@ static int init_rmode(struct kvm *kvm); static u64 construct_eptp(unsigned long root_hpa); static void kvm_cpu_vmxon(u64 addr); static void kvm_cpu_vmxoff(void); +static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); static DEFINE_PER_CPU(struct vmcs *, vmxarea); static DEFINE_PER_CPU(struct vmcs *, current_vmcs); @@ -188,6 +192,8 @@ static unsigned long *vmx_io_bitmap_b; static unsigned long *vmx_msr_bitmap_legacy; static unsigned long *vmx_msr_bitmap_longmode; +static bool cpu_has_load_ia32_efer; + static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); static DEFINE_SPINLOCK(vmx_vpid_lock); @@ -472,7 +478,7 @@ static void vmcs_clear(struct vmcs *vmcs) u8 error; asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr) : "cc", "memory"); if (error) printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n", @@ -485,7 +491,7 @@ static void vmcs_load(struct vmcs *vmcs) u8 error; asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr) : "cc", "memory"); if (error) printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n", @@ -565,10 +571,10 @@ static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa) static unsigned long vmcs_readl(unsigned long field) { - unsigned long value; + unsigned long value = 0; asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX) - : "=a"(value) : "d"(field) : "cc"); + : "+a"(value) : "d"(field) : "cc"); return value; } @@ -661,6 +667,12 @@ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) unsigned i; struct msr_autoload *m = &vmx->msr_autoload; + if (msr == MSR_EFER && cpu_has_load_ia32_efer) { + vmcs_clear_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER); + vmcs_clear_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER); + return; + } + for (i = 0; i < m->nr; ++i) if (m->guest[i].index == msr) break; @@ -680,6 +692,14 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, unsigned i; struct msr_autoload *m = &vmx->msr_autoload; + if (msr == MSR_EFER && cpu_has_load_ia32_efer) { + vmcs_write64(GUEST_IA32_EFER, guest_val); + vmcs_write64(HOST_IA32_EFER, host_val); + vmcs_set_bits(VM_ENTRY_CONTROLS, VM_ENTRY_LOAD_IA32_EFER); + vmcs_set_bits(VM_EXIT_CONTROLS, VM_EXIT_LOAD_IA32_EFER); + return; + } + for (i = 0; i < m->nr; ++i) if (m->guest[i].index == msr) break; @@ -1009,6 +1029,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) vmx_set_interrupt_shadow(vcpu, 0); } +static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* Ensure that we clear the HLT state in the VMCS. We don't need to + * explicitly skip the instruction because if the HLT state is set, then + * the instruction is already executing and RIP has already been + * advanced. */ + if (!yield_on_hlt && + vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) + vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); +} + static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, bool has_error_code, u32 error_code, bool reinject) @@ -1035,6 +1066,7 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, intr_info |= INTR_TYPE_HARD_EXCEPTION; vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); + vmx_clear_hlt(vcpu); } static bool vmx_rdtscp_supported(void) @@ -1305,8 +1337,11 @@ static __init int vmx_disabled_by_bios(void) && tboot_enabled()) return 1; if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX) - && !tboot_enabled()) + && !tboot_enabled()) { + printk(KERN_WARNING "kvm: disable TXT in the BIOS or " + " activate TXT before enabling KVM\n"); return 1; + } } return 0; @@ -1400,6 +1435,14 @@ static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, return 0; } +static __init bool allow_1_setting(u32 msr, u32 ctl) +{ + u32 vmx_msr_low, vmx_msr_high; + + rdmsr(msr, vmx_msr_low, vmx_msr_high); + return vmx_msr_high & ctl; +} + static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) { u32 vmx_msr_low, vmx_msr_high; @@ -1416,7 +1459,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) &_pin_based_exec_control) < 0) return -EIO; - min = CPU_BASED_HLT_EXITING | + min = #ifdef CONFIG_X86_64 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | @@ -1429,6 +1472,10 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) CPU_BASED_MWAIT_EXITING | CPU_BASED_MONITOR_EXITING | CPU_BASED_INVLPG_EXITING; + + if (yield_on_hlt) + min |= CPU_BASED_HLT_EXITING; + opt = CPU_BASED_TPR_SHADOW | CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; @@ -1510,6 +1557,12 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; + cpu_has_load_ia32_efer = + allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS, + VM_ENTRY_LOAD_IA32_EFER) + && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS, + VM_EXIT_LOAD_IA32_EFER); + return 0; } @@ -1683,9 +1736,13 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save) save->limit = vmcs_read32(sf->limit); save->ar = vmcs_read32(sf->ar_bytes); vmcs_write16(sf->selector, save->base >> 4); - vmcs_write32(sf->base, save->base & 0xfffff); + vmcs_write32(sf->base, save->base & 0xffff0); vmcs_write32(sf->limit, 0xffff); vmcs_write32(sf->ar_bytes, 0xf3); + if (save->base & 0xf) + printk_once(KERN_WARNING "kvm: segment base is not paragraph" + " aligned when entering protected mode (seg=%d)", + seg); } static void enter_rmode(struct kvm_vcpu *vcpu) @@ -1814,6 +1871,13 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits; } +static void vmx_decache_cr3(struct kvm_vcpu *vcpu) +{ + if (enable_ept && is_paging(vcpu)) + vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); +} + static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; @@ -1857,6 +1921,7 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0, unsigned long cr0, struct kvm_vcpu *vcpu) { + vmx_decache_cr3(vcpu); if (!(cr0 & X86_CR0_PG)) { /* From paging/starting to nonpaging */ vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, @@ -1937,7 +2002,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (enable_ept) { eptp = construct_eptp(cr3); vmcs_write64(EPT_POINTER, eptp); - guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 : + guest_cr3 = is_paging(vcpu) ? kvm_read_cr3(vcpu) : vcpu->kvm->arch.ept_identity_map_addr; ept_load_pdptrs(vcpu); } @@ -2725,7 +2790,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_IDTR_BASE, 0); vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); - vmcs_write32(GUEST_ACTIVITY_STATE, 0); + vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); @@ -2787,6 +2852,10 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu) return; } + if (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { + enable_irq_window(vcpu); + return; + } cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); @@ -2814,6 +2883,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu) } else intr |= INTR_TYPE_EXT_INTR; vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); + vmx_clear_hlt(vcpu); } static void vmx_inject_nmi(struct kvm_vcpu *vcpu) @@ -2841,6 +2911,7 @@ static void vmx_inject_nmi(struct kvm_vcpu *vcpu) } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); + vmx_clear_hlt(vcpu); } static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) @@ -2849,7 +2920,8 @@ static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) return 0; return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_NMI)); + (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI + | GUEST_INTR_STATE_NMI)); } static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) @@ -2910,7 +2982,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, * Cause the #SS fault with 0 error code in VM86 mode. */ if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) - if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE) + if (emulate_instruction(vcpu, 0) == EMULATE_DONE) return 1; /* * Forward all other exceptions that are valid in real mode. @@ -3007,7 +3079,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) } if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD); + er = emulate_instruction(vcpu, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(vcpu, UD_VECTOR); return 1; @@ -3026,7 +3098,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) if (kvm_event_needs_reinjection(vcpu)) kvm_mmu_unprotect_page_virt(vcpu, cr2); - return kvm_mmu_page_fault(vcpu, cr2, error_code); + return kvm_mmu_page_fault(vcpu, cr2, error_code, NULL, 0); } if (vmx->rmode.vm86_active && @@ -3098,7 +3170,7 @@ static int handle_io(struct kvm_vcpu *vcpu) ++vcpu->stat.io_exits; if (string || in) - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -3118,14 +3190,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[2] = 0xc1; } -static void complete_insn_gp(struct kvm_vcpu *vcpu, int err) -{ - if (err) - kvm_inject_gp(vcpu, 0); - else - skip_emulated_instruction(vcpu); -} - static int handle_cr(struct kvm_vcpu *vcpu) { unsigned long exit_qualification, val; @@ -3143,21 +3207,21 @@ static int handle_cr(struct kvm_vcpu *vcpu) switch (cr) { case 0: err = kvm_set_cr0(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 3: err = kvm_set_cr3(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 4: err = kvm_set_cr4(vcpu, val); - complete_insn_gp(vcpu, err); + kvm_complete_insn_gp(vcpu, err); return 1; case 8: { u8 cr8_prev = kvm_get_cr8(vcpu); u8 cr8 = kvm_register_read(vcpu, reg); - kvm_set_cr8(vcpu, cr8); - skip_emulated_instruction(vcpu); + err = kvm_set_cr8(vcpu, cr8); + kvm_complete_insn_gp(vcpu, err); if (irqchip_in_kernel(vcpu->kvm)) return 1; if (cr8_prev <= cr8) @@ -3176,8 +3240,9 @@ static int handle_cr(struct kvm_vcpu *vcpu) case 1: /*mov from cr*/ switch (cr) { case 3: - kvm_register_write(vcpu, reg, vcpu->arch.cr3); - trace_kvm_cr_read(cr, vcpu->arch.cr3); + val = kvm_read_cr3(vcpu); + kvm_register_write(vcpu, reg, val); + trace_kvm_cr_read(cr, val); skip_emulated_instruction(vcpu); return 1; case 8: @@ -3349,6 +3414,11 @@ static int handle_vmx_insn(struct kvm_vcpu *vcpu) return 1; } +static int handle_invd(struct kvm_vcpu *vcpu) +{ + return emulate_instruction(vcpu, 0) == EMULATE_DONE; +} + static int handle_invlpg(struct kvm_vcpu *vcpu) { unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); @@ -3377,7 +3447,7 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { - return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE; + return emulate_instruction(vcpu, 0) == EMULATE_DONE; } static int handle_task_switch(struct kvm_vcpu *vcpu) @@ -3476,7 +3546,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); trace_kvm_page_fault(gpa, exit_qualification); - return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0); + return kvm_mmu_page_fault(vcpu, gpa, exit_qualification & 0x3, NULL, 0); } static u64 ept_rsvd_mask(u64 spte, int level) @@ -3592,7 +3662,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) && (kvm_get_rflags(&vmx->vcpu) & X86_EFLAGS_IF)) return handle_interrupt_window(&vmx->vcpu); - err = emulate_instruction(vcpu, 0, 0, 0); + err = emulate_instruction(vcpu, 0); if (err == EMULATE_DO_MMIO) { ret = 0; @@ -3649,6 +3719,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_MSR_WRITE] = handle_wrmsr, [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, [EXIT_REASON_HLT] = handle_halt, + [EXIT_REASON_INVD] = handle_invd, [EXIT_REASON_INVLPG] = handle_invlpg, [EXIT_REASON_VMCALL] = handle_vmcall, [EXIT_REASON_VMCLEAR] = handle_vmx_insn, @@ -3676,6 +3747,12 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) +{ + *info1 = vmcs_readl(EXIT_QUALIFICATION); + *info2 = vmcs_read32(VM_EXIT_INTR_INFO); +} + /* * The guest has exited. See if we can fix it or if we need userspace * assistance. @@ -3686,17 +3763,12 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) u32 exit_reason = vmx->exit_reason; u32 vectoring_info = vmx->idt_vectoring_info; - trace_kvm_exit(exit_reason, vcpu); + trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX); /* If guest state is invalid, start emulating */ if (vmx->emulation_required && emulate_invalid_guest_state) return handle_invalid_guest_state(vcpu); - /* Access CR3 don't cause VMExit in paging mode, so we need - * to sync with guest real CR3. */ - if (enable_ept && is_paging(vcpu)) - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); - if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) { vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; vcpu->run->fail_entry.hardware_entry_failure_reason @@ -4013,7 +4085,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) ); vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) - | (1 << VCPU_EXREG_PDPTR)); + | (1 << VCPU_EXREG_PDPTR) + | (1 << VCPU_EXREG_CR3)); vcpu->arch.regs_dirty = 0; vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); @@ -4280,6 +4353,7 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, + .decache_cr3 = vmx_decache_cr3, .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr3 = vmx_set_cr3, @@ -4320,7 +4394,9 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_tdp_level = get_ept_level, .get_mt_mask = vmx_get_mt_mask, + .get_exit_info = vmx_get_exit_info, .exit_reasons_str = vmx_exit_reasons_str, + .get_lpage_level = vmx_get_lpage_level, .cpuid_update = vmx_cpuid_update, @@ -4396,8 +4472,6 @@ static int __init vmx_init(void) if (enable_ept) { bypass_guest_pf = 0; - kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK | - VMX_EPT_WRITABLE_MASK); kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull, VMX_EPT_EXECUTABLE_MASK); kvm_enable_tdp(); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 46a368c..bcc0efc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -43,6 +43,7 @@ #include <linux/slab.h> #include <linux/perf_event.h> #include <linux/uaccess.h> +#include <linux/hash.h> #include <trace/events/kvm.h> #define CREATE_TRACE_POINTS @@ -155,6 +156,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { u64 __read_mostly host_xcr0; +static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) +{ + int i; + for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++) + vcpu->arch.apf.gfns[i] = ~0; +} + static void kvm_on_user_return(struct user_return_notifier *urn) { unsigned slot; @@ -326,23 +334,28 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) } EXPORT_SYMBOL_GPL(kvm_requeue_exception); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu) +void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) { - unsigned error_code = vcpu->arch.fault.error_code; + if (err) + kvm_inject_gp(vcpu, 0); + else + kvm_x86_ops->skip_emulated_instruction(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) +{ ++vcpu->stat.pf_guest; - vcpu->arch.cr2 = vcpu->arch.fault.address; - kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); + vcpu->arch.cr2 = fault->address; + kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code); } -void kvm_propagate_fault(struct kvm_vcpu *vcpu) +void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { - if (mmu_is_nested(vcpu) && !vcpu->arch.fault.nested) - vcpu->arch.nested_mmu.inject_page_fault(vcpu); + if (mmu_is_nested(vcpu) && !fault->nested_page_fault) + vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); else - vcpu->arch.mmu.inject_page_fault(vcpu); - - vcpu->arch.fault.nested = false; + vcpu->arch.mmu.inject_page_fault(vcpu, fault); } void kvm_inject_nmi(struct kvm_vcpu *vcpu) @@ -460,8 +473,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) (unsigned long *)&vcpu->arch.regs_avail)) return true; - gfn = (vcpu->arch.cr3 & ~31u) >> PAGE_SHIFT; - offset = (vcpu->arch.cr3 & ~31u) & (PAGE_SIZE - 1); + gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT; + offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1); r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte), PFERR_USER_MASK | PFERR_WRITE_MASK); if (r < 0) @@ -506,12 +519,15 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } else #endif if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, - vcpu->arch.cr3)) + kvm_read_cr3(vcpu))) return 1; } kvm_x86_ops->set_cr0(vcpu, cr0); + if ((cr0 ^ old_cr0) & X86_CR0_PG) + kvm_clear_async_pf_completion_queue(vcpu); + if ((cr0 ^ old_cr0) & update_bits) kvm_mmu_reset_context(vcpu); return 0; @@ -595,7 +611,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return 1; } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) && ((cr4 ^ old_cr4) & pdptr_bits) - && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3)) + && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, + kvm_read_cr3(vcpu))) return 1; if (cr4 & X86_CR4_VMXE) @@ -615,7 +632,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4); int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { - if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { + if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { kvm_mmu_sync_roots(vcpu); kvm_mmu_flush_tlb(vcpu); return 0; @@ -650,12 +667,13 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) return 1; vcpu->arch.cr3 = cr3; + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); vcpu->arch.mmu.new_cr3(vcpu); return 0; } EXPORT_SYMBOL_GPL(kvm_set_cr3); -int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { if (cr8 & CR8_RESERVED_BITS) return 1; @@ -665,12 +683,6 @@ int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) vcpu->arch.cr8 = cr8; return 0; } - -void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (__kvm_set_cr8(vcpu, cr8)) - kvm_inject_gp(vcpu, 0); -} EXPORT_SYMBOL_GPL(kvm_set_cr8); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) @@ -775,12 +787,12 @@ EXPORT_SYMBOL_GPL(kvm_get_dr); * kvm-specific. Those are put in the beginning of the list. */ -#define KVM_SAVE_MSRS_BEGIN 7 +#define KVM_SAVE_MSRS_BEGIN 8 static u32 msrs_to_save[] = { MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, - HV_X64_MSR_APIC_ASSIST_PAGE, + HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, MSR_STAR, #ifdef CONFIG_X86_64 @@ -830,7 +842,6 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer) kvm_x86_ops->set_efer(vcpu, efer); vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled; - kvm_mmu_reset_context(vcpu); /* Update reserved bits */ if ((efer ^ old_efer) & EFER_NX) @@ -1418,6 +1429,30 @@ static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data) return 0; } +static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) +{ + gpa_t gpa = data & ~0x3f; + + /* Bits 2:5 are resrved, Should be zero */ + if (data & 0x3c) + return 1; + + vcpu->arch.apf.msr_val = data; + + if (!(data & KVM_ASYNC_PF_ENABLED)) { + kvm_clear_async_pf_completion_queue(vcpu); + kvm_async_pf_hash_reset(vcpu); + return 0; + } + + if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa)) + return 1; + + vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); + kvm_async_pf_wakeup_all(vcpu); + return 0; +} + int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) { switch (msr) { @@ -1499,6 +1534,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) } break; } + case MSR_KVM_ASYNC_PF_EN: + if (kvm_pv_enable_async_pf(vcpu, data)) + return 1; + break; case MSR_IA32_MCG_CTL: case MSR_IA32_MCG_STATUS: case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1: @@ -1775,6 +1814,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) case MSR_KVM_SYSTEM_TIME_NEW: data = vcpu->arch.time; break; + case MSR_KVM_ASYNC_PF_EN: + data = vcpu->arch.apf.msr_val; + break; case MSR_IA32_P5_MC_ADDR: case MSR_IA32_P5_MC_TYPE: case MSR_IA32_MCG_CAP: @@ -1904,6 +1946,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_NOP_IO_DELAY: case KVM_CAP_MP_STATE: case KVM_CAP_SYNC_MMU: + case KVM_CAP_USER_NMI: case KVM_CAP_REINJECT_CONTROL: case KVM_CAP_IRQ_INJECT_STATUS: case KVM_CAP_ASSIGN_DEV_IRQ: @@ -1922,6 +1965,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_DEBUGREGS: case KVM_CAP_X86_ROBUST_SINGLESTEP: case KVM_CAP_XSAVE: + case KVM_CAP_ASYNC_PF: r = 1; break; case KVM_CAP_COALESCED_MMIO: @@ -2185,6 +2229,11 @@ out: return r; } +static void cpuid_mask(u32 *word, int wordnum) +{ + *word &= boot_cpu_data.x86_capability[wordnum]; +} + static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, u32 index) { @@ -2259,7 +2308,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, break; case 1: entry->edx &= kvm_supported_word0_x86_features; + cpuid_mask(&entry->edx, 0); entry->ecx &= kvm_supported_word4_x86_features; + cpuid_mask(&entry->ecx, 4); /* we support x2apic emulation even if host does not support * it since we emulate x2apic in software */ entry->ecx |= F(X2APIC); @@ -2350,7 +2401,9 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, break; case 0x80000001: entry->edx &= kvm_supported_word1_x86_features; + cpuid_mask(&entry->edx, 1); entry->ecx &= kvm_supported_word6_x86_features; + cpuid_mask(&entry->ecx, 6); break; } @@ -3169,20 +3222,18 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_memslots *slots, *old_slots; unsigned long *dirty_bitmap; - r = -ENOMEM; - dirty_bitmap = vmalloc(n); - if (!dirty_bitmap) - goto out; + dirty_bitmap = memslot->dirty_bitmap_head; + if (memslot->dirty_bitmap == dirty_bitmap) + dirty_bitmap += n / sizeof(long); memset(dirty_bitmap, 0, n); r = -ENOMEM; slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL); - if (!slots) { - vfree(dirty_bitmap); + if (!slots) goto out; - } memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots)); slots->memslots[log->slot].dirty_bitmap = dirty_bitmap; + slots->generation++; old_slots = kvm->memslots; rcu_assign_pointer(kvm->memslots, slots); @@ -3195,11 +3246,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, spin_unlock(&kvm->mmu_lock); r = -EFAULT; - if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) { - vfree(dirty_bitmap); + if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) goto out; - } - vfree(dirty_bitmap); } else { r = -EFAULT; if (clear_user(log->dirty_bitmap, n)) @@ -3266,8 +3314,10 @@ long kvm_arch_vm_ioctl(struct file *filp, if (vpic) { r = kvm_ioapic_init(kvm); if (r) { + mutex_lock(&kvm->slots_lock); kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &vpic->dev); + mutex_unlock(&kvm->slots_lock); kfree(vpic); goto create_irqchip_unlock; } @@ -3278,10 +3328,12 @@ long kvm_arch_vm_ioctl(struct file *filp, smp_wmb(); r = kvm_setup_default_irq_routing(kvm); if (r) { + mutex_lock(&kvm->slots_lock); mutex_lock(&kvm->irq_lock); kvm_ioapic_destroy(kvm); kvm_destroy_pic(kvm); mutex_unlock(&kvm->irq_lock); + mutex_unlock(&kvm->slots_lock); } create_irqchip_unlock: mutex_unlock(&kvm->lock); @@ -3557,63 +3609,63 @@ static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) static gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access) { gpa_t t_gpa; - u32 error; + struct x86_exception exception; BUG_ON(!mmu_is_nested(vcpu)); /* NPT walks are always user-walks */ access |= PFERR_USER_MASK; - t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &error); - if (t_gpa == UNMAPPED_GVA) - vcpu->arch.fault.nested = true; + t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception); return t_gpa; } -gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } - gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) + gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_FETCH_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } -gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_WRITE_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); } /* uses this to access any guest's mapped memory without checking CPL */ -gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error) +gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, + struct x86_exception *exception) { - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, error); + return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception); } static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu, u32 access, - u32 *error) + struct x86_exception *exception) { void *data = val; int r = X86EMUL_CONTINUE; while (bytes) { gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access, - error); + exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; - if (gpa == UNMAPPED_GVA) { - r = X86EMUL_PROPAGATE_FAULT; - goto out; - } + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; ret = kvm_read_guest(vcpu->kvm, gpa, data, toread); if (ret < 0) { r = X86EMUL_IO_NEEDED; @@ -3630,31 +3682,35 @@ out: /* used for instruction fetching */ static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, - access | PFERR_FETCH_MASK, error); + access | PFERR_FETCH_MASK, + exception); } static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, - error); + exception); } static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 *error) + struct kvm_vcpu *vcpu, + struct x86_exception *exception) { - return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error); + return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception); } static int kvm_write_guest_virt_system(gva_t addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu, - u32 *error) + struct x86_exception *exception) { void *data = val; int r = X86EMUL_CONTINUE; @@ -3662,15 +3718,13 @@ static int kvm_write_guest_virt_system(gva_t addr, void *val, while (bytes) { gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, PFERR_WRITE_MASK, - error); + exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; - if (gpa == UNMAPPED_GVA) { - r = X86EMUL_PROPAGATE_FAULT; - goto out; - } + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite); if (ret < 0) { r = X86EMUL_IO_NEEDED; @@ -3688,7 +3742,7 @@ out: static int emulator_read_emulated(unsigned long addr, void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; @@ -3701,7 +3755,7 @@ static int emulator_read_emulated(unsigned long addr, return X86EMUL_CONTINUE; } - gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, error_code); + gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, exception); if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; @@ -3710,8 +3764,8 @@ static int emulator_read_emulated(unsigned long addr, if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) goto mmio; - if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL) - == X86EMUL_CONTINUE) + if (kvm_read_guest_virt(addr, val, bytes, vcpu, exception) + == X86EMUL_CONTINUE) return X86EMUL_CONTINUE; mmio: @@ -3735,7 +3789,7 @@ mmio: } int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, - const void *val, int bytes) + const void *val, int bytes) { int ret; @@ -3749,12 +3803,12 @@ int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, static int emulator_write_emulated_onepage(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; - gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error_code); + gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, exception); if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; @@ -3787,7 +3841,7 @@ mmio: int emulator_write_emulated(unsigned long addr, const void *val, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { /* Crossing a page boundary? */ @@ -3795,7 +3849,7 @@ int emulator_write_emulated(unsigned long addr, int rc, now; now = -addr & ~PAGE_MASK; - rc = emulator_write_emulated_onepage(addr, val, now, error_code, + rc = emulator_write_emulated_onepage(addr, val, now, exception, vcpu); if (rc != X86EMUL_CONTINUE) return rc; @@ -3803,7 +3857,7 @@ int emulator_write_emulated(unsigned long addr, val += now; bytes -= now; } - return emulator_write_emulated_onepage(addr, val, bytes, error_code, + return emulator_write_emulated_onepage(addr, val, bytes, exception, vcpu); } @@ -3821,7 +3875,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, const void *old, const void *new, unsigned int bytes, - unsigned int *error_code, + struct x86_exception *exception, struct kvm_vcpu *vcpu) { gpa_t gpa; @@ -3879,7 +3933,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, emul_write: printk_once(KERN_WARNING "kvm: emulating exchange as write\n"); - return emulator_write_emulated(addr, new, bytes, error_code, vcpu); + return emulator_write_emulated(addr, new, bytes, exception, vcpu); } static int kernel_pio(struct kvm_vcpu *vcpu, void *pd) @@ -3904,7 +3958,7 @@ static int emulator_pio_in_emulated(int size, unsigned short port, void *val, if (vcpu->arch.pio.count) goto data_avail; - trace_kvm_pio(0, port, size, 1); + trace_kvm_pio(0, port, size, count); vcpu->arch.pio.port = port; vcpu->arch.pio.in = 1; @@ -3932,7 +3986,7 @@ static int emulator_pio_out_emulated(int size, unsigned short port, const void *val, unsigned int count, struct kvm_vcpu *vcpu) { - trace_kvm_pio(1, port, size, 1); + trace_kvm_pio(1, port, size, count); vcpu->arch.pio.port = port; vcpu->arch.pio.in = 0; @@ -3973,13 +4027,15 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) return X86EMUL_CONTINUE; if (kvm_x86_ops->has_wbinvd_exit()) { - preempt_disable(); + int cpu = get_cpu(); + + cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); smp_call_function_many(vcpu->arch.wbinvd_dirty_mask, wbinvd_ipi, NULL, 1); - preempt_enable(); + put_cpu(); cpumask_clear(vcpu->arch.wbinvd_dirty_mask); - } - wbinvd(); + } else + wbinvd(); return X86EMUL_CONTINUE; } EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); @@ -4019,7 +4075,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu) value = vcpu->arch.cr2; break; case 3: - value = vcpu->arch.cr3; + value = kvm_read_cr3(vcpu); break; case 4: value = kvm_read_cr4(vcpu); @@ -4053,7 +4109,7 @@ static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu) res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); break; case 8: - res = __kvm_set_cr8(vcpu, val & 0xfUL); + res = kvm_set_cr8(vcpu, val); break; default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr); @@ -4206,12 +4262,13 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) static void inject_emulated_exception(struct kvm_vcpu *vcpu) { struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; - if (ctxt->exception == PF_VECTOR) - kvm_propagate_fault(vcpu); - else if (ctxt->error_code_valid) - kvm_queue_exception_e(vcpu, ctxt->exception, ctxt->error_code); + if (ctxt->exception.vector == PF_VECTOR) + kvm_propagate_fault(vcpu, &ctxt->exception); + else if (ctxt->exception.error_code_valid) + kvm_queue_exception_e(vcpu, ctxt->exception.vector, + ctxt->exception.error_code); else - kvm_queue_exception(vcpu, ctxt->exception); + kvm_queue_exception(vcpu, ctxt->exception.vector); } static void init_emulate_ctxt(struct kvm_vcpu *vcpu) @@ -4267,13 +4324,19 @@ EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); static int handle_emulation_failure(struct kvm_vcpu *vcpu) { + int r = EMULATE_DONE; + ++vcpu->stat.insn_emulation_fail; trace_kvm_emulate_insn_failed(vcpu); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (!is_guest_mode(vcpu)) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + r = EMULATE_FAIL; + } kvm_queue_exception(vcpu, UD_VECTOR); - return EMULATE_FAIL; + + return r; } static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) @@ -4302,10 +4365,11 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) return false; } -int emulate_instruction(struct kvm_vcpu *vcpu, - unsigned long cr2, - u16 error_code, - int emulation_type) +int x86_emulate_instruction(struct kvm_vcpu *vcpu, + unsigned long cr2, + int emulation_type, + void *insn, + int insn_len) { int r; struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode; @@ -4323,10 +4387,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu, if (!(emulation_type & EMULTYPE_NO_DECODE)) { init_emulate_ctxt(vcpu); vcpu->arch.emulate_ctxt.interruptibility = 0; - vcpu->arch.emulate_ctxt.exception = -1; + vcpu->arch.emulate_ctxt.have_exception = false; vcpu->arch.emulate_ctxt.perm_ok = false; - r = x86_decode_insn(&vcpu->arch.emulate_ctxt); + r = x86_decode_insn(&vcpu->arch.emulate_ctxt, insn, insn_len); if (r == X86EMUL_PROPAGATE_FAULT) goto done; @@ -4389,7 +4453,7 @@ restart: } done: - if (vcpu->arch.emulate_ctxt.exception >= 0) { + if (vcpu->arch.emulate_ctxt.have_exception) { inject_emulated_exception(vcpu); r = EMULATE_DONE; } else if (vcpu->arch.pio.count) { @@ -4413,7 +4477,7 @@ done: return r; } -EXPORT_SYMBOL_GPL(emulate_instruction); +EXPORT_SYMBOL_GPL(x86_emulate_instruction); int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) { @@ -4653,7 +4717,6 @@ int kvm_arch_init(void *opaque) kvm_x86_ops = ops; kvm_mmu_set_nonpresent_ptes(0ull, 0ull); - kvm_mmu_set_base_ptes(PT_PRESENT_MASK); kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, PT_DIRTY_MASK, PT64_NX_MASK, 0); @@ -5116,6 +5179,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->fpu_active = 0; kvm_x86_ops->fpu_deactivate(vcpu); } + if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { + /* Page is swapped out. Do synthetic halt */ + vcpu->arch.apf.halted = true; + r = 1; + goto out; + } } r = kvm_mmu_reload(vcpu); @@ -5244,7 +5313,8 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) r = 1; while (r > 0) { - if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && + !vcpu->arch.apf.halted) r = vcpu_enter_guest(vcpu); else { srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); @@ -5257,6 +5327,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; case KVM_MP_STATE_RUNNABLE: + vcpu->arch.apf.halted = false; break; case KVM_MP_STATE_SIPI_RECEIVED: default: @@ -5278,6 +5349,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->run->exit_reason = KVM_EXIT_INTR; ++vcpu->stat.request_irq_exits; } + + kvm_check_async_pf_completion(vcpu); + if (signal_pending(current)) { r = -EINTR; vcpu->run->exit_reason = KVM_EXIT_INTR; @@ -5302,6 +5376,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) int r; sigset_t sigsaved; + if (!tsk_used_math(current) && init_fpu(current)) + return -ENOMEM; + if (vcpu->sigset_active) sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); @@ -5313,8 +5390,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } /* re-sync apic's tpr */ - if (!irqchip_in_kernel(vcpu->kvm)) - kvm_set_cr8(vcpu, kvm_run->cr8); + if (!irqchip_in_kernel(vcpu->kvm)) { + if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { + r = -EINVAL; + goto out; + } + } if (vcpu->arch.pio.count || vcpu->mmio_needed) { if (vcpu->mmio_needed) { @@ -5323,7 +5404,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_needed = 0; } vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); - r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE); + r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); if (r != EMULATE_DONE) { r = 0; @@ -5436,7 +5517,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->cr0 = kvm_read_cr0(vcpu); sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = vcpu->arch.cr3; + sregs->cr3 = kvm_read_cr3(vcpu); sregs->cr4 = kvm_read_cr4(vcpu); sregs->cr8 = kvm_get_cr8(vcpu); sregs->efer = vcpu->arch.efer; @@ -5504,8 +5585,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->set_gdt(vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; + __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); kvm_set_cr8(vcpu, sregs->cr8); @@ -5522,7 +5604,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, if (sregs->cr4 & X86_CR4_OSXSAVE) update_cpuid(vcpu); if (!is_long_mode(vcpu) && is_pae(vcpu)) { - load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); mmu_reset_needed = 1; } @@ -5773,6 +5855,8 @@ free_vcpu: void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { + vcpu->arch.apf.msr_val = 0; + vcpu_load(vcpu); kvm_mmu_unload(vcpu); vcpu_put(vcpu); @@ -5792,6 +5876,11 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) vcpu->arch.dr7 = DR7_FIXED_1; kvm_make_request(KVM_REQ_EVENT, vcpu); + vcpu->arch.apf.msr_val = 0; + + kvm_clear_async_pf_completion_queue(vcpu); + kvm_async_pf_hash_reset(vcpu); + vcpu->arch.apf.halted = false; return kvm_x86_ops->vcpu_reset(vcpu); } @@ -5881,6 +5970,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) goto fail_free_mce_banks; + kvm_async_pf_hash_reset(vcpu); + return 0; fail_free_mce_banks: kfree(vcpu->arch.mce_banks); @@ -5906,13 +5997,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) free_page((unsigned long)vcpu->arch.pio_data); } -struct kvm *kvm_arch_create_vm(void) +int kvm_arch_init_vm(struct kvm *kvm) { - struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - - if (!kvm) - return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); @@ -5921,7 +6007,7 @@ struct kvm *kvm_arch_create_vm(void) spin_lock_init(&kvm->arch.tsc_write_lock); - return kvm; + return 0; } static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) @@ -5939,8 +6025,10 @@ static void kvm_free_vcpus(struct kvm *kvm) /* * Unpin any mmu pages first. */ - kvm_for_each_vcpu(i, vcpu, kvm) + kvm_for_each_vcpu(i, vcpu, kvm) { + kvm_clear_async_pf_completion_queue(vcpu); kvm_unload_vcpu_mmu(vcpu); + } kvm_for_each_vcpu(i, vcpu, kvm) kvm_arch_vcpu_free(vcpu); @@ -5964,13 +6052,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kfree(kvm->arch.vpic); kfree(kvm->arch.vioapic); kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); if (kvm->arch.apic_access_page) put_page(kvm->arch.apic_access_page); if (kvm->arch.ept_identity_pagetable) put_page(kvm->arch.ept_identity_pagetable); - cleanup_srcu_struct(&kvm->srcu); - kfree(kvm); } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -6051,7 +6136,9 @@ void kvm_arch_flush_shadow(struct kvm *kvm) int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { - return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE + return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && + !vcpu->arch.apf.halted) + || !list_empty_careful(&vcpu->async_pf.done) || vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED || vcpu->arch.nmi_pending || (kvm_arch_interrupt_allowed(vcpu) && @@ -6110,6 +6197,147 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) } EXPORT_SYMBOL_GPL(kvm_set_rflags); +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) +{ + int r; + + if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) || + is_error_page(work->page)) + return; + + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + return; + + if (!vcpu->arch.mmu.direct_map && + work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu)) + return; + + vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true); +} + +static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) +{ + return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); +} + +static inline u32 kvm_async_pf_next_probe(u32 key) +{ + return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1); +} + +static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + u32 key = kvm_async_pf_hash_fn(gfn); + + while (vcpu->arch.apf.gfns[key] != ~0) + key = kvm_async_pf_next_probe(key); + + vcpu->arch.apf.gfns[key] = gfn; +} + +static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + int i; + u32 key = kvm_async_pf_hash_fn(gfn); + + for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) && + (vcpu->arch.apf.gfns[key] != gfn && + vcpu->arch.apf.gfns[key] != ~0); i++) + key = kvm_async_pf_next_probe(key); + + return key; +} + +bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; +} + +static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + u32 i, j, k; + + i = j = kvm_async_pf_gfn_slot(vcpu, gfn); + while (true) { + vcpu->arch.apf.gfns[i] = ~0; + do { + j = kvm_async_pf_next_probe(j); + if (vcpu->arch.apf.gfns[j] == ~0) + return; + k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); + /* + * k lies cyclically in ]i,j] + * | i.k.j | + * |....j i.k.| or |.k..j i...| + */ + } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); + vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; + i = j; + } +} + +static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) +{ + + return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, + sizeof(val)); +} + +void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + struct x86_exception fault; + + trace_kvm_async_pf_not_present(work->arch.token, work->gva); + kvm_add_async_pf_gfn(vcpu, work->arch.gfn); + + if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || + (vcpu->arch.apf.send_user_only && + kvm_x86_ops->get_cpl(vcpu) == 0)) + kvm_make_request(KVM_REQ_APF_HALT, vcpu); + else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) { + fault.vector = PF_VECTOR; + fault.error_code_valid = true; + fault.error_code = 0; + fault.nested_page_fault = false; + fault.address = work->arch.token; + kvm_inject_page_fault(vcpu, &fault); + } +} + +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + struct x86_exception fault; + + trace_kvm_async_pf_ready(work->arch.token, work->gva); + if (is_error_page(work->page)) + work->arch.token = ~0; /* broadcast wakeup */ + else + kvm_del_async_pf_gfn(vcpu, work->arch.gfn); + + if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) && + !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { + fault.vector = PF_VECTOR; + fault.error_code_valid = true; + fault.error_code = 0; + fault.nested_page_fault = false; + fault.address = work->arch.token; + kvm_inject_page_fault(vcpu, &fault); + } + vcpu->arch.apf.halted = false; +} + +bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) +{ + if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) + return true; + else + return !kvm_event_needs_reinjection(vcpu) && + kvm_x86_ops->interrupt_allowed(vcpu); +} + EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index f89b5bb..c821074 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -45,6 +45,7 @@ #include <asm/bugs.h> #include <asm/tlb.h> #include <asm/tlbflush.h> +#include <asm/olpc_ofw.h> #include <asm/pgalloc.h> #include <asm/sections.h> #include <asm/paravirt.h> @@ -715,6 +716,7 @@ void __init paging_init(void) /* * NOTE: at this point the bootmem allocator is fully available. */ + olpc_dt_build_devicetree(); sparse_init(); zone_sizes_init(); } diff --git a/arch/x86/platform/mrst/early_printk_mrst.c b/arch/x86/platform/mrst/early_printk_mrst.c index 65df603..25bfdbb 100644 --- a/arch/x86/platform/mrst/early_printk_mrst.c +++ b/arch/x86/platform/mrst/early_printk_mrst.c @@ -103,7 +103,7 @@ struct dw_spi_reg { static unsigned long mrst_spi_paddr = MRST_REGBASE_SPI0; static u32 *pclk_spi0; -/* Always contains an accessable address, start with 0 */ +/* Always contains an accessible address, start with 0 */ static struct dw_spi_reg *pspi; static struct kmsg_dumper dw_dumper; diff --git a/arch/x86/platform/olpc/Makefile b/arch/x86/platform/olpc/Makefile index c31b8fc..e797428 100644 --- a/arch/x86/platform/olpc/Makefile +++ b/arch/x86/platform/olpc/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_OLPC) += olpc.o obj-$(CONFIG_OLPC_XO1) += olpc-xo1.o obj-$(CONFIG_OLPC_OPENFIRMWARE) += olpc_ofw.o +obj-$(CONFIG_OLPC_OPENFIRMWARE_DT) += olpc_dt.o diff --git a/arch/x86/platform/olpc/olpc_dt.c b/arch/x86/platform/olpc/olpc_dt.c new file mode 100644 index 0000000..dab8746 --- /dev/null +++ b/arch/x86/platform/olpc/olpc_dt.c @@ -0,0 +1,183 @@ +/* + * OLPC-specific OFW device tree support code. + * + * Paul Mackerras August 1996. + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner. + * {engebret|bergner}@us.ibm.com + * + * Adapted for sparc by David S. Miller davem@davemloft.net + * Adapted for x86/OLPC by Andres Salomon <dilinger@queued.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include <linux/kernel.h> +#include <linux/bootmem.h> +#include <linux/of.h> +#include <linux/of_pdt.h> +#include <asm/olpc_ofw.h> + +static phandle __init olpc_dt_getsibling(phandle node) +{ + const void *args[] = { (void *)node }; + void *res[] = { &node }; + + if ((s32)node == -1) + return 0; + + if (olpc_ofw("peer", args, res) || (s32)node == -1) + return 0; + + return node; +} + +static phandle __init olpc_dt_getchild(phandle node) +{ + const void *args[] = { (void *)node }; + void *res[] = { &node }; + + if ((s32)node == -1) + return 0; + + if (olpc_ofw("child", args, res) || (s32)node == -1) { + pr_err("PROM: %s: fetching child failed!\n", __func__); + return 0; + } + + return node; +} + +static int __init olpc_dt_getproplen(phandle node, const char *prop) +{ + const void *args[] = { (void *)node, prop }; + int len; + void *res[] = { &len }; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("getproplen", args, res)) { + pr_err("PROM: %s: getproplen failed!\n", __func__); + return -1; + } + + return len; +} + +static int __init olpc_dt_getproperty(phandle node, const char *prop, + char *buf, int bufsize) +{ + int plen; + + plen = olpc_dt_getproplen(node, prop); + if (plen > bufsize || plen < 1) { + return -1; + } else { + const void *args[] = { (void *)node, prop, buf, (void *)plen }; + void *res[] = { &plen }; + + if (olpc_ofw("getprop", args, res)) { + pr_err("PROM: %s: getprop failed!\n", __func__); + return -1; + } + } + + return plen; +} + +static int __init olpc_dt_nextprop(phandle node, char *prev, char *buf) +{ + const void *args[] = { (void *)node, prev, buf }; + int success; + void *res[] = { &success }; + + buf[0] = '\0'; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("nextprop", args, res) || success != 1) + return -1; + + return 0; +} + +static int __init olpc_dt_pkg2path(phandle node, char *buf, + const int buflen, int *len) +{ + const void *args[] = { (void *)node, buf, (void *)buflen }; + void *res[] = { len }; + + if ((s32)node == -1) + return -1; + + if (olpc_ofw("package-to-path", args, res) || *len < 1) + return -1; + + return 0; +} + +static unsigned int prom_early_allocated __initdata; + +void * __init prom_early_alloc(unsigned long size) +{ + static u8 *mem; + static size_t free_mem; + void *res; + + if (free_mem < size) { + const size_t chunk_size = max(PAGE_SIZE, size); + + /* + * To mimimize the number of allocations, grab at least + * PAGE_SIZE of memory (that's an arbitrary choice that's + * fast enough on the platforms we care about while minimizing + * wasted bootmem) and hand off chunks of it to callers. + */ + res = alloc_bootmem(chunk_size); + if (!res) + return NULL; + prom_early_allocated += chunk_size; + memset(res, 0, chunk_size); + free_mem = chunk_size; + mem = res; + } + + /* allocate from the local cache */ + free_mem -= size; + res = mem; + mem += size; + return res; +} + +static struct of_pdt_ops prom_olpc_ops __initdata = { + .nextprop = olpc_dt_nextprop, + .getproplen = olpc_dt_getproplen, + .getproperty = olpc_dt_getproperty, + .getchild = olpc_dt_getchild, + .getsibling = olpc_dt_getsibling, + .pkg2path = olpc_dt_pkg2path, +}; + +void __init olpc_dt_build_devicetree(void) +{ + phandle root; + + if (!olpc_ofw_is_installed()) + return; + + root = olpc_dt_getsibling(0); + if (!root) { + pr_err("PROM: unable to get root node from OFW!\n"); + return; + } + of_pdt_build_devicetree(root, &prom_olpc_ops); + + pr_info("PROM DT: Built device tree with %u bytes of memory.\n", + prom_early_allocated); +} diff --git a/arch/x86/platform/olpc/olpc_ofw.c b/arch/x86/platform/olpc/olpc_ofw.c index 7873204..e7604f6 100644 --- a/arch/x86/platform/olpc/olpc_ofw.c +++ b/arch/x86/platform/olpc/olpc_ofw.c @@ -110,3 +110,8 @@ void __init olpc_ofw_detect(void) (unsigned long)olpc_ofw_cif, (-start) >> 20); reserve_top_address(-start); } + +bool __init olpc_ofw_is_installed(void) +{ + return olpc_ofw_cif != NULL; +} |