From b6f42a4a3c886bd18baf319d433a841ac9942c02 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:31 -0700 Subject: x86/xsaves: Add a kernel parameter noxsaves to disable xsaves/xrstors This patch adds a kernel parameter noxsaves to disable xsaves/xrstors feature. The kernel will fall back to use xsaveopt and xrstor to save and restor xstates. By using this parameter, xsave area occupies more memory because standard form of xsave area in xsaveopt/xrstor occupies more memory than compacted form of xsave area. This patch adds a description of the kernel parameter noxsaveopt in doc. The code to support the parameter noxsaveopt has been in the kernel before. This patch just adds the description of this parameter in the doc. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-4-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 30a8ad0d..0ebd952 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -2124,6 +2124,21 @@ bytes respectively. Such letter suffixes can also be entirely omitted. and restore using xsave. The kernel will fallback to enabling legacy floating-point and sse state. + noxsaveopt [X86] Disables xsaveopt used in saving x86 extended + register states. The kernel will fall back to use + xsave to save the states. By using this parameter, + performance of saving the states is degraded because + xsave doesn't support modified optimization while + xsaveopt supports it on xsaveopt enabled systems. + + noxsaves [X86] Disables xsaves and xrstors used in saving and + restoring x86 extended register state in compacted + form of xsave area. The kernel will fall back to use + xsaveopt and xrstor to save and restore the states + in standard form of xsave area. By using this + parameter, xsave area per process might occupy more + memory on xsaves enabled systems. + eagerfpu= [X86] on enable eager fpu restore off disable eager fpu restore diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index e7c4b97..cdc9585 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -146,6 +146,7 @@ static int __init x86_xsave_setup(char *s) { setup_clear_cpu_cap(X86_FEATURE_XSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); + setup_clear_cpu_cap(X86_FEATURE_XSAVES); setup_clear_cpu_cap(X86_FEATURE_AVX); setup_clear_cpu_cap(X86_FEATURE_AVX2); return 1; @@ -159,6 +160,13 @@ static int __init x86_xsaveopt_setup(char *s) } __setup("noxsaveopt", x86_xsaveopt_setup); +static int __init x86_xsaves_setup(char *s) +{ + setup_clear_cpu_cap(X86_FEATURE_XSAVES); + return 1; +} +__setup("noxsaves", x86_xsaves_setup); + #ifdef CONFIG_X86_32 static int cachesize_override = -1; static int disable_x86_serial_nr = 1; -- cgit v0.10.2 From 5b3e83f46a2a7e8625258dbf84a26e7f4032bfa8 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:32 -0700 Subject: x86/alternative: Add alternative_input_2 to support alternative with two features and input alternative_input_2() replaces old instruction with new instructions with input based on two features. In alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, feature2, input...), feature2 has higher priority to replace oldinstr than feature1. If CPU has feature2, newinstr2 replaces oldinstr and newinstr2 is executed during run time. If CPU doesn't have feature2, but it has feature1, newinstr1 replaces oldinstr and newinstr1 is executed during run time. If CPU doesn't have feature2 and feature1, oldinstr is executed during run time. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-5-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 0a3f9c9..473bdbe 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -161,6 +161,20 @@ static inline int alternatives_text_reserved(void *start, void *end) asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ : : "i" (0), ## input) +/* + * This is similar to alternative_input. But it has two features and + * respective instructions. + * + * If CPU has feature2, newinstr2 is used. + * Otherwise, if CPU has feature1, newinstr1 is used. + * Otherwise, oldinstr is used. + */ +#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ + feature2, input...) \ + asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ + newinstr2, feature2) \ + : : "i" (0), ## input) + /* Like alternative_input, but with a single output argument */ #define alternative_io(oldinstr, newinstr, feature, output, input...) \ asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ -- cgit v0.10.2 From 0b29643a58439dc9a8b0c0cacad0e7cb608c8199 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:33 -0700 Subject: x86/xsaves: Change compacted format xsave area header The XSAVE area header is changed to support both compacted format and standard format of xsave area. The XSAVE header of an xsave area comprises the 64 bytes starting at offset 512 from the area base address: - Bytes 7:0 of the xsave header is a state-component bitmap called xstate_bv. It identifies the state components in the xsave area. - Bytes 15:8 of the xsave header is a state-component bitmap called xcomp_bv. It is used as follows: - xcomp_bv[63] indicates the format of the extended region of the xsave area. If it is clear, the standard format is used. If it is set, the compacted format is used. - xcomp_bv[62:0] indicate which features (starting at feature 2) have space allocated for them in the compacted format. - Bytes 63:16 of the xsave header are reserved. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-6-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index a4ea023..2c8d3b8 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -386,8 +386,8 @@ struct bndcsr_struct { struct xsave_hdr_struct { u64 xstate_bv; - u64 reserved1[2]; - u64 reserved2[5]; + u64 xcomp_bv; + u64 reserved[6]; } __attribute__((packed)); struct xsave_struct { -- cgit v0.10.2 From 200b08a970b2ae764b670a326088ab8bc0a989cc Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:34 -0700 Subject: x86/xsaves: Define macros for xsave instructions Define macros for xsave, xsaveopt, xsaves, xrstor, and xrstors inline instructions. The instructions will be used for saving and restoring xstate. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-7-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index d949ef2..71bdde4 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -52,6 +52,12 @@ extern void xsave_init(void); extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); extern int init_fpu(struct task_struct *child); +#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" +#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" +#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" +#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" +#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; -- cgit v0.10.2 From b84e70552e5aad71a1c14536e6ffcfe7934b73e4 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:35 -0700 Subject: x86/xsaves: Define a macro for handling xsave/xrstor instruction fault Define a macro to handle fault generated by xsave, xsaveopt, xsaves, xrstor, and xrstors instructions. It is used in functions like xsave_state() etc. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-8-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 71bdde4..76c2459 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -58,6 +58,13 @@ extern int init_fpu(struct task_struct *child); #define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" #define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" +#define xstate_fault ".section .fixup,\"ax\"\n" \ + "3: movl $-1,%[err]\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err) + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; -- cgit v0.10.2 From f31a9f7c71691569359fa7fb8b0acaa44bce0324 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:36 -0700 Subject: x86/xsaves: Use xsaves/xrstors to save and restore xsave area If xsaves is eanbled, use xsaves/xrstors instrucitons to save and restore xstate. xsaves and xrstors support compacted format, init optimization, modified optimization, and supervisor states. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-9-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 76c2459..f9177a2 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -65,6 +65,70 @@ extern int init_fpu(struct task_struct *child); _ASM_EXTABLE(1b, 3b) \ : [err] "=r" (err) +/* + * Save processor xstate to xsave area. + */ +static inline int xsave_state(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + /* + * If xsaves is enabled, xsaves replaces xsaveopt because + * it supports compact format and supervisor states in addition to + * modified optimization in xsaveopt. + * + * Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave + * because xsaveopt supports modified optimization which is not + * supported by xsave. + * + * If none of xsaves and xsaveopt is enabled, use xsave. + */ + alternative_input_2( + "1:"XSAVE, + "1:"XSAVEOPT, + X86_FEATURE_XSAVEOPT, + "1:"XSAVES, + X86_FEATURE_XSAVES, + [fx] "D" (fx), "a" (lmask), "d" (hmask) : + "memory"); + asm volatile("2:\n\t" + xstate_fault + : "0" (0) + : "memory"); + + return err; +} + +/* + * Restore processor xstate from xsave area. + */ +static inline int xrstor_state(struct xsave_struct *fx, u64 mask) +{ + int err = 0; + u32 lmask = mask; + u32 hmask = mask >> 32; + + /* + * Use xrstors to restore context if it is enabled. xrstors supports + * compacted format of xsave area which is not supported by xrstor. + */ + alternative_input( + "1: " XRSTOR, + "1: " XRSTORS, + X86_FEATURE_XSAVES, + "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile("2:\n" + xstate_fault + : "0" (0) + : "memory"); + + return err; +} + static inline int fpu_xrstor_checking(struct xsave_struct *fx) { int err; @@ -130,26 +194,6 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } -static inline void xrstor_state(struct xsave_struct *fx, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - - asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" - : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) - : "memory"); -} - -static inline void xsave_state(struct xsave_struct *fx, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - - asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t" - : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) - : "memory"); -} - static inline void fpu_xsave(struct fpu *fpu) { /* This, however, we can work around by forcing the compiler to select -- cgit v0.10.2 From f9de314b340f4816671f037e79ed01f685ac9787 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:37 -0700 Subject: x86/xsaves: Use xsaves/xrstors for context switch If xsaves is eanbled, use xsaves/xrstors for context switch to support compacted format xsave area to occupy less memory and modified optimization to improve saving performance. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-10-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index f9177a2..8b75824 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -129,22 +129,20 @@ static inline int xrstor_state(struct xsave_struct *fx, u64 mask) return err; } -static inline int fpu_xrstor_checking(struct xsave_struct *fx) +/* + * Save xstate context for old process during context switch. + */ +static inline void fpu_xsave(struct fpu *fpu) { - int err; - - asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" - "2:\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - : [err] "=r" (err) - : "D" (fx), "m" (*fx), "a" (-1), "d" (-1), "0" (0) - : "memory"); + xsave_state(&fpu->state->xsave, -1); +} - return err; +/* + * Restore xstate context for new process during context switch. + */ +static inline int fpu_xrstor_checking(struct xsave_struct *fx) +{ + return xrstor_state(fx, -1); } static inline int xsave_user(struct xsave_struct __user *buf) @@ -194,15 +192,4 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } -static inline void fpu_xsave(struct fpu *fpu) -{ - /* This, however, we can work around by forcing the compiler to select - an addressing mode that doesn't require extended registers. */ - alternative_input( - ".byte " REX_PREFIX "0x0f,0xae,0x27", - ".byte " REX_PREFIX "0x0f,0xae,0x37", - X86_FEATURE_XSAVEOPT, - [fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) : - "memory"); -} #endif -- cgit v0.10.2 From facbf4d91ae64f84ef93a00e4037135cd9f4b2ab Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:38 -0700 Subject: x86/xsaves: Use xsave/xrstor for saving and restoring user space context We use legacy xsave/xrstor to save and restore standard form of xsave area in user space context. No xsaveopt or xsaves is used here for two reasons. First, we don't want to use modified optimization which is implemented in xsaveopt and xsaves because xrstor/xrstors might track a wrong user space application. Secondly, we don't use compacted format of xsave area for backward compatibility because legacy user space applications only don't understand the compacted format of the xsave area. Using standard form of the xsave area may allocate more memory for user context than compacted form, but preserves compatibility with legacy applications. Furthermore, even with holes, the relevant cache lines don't get touched and thus the performance impact is limited. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-11-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 8b75824..0d15231 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -145,6 +145,16 @@ static inline int fpu_xrstor_checking(struct xsave_struct *fx) return xrstor_state(fx, -1); } +/* + * Save xstate to user space xsave area. + * + * We don't use modified optimization because xrstor/xrstors might track + * a different application. + * + * We don't use compacted format xsave area for + * backward compatibility for old applications which don't understand + * compacted format of xsave area. + */ static inline int xsave_user(struct xsave_struct __user *buf) { int err; @@ -158,35 +168,28 @@ static inline int xsave_user(struct xsave_struct __user *buf) return -EFAULT; __asm__ __volatile__(ASM_STAC "\n" - "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" + "1:"XSAVE"\n" "2: " ASM_CLAC "\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b,3b) - : [err] "=r" (err) + xstate_fault : "D" (buf), "a" (-1), "d" (-1), "0" (0) : "memory"); return err; } +/* + * Restore xstate from user space xsave area. + */ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) { - int err; + int err = 0; struct xsave_struct *xstate = ((__force struct xsave_struct *)buf); u32 lmask = mask; u32 hmask = mask >> 32; __asm__ __volatile__(ASM_STAC "\n" - "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n" + "1:"XRSTOR"\n" "2: " ASM_CLAC "\n" - ".section .fixup,\"ax\"\n" - "3: movl $-1,%[err]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b,3b) - : [err] "=r" (err) + xstate_fault : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0) : "memory"); /* memory required? */ return err; -- cgit v0.10.2 From 21e726c4a3625a1038e97795b7aad97109ba7e19 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:39 -0700 Subject: x86/xsaves: Clear reserved bits in xsave header The reserved bits (128~511) in the xsave header must be zero according to X86 SDM. Clear the bits in this patch. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-12-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index d5dd808..a9a4229 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -375,7 +375,7 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, /* * These bits must be zero. */ - xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0; + memset(xsave_hdr->reserved, 0, 48); return ret; } -- cgit v0.10.2 From adb9d526e98268b647a74726346e1c40e6a37d2e Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:40 -0700 Subject: x86/xsaves: Add xsaves and xrstors support for booting time Since boot_cpu_data and cpu capabilities are not enabled yet during early booting time, alternative can not be used in some functions to access xsave area. Therefore, we define two new functions xrstor_state_booting() and xsave_state_booting() to access xsave area just during early booting time. xrstor_state_booting restores xstate from xsave area during early booting time. xsave_state_booting saves xstate to xsave area during early booting time. The two functions are similar to xrstor_state and xsave_state respectively. But the two functions don't use alternatives because alternatives are not enabled when they are called in such early booting time. xrstor_state_booting is called only by functions defined as __init. So it's defined as __init and will be removed from memory after booting time. There is no extra memory cost caused by this function during running time. But because xsave_state_booting can be called by run-time function __save_fpu(), it's not defined as __init and will stay in memory during running time although it will not be called anymore during running time. It is not ideal to have this function stay in memory during running time. But it's a pretty small function and the memory cost will be small. By doing in this way, we can avoid to change a lot of code to just remove this small function and save a bit memory for running time. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-13-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 0d15231..aa3ff0c 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -66,6 +66,66 @@ extern int init_fpu(struct task_struct *child); : [err] "=r" (err) /* + * This function is called only during boot time when x86 caps are not set + * up and alternative can not be used yet. + */ +static int xsave_state_booting(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + WARN_ON(system_state != SYSTEM_BOOTING); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + asm volatile("1:"XSAVES"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + else + asm volatile("1:"XSAVE"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile(xstate_fault + : "0" (0) + : "memory"); + + return err; +} + +/* + * This function is called only during boot time when x86 caps are not set + * up and alternative can not be used yet. + */ +static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask) +{ + u32 lmask = mask; + u32 hmask = mask >> 32; + int err = 0; + + WARN_ON(system_state != SYSTEM_BOOTING); + + if (boot_cpu_has(X86_FEATURE_XSAVES)) + asm volatile("1:"XRSTORS"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + else + asm volatile("1:"XRSTOR"\n\t" + "2:\n\t" + : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) + : "memory"); + + asm volatile(xstate_fault + : "0" (0) + : "memory"); + + return err; +} + +/* * Save processor xstate to xsave area. */ static inline int xsave_state(struct xsave_struct *fx, u64 mask) -- cgit v0.10.2 From f41d830fa890044cb60f6bb39fc8f6493ffebb47 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:41 -0700 Subject: x86/xsaves: Save xstate to task's xsave area in __save_fpu during booting time __save_fpu() can be called during early booting time when cpu caps are not enabled and alternative can not be used yet. Therefore, it calls xsave_state_booting() during booting time to save xstate to task's xsave area. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-14-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index cea1c76..6099c0e 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -508,9 +508,12 @@ static inline void user_fpu_begin(void) static inline void __save_fpu(struct task_struct *tsk) { - if (use_xsave()) - xsave_state(&tsk->thread.fpu.state->xsave, -1); - else + if (use_xsave()) { + if (unlikely(system_state == SYSTEM_BOOTING)) + xsave_state_booting(&tsk->thread.fpu.state->xsave, -1); + else + xsave_state(&tsk->thread.fpu.state->xsave, -1); + } else fpu_fxsave(&tsk->thread.fpu); } -- cgit v0.10.2 From 47c2f292cc8669f70644a949cadd5fa5ee0e0e07 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:42 -0700 Subject: x86/xsaves: Call booting time xsaves and xrstors in setup_init_fpu_buf setup_init_fpu_buf() calls booting time xsaves and xrstors to save and restore xstate in xsave area. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-15-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index a4b451c..8fa7c7d 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -496,15 +496,21 @@ static void __init setup_init_fpu_buf(void) setup_xstate_features(); + if (cpu_has_xsaves) { + init_xstate_buf->xsave_hdr.xcomp_bv = + (u64)1 << 63 | pcntxt_mask; + init_xstate_buf->xsave_hdr.xstate_bv = pcntxt_mask; + } + /* * Init all the features state with header_bv being 0x0 */ - xrstor_state(init_xstate_buf, -1); + xrstor_state_booting(init_xstate_buf, -1); /* * Dump the init state again. This is to identify the init state * of any feature which is not represented by all zero's. */ - xsave_state(init_xstate_buf, -1); + xsave_state_booting(init_xstate_buf, -1); } static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO; -- cgit v0.10.2 From 7e7ce87f6ad4e1730364e5e76628b43c5759b700 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:43 -0700 Subject: x86/xsaves: Enable xsaves/xrstors If xsaves/xrstors is enabled, compacted format of xsave area will be used and less memory may be used for context per process. And modified optimization implemented in xsaves/xrstors improves performance of saving xstate. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-16-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index 8fa7c7d..f930f8a 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,9 @@ u64 pcntxt_mask; struct xsave_struct *init_xstate_buf; static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32; -static unsigned int *xstate_offsets, *xstate_sizes, xstate_features; +static unsigned int *xstate_offsets, *xstate_sizes; +static unsigned int *xstate_comp_offsets, *xstate_comp_sizes; +static unsigned int xstate_features; /* * If a processor implementation discern that a processor state component is @@ -283,7 +286,7 @@ sanitize_restored_xstate(struct task_struct *tsk, if (use_xsave()) { /* These bits must be zero. */ - xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0; + memset(xsave_hdr->reserved, 0, 48); /* * Init the state that is not present in the memory @@ -526,6 +529,30 @@ static int __init eager_fpu_setup(char *s) } __setup("eagerfpu=", eager_fpu_setup); + +/* + * Calculate total size of enabled xstates in XCR0/pcntxt_mask. + */ +static void __init init_xstate_size(void) +{ + unsigned int eax, ebx, ecx, edx; + int i; + + if (!cpu_has_xsaves) { + cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); + xstate_size = ebx; + return; + } + + xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; + for (i = 2; i < 64; i++) { + if (test_bit(i, (unsigned long *)&pcntxt_mask)) { + cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); + xstate_size += eax; + } + } +} + /* * Enable and initialize the xsave feature. */ @@ -557,8 +584,7 @@ static void __init xstate_enable_boot_cpu(void) /* * Recompute the context size for enabled features */ - cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); - xstate_size = ebx; + init_xstate_size(); update_regset_xstate_info(xstate_size, pcntxt_mask); prepare_fx_sw_frame(); @@ -578,8 +604,9 @@ static void __init xstate_enable_boot_cpu(void) } } - pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n", - pcntxt_mask, xstate_size); + pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x using %s\n", + pcntxt_mask, xstate_size, + cpu_has_xsaves ? "compacted form" : "standard form"); } /* -- cgit v0.10.2 From 7496d6458fe3219d63848ce4a9afbd86245cab22 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 29 May 2014 11:12:44 -0700 Subject: Define kernel API to get address of each state in xsave area In standard form, each state is saved in the xsave area in fixed offset. But in compacted form, offset of each saved state only can be calculated during run time because some xstates may not be enabled and saved. We define kernel API get_xsave_addr() returns address of a given state saved in a xsave area. It can be called in kernel to get address of each xstate in xsave area in either standard format or compacted format. It's useful when kernel wants to directly access each state in xsave area. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-17-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index aa3ff0c..1ba577c 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -255,4 +255,7 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask) return err; } +void *get_xsave_addr(struct xsave_struct *xsave, int xstate); +void setup_xstate_comp(void); + #endif diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 4505e2a..f804dc9 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -93,6 +93,7 @@ void arch_task_cache_init(void) kmem_cache_create("task_xstate", xstate_size, __alignof__(union thread_xstate), SLAB_PANIC | SLAB_NOTRACK, NULL); + setup_xstate_comp(); } /* diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index f930f8a..a6cb823 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -482,6 +482,47 @@ static void __init setup_xstate_features(void) } /* + * This function sets up offsets and sizes of all extended states in + * xsave area. This supports both standard format and compacted format + * of the xsave aread. + * + * Input: void + * Output: void + */ +void setup_xstate_comp(void) +{ + int i; + + xstate_comp_offsets = kmalloc(xstate_features * sizeof(int), + GFP_KERNEL); + xstate_comp_sizes = kmalloc(xstate_features * sizeof(int), GFP_KERNEL); + + if (!cpu_has_xsaves) { + for (i = 2; i < xstate_features; i++) { + if (test_bit(i, (unsigned long *)&pcntxt_mask)) { + xstate_comp_offsets[i] = xstate_offsets[i]; + xstate_comp_sizes[i] = xstate_sizes[i]; + } + } + return; + } + + xstate_comp_offsets[2] = FXSAVE_SIZE + XSAVE_HDR_SIZE; + + for (i = 2; i < xstate_features; i++) { + if (test_bit(i, (unsigned long *)&pcntxt_mask)) + xstate_comp_sizes[i] = xstate_sizes[i]; + else + xstate_comp_sizes[i] = 0; + + if (i > 2) + xstate_comp_offsets[i] = xstate_comp_offsets[i-1] + + xstate_comp_sizes[i-1]; + + } +} + +/* * setup the xstate image representing the init state */ static void __init setup_init_fpu_buf(void) @@ -668,3 +709,26 @@ void eager_fpu_init(void) else fxrstor_checking(&init_xstate_buf->i387); } + +/* + * Given the xsave area and a state inside, this function returns the + * address of the state. + * + * This is the API that is called to get xstate address in either + * standard format or compacted format of xsave area. + * + * Inputs: + * xsave: base address of the xsave area; + * xstate: state which is defined in xsave.h (e.g. XSTATE_FP, XSTATE_SSE, + * etc.) + * Output: + * address of the state in the xsave area. + */ +void *get_xsave_addr(struct xsave_struct *xsave, int xstate) +{ + int feature = fls64(xstate) - 1; + if (!test_bit(feature, (unsigned long *)&pcntxt_mask)) + return NULL; + + return (void *)xsave + xstate_comp_offsets[feature]; +} -- cgit v0.10.2 From c9e5a5a7034146493386d985ff432aed8059929a Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 30 May 2014 08:19:21 -0700 Subject: x86/xsave: Make it clear that the XSAVE macros use (%edi)/(%rdi) The XSAVE instruction family takes a memory argment. The macros use (%edi)/(%rdi) as that memory argument - make that clear to the reader. Signed-off-by: H. Peter Anvin Cc: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-7-git-send-email-fenghua.yu@intel.com diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index 1ba577c..bbebd6e 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -52,6 +52,7 @@ extern void xsave_init(void); extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); extern int init_fpu(struct task_struct *child); +/* These macros all use (%edi)/(%rdi) as the single memory argument. */ #define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" #define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" #define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" -- cgit v0.10.2 From 8ff925e10f2c72680918b95173ef4f8bb982d59e Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 30 May 2014 14:59:24 -0700 Subject: x86/xsaves: Clean up code in xstate offsets computation in xsave area This patch cleans up some code in xstate offsets computation in xsave area: 1. It changes xstate_comp_offsets as an array. This avoids possible NULL pointer caused by possible kmalloc() failure during boot time. 2. It changes the global variable xstate_comp_sizes to a local variable because it is used only in setup_xstate_comp(). 3. It adds missing offsets for FP and SSE in xsave area. Signed-off-by: Fenghua Yu Link: http://lkml.kernel.org/r/1401387164-43416-17-git-send-email-fenghua.yu@intel.com Signed-off-by: H. Peter Anvin diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index a6cb823..940b142 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -26,7 +26,7 @@ struct xsave_struct *init_xstate_buf; static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32; static unsigned int *xstate_offsets, *xstate_sizes; -static unsigned int *xstate_comp_offsets, *xstate_comp_sizes; +static unsigned int xstate_comp_offsets[sizeof(pcntxt_mask)*8]; static unsigned int xstate_features; /* @@ -491,11 +491,16 @@ static void __init setup_xstate_features(void) */ void setup_xstate_comp(void) { + unsigned int xstate_comp_sizes[sizeof(pcntxt_mask)*8]; int i; - xstate_comp_offsets = kmalloc(xstate_features * sizeof(int), - GFP_KERNEL); - xstate_comp_sizes = kmalloc(xstate_features * sizeof(int), GFP_KERNEL); + /* + * The FP xstates and SSE xstates are legacy states. They are always + * in the fixed offsets in the xsave area in either compacted form + * or standard form. + */ + xstate_comp_offsets[0] = 0; + xstate_comp_offsets[1] = offsetof(struct i387_fxsave_struct, xmm_space); if (!cpu_has_xsaves) { for (i = 2; i < xstate_features; i++) { -- cgit v0.10.2 From d0f2dd186133a0241a2ccefb188a0e49e8187859 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Tue, 17 Jun 2014 12:21:08 +0200 Subject: x86, xsave: Add forgotten inline annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a missing inline annotation on a static function, in order to shut up a bunch of warnings like: In file included from arch/x86/crypto/camellia_aesni_avx_glue.c:23:0: ./arch/x86/include/asm/xsave.h:73:12: warning: ‘xsave_state_booting’ defined but not used [-Wunused-function] static int xsave_state_booting(struct xsave_struct *fx, u64 mask) ^ In file included from arch/x86/crypto/camellia_aesni_avx2_glue.c:23:0: ./arch/x86/include/asm/xsave.h:73:12: warning: ‘xsave_state_booting’ defined but not used [-Wunused-function] static int xsave_state_booting(struct xsave_struct *fx, u64 mask) ^ ... Cc: Fenghua Yu Signed-off-by: Borislav Petkov Link: http://lkml.kernel.org/r/1403000468-30094-1-git-send-email-bp@alien8.de Signed-off-by: H. Peter Anvin diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h index bbebd6e..7e7a79a 100644 --- a/arch/x86/include/asm/xsave.h +++ b/arch/x86/include/asm/xsave.h @@ -70,7 +70,7 @@ extern int init_fpu(struct task_struct *child); * This function is called only during boot time when x86 caps are not set * up and alternative can not be used yet. */ -static int xsave_state_booting(struct xsave_struct *fx, u64 mask) +static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask) { u32 lmask = mask; u32 hmask = mask >> 32; -- cgit v0.10.2