1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
|
From e57b5ea6fa2b2d93fde033cec11d6ec6d524ade3 Mon Sep 17 00:00:00 2001
From: Andrew Cooper <andrew.cooper3@citrix.com>
Date: Tue, 17 Apr 2018 14:15:04 +0100
Subject: [PATCH] x86/spec_ctrl: Merge bti_ist_info and use_shadow_spec_ctrl
into spec_ctrl_flags
All 3 bits of information here are control flags for the entry/exit code
behaviour. Treat them as such, rather than having two different variables.
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Wei Liu <wei.liu2@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Release-acked-by: Juergen Gross <jgross@suse.com>
(cherry picked from commit 5262ba2e7799001402dfe139ff944e035dfff928)
---
xen/arch/x86/acpi/power.c | 4 +--
xen/arch/x86/spec_ctrl.c | 10 ++++---
xen/arch/x86/x86_64/asm-offsets.c | 3 +--
xen/include/asm-x86/current.h | 3 +--
xen/include/asm-x86/nops.h | 5 ++--
xen/include/asm-x86/spec_ctrl.h | 10 +++----
xen/include/asm-x86/spec_ctrl_asm.h | 52 ++++++++++++++++++++-----------------
7 files changed, 45 insertions(+), 42 deletions(-)
diff --git a/xen/arch/x86/acpi/power.c b/xen/arch/x86/acpi/power.c
index f7085d3..f3480aa 100644
--- a/xen/arch/x86/acpi/power.c
+++ b/xen/arch/x86/acpi/power.c
@@ -215,7 +215,7 @@ static int enter_state(u32 state)
ci = get_cpu_info();
spec_ctrl_enter_idle(ci);
/* Avoid NMI/#MC using MSR_SPEC_CTRL until we've reloaded microcode. */
- ci->bti_ist_info = 0;
+ ci->spec_ctrl_flags &= ~SCF_ist_wrmsr;
ACPI_FLUSH_CPU_CACHE();
@@ -256,7 +256,7 @@ static int enter_state(u32 state)
microcode_resume_cpu(0);
/* Re-enabled default NMI/#MC use of MSR_SPEC_CTRL. */
- ci->bti_ist_info = default_bti_ist_info;
+ ci->spec_ctrl_flags |= (default_spec_ctrl_flags & SCF_ist_wrmsr);
spec_ctrl_exit_idle(ci);
done:
diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c
index 1143521..2d69910 100644
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -39,7 +39,7 @@ static bool __initdata opt_rsb_native = true;
static bool __initdata opt_rsb_vmexit = true;
bool __read_mostly opt_ibpb = true;
uint8_t __read_mostly default_xen_spec_ctrl;
-uint8_t __read_mostly default_bti_ist_info;
+uint8_t __read_mostly default_spec_ctrl_flags;
static int __init parse_bti(const char *s)
{
@@ -293,7 +293,7 @@ void __init init_speculation_mitigations(void)
else
setup_force_cpu_cap(X86_FEATURE_XEN_IBRS_CLEAR);
- default_bti_ist_info |= BTI_IST_WRMSR;
+ default_spec_ctrl_flags |= SCF_ist_wrmsr;
}
/*
@@ -312,7 +312,7 @@ void __init init_speculation_mitigations(void)
if ( opt_rsb_native )
{
setup_force_cpu_cap(X86_FEATURE_RSB_NATIVE);
- default_bti_ist_info |= BTI_IST_RSB;
+ default_spec_ctrl_flags |= SCF_ist_rsb;
}
/*
@@ -326,7 +326,7 @@ void __init init_speculation_mitigations(void)
if ( !boot_cpu_has(X86_FEATURE_IBRSB) && !boot_cpu_has(X86_FEATURE_IBPB) )
opt_ibpb = false;
- /* (Re)init BSP state now that default_bti_ist_info has been calculated. */
+ /* (Re)init BSP state now that default_spec_ctrl_flags has been calculated. */
init_shadow_spec_ctrl_state();
print_details(thunk, caps);
@@ -334,6 +334,8 @@ void __init init_speculation_mitigations(void)
static void __init __maybe_unused build_assertions(void)
{
+ /* The optimised assembly relies on this alias. */
+ BUILD_BUG_ON(SCF_use_shadow != 1);
}
/*
diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c
index 6dd0476..cc97d75 100644
--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -143,8 +143,7 @@ void __dummy__(void)
OFFSET(CPUINFO_pv_cr3, struct cpu_info, pv_cr3);
OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl);
OFFSET(CPUINFO_xen_spec_ctrl, struct cpu_info, xen_spec_ctrl);
- OFFSET(CPUINFO_use_shadow_spec_ctrl, struct cpu_info, use_shadow_spec_ctrl);
- OFFSET(CPUINFO_bti_ist_info, struct cpu_info, bti_ist_info);
+ OFFSET(CPUINFO_spec_ctrl_flags, struct cpu_info, spec_ctrl_flags);
DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
BLANK();
diff --git a/xen/include/asm-x86/current.h b/xen/include/asm-x86/current.h
index d10b13c..7afff0e 100644
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -57,8 +57,7 @@ struct cpu_info {
/* See asm-x86/spec_ctrl_asm.h for usage. */
unsigned int shadow_spec_ctrl;
uint8_t xen_spec_ctrl;
- bool use_shadow_spec_ctrl;
- uint8_t bti_ist_info;
+ uint8_t spec_ctrl_flags;
unsigned long __pad;
/* get_stack_bottom() must be 16-byte aligned */
diff --git a/xen/include/asm-x86/nops.h b/xen/include/asm-x86/nops.h
index 37f9819..b744895 100644
--- a/xen/include/asm-x86/nops.h
+++ b/xen/include/asm-x86/nops.h
@@ -62,10 +62,9 @@
#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8)
#define ASM_NOP17 ASM_NOP8; ASM_NOP7; ASM_NOP2
-#define ASM_NOP21 ASM_NOP8; ASM_NOP8; ASM_NOP5
+#define ASM_NOP22 ASM_NOP8; ASM_NOP8; ASM_NOP6
#define ASM_NOP24 ASM_NOP8; ASM_NOP8; ASM_NOP8
-#define ASM_NOP29 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP5
-#define ASM_NOP32 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8
+#define ASM_NOP33 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP7; ASM_NOP2
#define ASM_NOP40 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8
#define ASM_NOP_MAX 8
diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h
index 5e4fc84..059e291 100644
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -28,15 +28,15 @@ void init_speculation_mitigations(void);
extern bool opt_ibpb;
extern uint8_t default_xen_spec_ctrl;
-extern uint8_t default_bti_ist_info;
+extern uint8_t default_spec_ctrl_flags;
static inline void init_shadow_spec_ctrl_state(void)
{
struct cpu_info *info = get_cpu_info();
- info->shadow_spec_ctrl = info->use_shadow_spec_ctrl = 0;
+ info->shadow_spec_ctrl = 0;
info->xen_spec_ctrl = default_xen_spec_ctrl;
- info->bti_ist_info = default_bti_ist_info;
+ info->spec_ctrl_flags = default_spec_ctrl_flags;
}
/* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */
@@ -50,7 +50,7 @@ static always_inline void spec_ctrl_enter_idle(struct cpu_info *info)
*/
info->shadow_spec_ctrl = val;
barrier();
- info->use_shadow_spec_ctrl = true;
+ info->spec_ctrl_flags |= SCF_use_shadow;
barrier();
asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET)
:: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" );
@@ -65,7 +65,7 @@ static always_inline void spec_ctrl_exit_idle(struct cpu_info *info)
* Disable shadowing before updating the MSR. There are no SMP issues
* here; only local processor ordering concerns.
*/
- info->use_shadow_spec_ctrl = false;
+ info->spec_ctrl_flags &= ~SCF_use_shadow;
barrier();
asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET)
:: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" );
diff --git a/xen/include/asm-x86/spec_ctrl_asm.h b/xen/include/asm-x86/spec_ctrl_asm.h
index 9c16945..582403a 100644
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -20,9 +20,10 @@
#ifndef __X86_SPEC_CTRL_ASM_H__
#define __X86_SPEC_CTRL_ASM_H__
-/* Encoding of the bottom bits in cpuinfo.bti_ist_info */
-#define BTI_IST_WRMSR (1 << 1)
-#define BTI_IST_RSB (1 << 2)
+/* Encoding of cpuinfo.spec_ctrl_flags */
+#define SCF_use_shadow (1 << 0)
+#define SCF_ist_wrmsr (1 << 1)
+#define SCF_ist_rsb (1 << 2)
#ifdef __ASSEMBLY__
#include <asm/msr-index.h>
@@ -49,20 +50,20 @@
* after VMEXIT. The VMEXIT-specific code reads MSR_SPEC_CTRL and updates
* current before loading Xen's MSR_SPEC_CTRL setting.
*
- * Factor 2 is harder. We maintain a shadow_spec_ctrl value, and
- * use_shadow_spec_ctrl boolean per cpu. The synchronous use is:
+ * Factor 2 is harder. We maintain a shadow_spec_ctrl value, and a use_shadow
+ * boolean in the per cpu spec_ctrl_flags. The synchronous use is:
*
* 1) Store guest value in shadow_spec_ctrl
- * 2) Set use_shadow_spec_ctrl boolean
+ * 2) Set the use_shadow boolean
* 3) Load guest value into MSR_SPEC_CTRL
* 4) Exit to guest
* 5) Entry from guest
- * 6) Clear use_shadow_spec_ctrl boolean
+ * 6) Clear the use_shadow boolean
* 7) Load Xen's value into MSR_SPEC_CTRL
*
* The asynchronous use for interrupts/exceptions is:
* - Set/clear IBRS on entry to Xen
- * - On exit to Xen, check use_shadow_spec_ctrl
+ * - On exit to Xen, check use_shadow
* - If set, load shadow_spec_ctrl
*
* Therefore, an interrupt/exception which hits the synchronous path between
@@ -133,7 +134,7 @@
xor %edx, %edx
/* Clear SPEC_CTRL shadowing *before* loading Xen's value. */
- movb %dl, CPUINFO_use_shadow_spec_ctrl(%rsp)
+ andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp)
/* Load Xen's intended value. */
mov $\ibrs_val, %eax
@@ -159,12 +160,14 @@
* block so calculate the position directly.
*/
.if \maybexen
+ xor %eax, %eax
/* Branchless `if ( !xen ) clear_shadowing` */
testb $3, UREGS_cs(%rsp)
- setz %al
- and %al, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%r14)
+ setnz %al
+ not %eax
+ and %al, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14)
.else
- movb %dl, CPUINFO_use_shadow_spec_ctrl(%rsp)
+ andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp)
.endif
/* Load Xen's intended value. */
@@ -183,8 +186,8 @@
*/
xor %edx, %edx
- cmpb %dl, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%rbx)
- je .L\@_skip
+ testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx)
+ jz .L\@_skip
mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax
mov $MSR_SPEC_CTRL, %ecx
@@ -205,7 +208,7 @@
mov %eax, CPUINFO_shadow_spec_ctrl(%rsp)
/* Set SPEC_CTRL shadowing *before* loading the guest value. */
- movb $1, CPUINFO_use_shadow_spec_ctrl(%rsp)
+ orb $SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp)
mov $MSR_SPEC_CTRL, %ecx
xor %edx, %edx
@@ -216,7 +219,7 @@
#define SPEC_CTRL_ENTRY_FROM_VMEXIT \
ALTERNATIVE __stringify(ASM_NOP40), \
DO_OVERWRITE_RSB, X86_FEATURE_RSB_VMEXIT; \
- ALTERNATIVE_2 __stringify(ASM_NOP32), \
+ ALTERNATIVE_2 __stringify(ASM_NOP33), \
__stringify(DO_SPEC_CTRL_ENTRY_FROM_VMEXIT \
ibrs_val=SPEC_CTRL_IBRS), \
X86_FEATURE_XEN_IBRS_SET, \
@@ -228,7 +231,7 @@
#define SPEC_CTRL_ENTRY_FROM_PV \
ALTERNATIVE __stringify(ASM_NOP40), \
DO_OVERWRITE_RSB, X86_FEATURE_RSB_NATIVE; \
- ALTERNATIVE_2 __stringify(ASM_NOP21), \
+ ALTERNATIVE_2 __stringify(ASM_NOP22), \
__stringify(DO_SPEC_CTRL_ENTRY maybexen=0 \
ibrs_val=SPEC_CTRL_IBRS), \
X86_FEATURE_XEN_IBRS_SET, \
@@ -239,7 +242,7 @@
#define SPEC_CTRL_ENTRY_FROM_INTR \
ALTERNATIVE __stringify(ASM_NOP40), \
DO_OVERWRITE_RSB, X86_FEATURE_RSB_NATIVE; \
- ALTERNATIVE_2 __stringify(ASM_NOP29), \
+ ALTERNATIVE_2 __stringify(ASM_NOP33), \
__stringify(DO_SPEC_CTRL_ENTRY maybexen=1 \
ibrs_val=SPEC_CTRL_IBRS), \
X86_FEATURE_XEN_IBRS_SET, \
@@ -267,22 +270,23 @@
* This is logical merge of DO_OVERWRITE_RSB and DO_SPEC_CTRL_ENTRY
* maybexen=1, but with conditionals rather than alternatives.
*/
- movzbl STACK_CPUINFO_FIELD(bti_ist_info)(%r14), %eax
+ movzbl STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14), %eax
- testb $BTI_IST_RSB, %al
+ test $SCF_ist_rsb, %al
jz .L\@_skip_rsb
DO_OVERWRITE_RSB tmp=rdx /* Clobbers %rcx/%rdx */
.L\@_skip_rsb:
- testb $BTI_IST_WRMSR, %al
+ test $SCF_ist_wrmsr, %al
jz .L\@_skip_wrmsr
xor %edx, %edx
testb $3, UREGS_cs(%rsp)
- setz %dl
- and %dl, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%r14)
+ setnz %dl
+ not %edx
+ and %dl, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14)
/* Load Xen's intended value. */
mov $MSR_SPEC_CTRL, %ecx
@@ -309,7 +313,7 @@ UNLIKELY_DISPATCH_LABEL(\@_serialise):
* Requires %rbx=stack_end
* Clobbers %rax, %rcx, %rdx
*/
- testb $BTI_IST_WRMSR, STACK_CPUINFO_FIELD(bti_ist_info)(%rbx)
+ testb $SCF_ist_wrmsr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx)
jz .L\@_skip
DO_SPEC_CTRL_EXIT_TO_XEN
--
2.1.4
|
