diff options
| author | Henrik Riomar <henrik.riomar@gmail.com> | 2018-09-26 22:57:24 +0200 |
|---|---|---|
| committer | Leonardo Arena <rnalrd@alpinelinux.org> | 2018-10-01 12:22:30 +0000 |
| commit | 35dc7cff76a711450db5044179e53d074ad7257a (patch) | |
| tree | 268baa27d9ae1a6b3cff9a173bedef0fcfbdac9a | |
| parent | 96e1e57fed146a3449b7070143861b7e22ba57f4 (diff) | |
| download | aports-35dc7cff76a711450db5044179e53d074ad7257a.tar.bz2 aports-35dc7cff76a711450db5044179e53d074ad7257a.tar.xz | |
main/xen: upgrade to 4.10.2
Remove XSA-270 and XSA-274 from secfix list as they are not in XEN but in
the Linux kernel.
Remove XSA-271 for the list due to that tha bug is in XAPI and not in XEN.
| -rw-r--r-- | main/xen/APKBUILD | 11 | ||||
| -rw-r--r-- | main/xen/git.patch | 8547 |
2 files changed, 3 insertions, 8555 deletions
diff --git a/main/xen/APKBUILD b/main/xen/APKBUILD index 3385370b08..e413a045ed 100644 --- a/main/xen/APKBUILD +++ b/main/xen/APKBUILD @@ -2,8 +2,8 @@ # Contributor: Roger Pau Monne <roger.pau@entel.upc.edu> # Maintainer: William Pitcock <nenolod@dereferenced.org> pkgname=xen -pkgver=4.10.1 -pkgrel=3 +pkgver=4.10.2 +pkgrel=0 pkgdesc="Xen hypervisor" url="http://www.xen.org/" arch="x86_64 armhf aarch64" @@ -120,11 +120,8 @@ options="!strip" # - CVE-2018-10982 XSA-261 # - CVE-2018-10981 XSA-262 # 4.10.1-r3: -# - CVE-2018-14678 XSA-274 # - CVE-2018-3646 XSA-273 # - CVE-2018-15470 XSA-272 -# - CVE-2018-14007 XSA-271 -# - CVE-2018-15471 XSA-270 # - CVE-2018-15468 XSA-269 # - CVE-2018-15469 XSA-268 # - CVE-2018-3665 XSA-267 @@ -178,7 +175,6 @@ source="https://downloads.xenproject.org/release/$pkgname/$pkgver/$pkgname-$pkgv http://xenbits.xen.org/xen-extfiles/tpm_emulator-$_TPMEMU_VERSION.tar.gz http://xenbits.xen.org/xen-extfiles/zlib-$_ZLIB_VERSION.tar.gz http://xenbits.xen.org/xen-extfiles/ipxe-git-$_IPXE_GIT_TAG.tar.gz - git.patch qemu-xen_paths.patch @@ -430,7 +426,7 @@ EOF } -sha512sums="236c02bee69e33644703ed26d323d4c491a91fc05bd0ee0990a7368579f7c82f5bb4510845bf80348fd923024d7d60d521f593dfd0365d971dc592f8ef10fbea xen-4.10.1.tar.gz +sha512sums="2e301b4d6eb106fcb52203bc1208c8fd1fc7bb5d795166c0f86e4085227050301f7daa6f21db440c47bc2ca2c7f87d9b96753cfdf3fa724e0416b5679b84c163 xen-4.10.2.tar.gz 2e0b0fd23e6f10742a5517981e5171c6e88b0a93c83da701b296f5c0861d72c19782daab589a7eac3f9032152a0fc7eff7f5362db8fccc4859564a9aa82329cf gmp-4.3.2.tar.bz2 c2bc9ffc8583aeae71cee9ddcc4418969768d4e3764d47307da54f93981c0109fb07d84b061b3a3628bd00ba4d14a54742bc04848110eb3ae8ca25dbfbaabadb grub-0.97.tar.gz 1465b58279af1647f909450e394fe002ca165f0ff4a0254bfa9fe0e64316f50facdde2729d79a4e632565b4500cf4d6c74192ac0dd3bc9fe09129bbd67ba089d lwip-1.3.0.tar.gz @@ -440,7 +436,6 @@ c2bc9ffc8583aeae71cee9ddcc4418969768d4e3764d47307da54f93981c0109fb07d84b061b3a36 4928b5b82f57645be9408362706ff2c4d9baa635b21b0d41b1c82930e8c60a759b1ea4fa74d7e6c7cae1b7692d006aa5cb72df0c3b88bf049779aa2b566f9d35 tpm_emulator-0.7.4.tar.gz 021b958fcd0d346c4ba761bcf0cc40f3522de6186cf5a0a6ea34a70504ce9622b1c2626fce40675bc8282cf5f5ade18473656abc38050f72f5d6480507a2106e zlib-1.2.3.tar.gz bbcce5e55040e7e29adebd4a5253a046016a6e2e7ff34cf801a42d147e1ec1af57e0297318249bfa9c5bbeac969fe4b37c18cbf845a80b2136d65387a4fc31da ipxe-git-356f6c1b64d7a97746d1816cef8ca22bdd8d0b5d.tar.gz -0fd2622469f3ff136b33a66576319920e050aac3fefa41c06306661eb6f6792fc21a4c15c8928febd10b1a14b4c712a2918532cdb23ccbddba9f1ba55d7d4478 git.patch 1936ab39a1867957fa640eb81c4070214ca4856a2743ba7e49c0cd017917071a9680d015f002c57fa7b9600dbadd29dcea5887f50e6c133305df2669a7a933f3 qemu-xen_paths.patch f095ea373f36381491ad36f0662fb4f53665031973721256b23166e596318581da7cbb0146d0beb2446729adfdb321e01468e377793f6563a67d68b8b0f7ffe3 hotplug-vif-vtrill.patch 77b08e9655e091b0352e4630d520b54c6ca6d659d1d38fbb4b3bfc9ff3e66db433a2e194ead32bb10ff962c382d800a670e82b7a62835b238e294b22808290ea musl-hvmloader-fix-stdint.patch diff --git a/main/xen/git.patch b/main/xen/git.patch deleted file mode 100644 index b4224d8d67..0000000000 --- a/main/xen/git.patch +++ /dev/null @@ -1,8547 +0,0 @@ -diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5 -index da91b8626c..37262a7ef8 100644 ---- a/docs/man/xl.conf.pod.5 -+++ b/docs/man/xl.conf.pod.5 -@@ -185,6 +185,28 @@ massively huge guests). - - =back - -+=item B<vm.cpumask>="CPULIST" -+ -+=item B<vm.hvm.cpumask>="CPULIST" -+ -+=item B<vm.pv.cpumask>="CPULIST" -+ -+Global masks that are applied when creating guests and pinning vcpus -+to indicate which cpus they are allowed to run on. Specifically, -+C<vm.cpumask> applies to all guest types, C<vm.hvm.cpumask> applies to -+both HVM and PVH guests and C<vm.pv.cpumask> applies to PV guests. -+ -+The hard affinity of guest's vcpus are logical-AND'ed with respective -+masks. If the resulting affinity mask is empty, operation will fail. -+ -+Use --ignore-global-affinity-masks to skip applying global masks. -+ -+The default value for these masks are all 1's, i.e. all cpus are allowed. -+ -+Due to bug(s), these options may not interact well with other options -+concerning CPU affinity. One example is CPU pools. Users should always double -+check that the required affinity has taken effect. -+ - =back - - =head1 SEE ALSO -diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown -index 6c673eedc8..470da80174 100644 ---- a/docs/misc/xen-command-line.markdown -+++ b/docs/misc/xen-command-line.markdown -@@ -248,6 +248,9 @@ the NMI watchdog is also enabled. - ### bti (x86) - > `= List of [ thunk=retpoline|lfence|jmp, ibrs=<bool>, ibpb=<bool>, rsb_{vmexit,native}=<bool> ]` - -+**WARNING: This command line option is deprecated, and superseded by -+_spec-ctrl=_ - using both options in combination is undefined.** -+ - Branch Target Injection controls. By default, Xen will pick the most - appropriate BTI mitigations based on compiled in support, loaded microcode, - and hardware details. -@@ -493,9 +496,10 @@ accounting for hardware capabilities as enumerated via CPUID. - - Currently accepted: - --The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb` are used by --default if avaiable. They can be ignored, e.g. `no-ibrsb`, at which point Xen --won't use them itself, and won't offer them to guests. -+The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`, -+`l1d-flush` and `ssbd` are used by default if available and applicable. They can -+be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and -+won't offer them to guests. - - ### cpuid\_mask\_cpu (AMD only) - > `= fam_0f_rev_c | fam_0f_rev_d | fam_0f_rev_e | fam_0f_rev_f | fam_0f_rev_g | fam_10_rev_b | fam_10_rev_c | fam_11_rev_b` -@@ -916,6 +920,21 @@ Controls EPT related features. - - Specify which console gdbstub should use. See **console**. - -+### gnttab -+> `= List of [ max-ver:<integer>, transitive=<bool> ]` -+ -+> Default: `gnttab=max-ver:2,transitive` -+ -+Control various aspects of the grant table behaviour available to guests. -+ -+* `max-ver` Select the maximum grant table version to offer to guests. Valid -+version are 1 and 2. -+* `transitive` Permit or disallow the use of transitive grants. Note that the -+use of grant table v2 without transitive grants is an ABI breakage from the -+guests point of view. -+ -+The usage of gnttab v2 is not security supported on ARM platforms. -+ - ### gnttab\_max\_frames - > `= <integer>` - -@@ -1348,6 +1367,15 @@ Because responsibility for APIC setup is shared between Xen and the - domain 0 kernel this option is automatically propagated to the domain - 0 command line. - -+### invpcid (x86) -+> `= <boolean>` -+ -+> Default: `true` -+ -+By default, Xen will use the INVPCID instruction for TLB management if -+it is available. This option can be used to cause Xen to fall back to -+older mechanisms, which are generally slower. -+ - ### noirqbalance - > `= <boolean>` - -@@ -1426,6 +1454,20 @@ Flag to enable Memory Protection Keys. - The protection-key feature provides an additional mechanism by which IA-32e - paging controls access to usermode addresses. - -+### pcid (x86) -+> `= <boolean> | xpti=<bool>` -+ -+> Default: `xpti` -+ -+> Can be modified at runtime (change takes effect only for domains created -+ afterwards) -+ -+If available, control usage of the PCID feature of the processor for -+64-bit pv-domains. PCID can be used either for no domain at all (`false`), -+for all of them (`true`), only for those subject to XPTI (`xpti`) or for -+those not subject to XPTI (`no-xpti`). The feature is used only in case -+INVPCID is supported and not disabled via `invpcid=false`. -+ - ### psr (Intel) - > `= List of ( cmt:<boolean> | rmid_max:<integer> | cat:<boolean> | cos_max:<integer> | cdp:<boolean> )` - -@@ -1486,6 +1528,30 @@ do; there may be other custom operating systems which do. If you're - certain you don't plan on having PV guests which use this feature, - turning it off can reduce the attack surface. - -+### pv-l1tf (x86) -+> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]` -+ -+> Default: `false` on believed-unaffected hardware, or in pv-shim mode. -+> `domu` on believed-affected hardware. -+ -+Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests. -+ -+For backwards compatibility, we may not alter an architecturally-legitimate -+pagetable entry a PV guest chooses to write. We can however force such a -+guest into shadow mode so that Xen controls the PTEs which are reachable by -+the CPU pagewalk. -+ -+Shadowing is performed at the point where a PV guest first tries to write an -+L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with -+its own L1TF mitigations will not trigger shadow mode if it is well behaved. -+ -+If CONFIG\_SHADOW\_PAGING is not compiled in, this mitigation instead crashes -+the guest when an L1TF-vulnerable PTE is written, which still allows updated, -+well-behaved PV guests to run, despite Shadow being compiled out. -+ -+In the pv-shim case, Shadow is expected to be compiled out, and a malicious -+guest kernel can only leak data from the shim Xen, rather than the host Xen. -+ - ### pv-shim (x86) - > `= <boolean>` - -@@ -1690,6 +1756,13 @@ Use `smap=hvm` to allow SMAP use by HVM guests only. - Flag to enable Supervisor Mode Execution Protection - Use `smep=hvm` to allow SMEP use by HVM guests only. - -+### smt (x86) -+> `= <boolean>` -+ -+Default: `true` -+ -+Control bring up of multiple hyper-threads per CPU core. -+ - ### snb\_igd\_quirk - > `= <boolean> | cap | <integer>` - -@@ -1698,6 +1771,75 @@ enforces the maximum theoretically necessary timeout of 670ms. Any number - is being interpreted as a custom timeout in milliseconds. Zero or boolean - false disable the quirk workaround, which is also the default. - -+### spec-ctrl (x86) -+> `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>, -+> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu, -+> l1d-flush}=<bool> ]` -+ -+Controls for speculative execution sidechannel mitigations. By default, Xen -+will pick the most appropriate mitigations based on compiled in support, -+loaded microcode, and hardware details, and will virtualise appropriate -+mitigations for guests to use. -+ -+**WARNING: Any use of this option may interfere with heuristics. Use with -+extreme care.** -+ -+An overall boolean value, `spec-ctrl=no`, can be specified to turn off all -+mitigations, including pieces of infrastructure used to virtualise certain -+mitigation features for guests. This also includes settings which `xpti`, -+`smt`, `pv-l1tf` control, unless the respective option(s) have been -+specified earlier on the command line. -+ -+Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to -+turn off all of Xen's mitigations, while leaving the virtualisation support -+in place for guests to use. -+ -+Use of a positive boolean value for either of these options is invalid. -+ -+The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control -+over the alternative blocks used by Xen. These impact Xen's ability to -+protect itself, and Xen's ability to virtualise support for guests to use. -+ -+* `pv=` and `hvm=` offer control over all suboptions for PV and HVM guests -+ respectively. -+* `msr-sc=` offers control over Xen's support for manipulating MSR\_SPEC\_CTRL -+ on entry and exit. These blocks are necessary to virtualise support for -+ guests and if disabled, guests will be unable to use IBRS/STIBP/SSBD/etc. -+* `rsb=` offers control over whether to overwrite the Return Stack Buffer / -+ Return Address Stack on entry to Xen. -+ -+If Xen was compiled with INDIRECT\_THUNK support, `bti-thunk=` can be used to -+select which of the thunks gets patched into the `__x86_indirect_thunk_%reg` -+locations. The default thunk is `retpoline` (generally preferred for Intel -+hardware), with the alternatives being `jmp` (a `jmp *%reg` gadget, minimal -+overhead), and `lfence` (an `lfence; jmp *%reg` gadget, preferred for AMD). -+ -+On hardware supporting IBRS (Indirect Branch Restricted Speculation), the -+`ibrs=` option can be used to force or prevent Xen using the feature itself. -+If Xen is not using IBRS itself, functionality is still set up so IBRS can be -+virtualised for guests. -+ -+On hardware supporting IBPB (Indirect Branch Prediction Barrier), the `ibpb=` -+option can be used to force (the default) or prevent Xen from issuing branch -+prediction barriers on vcpu context switches. -+ -+On hardware supporting SSBD (Speculative Store Bypass Disable), the `ssbd=` -+option can be used to force or prevent Xen using the feature itself. On AMD -+hardware, this is a global option applied at boot, and not virtualised for -+guest use. On Intel hardware, the feature is virtualised for guests, -+independently of Xen's choice of setting. -+ -+On all hardware, the `eager-fpu=` option can be used to force or prevent Xen -+from using fully eager FPU context switches. This is currently implemented as -+a global control. By default, Xen will choose to use fully eager context -+switches on hardware believed to speculate past #NM exceptions. -+ -+On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force -+or prevent Xen from issuing an L1 data cache flush on each VMEntry. -+Irrespective of Xen's setting, the feature is virtualised for HVM guests to -+use. By default, Xen will enable this mitigation on hardware believed to be -+vulnerable to L1TF. -+ - ### sync\_console - > `= <boolean>` - -@@ -1923,14 +2065,24 @@ clustered mode. The default, given no hint from the **FADT**, is cluster - mode. - - ### xpti --> `= <boolean>` -+> `= List of [ default | <boolean> | dom0=<bool> | domu=<bool> ]` - --> Default: `false` on AMD hardware -+> Default: `false` on hardware not to be vulnerable to Meltdown (e.g. AMD) - > Default: `true` everywhere else - - Override default selection of whether to isolate 64-bit PV guest page - tables. - -+`true` activates page table isolation even on hardware not vulnerable by -+Meltdown for all domains. -+ -+`false` deactivates page table isolation on all systems for all domains. -+ -+`default` sets the default behaviour. -+ -+With `dom0` and `domu` it is possible to control page table isolation -+for dom0 or guest domains only. -+ - ### xsave - > `= <boolean>` - -diff --git a/tools/Makefile b/tools/Makefile -index ab7a01ee1b..67977ad850 100644 ---- a/tools/Makefile -+++ b/tools/Makefile -@@ -232,7 +232,7 @@ subdir-all-qemu-xen-dir: qemu-xen-dir-find - else \ - enable_trace_backend='' ; \ - fi ; \ -- PKG_CONFIG_PATH=$(XEN_ROOT)/tools/pkg-config \ -+ PKG_CONFIG_PATH=$(XEN_ROOT)/tools/pkg-config$${PKG_CONFIG_PATH:+:$${PKG_CONFIG_PATH}} \ - $$source/configure --enable-xen --target-list=i386-softmmu \ - $(QEMU_XEN_ENABLE_DEBUG) \ - $$enable_trace_backend \ -diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf -index 374b6bbc2e..0446deb304 100644 ---- a/tools/examples/xl.conf -+++ b/tools/examples/xl.conf -@@ -37,3 +37,8 @@ - # (which can take a long time to find out if launching huge guests). - # see xl.conf(5) for details. - #claim_mode=1 -+ -+# Specify global vcpu hard affinity masks. See xl.conf(5) for details. -+#vm.cpumask="0-7" -+#vm.pv.cpumask="0-3" -+#vm.hvm.cpumask="3-7" -diff --git a/tools/libacpi/Makefile b/tools/libacpi/Makefile -index a47a658a25..c17f3924cc 100644 ---- a/tools/libacpi/Makefile -+++ b/tools/libacpi/Makefile -@@ -43,7 +43,7 @@ all: $(C_SRC) $(H_SRC) - - $(H_SRC): $(ACPI_BUILD_DIR)/%.h: %.asl iasl - iasl -vs -p $(ACPI_BUILD_DIR)/$*.$(TMP_SUFFIX) -tc $< -- sed -e 's/AmlCode/$*/g' $(ACPI_BUILD_DIR)/$*.hex >$@ -+ sed -e 's/AmlCode/$*/g' -e 's/_aml_code//g' $(ACPI_BUILD_DIR)/$*.hex >$@ - rm -f $(addprefix $(ACPI_BUILD_DIR)/, $*.aml $*.hex) - - $(MK_DSDT): mk_dsdt.c -@@ -76,7 +76,7 @@ $(ACPI_BUILD_DIR)/dsdt_anycpu_arm.asl: $(MK_DSDT) - - $(C_SRC): $(ACPI_BUILD_DIR)/%.c: iasl $(ACPI_BUILD_DIR)/%.asl - iasl -vs -p $(ACPI_BUILD_DIR)/$*.$(TMP_SUFFIX) -tc $(ACPI_BUILD_DIR)/$*.asl -- sed -e 's/AmlCode/$*/g' $(ACPI_BUILD_DIR)/$*.hex > $@.$(TMP_SUFFIX) -+ sed -e 's/AmlCode/$*/g' -e 's/_aml_code//g' $(ACPI_BUILD_DIR)/$*.hex > $@.$(TMP_SUFFIX) - echo "int $*_len=sizeof($*);" >> $@.$(TMP_SUFFIX) - mv -f $@.$(TMP_SUFFIX) $@ - rm -f $(addprefix $(ACPI_BUILD_DIR)/, $*.aml $*.hex) -diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c -index 9fa2f7c360..21537f06f1 100644 ---- a/tools/libxc/xc_cpuid_x86.c -+++ b/tools/libxc/xc_cpuid_x86.c -@@ -575,6 +575,12 @@ static void xc_cpuid_pv_policy(xc_interface *xch, - break; - } - -+ case 0x80000008: -+ regs[0] &= 0x0000ffffu; -+ regs[1] = info->featureset[featureword_of(X86_FEATURE_CLZERO)]; -+ regs[2] = regs[3] = 0; -+ break; -+ - case 0x00000005: /* MONITOR/MWAIT */ - case 0x0000000b: /* Extended Topology Enumeration */ - case 0x8000000a: /* SVM revision and features */ -diff --git a/tools/libxl/libxl_cpuid.c b/tools/libxl/libxl_cpuid.c -index 3a21f4e7da..52e16c20ed 100644 ---- a/tools/libxl/libxl_cpuid.c -+++ b/tools/libxl/libxl_cpuid.c -@@ -204,7 +204,9 @@ int libxl_cpuid_parse_config(libxl_cpuid_policy_list *cpuid, const char* str) - {"avx512-4fmaps",0x00000007, 0, CPUID_REG_EDX, 3, 1}, - {"ibrsb", 0x00000007, 0, CPUID_REG_EDX, 26, 1}, - {"stibp", 0x00000007, 0, CPUID_REG_EDX, 27, 1}, -+ {"l1d-flush", 0x00000007, 0, CPUID_REG_EDX, 28, 1}, - {"arch-caps", 0x00000007, 0, CPUID_REG_EDX, 29, 1}, -+ {"ssbd", 0x00000007, 0, CPUID_REG_EDX, 31, 1}, - - {"lahfsahf", 0x80000001, NA, CPUID_REG_ECX, 0, 1}, - {"cmplegacy", 0x80000001, NA, CPUID_REG_ECX, 1, 1}, -diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c -index b51178b9fd..07399bb8e0 100644 ---- a/tools/libxl/libxl_dm.c -+++ b/tools/libxl/libxl_dm.c -@@ -798,6 +798,8 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *target_path, - int colo_mode) - { - char *drive = NULL; -+ char *common = GCSPRINTF("cache=writeback,readonly=%s", -+ disk->readwrite ? "off" : "on"); - const char *exportname = disk->colo_export; - const char *active_disk = disk->active_disk; - const char *hidden_disk = disk->hidden_disk; -@@ -805,8 +807,8 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *target_path, - switch (colo_mode) { - case LIBXL__COLO_NONE: - drive = libxl__sprintf -- (gc, "file=%s,if=scsi,bus=0,unit=%d,format=%s,cache=writeback", -- target_path, unit, format); -+ (gc, "%s,file=%s,if=scsi,bus=0,unit=%d,format=%s", -+ common, target_path, unit, format); - break; - case LIBXL__COLO_PRIMARY: - /* -@@ -819,13 +821,13 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *target_path, - * vote-threshold=1 - */ - drive = GCSPRINTF( -- "if=scsi,bus=0,unit=%d,cache=writeback,driver=quorum," -+ "%s,if=scsi,bus=0,unit=%d,,driver=quorum," - "id=%s," - "children.0.file.filename=%s," - "children.0.driver=%s," - "read-pattern=fifo," - "vote-threshold=1", -- unit, exportname, target_path, format); -+ common, unit, exportname, target_path, format); - break; - case LIBXL__COLO_SECONDARY: - /* -@@ -839,7 +841,7 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *target_path, - * file.backing.backing=exportname, - */ - drive = GCSPRINTF( -- "if=scsi,id=top-colo,bus=0,unit=%d,cache=writeback," -+ "%s,if=scsi,id=top-colo,bus=0,unit=%d," - "driver=replication," - "mode=secondary," - "top-id=top-colo," -@@ -848,7 +850,7 @@ static char *qemu_disk_scsi_drive_string(libxl__gc *gc, const char *target_path, - "file.backing.driver=qcow2," - "file.backing.file.filename=%s," - "file.backing.backing=%s", -- unit, active_disk, hidden_disk, exportname); -+ common, unit, active_disk, hidden_disk, exportname); - break; - default: - abort(); -@@ -866,6 +868,8 @@ static char *qemu_disk_ide_drive_string(libxl__gc *gc, const char *target_path, - const char *exportname = disk->colo_export; - const char *active_disk = disk->active_disk; - const char *hidden_disk = disk->hidden_disk; -+ -+ assert(disk->readwrite); /* should have been checked earlier */ - - switch (colo_mode) { - case LIBXL__COLO_NONE: -@@ -1575,8 +1579,9 @@ static int libxl__build_device_model_args_new(libxl__gc *gc, - if (strncmp(disks[i].vdev, "sd", 2) == 0) { - if (colo_mode == LIBXL__COLO_SECONDARY) { - drive = libxl__sprintf -- (gc, "if=none,driver=%s,file=%s,id=%s", -- format, target_path, disks[i].colo_export); -+ (gc, "if=none,driver=%s,file=%s,id=%s,readonly=%s", -+ format, target_path, disks[i].colo_export, -+ disks[i].readwrite ? "off" : "on"); - - flexarray_append(dm_args, "-drive"); - flexarray_append(dm_args, drive); -@@ -2586,7 +2591,7 @@ int libxl__need_xenpv_qemu(libxl__gc *gc, libxl_domain_config *d_config) - goto out; - } - -- if (d_config->num_vfbs > 0) { -+ if (d_config->num_vfbs > 0 || d_config->num_p9s > 0) { - ret = 1; - goto out; - } -diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c -index b1a46c667d..85298d277d 100644 ---- a/tools/misc/xen-cpuid.c -+++ b/tools/misc/xen-cpuid.c -@@ -165,9 +165,8 @@ static const char *str_7d0[32] = - [4 ... 25] = "REZ", - - [26] = "ibrsb", [27] = "stibp", -- [28] = "REZ", [29] = "arch_caps", -- -- [30 ... 31] = "REZ", -+ [28] = "l1d_flush", [29] = "arch_caps", -+ [30] = "REZ", [31] = "ssbd", - }; - - static struct { -diff --git a/tools/ocaml/xenstored/store.ml b/tools/ocaml/xenstored/store.ml -index 13cf3b5bf4..5a8c377603 100644 ---- a/tools/ocaml/xenstored/store.ml -+++ b/tools/ocaml/xenstored/store.ml -@@ -262,7 +262,8 @@ let path_write store perm path value = - Node.check_perm store.root perm Perms.WRITE; - Node.set_value store.root value, false - ) else -- Path.apply_modify store.root path do_write, !node_created -+ let root = Path.apply_modify store.root path do_write in -+ root, !node_created - - let path_rm store perm path = - let do_rm node name = -diff --git a/tools/xl/xl.c b/tools/xl/xl.c -index 179908b4f6..7d2142f16f 100644 ---- a/tools/xl/xl.c -+++ b/tools/xl/xl.c -@@ -28,6 +28,9 @@ - #include <libxl_utils.h> - #include <libxlutil.h> - #include "xl.h" -+#include "xl_parse.h" -+ -+#include "xl_utils.h" - - xentoollog_logger_stdiostream *logger; - int dryrun_only; -@@ -42,6 +45,9 @@ char *default_gatewaydev = NULL; - char *default_vifbackend = NULL; - char *default_remus_netbufscript = NULL; - char *default_colo_proxy_script = NULL; -+libxl_bitmap global_vm_affinity_mask; -+libxl_bitmap global_hvm_affinity_mask; -+libxl_bitmap global_pv_affinity_mask; - enum output_format default_output_format = OUTPUT_FORMAT_JSON; - int claim_mode = 1; - bool progress_use_cr = 0; -@@ -203,6 +209,26 @@ static void parse_global_config(const char *configfile, - if (!xlu_cfg_get_long (config, "max_maptrack_frames", &l, 0)) - max_maptrack_frames = l; - -+ libxl_bitmap_init(&global_vm_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_vm_affinity_mask, 0); -+ libxl_bitmap_init(&global_hvm_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_hvm_affinity_mask, 0); -+ libxl_bitmap_init(&global_pv_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_pv_affinity_mask, 0); -+ -+ if (!xlu_cfg_get_string (config, "vm.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_vm_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_vm_affinity_mask); -+ if (!xlu_cfg_get_string (config, "vm.hvm.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_hvm_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_hvm_affinity_mask); -+ if (!xlu_cfg_get_string (config, "vm.pv.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_pv_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_pv_affinity_mask); -+ - xlu_cfg_destroy(config); - } - -diff --git a/tools/xl/xl.h b/tools/xl/xl.h -index 6b60d1db50..7b9f58fc6c 100644 ---- a/tools/xl/xl.h -+++ b/tools/xl/xl.h -@@ -41,6 +41,7 @@ struct domain_create { - int vncautopass; - int console_autoconnect; - int checkpointed_stream; -+ int ignore_global_affinity_masks; - const char *config_file; - char *extra_config; /* extra config string */ - const char *restore_file; -@@ -277,6 +278,9 @@ extern char *default_colo_proxy_script; - extern char *blkdev_start; - extern int max_grant_frames; - extern int max_maptrack_frames; -+extern libxl_bitmap global_vm_affinity_mask; -+extern libxl_bitmap global_hvm_affinity_mask; -+extern libxl_bitmap global_pv_affinity_mask; - - enum output_format { - OUTPUT_FORMAT_JSON, -@@ -292,6 +296,9 @@ typedef enum { - } domain_restart_type; - - extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh); -+extern void apply_global_affinity_masks(libxl_domain_type type, -+ libxl_bitmap *vcpu_affinity_array, -+ unsigned int size); - - #define XL_GLOBAL_CONFIG XEN_CONFIG_DIR "/xl.conf" - #define XL_LOCK_FILE XEN_LOCK_DIR "/xl" -diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c -index 5546cf66e7..1a6c28dfdc 100644 ---- a/tools/xl/xl_cmdtable.c -+++ b/tools/xl/xl_cmdtable.c -@@ -34,7 +34,8 @@ struct cmd_spec cmd_table[] = { - "-e Do not wait in the background for the death of the domain.\n" - "-V, --vncviewer Connect to the VNC display after the domain is created.\n" - "-A, --vncviewer-autopass\n" -- " Pass VNC password to viewer via stdin." -+ " Pass VNC password to viewer via stdin.\n" -+ "--ignore-global-affinity-masks Ignore global masks in xl.conf." - }, - { "config-update", - &main_config_update, 1, 1, -@@ -224,7 +225,8 @@ struct cmd_spec cmd_table[] = { - &main_vcpupin, 1, 1, - "Set which CPUs a VCPU can use", - "[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>", -- "-f, --force undo an override pinning done by the kernel", -+ "-f, --force undo an override pinning done by the kernel\n" -+ "--ignore-global-affinity-masks Ignore global masks in xl.conf", - }, - { "vcpu-set", - &main_vcpuset, 0, 1, -diff --git a/tools/xl/xl_vcpu.c b/tools/xl/xl_vcpu.c -index 8e735b38c1..3384eeed06 100644 ---- a/tools/xl/xl_vcpu.c -+++ b/tools/xl/xl_vcpu.c -@@ -68,6 +68,61 @@ static void print_domain_vcpuinfo(uint32_t domid, uint32_t nr_cpus) - libxl_vcpuinfo_list_free(vcpuinfo, nb_vcpu); - } - -+void apply_global_affinity_masks(libxl_domain_type type, -+ libxl_bitmap *vcpu_affinity_array, -+ unsigned int size) -+{ -+ libxl_bitmap *mask = &global_vm_affinity_mask; -+ libxl_bitmap *type_mask; -+ unsigned int i; -+ -+ switch (type) { -+ case LIBXL_DOMAIN_TYPE_HVM: -+ case LIBXL_DOMAIN_TYPE_PVH: -+ type_mask = &global_hvm_affinity_mask; -+ break; -+ case LIBXL_DOMAIN_TYPE_PV: -+ type_mask = &global_pv_affinity_mask; -+ break; -+ default: -+ fprintf(stderr, "Unknown guest type\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ for (i = 0; i < size; i++) { -+ int rc; -+ libxl_bitmap *t = &vcpu_affinity_array[i]; -+ libxl_bitmap b1, b2; -+ -+ libxl_bitmap_init(&b1); -+ libxl_bitmap_init(&b2); -+ -+ rc = libxl_bitmap_and(ctx, &b1, t, mask); -+ if (rc) { -+ fprintf(stderr, "libxl_bitmap_and errored\n"); -+ exit(EXIT_FAILURE); -+ } -+ rc = libxl_bitmap_and(ctx, &b2, &b1, type_mask); -+ if (rc) { -+ fprintf(stderr, "libxl_bitmap_and errored\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (libxl_bitmap_is_empty(&b2)) { -+ fprintf(stderr, "vcpu hard affinity map is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ /* Replace target bitmap with the result */ -+ libxl_bitmap_dispose(t); -+ libxl_bitmap_init(t); -+ libxl_bitmap_copy_alloc(ctx, t, &b2); -+ -+ libxl_bitmap_dispose(&b1); -+ libxl_bitmap_dispose(&b2); -+ } -+} -+ - static void vcpulist(int argc, char **argv) - { - libxl_dominfo *dominfo; -@@ -118,6 +173,7 @@ int main_vcpupin(int argc, char **argv) - { - static struct option opts[] = { - {"force", 0, 0, 'f'}, -+ {"ignore-global-affinity-masks", 0, 0, 'i'}, - COMMON_LONG_OPTS - }; - libxl_vcpuinfo *vcpuinfo; -@@ -132,15 +188,18 @@ int main_vcpupin(int argc, char **argv) - const char *vcpu, *hard_str, *soft_str; - char *endptr; - int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE; -- bool force = false; -+ bool force = false, ignore_masks = false; - - libxl_bitmap_init(&cpumap_hard); - libxl_bitmap_init(&cpumap_soft); - -- SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) { -+ SWITCH_FOREACH_OPT(opt, "fi", opts, "vcpu-pin", 3) { - case 'f': - force = true; - break; -+ case 'i': -+ ignore_masks = true; -+ break; - default: - break; - } -@@ -222,6 +281,23 @@ int main_vcpupin(int argc, char **argv) - goto out; - } - -+ /* Only hard affinity matters here */ -+ if (!ignore_masks) { -+ libxl_domain_config d_config; -+ -+ libxl_domain_config_init(&d_config); -+ rc = libxl_retrieve_domain_configuration(ctx, domid, &d_config); -+ if (rc) { -+ fprintf(stderr, "Could not retrieve domain configuration\n"); -+ libxl_domain_config_dispose(&d_config); -+ goto out; -+ } -+ -+ apply_global_affinity_masks(d_config.b_info.type, hard, 1); -+ -+ libxl_domain_config_dispose(&d_config); -+ } -+ - if (force) { - if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) { - fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n", -diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c -index 89c2b25ded..a1d633795c 100644 ---- a/tools/xl/xl_vmcontrol.c -+++ b/tools/xl/xl_vmcontrol.c -@@ -804,6 +804,36 @@ int create_domain(struct domain_create *dom_info) - parse_config_data(config_source, config_data, config_len, &d_config); - } - -+ if (!dom_info->ignore_global_affinity_masks) { -+ libxl_domain_build_info *b_info = &d_config.b_info; -+ -+ /* It is possible that no hard affinity is specified in config file. -+ * Generate hard affinity maps now if we care about those. -+ */ -+ if (b_info->num_vcpu_hard_affinity == 0 && -+ (!libxl_bitmap_is_full(&global_vm_affinity_mask) || -+ (b_info->type == LIBXL_DOMAIN_TYPE_PV && -+ !libxl_bitmap_is_full(&global_pv_affinity_mask)) || -+ (b_info->type != LIBXL_DOMAIN_TYPE_PV && -+ !libxl_bitmap_is_full(&global_hvm_affinity_mask)) -+ )) { -+ b_info->num_vcpu_hard_affinity = b_info->max_vcpus; -+ b_info->vcpu_hard_affinity = -+ xmalloc(b_info->max_vcpus * sizeof(libxl_bitmap)); -+ -+ for (i = 0; i < b_info->num_vcpu_hard_affinity; i++) { -+ libxl_bitmap *m = &b_info->vcpu_hard_affinity[i]; -+ libxl_bitmap_init(m); -+ libxl_cpu_bitmap_alloc(ctx, m, 0); -+ libxl_bitmap_set_any(m); -+ } -+ } -+ -+ apply_global_affinity_masks(b_info->type, -+ b_info->vcpu_hard_affinity, -+ b_info->num_vcpu_hard_affinity); -+ } -+ - if (migrate_fd >= 0) { - if (d_config.c_info.name) { - /* when we receive a domain we get its name from the config -@@ -1124,7 +1154,7 @@ int main_create(int argc, char **argv) - const char *filename = NULL; - struct domain_create dom_info; - int paused = 0, debug = 0, daemonize = 1, console_autoconnect = 0, -- quiet = 0, monitor = 1, vnc = 0, vncautopass = 0; -+ quiet = 0, monitor = 1, vnc = 0, vncautopass = 0, ignore_masks = 0; - int opt, rc; - static struct option opts[] = { - {"dryrun", 0, 0, 'n'}, -@@ -1132,6 +1162,7 @@ int main_create(int argc, char **argv) - {"defconfig", 1, 0, 'f'}, - {"vncviewer", 0, 0, 'V'}, - {"vncviewer-autopass", 0, 0, 'A'}, -+ {"ignore-global-affinity-masks", 0, 0, 'i'}, - COMMON_LONG_OPTS - }; - -@@ -1142,7 +1173,7 @@ int main_create(int argc, char **argv) - argc--; argv++; - } - -- SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVA", opts, "create", 0) { -+ SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVAi", opts, "create", 0) { - case 'f': - filename = optarg; - break; -@@ -1174,6 +1205,9 @@ int main_create(int argc, char **argv) - case 'A': - vnc = vncautopass = 1; - break; -+ case 'i': -+ ignore_masks = 1; -+ break; - } - - memset(&dom_info, 0, sizeof(dom_info)); -@@ -1203,6 +1237,7 @@ int main_create(int argc, char **argv) - dom_info.vnc = vnc; - dom_info.vncautopass = vncautopass; - dom_info.console_autoconnect = console_autoconnect; -+ dom_info.ignore_global_affinity_masks = ignore_masks; - - rc = create_domain(&dom_info); - if (rc < 0) { -diff --git a/xen/Makefile b/xen/Makefile -index ecec297b9b..580af86931 100644 ---- a/xen/Makefile -+++ b/xen/Makefile -@@ -2,7 +2,7 @@ - # All other places this is stored (eg. compile.h) should be autogenerated. - export XEN_VERSION = 4 - export XEN_SUBVERSION = 10 --export XEN_EXTRAVERSION ?= .1$(XEN_VENDORVERSION) -+export XEN_EXTRAVERSION ?= .2-pre$(XEN_VENDORVERSION) - export XEN_FULLVERSION = $(XEN_VERSION).$(XEN_SUBVERSION)$(XEN_EXTRAVERSION) - -include xen-version - -diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig -index f621e799ed..33535ca9aa 100644 ---- a/xen/arch/x86/Kconfig -+++ b/xen/arch/x86/Kconfig -@@ -71,6 +71,7 @@ config SHADOW_PAGING - * Running HVM guests on hardware lacking hardware paging support - (First-generation Intel VT-x or AMD SVM). - * Live migration of PV guests. -+ * L1TF sidechannel mitigation for PV guests. - - Under a small number of specific workloads, shadow paging may be - deliberately used as a performance optimisation. -diff --git a/xen/arch/x86/Rules.mk b/xen/arch/x86/Rules.mk -index 70e9d8f5cf..03b1b581f3 100644 ---- a/xen/arch/x86/Rules.mk -+++ b/xen/arch/x86/Rules.mk -@@ -23,6 +23,7 @@ $(call as-insn-check,CFLAGS,CC,"rdseed %eax",-DHAVE_GAS_RDSEED) - $(call as-insn-check,CFLAGS,CC,".equ \"x\"$$(comma)1", \ - -U__OBJECT_LABEL__ -DHAVE_GAS_QUOTED_SYM \ - '-D__OBJECT_LABEL__=$(subst $(BASEDIR)/,,$(CURDIR))/$$@') -+$(call as-insn-check,CFLAGS,CC,"invpcid (%rax)$$(comma)%rax",-DHAVE_AS_INVPCID) - - CFLAGS += -mno-red-zone -mno-sse -fpic - CFLAGS += -fno-asynchronous-unwind-tables -diff --git a/xen/arch/x86/acpi/power.c b/xen/arch/x86/acpi/power.c -index 1e4e5680a7..f3480aa800 100644 ---- a/xen/arch/x86/acpi/power.c -+++ b/xen/arch/x86/acpi/power.c -@@ -28,6 +28,7 @@ - #include <asm/tboot.h> - #include <asm/apic.h> - #include <asm/io_apic.h> -+#include <asm/spec_ctrl.h> - #include <acpi/cpufreq/cpufreq.h> - - uint32_t system_reset_counter = 1; -@@ -163,6 +164,7 @@ static int enter_state(u32 state) - { - unsigned long flags; - int error; -+ struct cpu_info *ci; - unsigned long cr4; - - if ( (state <= ACPI_STATE_S0) || (state > ACPI_S_STATES_MAX) ) -@@ -203,12 +205,18 @@ static int enter_state(u32 state) - printk(XENLOG_ERR "Some devices failed to power down."); - system_state = SYS_STATE_resume; - device_power_up(error); -+ console_end_sync(); - error = -EIO; - goto done; - } - else - error = 0; - -+ ci = get_cpu_info(); -+ spec_ctrl_enter_idle(ci); -+ /* Avoid NMI/#MC using MSR_SPEC_CTRL until we've reloaded microcode. */ -+ ci->spec_ctrl_flags &= ~SCF_ist_wrmsr; -+ - ACPI_FLUSH_CPU_CACHE(); - - switch ( state ) -@@ -243,17 +251,23 @@ static int enter_state(u32 state) - if ( (state == ACPI_STATE_S3) && error ) - tboot_s3_error(error); - -+ console_end_sync(); -+ -+ microcode_resume_cpu(0); -+ -+ /* Re-enabled default NMI/#MC use of MSR_SPEC_CTRL. */ -+ ci->spec_ctrl_flags |= (default_spec_ctrl_flags & SCF_ist_wrmsr); -+ spec_ctrl_exit_idle(ci); -+ - done: - spin_debug_enable(); - local_irq_restore(flags); -- console_end_sync(); - acpi_sleep_post(state); - if ( hvm_cpu_up() ) - BUG(); -+ cpufreq_add_cpu(0); - - enable_cpu: -- cpufreq_add_cpu(0); -- microcode_resume_cpu(0); - rcu_barrier(); - mtrr_aps_sync_begin(); - enable_nonboot_cpus(); -diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c -index fc9677f020..76078b55b2 100644 ---- a/xen/arch/x86/cpu/amd.c -+++ b/xen/arch/x86/cpu/amd.c -@@ -9,6 +9,7 @@ - #include <asm/amd.h> - #include <asm/hvm/support.h> - #include <asm/setup.h> /* amd_init_cpu */ -+#include <asm/spec_ctrl.h> - #include <asm/acpi.h> - #include <asm/apic.h> - -@@ -504,17 +505,23 @@ static void amd_get_topology(struct cpuinfo_x86 *c) - u32 eax, ebx, ecx, edx; - - cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); -- c->compute_unit_id = ebx & 0xFF; - c->x86_num_siblings = ((ebx >> 8) & 0x3) + 1; -+ -+ if (c->x86 < 0x17) -+ c->compute_unit_id = ebx & 0xFF; -+ else { -+ c->cpu_core_id = ebx & 0xFF; -+ c->x86_max_cores /= c->x86_num_siblings; -+ } - } - - if (opt_cpu_info) - printk("CPU %d(%d) -> Processor %d, %s %d\n", - cpu, c->x86_max_cores, c->phys_proc_id, -- cpu_has(c, X86_FEATURE_TOPOEXT) ? "Compute Unit" : -- "Core", -- cpu_has(c, X86_FEATURE_TOPOEXT) ? c->compute_unit_id : -- c->cpu_core_id); -+ c->compute_unit_id != INVALID_CUID ? "Compute Unit" -+ : "Core", -+ c->compute_unit_id != INVALID_CUID ? c->compute_unit_id -+ : c->cpu_core_id); - } - - static void early_init_amd(struct cpuinfo_x86 *c) -@@ -594,6 +601,25 @@ static void init_amd(struct cpuinfo_x86 *c) - c->x86_capability); - } - -+ /* -+ * If the user has explicitly chosen to disable Memory Disambiguation -+ * to mitigiate Speculative Store Bypass, poke the appropriate MSR. -+ */ -+ if (opt_ssbd) { -+ int bit = -1; -+ -+ switch (c->x86) { -+ case 0x15: bit = 54; break; -+ case 0x16: bit = 33; break; -+ case 0x17: bit = 10; break; -+ } -+ -+ if (bit >= 0 && !rdmsr_safe(MSR_AMD64_LS_CFG, value)) { -+ value |= 1ull << bit; -+ wrmsr_safe(MSR_AMD64_LS_CFG, value); -+ } -+ } -+ - /* MFENCE stops RDTSC speculation */ - if (!cpu_has_lfence_dispatch) - __set_bit(X86_FEATURE_MFENCE_RDTSC, c->x86_capability); -diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c -index fdb2bf1779..eb266c5ba6 100644 ---- a/xen/arch/x86/cpu/common.c -+++ b/xen/arch/x86/cpu/common.c -@@ -14,6 +14,7 @@ - #include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */ - - #include "cpu.h" -+#include "mcheck/x86_mca.h" - - bool_t opt_arat = 1; - boolean_param("arat", opt_arat); -@@ -345,6 +346,9 @@ static void __init early_cpu_detect(void) - hap_paddr_bits = PADDR_BITS; - } - -+ if (c->x86_vendor != X86_VENDOR_AMD) -+ park_offline_cpus = opt_mce; -+ - initialize_cpu_data(0); - } - -@@ -747,6 +751,7 @@ void load_system_tables(void) - [IST_MCE - 1] = stack_top + IST_MCE * PAGE_SIZE, - [IST_DF - 1] = stack_top + IST_DF * PAGE_SIZE, - [IST_NMI - 1] = stack_top + IST_NMI * PAGE_SIZE, -+ [IST_DB - 1] = stack_top + IST_DB * PAGE_SIZE, - - [IST_MAX ... ARRAY_SIZE(tss->ist) - 1] = - 0x8600111111111111ul, -@@ -774,6 +779,7 @@ void load_system_tables(void) - set_ist(&idt_tables[cpu][TRAP_double_fault], IST_DF); - set_ist(&idt_tables[cpu][TRAP_nmi], IST_NMI); - set_ist(&idt_tables[cpu][TRAP_machine_check], IST_MCE); -+ set_ist(&idt_tables[cpu][TRAP_debug], IST_DB); - - /* - * Bottom-of-stack must be 16-byte aligned! -diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c -index df0106ec3c..1ee1870de9 100644 ---- a/xen/arch/x86/cpu/mcheck/mce.c -+++ b/xen/arch/x86/cpu/mcheck/mce.c -@@ -695,12 +695,15 @@ static void cpu_bank_free(unsigned int cpu) - - mcabanks_free(poll); - mcabanks_free(clr); -+ -+ per_cpu(poll_bankmask, cpu) = NULL; -+ per_cpu(mce_clear_banks, cpu) = NULL; - } - - static int cpu_bank_alloc(unsigned int cpu) - { -- struct mca_banks *poll = mcabanks_alloc(); -- struct mca_banks *clr = mcabanks_alloc(); -+ struct mca_banks *poll = per_cpu(poll_bankmask, cpu) ?: mcabanks_alloc(); -+ struct mca_banks *clr = per_cpu(mce_clear_banks, cpu) ?: mcabanks_alloc(); - - if ( !poll || !clr ) - { -@@ -728,7 +731,13 @@ static int cpu_callback( - - case CPU_UP_CANCELED: - case CPU_DEAD: -- cpu_bank_free(cpu); -+ if ( !park_offline_cpus ) -+ cpu_bank_free(cpu); -+ break; -+ -+ case CPU_REMOVE: -+ if ( park_offline_cpus ) -+ cpu_bank_free(cpu); - break; - } - -diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c -index e5dd956a24..4474a34e34 100644 ---- a/xen/arch/x86/cpu/mcheck/mce_intel.c -+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c -@@ -636,8 +636,6 @@ static void clear_cmci(void) - - static void cpu_mcheck_disable(void) - { -- clear_in_cr4(X86_CR4_MCE); -- - if ( cmci_support && opt_mce ) - clear_cmci(); - } -diff --git a/xen/arch/x86/cpu/mtrr/generic.c b/xen/arch/x86/cpu/mtrr/generic.c -index e9c0e5e059..7ba0c3f0fe 100644 ---- a/xen/arch/x86/cpu/mtrr/generic.c -+++ b/xen/arch/x86/cpu/mtrr/generic.c -@@ -5,6 +5,7 @@ - #include <xen/mm.h> - #include <xen/stdbool.h> - #include <asm/flushtlb.h> -+#include <asm/invpcid.h> - #include <asm/io.h> - #include <asm/mtrr.h> - #include <asm/msr.h> -@@ -400,8 +401,10 @@ static DEFINE_SPINLOCK(set_atomicity_lock); - * has been called. - */ - --static void prepare_set(void) -+static bool prepare_set(void) - { -+ unsigned long cr4; -+ - /* Note that this is not ideal, since the cache is only flushed/disabled - for this CPU while the MTRRs are changed, but changing this requires - more invasive changes to the way the kernel boots */ -@@ -412,18 +415,24 @@ static void prepare_set(void) - write_cr0(read_cr0() | X86_CR0_CD); - wbinvd(); - -- /* TLB flushing here relies on Xen always using CR4.PGE. */ -- BUILD_BUG_ON(!(XEN_MINIMAL_CR4 & X86_CR4_PGE)); -- write_cr4(read_cr4() & ~X86_CR4_PGE); -+ cr4 = read_cr4(); -+ if (cr4 & X86_CR4_PGE) -+ write_cr4(cr4 & ~X86_CR4_PGE); -+ else if (use_invpcid) -+ invpcid_flush_all(); -+ else -+ write_cr3(read_cr3()); - - /* Save MTRR state */ - rdmsrl(MSR_MTRRdefType, deftype); - - /* Disable MTRRs, and set the default type to uncached */ - mtrr_wrmsr(MSR_MTRRdefType, deftype & ~0xcff); -+ -+ return cr4 & X86_CR4_PGE; - } - --static void post_set(void) -+static void post_set(bool pge) - { - /* Intel (P6) standard MTRRs */ - mtrr_wrmsr(MSR_MTRRdefType, deftype); -@@ -432,7 +441,12 @@ static void post_set(void) - write_cr0(read_cr0() & ~X86_CR0_CD); - - /* Reenable CR4.PGE (also flushes the TLB) */ -- write_cr4(read_cr4() | X86_CR4_PGE); -+ if (pge) -+ write_cr4(read_cr4() | X86_CR4_PGE); -+ else if (use_invpcid) -+ invpcid_flush_all(); -+ else -+ write_cr3(read_cr3()); - - spin_unlock(&set_atomicity_lock); - } -@@ -441,14 +455,15 @@ static void generic_set_all(void) - { - unsigned long mask, count; - unsigned long flags; -+ bool pge; - - local_irq_save(flags); -- prepare_set(); -+ pge = prepare_set(); - - /* Actually set the state */ - mask = set_mtrr_state(); - -- post_set(); -+ post_set(pge); - local_irq_restore(flags); - - /* Use the atomic bitops to update the global mask */ -@@ -457,7 +472,6 @@ static void generic_set_all(void) - set_bit(count, &smp_changes_mask); - mask >>= 1; - } -- - } - - static void generic_set_mtrr(unsigned int reg, unsigned long base, -@@ -474,11 +488,12 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base, - { - unsigned long flags; - struct mtrr_var_range *vr; -+ bool pge; - - vr = &mtrr_state.var_ranges[reg]; - - local_irq_save(flags); -- prepare_set(); -+ pge = prepare_set(); - - if (size == 0) { - /* The invalid bit is kept in the mask, so we simply clear the -@@ -499,7 +514,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base, - mtrr_wrmsr(MSR_IA32_MTRR_PHYSMASK(reg), vr->mask); - } - -- post_set(); -+ post_set(pge); - local_irq_restore(flags); - } - -diff --git a/xen/arch/x86/cpu/vpmu_intel.c b/xen/arch/x86/cpu/vpmu_intel.c -index 207e2e712c..6e27f6ec8e 100644 ---- a/xen/arch/x86/cpu/vpmu_intel.c -+++ b/xen/arch/x86/cpu/vpmu_intel.c -@@ -454,13 +454,11 @@ static int core2_vpmu_alloc_resource(struct vcpu *v) - - if ( is_hvm_vcpu(v) ) - { -- wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); -- if ( vmx_add_host_load_msr(MSR_CORE_PERF_GLOBAL_CTRL) ) -+ if ( vmx_add_host_load_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) ) - goto out_err; - -- if ( vmx_add_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL) ) -+ if ( vmx_add_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) ) - goto out_err; -- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, 0); - } - - core2_vpmu_cxt = xzalloc_bytes(sizeof(*core2_vpmu_cxt) + -@@ -535,27 +533,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - uint64_t *enabled_cntrs; - - if ( !core2_vpmu_msr_common_check(msr, &type, &index) ) -- { -- /* Special handling for BTS */ -- if ( msr == MSR_IA32_DEBUGCTLMSR ) -- { -- supported |= IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS | -- IA32_DEBUGCTLMSR_BTINT; -- -- if ( cpu_has(¤t_cpu_data, X86_FEATURE_DSCPL) ) -- supported |= IA32_DEBUGCTLMSR_BTS_OFF_OS | -- IA32_DEBUGCTLMSR_BTS_OFF_USR; -- if ( !(msr_content & ~supported) && -- vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) ) -- return 0; -- if ( (msr_content & supported) && -- !vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) ) -- printk(XENLOG_G_WARNING -- "%pv: Debug Store unsupported on this CPU\n", -- current); -- } - return -EINVAL; -- } - - ASSERT(!supported); - -@@ -613,7 +591,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - return -EINVAL; - - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, - &core2_vpmu_cxt->global_ctrl); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl); -@@ -682,7 +660,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - return -EINVAL; - - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, - &core2_vpmu_cxt->global_ctrl); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl); -@@ -701,7 +679,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - else - { - if ( is_hvm_vcpu(v) ) -- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); -+ vmx_write_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - else - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - } -@@ -735,7 +713,7 @@ static int core2_vpmu_do_rdmsr(unsigned int msr, uint64_t *msr_content) - break; - case MSR_CORE_PERF_GLOBAL_CTRL: - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, *msr_content); - break; -diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c -index b3c9ac6c48..24b9495faa 100644 ---- a/xen/arch/x86/cpuid.c -+++ b/xen/arch/x86/cpuid.c -@@ -43,6 +43,16 @@ static int __init parse_xen_cpuid(const char *s) - if ( !val ) - setup_clear_cpu_cap(X86_FEATURE_STIBP); - } -+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 ) -+ { -+ if ( !val ) -+ setup_clear_cpu_cap(X86_FEATURE_L1D_FLUSH); -+ } -+ else if ( (val = parse_boolean("ssbd", s, ss)) >= 0 ) -+ { -+ if ( !val ) -+ setup_clear_cpu_cap(X86_FEATURE_SSBD); -+ } - else - rc = -EINVAL; - -@@ -368,6 +378,28 @@ static void __init calculate_host_policy(void) - } - } - -+static void __init guest_common_feature_adjustments(uint32_t *fs) -+{ -+ /* Unconditionally claim to be able to set the hypervisor bit. */ -+ __set_bit(X86_FEATURE_HYPERVISOR, fs); -+ -+ /* -+ * If IBRS is offered to the guest, unconditionally offer STIBP. It is a -+ * nop on non-HT hardware, and has this behaviour to make heterogeneous -+ * setups easier to manage. -+ */ -+ if ( test_bit(X86_FEATURE_IBRSB, fs) ) -+ __set_bit(X86_FEATURE_STIBP, fs); -+ -+ /* -+ * On hardware which supports IBRS/IBPB, we can offer IBPB independently -+ * of IBRS by using the AMD feature bit. An administrator may wish for -+ * performance reasons to offer IBPB without IBRS. -+ */ -+ if ( host_cpuid_policy.feat.ibrsb ) -+ __set_bit(X86_FEATURE_IBPB, fs); -+} -+ - static void __init calculate_pv_max_policy(void) - { - struct cpuid_policy *p = &pv_max_cpuid_policy; -@@ -380,18 +412,14 @@ static void __init calculate_pv_max_policy(void) - for ( i = 0; i < ARRAY_SIZE(pv_featureset); ++i ) - pv_featureset[i] &= pv_featuremask[i]; - -- /* Unconditionally claim to be able to set the hypervisor bit. */ -- __set_bit(X86_FEATURE_HYPERVISOR, pv_featureset); -- -- /* On hardware with IBRS/IBPB support, there are further adjustments. */ -- if ( test_bit(X86_FEATURE_IBRSB, pv_featureset) ) -- { -- /* Offer STIBP unconditionally. It is a nop on non-HT hardware. */ -- __set_bit(X86_FEATURE_STIBP, pv_featureset); -+ /* -+ * If Xen isn't virtualising MSR_SPEC_CTRL for PV guests because of -+ * administrator choice, hide the feature. -+ */ -+ if ( !boot_cpu_has(X86_FEATURE_SC_MSR_PV) ) -+ __clear_bit(X86_FEATURE_IBRSB, pv_featureset); - -- /* AMD's IBPB is a subset of IBRS/IBPB. */ -- __set_bit(X86_FEATURE_IBPB, pv_featureset); -- } -+ guest_common_feature_adjustments(pv_featureset); - - sanitise_featureset(pv_featureset); - cpuid_featureset_to_policy(pv_featureset, p); -@@ -419,9 +447,6 @@ static void __init calculate_hvm_max_policy(void) - for ( i = 0; i < ARRAY_SIZE(hvm_featureset); ++i ) - hvm_featureset[i] &= hvm_featuremask[i]; - -- /* Unconditionally claim to be able to set the hypervisor bit. */ -- __set_bit(X86_FEATURE_HYPERVISOR, hvm_featureset); -- - /* - * Xen can provide an APIC emulation to HVM guests even if the host's APIC - * isn't enabled. -@@ -437,6 +462,13 @@ static void __init calculate_hvm_max_policy(void) - raw_cpuid_policy.basic.sep ) - __set_bit(X86_FEATURE_SEP, hvm_featureset); - -+ /* -+ * If Xen isn't virtualising MSR_SPEC_CTRL for HVM guests because of -+ * administrator choice, hide the feature. -+ */ -+ if ( !boot_cpu_has(X86_FEATURE_SC_MSR_HVM) ) -+ __clear_bit(X86_FEATURE_IBRSB, hvm_featureset); -+ - /* - * With VT-x, some features are only supported by Xen if dedicated - * hardware support is also available. -@@ -450,15 +482,7 @@ static void __init calculate_hvm_max_policy(void) - __clear_bit(X86_FEATURE_XSAVES, hvm_featureset); - } - -- /* On hardware with IBRS/IBPB support, there are further adjustments. */ -- if ( test_bit(X86_FEATURE_IBRSB, hvm_featureset) ) -- { -- /* Offer STIBP unconditionally. It is a nop on non-HT hardware. */ -- __set_bit(X86_FEATURE_STIBP, hvm_featureset); -- -- /* AMD's IBPB is a subset of IBRS/IBPB. */ -- __set_bit(X86_FEATURE_IBPB, hvm_featureset); -- } -+ guest_common_feature_adjustments(hvm_featureset); - - sanitise_featureset(hvm_featureset); - cpuid_featureset_to_policy(hvm_featureset, p); -@@ -601,14 +625,6 @@ void recalculate_cpuid_policy(struct domain *d) - recalculate_xstate(p); - recalculate_misc(p); - -- /* -- * Override STIBP to match IBRS. Guests can safely use STIBP -- * functionality on non-HT hardware, but can't necesserily protect -- * themselves from SP2/Spectre/Branch Target Injection if STIBP is hidden -- * on HT-capable hardware. -- */ -- p->feat.stibp = p->feat.ibrsb; -- - for ( i = 0; i < ARRAY_SIZE(p->cache.raw); ++i ) - { - if ( p->cache.subleaf[i].type >= 1 && -diff --git a/xen/arch/x86/debug.c b/xen/arch/x86/debug.c -index 9159f32db4..a500df01ac 100644 ---- a/xen/arch/x86/debug.c -+++ b/xen/arch/x86/debug.c -@@ -98,7 +98,7 @@ dbg_pv_va2mfn(dbgva_t vaddr, struct domain *dp, uint64_t pgd3val) - l2_pgentry_t l2e, *l2t; - l1_pgentry_t l1e, *l1t; - unsigned long cr3 = (pgd3val ? pgd3val : dp->vcpu[0]->arch.cr3); -- mfn_t mfn = maddr_to_mfn(cr3); -+ mfn_t mfn = maddr_to_mfn(cr3_pa(cr3)); - - DBGP2("vaddr:%lx domid:%d cr3:%lx pgd3:%lx\n", vaddr, dp->domain_id, - cr3, pgd3val); -diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c -index 5a4d5c3bfc..2020e0b682 100644 ---- a/xen/arch/x86/domain.c -+++ b/xen/arch/x86/domain.c -@@ -107,10 +107,11 @@ static void play_dead(void) - local_irq_disable(); - - /* -- * NOTE: After cpu_exit_clear, per-cpu variables are no longer accessible, -- * as they may be freed at any time. In this case, heap corruption or -- * #PF can occur (when heap debugging is enabled). For example, even -- * printk() can involve tasklet scheduling, which touches per-cpu vars. -+ * NOTE: After cpu_exit_clear, per-cpu variables may no longer accessible, -+ * as they may be freed at any time if offline CPUs don't get parked. In -+ * this case, heap corruption or #PF can occur (when heap debugging is -+ * enabled). For example, even printk() can involve tasklet scheduling, -+ * which touches per-cpu vars. - * - * Consider very carefully when adding code to *dead_idle. Most hypervisor - * subsystems are unsafe to call. -@@ -1517,17 +1518,12 @@ void paravirt_ctxt_switch_from(struct vcpu *v) - void paravirt_ctxt_switch_to(struct vcpu *v) - { - root_pgentry_t *root_pgt = this_cpu(root_pgt); -- unsigned long cr4; - - if ( root_pgt ) - root_pgt[root_table_offset(PERDOMAIN_VIRT_START)] = - l4e_from_page(v->domain->arch.perdomain_l3_pg, - __PAGE_HYPERVISOR_RW); - -- cr4 = pv_guest_cr4_to_real_cr4(v); -- if ( unlikely(cr4 != read_cr4()) ) -- write_cr4(cr4); -- - if ( unlikely(v->arch.debugreg[7] & DR7_ACTIVE_MASK) ) - activate_debugregs(v); - -@@ -1640,7 +1636,7 @@ static void __context_switch(void) - if ( cpu_has_xsaves && is_hvm_vcpu(n) ) - set_msr_xss(n->arch.hvm_vcpu.msr_xss); - } -- vcpu_restore_fpu_eager(n); -+ vcpu_restore_fpu_nonlazy(n, false); - nd->arch.ctxt_switch->to(n); - } - -@@ -1693,6 +1689,7 @@ void context_switch(struct vcpu *prev, struct vcpu *next) - - ASSERT(local_irq_is_enabled()); - -+ get_cpu_info()->use_pv_cr3 = false; - get_cpu_info()->xen_cr3 = 0; - - cpumask_copy(&dirty_mask, next->vcpu_dirty_cpumask); -diff --git a/xen/arch/x86/domain_page.c b/xen/arch/x86/domain_page.c -index 3432a854dd..9a52276866 100644 ---- a/xen/arch/x86/domain_page.c -+++ b/xen/arch/x86/domain_page.c -@@ -51,7 +51,7 @@ static inline struct vcpu *mapcache_current_vcpu(void) - if ( (v = idle_vcpu[smp_processor_id()]) == current ) - sync_local_execstate(); - /* We must now be running on the idle page table. */ -- ASSERT(read_cr3() == __pa(idle_pg_table)); -+ ASSERT(cr3_pa(read_cr3()) == __pa(idle_pg_table)); - } - - return v; -diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c -index 74e9749d89..eefd94eb92 100644 ---- a/xen/arch/x86/domctl.c -+++ b/xen/arch/x86/domctl.c -@@ -226,7 +226,8 @@ static int update_domain_cpuid_info(struct domain *d, - */ - call_policy_changed = (is_hvm_domain(d) && - ((old_7d0 ^ p->feat.raw[0].d) & -- cpufeat_mask(X86_FEATURE_IBRSB))); -+ (cpufeat_mask(X86_FEATURE_IBRSB) | -+ cpufeat_mask(X86_FEATURE_L1D_FLUSH)))); - break; - - case 0xa: -@@ -1227,7 +1228,7 @@ long arch_do_domctl( - if ( _xcr0_accum ) - { - if ( evc->size >= PV_XSAVE_HDR_SIZE + XSTATE_AREA_MIN_SIZE ) -- ret = validate_xstate(_xcr0, _xcr0_accum, -+ ret = validate_xstate(d, _xcr0, _xcr0_accum, - &_xsave_area->xsave_hdr); - } - else if ( !_xcr0 ) -@@ -1251,8 +1252,7 @@ long arch_do_domctl( - vcpu_pause(v); - v->arch.xcr0 = _xcr0; - v->arch.xcr0_accum = _xcr0_accum; -- if ( _xcr0_accum & XSTATE_NONLAZY ) -- v->arch.nonlazy_xstate_used = 1; -+ v->arch.nonlazy_xstate_used = _xcr0_accum & XSTATE_NONLAZY; - compress_xsave_states(v, _xsave_area, - evc->size - PV_XSAVE_HDR_SIZE); - vcpu_unpause(v); -diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c -index f6d7ad1650..797c5d52cc 100644 ---- a/xen/arch/x86/flushtlb.c -+++ b/xen/arch/x86/flushtlb.c -@@ -8,9 +8,12 @@ - */ - - #include <xen/sched.h> -+#include <xen/smp.h> - #include <xen/softirq.h> - #include <asm/flushtlb.h> -+#include <asm/invpcid.h> - #include <asm/page.h> -+#include <asm/pv/domain.h> - - /* Debug builds: Wrap frequently to stress-test the wrap logic. */ - #ifdef NDEBUG -@@ -49,6 +52,8 @@ static u32 pre_flush(void) - raise_softirq(NEW_TLBFLUSH_CLOCK_PERIOD_SOFTIRQ); - - skip_clocktick: -+ hvm_flush_guest_tlbs(); -+ - return t2; - } - -@@ -69,21 +74,65 @@ static void post_flush(u32 t) - this_cpu(tlbflush_time) = t; - } - --void write_cr3(unsigned long cr3) -+static void do_tlb_flush(void) -+{ -+ u32 t = pre_flush(); -+ -+ if ( use_invpcid ) -+ invpcid_flush_all(); -+ else -+ { -+ unsigned long cr4 = read_cr4(); -+ -+ write_cr4(cr4 ^ X86_CR4_PGE); -+ write_cr4(cr4); -+ } -+ -+ post_flush(t); -+} -+ -+void switch_cr3_cr4(unsigned long cr3, unsigned long cr4) - { -- unsigned long flags, cr4 = read_cr4(); -+ unsigned long flags, old_cr4; - u32 t; -+ unsigned long old_pcid = cr3_pcid(read_cr3()); - - /* This non-reentrant function is sometimes called in interrupt context. */ - local_irq_save(flags); - - t = pre_flush(); - -- hvm_flush_guest_tlbs(); -- -- write_cr4(cr4 & ~X86_CR4_PGE); -- asm volatile ( "mov %0, %%cr3" : : "r" (cr3) : "memory" ); -- write_cr4(cr4); -+ old_cr4 = read_cr4(); -+ if ( old_cr4 & X86_CR4_PGE ) -+ { -+ /* -+ * X86_CR4_PGE set means PCID is inactive. -+ * We have to purge the TLB via flipping cr4.pge. -+ */ -+ old_cr4 = cr4 & ~X86_CR4_PGE; -+ write_cr4(old_cr4); -+ } -+ else if ( use_invpcid ) -+ /* -+ * Flushing the TLB via INVPCID is necessary only in case PCIDs are -+ * in use, which is true only with INVPCID being available. -+ * Without PCID usage the following write_cr3() will purge the TLB -+ * (we are in the cr4.pge off path) of all entries. -+ * Using invpcid_flush_all_nonglobals() seems to be faster than -+ * invpcid_flush_all(), so use that. -+ */ -+ invpcid_flush_all_nonglobals(); -+ -+ write_cr3(cr3); -+ -+ if ( old_cr4 != cr4 ) -+ write_cr4(cr4); -+ else if ( old_pcid != cr3_pcid(cr3) ) -+ /* -+ * Make sure no TLB entries related to the old PCID created between -+ * flushing the TLB and writing the new %cr3 value remain in the TLB. -+ */ -+ invpcid_flush_single_context(old_pcid); - - post_flush(t); - -@@ -113,22 +162,32 @@ unsigned int flush_area_local(const void *va, unsigned int flags) - * are various errata surrounding INVLPG usage on superpages, and - * a full flush is in any case not *that* expensive. - */ -- asm volatile ( "invlpg %0" -- : : "m" (*(const char *)(va)) : "memory" ); -+ if ( read_cr4() & X86_CR4_PCIDE ) -+ { -+ unsigned long addr = (unsigned long)va; -+ -+ /* -+ * Flush the addresses for all potential address spaces. -+ * We can't check the current domain for being subject to -+ * XPTI as current might be the idle vcpu while we still have -+ * some XPTI domain TLB entries. -+ * Using invpcid is okay here, as with PCID enabled we always -+ * have global pages disabled. -+ */ -+ invpcid_flush_one(PCID_PV_PRIV, addr); -+ invpcid_flush_one(PCID_PV_USER, addr); -+ if ( !cpu_has_no_xpti ) -+ { -+ invpcid_flush_one(PCID_PV_PRIV | PCID_PV_XPTI, addr); -+ invpcid_flush_one(PCID_PV_USER | PCID_PV_XPTI, addr); -+ } -+ } -+ else -+ asm volatile ( "invlpg %0" -+ : : "m" (*(const char *)(va)) : "memory" ); - } - else -- { -- u32 t = pre_flush(); -- unsigned long cr4 = read_cr4(); -- -- hvm_flush_guest_tlbs(); -- -- write_cr4(cr4 & ~X86_CR4_PGE); -- barrier(); -- write_cr4(cr4); -- -- post_flush(t); -- } -+ do_tlb_flush(); - } - - if ( flags & FLUSH_CACHE ) -@@ -161,5 +220,8 @@ unsigned int flush_area_local(const void *va, unsigned int flags) - - local_irq_restore(irqfl); - -+ if ( flags & FLUSH_ROOT_PGTBL ) -+ get_cpu_info()->root_pgt_changed = true; -+ - return flags; - } -diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c -index 5fffb317d9..4fb9a2225d 100644 ---- a/xen/arch/x86/genapic/x2apic.c -+++ b/xen/arch/x86/genapic/x2apic.c -@@ -201,18 +201,21 @@ static int update_clusterinfo( - if ( !cluster_cpus_spare ) - cluster_cpus_spare = xzalloc(cpumask_t); - if ( !cluster_cpus_spare || -- !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) ) -+ !cond_alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) ) - err = -ENOMEM; - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -+ case CPU_REMOVE: -+ if ( park_offline_cpus == (action != CPU_REMOVE) ) -+ break; - if ( per_cpu(cluster_cpus, cpu) ) - { - cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu)); - if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) ) -- xfree(per_cpu(cluster_cpus, cpu)); -+ XFREE(per_cpu(cluster_cpus, cpu)); - } -- free_cpumask_var(per_cpu(scratch_mask, cpu)); -+ FREE_CPUMASK_VAR(per_cpu(scratch_mask, cpu)); - break; - } - -diff --git a/xen/arch/x86/hpet.c b/xen/arch/x86/hpet.c -index 8229c635e4..f18cbbd55a 100644 ---- a/xen/arch/x86/hpet.c -+++ b/xen/arch/x86/hpet.c -@@ -509,6 +509,8 @@ static void hpet_attach_channel(unsigned int cpu, - static void hpet_detach_channel(unsigned int cpu, - struct hpet_event_channel *ch) - { -+ unsigned int next; -+ - spin_lock_irq(&ch->lock); - - ASSERT(ch == per_cpu(cpu_bc_channel, cpu)); -@@ -517,7 +519,7 @@ static void hpet_detach_channel(unsigned int cpu, - - if ( cpu != ch->cpu ) - spin_unlock_irq(&ch->lock); -- else if ( cpumask_empty(ch->cpumask) ) -+ else if ( (next = cpumask_first(ch->cpumask)) >= nr_cpu_ids ) - { - ch->cpu = -1; - clear_bit(HPET_EVT_USED_BIT, &ch->flags); -@@ -525,7 +527,7 @@ static void hpet_detach_channel(unsigned int cpu, - } - else - { -- ch->cpu = cpumask_first(ch->cpumask); -+ ch->cpu = next; - set_channel_irq_affinity(ch); - local_irq_enable(); - } -diff --git a/xen/arch/x86/hvm/emulate.c b/xen/arch/x86/hvm/emulate.c -index b282089e03..03db6b3d31 100644 ---- a/xen/arch/x86/hvm/emulate.c -+++ b/xen/arch/x86/hvm/emulate.c -@@ -1900,6 +1900,7 @@ static int hvmemul_get_fpu( - * masking of all exceptions by FNSTENV.) - */ - save_fpu_enable(); -+ curr->fpu_initialised = true; - curr->fpu_dirtied = true; - if ( (fpu_ctxt->fcw & 0x3f) != 0x3f ) - { -@@ -1991,13 +1992,20 @@ static void hvmemul_put_fpu( - if ( backout == X86EMUL_FPU_fpu ) - { - /* -- * To back out changes to the register file simply adjust state such -- * that upon next FPU insn use by the guest we'll reload the state -- * saved (or freshly loaded) by hvmemul_get_fpu(). -+ * To back out changes to the register file -+ * - in fully eager mode, restore original state immediately, -+ * - in lazy mode, simply adjust state such that upon next FPU insn -+ * use by the guest we'll reload the state saved (or freshly loaded) -+ * by hvmemul_get_fpu(). - */ -- curr->fpu_dirtied = false; -- stts(); -- hvm_funcs.fpu_leave(curr); -+ if ( curr->arch.fully_eager_fpu ) -+ vcpu_restore_fpu_nonlazy(curr, false); -+ else -+ { -+ curr->fpu_dirtied = false; -+ stts(); -+ hvm_funcs.fpu_leave(curr); -+ } - } - } - -@@ -2113,22 +2121,20 @@ static int _hvm_emulate_one(struct hvm_emulate_ctxt *hvmemul_ctxt, - - vio->mmio_retry = 0; - -- switch ( rc = x86_emulate(&hvmemul_ctxt->ctxt, ops) ) -+ rc = x86_emulate(&hvmemul_ctxt->ctxt, ops); -+ if ( rc == X86EMUL_OKAY && vio->mmio_retry ) -+ rc = X86EMUL_RETRY; -+ -+ if ( !hvm_vcpu_io_need_completion(vio) ) - { -- case X86EMUL_OKAY: -- if ( vio->mmio_retry ) -- rc = X86EMUL_RETRY; -- /* fall through */ -- default: - vio->mmio_cache_count = 0; - vio->mmio_insn_bytes = 0; -- break; -- -- case X86EMUL_RETRY: -+ } -+ else -+ { - BUILD_BUG_ON(sizeof(vio->mmio_insn) < sizeof(hvmemul_ctxt->insn_buf)); - vio->mmio_insn_bytes = hvmemul_ctxt->insn_buf_bytes; - memcpy(vio->mmio_insn, hvmemul_ctxt->insn_buf, vio->mmio_insn_bytes); -- break; - } - - if ( hvmemul_ctxt->ctxt.retire.singlestep ) -diff --git a/xen/arch/x86/hvm/hpet.c b/xen/arch/x86/hvm/hpet.c -index f7aed7f69e..28377091ca 100644 ---- a/xen/arch/x86/hvm/hpet.c -+++ b/xen/arch/x86/hvm/hpet.c -@@ -264,13 +264,20 @@ static void hpet_set_timer(HPETState *h, unsigned int tn, - diff = (timer_is_32bit(h, tn) && (-diff > HPET_TINY_TIME_SPAN)) - ? (uint32_t)diff : 0; - -+ destroy_periodic_time(&h->pt[tn]); - if ( (tn <= 1) && (h->hpet.config & HPET_CFG_LEGACY) ) -+ { - /* if LegacyReplacementRoute bit is set, HPET specification requires - timer0 be routed to IRQ0 in NON-APIC or IRQ2 in the I/O APIC, - timer1 be routed to IRQ8 in NON-APIC or IRQ8 in the I/O APIC. */ - irq = (tn == 0) ? 0 : 8; -+ h->pt[tn].source = PTSRC_isa; -+ } - else -+ { - irq = timer_int_route(h, tn); -+ h->pt[tn].source = PTSRC_ioapic; -+ } - - /* - * diff is the time from now when the timer should fire, for a periodic -diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c -index 18adec5ad8..8e237eb1ac 100644 ---- a/xen/arch/x86/hvm/hvm.c -+++ b/xen/arch/x86/hvm/hvm.c -@@ -895,6 +895,9 @@ const char *hvm_efer_valid(const struct vcpu *v, uint64_t value, - else - p = &host_cpuid_policy; - -+ if ( value & ~EFER_KNOWN_MASK ) -+ return "Unknown bits set"; -+ - if ( (value & EFER_SCE) && !p->extd.syscall ) - return "SCE without feature"; - -@@ -1258,7 +1261,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h) - ctxt = (struct hvm_hw_cpu_xsave *)&h->data[h->cur]; - h->cur += desc->length; - -- err = validate_xstate(ctxt->xcr0, ctxt->xcr0_accum, -+ err = validate_xstate(d, ctxt->xcr0, ctxt->xcr0_accum, - (const void *)&ctxt->save_area.xsave_hdr); - if ( err ) - { -@@ -1313,8 +1316,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h) - - v->arch.xcr0 = ctxt->xcr0; - v->arch.xcr0_accum = ctxt->xcr0_accum; -- if ( ctxt->xcr0_accum & XSTATE_NONLAZY ) -- v->arch.nonlazy_xstate_used = 1; -+ v->arch.nonlazy_xstate_used = ctxt->xcr0_accum & XSTATE_NONLAZY; - compress_xsave_states(v, &ctxt->save_area, - size - offsetof(struct hvm_hw_cpu_xsave, save_area)); - -diff --git a/xen/arch/x86/hvm/ioreq.c b/xen/arch/x86/hvm/ioreq.c -index d5afe20cc8..25b2445429 100644 ---- a/xen/arch/x86/hvm/ioreq.c -+++ b/xen/arch/x86/hvm/ioreq.c -@@ -87,14 +87,17 @@ static void hvm_io_assist(struct hvm_ioreq_vcpu *sv, uint64_t data) - - static bool hvm_wait_for_io(struct hvm_ioreq_vcpu *sv, ioreq_t *p) - { -+ unsigned int prev_state = STATE_IOREQ_NONE; -+ - while ( sv->pending ) - { - unsigned int state = p->state; - -- rmb(); -- switch ( state ) -+ smp_rmb(); -+ -+ recheck: -+ if ( unlikely(state == STATE_IOREQ_NONE) ) - { -- case STATE_IOREQ_NONE: - /* - * The only reason we should see this case is when an - * emulator is dying and it races with an I/O being -@@ -102,14 +105,30 @@ static bool hvm_wait_for_io(struct hvm_ioreq_vcpu *sv, ioreq_t *p) - */ - hvm_io_assist(sv, ~0ul); - break; -+ } -+ -+ if ( unlikely(state < prev_state) ) -+ { -+ gdprintk(XENLOG_ERR, "Weird HVM ioreq state transition %u -> %u\n", -+ prev_state, state); -+ sv->pending = false; -+ domain_crash(sv->vcpu->domain); -+ return false; /* bail */ -+ } -+ -+ switch ( prev_state = state ) -+ { - case STATE_IORESP_READY: /* IORESP_READY -> NONE */ - p->state = STATE_IOREQ_NONE; - hvm_io_assist(sv, p->data); - break; - case STATE_IOREQ_READY: /* IOREQ_{READY,INPROCESS} -> IORESP_READY */ - case STATE_IOREQ_INPROCESS: -- wait_on_xen_event_channel(sv->ioreq_evtchn, p->state != state); -- break; -+ wait_on_xen_event_channel(sv->ioreq_evtchn, -+ ({ state = p->state; -+ smp_rmb(); -+ state != prev_state; })); -+ goto recheck; - default: - gdprintk(XENLOG_ERR, "Weird HVM iorequest state %u\n", state); - sv->pending = false; -diff --git a/xen/arch/x86/hvm/irq.c b/xen/arch/x86/hvm/irq.c -index f528e2d081..c85d004402 100644 ---- a/xen/arch/x86/hvm/irq.c -+++ b/xen/arch/x86/hvm/irq.c -@@ -41,6 +41,26 @@ static void assert_gsi(struct domain *d, unsigned ioapic_gsi) - vioapic_irq_positive_edge(d, ioapic_gsi); - } - -+int hvm_ioapic_assert(struct domain *d, unsigned int gsi, bool level) -+{ -+ struct hvm_irq *hvm_irq = hvm_domain_irq(d); -+ int vector; -+ -+ if ( gsi >= hvm_irq->nr_gsis ) -+ { -+ ASSERT_UNREACHABLE(); -+ return -1; -+ } -+ -+ spin_lock(&d->arch.hvm_domain.irq_lock); -+ if ( !level || hvm_irq->gsi_assert_count[gsi]++ == 0 ) -+ assert_gsi(d, gsi); -+ vector = vioapic_get_vector(d, gsi); -+ spin_unlock(&d->arch.hvm_domain.irq_lock); -+ -+ return vector; -+} -+ - static void assert_irq(struct domain *d, unsigned ioapic_gsi, unsigned pic_irq) - { - assert_gsi(d, ioapic_gsi); -diff --git a/xen/arch/x86/hvm/svm/entry.S b/xen/arch/x86/hvm/svm/entry.S -index bf092fe071..5e7c080c7c 100644 ---- a/xen/arch/x86/hvm/svm/entry.S -+++ b/xen/arch/x86/hvm/svm/entry.S -@@ -83,7 +83,7 @@ UNLIKELY_END(svm_trace) - mov VCPUMSR_spec_ctrl_raw(%rax), %eax - - /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ -- SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ -+ SPEC_CTRL_EXIT_TO_HVM /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ - - pop %r15 - pop %r14 -@@ -108,7 +108,7 @@ UNLIKELY_END(svm_trace) - - GET_CURRENT(bx) - -- SPEC_CTRL_ENTRY_FROM_VMEXIT /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */ -+ SPEC_CTRL_ENTRY_FROM_HVM /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */ - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - - mov VCPU_svm_vmcb(%rbx),%rcx -diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c -index dedec5752d..aad3655855 100644 ---- a/xen/arch/x86/hvm/svm/svm.c -+++ b/xen/arch/x86/hvm/svm/svm.c -@@ -546,7 +546,10 @@ void svm_update_guest_cr(struct vcpu *v, unsigned int cr) - if ( !(v->arch.hvm_vcpu.guest_cr[0] & X86_CR0_TS) ) - { - if ( v != current ) -- hw_cr0_mask |= X86_CR0_TS; -+ { -+ if ( !v->arch.fully_eager_fpu ) -+ hw_cr0_mask |= X86_CR0_TS; -+ } - else if ( vmcb_get_cr0(vmcb) & X86_CR0_TS ) - svm_fpu_enter(v); - } -@@ -1033,7 +1036,8 @@ static void svm_ctxt_switch_from(struct vcpu *v) - if ( unlikely((read_efer() & EFER_SVME) == 0) ) - return; - -- svm_fpu_leave(v); -+ if ( !v->arch.fully_eager_fpu ) -+ svm_fpu_leave(v); - - svm_save_dr(v); - svm_lwp_save(v); -@@ -1046,6 +1050,7 @@ static void svm_ctxt_switch_from(struct vcpu *v) - set_ist(&idt_tables[cpu][TRAP_double_fault], IST_DF); - set_ist(&idt_tables[cpu][TRAP_nmi], IST_NMI); - set_ist(&idt_tables[cpu][TRAP_machine_check], IST_MCE); -+ set_ist(&idt_tables[cpu][TRAP_debug], IST_DB); - } - - static void svm_ctxt_switch_to(struct vcpu *v) -@@ -1067,6 +1072,7 @@ static void svm_ctxt_switch_to(struct vcpu *v) - set_ist(&idt_tables[cpu][TRAP_double_fault], IST_NONE); - set_ist(&idt_tables[cpu][TRAP_nmi], IST_NONE); - set_ist(&idt_tables[cpu][TRAP_machine_check], IST_NONE); -+ set_ist(&idt_tables[cpu][TRAP_debug], IST_NONE); - - svm_restore_dr(v); - -@@ -1361,24 +1367,18 @@ static void svm_inject_event(const struct x86_event *event) - * Xen must emulate enough of the event injection to be sure that a - * further fault shouldn't occur during delivery. This covers the fact - * that hardware doesn't perform DPL checking on injection. -- * -- * Also, it accounts for proper positioning of %rip for an event with trap -- * semantics (where %rip should point after the instruction) which suffers -- * a fault during injection (at which point %rip should point at the -- * instruction). - */ - if ( event->type == X86_EVENTTYPE_PRI_SW_EXCEPTION || -- (!cpu_has_svm_nrips && (event->type == X86_EVENTTYPE_SW_INTERRUPT || -- event->type == X86_EVENTTYPE_SW_EXCEPTION)) ) -+ (!cpu_has_svm_nrips && (event->type >= X86_EVENTTYPE_SW_INTERRUPT)) ) - svm_emul_swint_injection(&_event); - -- switch ( _event.vector ) -+ switch ( _event.vector | -(_event.type == X86_EVENTTYPE_SW_INTERRUPT) ) - { - case TRAP_debug: - if ( regs->eflags & X86_EFLAGS_TF ) - { - __restore_debug_registers(vmcb, curr); -- vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | 0x4000); -+ vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | DR_STEP); - } - /* fall through */ - case TRAP_int3: -@@ -1388,6 +1388,13 @@ static void svm_inject_event(const struct x86_event *event) - domain_pause_for_debugger(); - return; - } -+ break; -+ -+ case TRAP_page_fault: -+ ASSERT(_event.type == X86_EVENTTYPE_HW_EXCEPTION); -+ curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2; -+ vmcb_set_cr2(vmcb, _event.cr2); -+ break; - } - - if ( unlikely(eventinj.fields.v) && -@@ -1410,13 +1417,9 @@ static void svm_inject_event(const struct x86_event *event) - * icebp, software events with trap semantics need emulating, so %rip in - * the trap frame points after the instruction. - * -- * The x86 emulator (if requested by the x86_swint_emulate_* choice) will -- * have performed checks such as presence/dpl/etc and believes that the -- * event injection will succeed without faulting. -- * -- * The x86 emulator will always provide fault semantics for software -- * events, with _trap.insn_len set appropriately. If the injection -- * requires emulation, move %rip forwards at this point. -+ * svm_emul_swint_injection() has already confirmed that events with trap -+ * semantics won't fault on injection. Position %rip/NextRIP suitably, -+ * and restrict the event type to what hardware will tolerate. - */ - switch ( _event.type ) - { -@@ -1473,16 +1476,12 @@ static void svm_inject_event(const struct x86_event *event) - eventinj.fields.errorcode == (uint16_t)eventinj.fields.errorcode); - vmcb->eventinj = eventinj; - -- if ( _event.vector == TRAP_page_fault ) -- { -- curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2; -- vmcb_set_cr2(vmcb, _event.cr2); -- HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, TRC_PAR_LONG(_event.cr2)); -- } -+ if ( _event.vector == TRAP_page_fault && -+ _event.type == X86_EVENTTYPE_HW_EXCEPTION ) -+ HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, -+ TRC_PAR_LONG(_event.cr2)); - else -- { - HVMTRACE_2D(INJ_EXC, _event.vector, _event.error_code); -- } - } - - static int svm_event_pending(struct vcpu *v) -@@ -1836,6 +1835,25 @@ static int svm_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - struct vcpu *v = current; - struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb; - -+ switch ( msr ) -+ { -+ /* -+ * Sync not needed while the cross-vendor logic is in unilateral effect. -+ case MSR_IA32_SYSENTER_CS: -+ case MSR_IA32_SYSENTER_ESP: -+ case MSR_IA32_SYSENTER_EIP: -+ */ -+ case MSR_STAR: -+ case MSR_LSTAR: -+ case MSR_CSTAR: -+ case MSR_SYSCALL_MASK: -+ case MSR_FS_BASE: -+ case MSR_GS_BASE: -+ case MSR_SHADOW_GS_BASE: -+ svm_sync_vmcb(v); -+ break; -+ } -+ - switch ( msr ) - { - case MSR_IA32_SYSENTER_CS: -@@ -1848,6 +1866,34 @@ static int svm_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - *msr_content = v->arch.hvm_svm.guest_sysenter_eip; - break; - -+ case MSR_STAR: -+ *msr_content = vmcb->star; -+ break; -+ -+ case MSR_LSTAR: -+ *msr_content = vmcb->lstar; -+ break; -+ -+ case MSR_CSTAR: -+ *msr_content = vmcb->cstar; -+ break; -+ -+ case MSR_SYSCALL_MASK: -+ *msr_content = vmcb->sfmask; -+ break; -+ -+ case MSR_FS_BASE: -+ *msr_content = vmcb->fs.base; -+ break; -+ -+ case MSR_GS_BASE: -+ *msr_content = vmcb->gs.base; -+ break; -+ -+ case MSR_SHADOW_GS_BASE: -+ *msr_content = vmcb->kerngsbase; -+ break; -+ - case MSR_IA32_MCx_MISC(4): /* Threshold register */ - case MSR_F10_MC4_MISC1 ... MSR_F10_MC4_MISC3: - /* -@@ -1976,32 +2022,81 @@ static int svm_msr_write_intercept(unsigned int msr, uint64_t msr_content) - int ret, result = X86EMUL_OKAY; - struct vcpu *v = current; - struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb; -- int sync = 0; -+ bool sync = false; - - switch ( msr ) - { - case MSR_IA32_SYSENTER_CS: - case MSR_IA32_SYSENTER_ESP: - case MSR_IA32_SYSENTER_EIP: -- sync = 1; -- break; -- default: -+ case MSR_STAR: -+ case MSR_LSTAR: -+ case MSR_CSTAR: -+ case MSR_SYSCALL_MASK: -+ case MSR_FS_BASE: -+ case MSR_GS_BASE: -+ case MSR_SHADOW_GS_BASE: -+ sync = true; - break; - } - - if ( sync ) -- svm_sync_vmcb(v); -+ svm_sync_vmcb(v); - - switch ( msr ) - { -+ case MSR_IA32_SYSENTER_ESP: -+ case MSR_IA32_SYSENTER_EIP: -+ case MSR_LSTAR: -+ case MSR_CSTAR: -+ case MSR_FS_BASE: -+ case MSR_GS_BASE: -+ case MSR_SHADOW_GS_BASE: -+ if ( !is_canonical_address(msr_content) ) -+ goto gpf; -+ -+ switch ( msr ) -+ { -+ case MSR_IA32_SYSENTER_ESP: -+ vmcb->sysenter_esp = v->arch.hvm_svm.guest_sysenter_esp = msr_content; -+ break; -+ -+ case MSR_IA32_SYSENTER_EIP: -+ vmcb->sysenter_eip = v->arch.hvm_svm.guest_sysenter_eip = msr_content; -+ break; -+ -+ case MSR_LSTAR: -+ vmcb->lstar = msr_content; -+ break; -+ -+ case MSR_CSTAR: -+ vmcb->cstar = msr_content; -+ break; -+ -+ case MSR_FS_BASE: -+ vmcb->fs.base = msr_content; -+ break; -+ -+ case MSR_GS_BASE: -+ vmcb->gs.base = msr_content; -+ break; -+ -+ case MSR_SHADOW_GS_BASE: -+ vmcb->kerngsbase = msr_content; -+ break; -+ } -+ break; -+ - case MSR_IA32_SYSENTER_CS: - vmcb->sysenter_cs = v->arch.hvm_svm.guest_sysenter_cs = msr_content; - break; -- case MSR_IA32_SYSENTER_ESP: -- vmcb->sysenter_esp = v->arch.hvm_svm.guest_sysenter_esp = msr_content; -+ -+ case MSR_STAR: -+ vmcb->star = msr_content; - break; -- case MSR_IA32_SYSENTER_EIP: -- vmcb->sysenter_eip = v->arch.hvm_svm.guest_sysenter_eip = msr_content; -+ -+ case MSR_SYSCALL_MASK: -+ vmcb->sfmask = msr_content; - break; - - case MSR_IA32_DEBUGCTLMSR: -diff --git a/xen/arch/x86/hvm/svm/svmdebug.c b/xen/arch/x86/hvm/svm/svmdebug.c -index 89ef2db932..b5b946aa94 100644 ---- a/xen/arch/x86/hvm/svm/svmdebug.c -+++ b/xen/arch/x86/hvm/svm/svmdebug.c -@@ -131,9 +131,8 @@ bool svm_vmcb_isvalid(const char *from, const struct vmcb_struct *vmcb, - PRINTF("DR7: bits [63:32] are not zero (%#"PRIx64")\n", - vmcb_get_dr7(vmcb)); - -- if ( efer & ~(EFER_SCE | EFER_LME | EFER_LMA | EFER_NX | EFER_SVME | -- EFER_LMSLE | EFER_FFXSE) ) -- PRINTF("EFER: undefined bits are not zero (%#"PRIx64")\n", efer); -+ if ( efer & ~EFER_KNOWN_MASK ) -+ PRINTF("EFER: unknown bits are not zero (%#"PRIx64")\n", efer); - - if ( hvm_efer_valid(v, efer, -1) ) - PRINTF("EFER: %s (%"PRIx64")\n", hvm_efer_valid(v, efer, -1), efer); -diff --git a/xen/arch/x86/hvm/svm/vmcb.c b/xen/arch/x86/hvm/svm/vmcb.c -index 997e7597e0..612ced2f0d 100644 ---- a/xen/arch/x86/hvm/svm/vmcb.c -+++ b/xen/arch/x86/hvm/svm/vmcb.c -@@ -178,8 +178,8 @@ static int construct_vmcb(struct vcpu *v) - paging_update_paging_modes(v); - - vmcb->_exception_intercepts = -- HVM_TRAP_MASK -- | (1U << TRAP_no_device); -+ HVM_TRAP_MASK | -+ (v->arch.fully_eager_fpu ? 0 : (1U << TRAP_no_device)); - - if ( paging_mode_hap(v->domain) ) - { -diff --git a/xen/arch/x86/hvm/viridian.c b/xen/arch/x86/hvm/viridian.c -index f0fa59d7d5..b02a70d086 100644 ---- a/xen/arch/x86/hvm/viridian.c -+++ b/xen/arch/x86/hvm/viridian.c -@@ -245,7 +245,7 @@ void cpuid_viridian_leaves(const struct vcpu *v, uint32_t leaf, - }; - union { - HV_PARTITION_PRIVILEGE_MASK mask; -- uint32_t lo, hi; -+ struct { uint32_t lo, hi; }; - } u; - - if ( !(viridian_feature_mask(d) & HVMPV_no_freq) ) -@@ -966,12 +966,10 @@ int viridian_hypercall(struct cpu_user_regs *regs) - gprintk(XENLOG_WARNING, "unimplemented hypercall %04x\n", - input.call_code); - /* Fallthrough. */ -- case HvGetPartitionId: - case HvExtCallQueryCapabilities: - /* -- * These hypercalls seem to be erroneously issued by Windows -- * despite neither AccessPartitionId nor EnableExtendedHypercalls -- * being set in CPUID leaf 2. -+ * This hypercall seems to be erroneously issued by Windows -+ * despite EnableExtendedHypercalls not being set in CPUID leaf 2. - * Given that return a status of 'invalid code' has not so far - * caused any problems it's not worth logging. - */ -diff --git a/xen/arch/x86/hvm/vmx/entry.S b/xen/arch/x86/hvm/vmx/entry.S -index e750544b4b..c6504e3207 100644 ---- a/xen/arch/x86/hvm/vmx/entry.S -+++ b/xen/arch/x86/hvm/vmx/entry.S -@@ -38,9 +38,18 @@ ENTRY(vmx_asm_vmexit_handler) - movb $1,VCPU_vmx_launched(%rbx) - mov %rax,VCPU_hvm_guest_cr2(%rbx) - -- SPEC_CTRL_ENTRY_FROM_VMEXIT /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */ -+ SPEC_CTRL_ENTRY_FROM_HVM /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */ - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - -+ /* Hardware clears MSR_DEBUGCTL on VMExit. Reinstate it if debugging Xen. */ -+ .macro restore_lbr -+ mov $IA32_DEBUGCTLMSR_LBR, %eax -+ mov $MSR_IA32_DEBUGCTLMSR, %ecx -+ xor %edx, %edx -+ wrmsr -+ .endm -+ ALTERNATIVE __stringify(ASM_NOP14), restore_lbr, X86_FEATURE_XEN_LBR -+ - mov %rsp,%rdi - call vmx_vmexit_handler - -@@ -76,7 +85,7 @@ UNLIKELY_END(realmode) - mov VCPUMSR_spec_ctrl_raw(%rax), %eax - - /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ -- SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ -+ SPEC_CTRL_EXIT_TO_HVM /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ - - mov VCPU_hvm_guest_cr2(%rbx),%rax - -diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c -index a642118eab..2b223a16fe 100644 ---- a/xen/arch/x86/hvm/vmx/vmcs.c -+++ b/xen/arch/x86/hvm/vmx/vmcs.c -@@ -38,6 +38,7 @@ - #include <asm/flushtlb.h> - #include <asm/monitor.h> - #include <asm/shadow.h> -+#include <asm/spec_ctrl.h> - #include <asm/tboot.h> - #include <asm/apic.h> - -@@ -999,6 +1000,7 @@ static int construct_vmcs(struct vcpu *v) - unsigned long sysenter_eip; - u32 vmexit_ctl = vmx_vmexit_control; - u32 vmentry_ctl = vmx_vmentry_control; -+ int rc = 0; - - vmx_vmcs_enter(v); - -@@ -1086,8 +1088,8 @@ static int construct_vmcs(struct vcpu *v) - - if ( msr_bitmap == NULL ) - { -- vmx_vmcs_exit(v); -- return -ENOMEM; -+ rc = -ENOMEM; -+ goto out; - } - - memset(msr_bitmap, ~0, PAGE_SIZE); -@@ -1146,7 +1148,9 @@ static int construct_vmcs(struct vcpu *v) - __vmwrite(HOST_GS_BASE, 0); - - /* Host control registers. */ -- v->arch.hvm_vmx.host_cr0 = read_cr0() | X86_CR0_TS; -+ v->arch.hvm_vmx.host_cr0 = read_cr0() & ~X86_CR0_TS; -+ if ( !v->arch.fully_eager_fpu ) -+ v->arch.hvm_vmx.host_cr0 |= X86_CR0_TS; - __vmwrite(HOST_CR0, v->arch.hvm_vmx.host_cr0); - __vmwrite(HOST_CR4, mmu_cr4_features); - -@@ -1226,7 +1230,7 @@ static int construct_vmcs(struct vcpu *v) - - v->arch.hvm_vmx.exception_bitmap = HVM_TRAP_MASK - | (paging_mode_hap(d) ? 0 : (1U << TRAP_page_fault)) -- | (1U << TRAP_no_device); -+ | (v->arch.fully_eager_fpu ? 0 : (1U << TRAP_no_device)); - vmx_update_exception_bitmap(v); - - v->arch.hvm_vcpu.guest_cr[0] = X86_CR0_PE | X86_CR0_ET; -@@ -1269,141 +1273,197 @@ static int construct_vmcs(struct vcpu *v) - if ( cpu_has_vmx_tsc_scaling ) - __vmwrite(TSC_MULTIPLIER, d->arch.hvm_domain.tsc_scaling_ratio); - -- vmx_vmcs_exit(v); -- - /* will update HOST & GUEST_CR3 as reqd */ - paging_update_paging_modes(v); - - vmx_vlapic_msr_changed(v); - -- return 0; -+ if ( opt_l1d_flush && paging_mode_hap(d) ) -+ rc = vmx_add_msr(v, MSR_FLUSH_CMD, FLUSH_CMD_L1D, -+ VMX_MSR_GUEST_LOADONLY); -+ -+ out: -+ vmx_vmcs_exit(v); -+ -+ return rc; - } - --static int vmx_msr_entry_key_cmp(const void *key, const void *elt) -+/* -+ * Search an MSR list looking for an MSR entry, or the slot in which it should -+ * live (to keep the data sorted) if an entry is not found. -+ * -+ * The return pointer is guaranteed to be bounded by start and end. However, -+ * it may point at end, and may be invalid for the caller to dereference. -+ */ -+static struct vmx_msr_entry *locate_msr_entry( -+ struct vmx_msr_entry *start, struct vmx_msr_entry *end, uint32_t msr) - { -- const u32 *msr = key; -- const struct vmx_msr_entry *entry = elt; -+ while ( start < end ) -+ { -+ struct vmx_msr_entry *mid = start + (end - start) / 2; - -- if ( *msr > entry->index ) -- return 1; -- if ( *msr < entry->index ) -- return -1; -+ if ( msr < mid->index ) -+ end = mid; -+ else if ( msr > mid->index ) -+ start = mid + 1; -+ else -+ return mid; -+ } - -- return 0; -+ return start; - } - --struct vmx_msr_entry *vmx_find_msr(u32 msr, int type) -+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr, -+ enum vmx_msr_list_type type) - { -- struct vcpu *curr = current; -- unsigned int msr_count; -- struct vmx_msr_entry *msr_area; -+ const struct arch_vmx_struct *vmx = &v->arch.hvm_vmx; -+ struct vmx_msr_entry *start = NULL, *ent, *end; -+ unsigned int substart = 0, subend = vmx->msr_save_count; -+ unsigned int total = vmx->msr_load_count; - -- if ( type == VMX_GUEST_MSR ) -- { -- msr_count = curr->arch.hvm_vmx.msr_count; -- msr_area = curr->arch.hvm_vmx.msr_area; -- } -- else -+ ASSERT(v == current || !vcpu_runnable(v)); -+ -+ switch ( type ) - { -- ASSERT(type == VMX_HOST_MSR); -- msr_count = curr->arch.hvm_vmx.host_msr_count; -- msr_area = curr->arch.hvm_vmx.host_msr_area; -+ case VMX_MSR_HOST: -+ start = vmx->host_msr_area; -+ subend = vmx->host_msr_count; -+ total = subend; -+ break; -+ -+ case VMX_MSR_GUEST: -+ start = vmx->msr_area; -+ break; -+ -+ case VMX_MSR_GUEST_LOADONLY: -+ start = vmx->msr_area; -+ substart = subend; -+ subend = total; -+ break; -+ -+ default: -+ ASSERT_UNREACHABLE(); - } - -- if ( msr_area == NULL ) -+ if ( !start ) - return NULL; - -- return bsearch(&msr, msr_area, msr_count, sizeof(struct vmx_msr_entry), -- vmx_msr_entry_key_cmp); -+ end = start + total; -+ ent = locate_msr_entry(start + substart, start + subend, msr); -+ -+ return ((ent < end) && (ent->index == msr)) ? ent : NULL; - } - --int vmx_read_guest_msr(u32 msr, u64 *val) -+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val, -+ enum vmx_msr_list_type type) - { -- struct vmx_msr_entry *ent; -+ struct arch_vmx_struct *vmx = &v->arch.hvm_vmx; -+ struct vmx_msr_entry **ptr, *start = NULL, *ent, *end; -+ unsigned int substart, subend, total; -+ int rc; -+ -+ ASSERT(v == current || !vcpu_runnable(v)); - -- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL ) -+ switch ( type ) - { -- *val = ent->data; -- return 0; -- } -+ case VMX_MSR_HOST: -+ ptr = &vmx->host_msr_area; -+ substart = 0; -+ subend = vmx->host_msr_count; -+ total = subend; -+ break; - -- return -ESRCH; --} -+ case VMX_MSR_GUEST: -+ ptr = &vmx->msr_area; -+ substart = 0; -+ subend = vmx->msr_save_count; -+ total = vmx->msr_load_count; -+ break; - --int vmx_write_guest_msr(u32 msr, u64 val) --{ -- struct vmx_msr_entry *ent; -+ case VMX_MSR_GUEST_LOADONLY: -+ ptr = &vmx->msr_area; -+ substart = vmx->msr_save_count; -+ subend = vmx->msr_load_count; -+ total = subend; -+ break; - -- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL ) -- { -- ent->data = val; -- return 0; -+ default: -+ ASSERT_UNREACHABLE(); -+ return -EINVAL; - } - -- return -ESRCH; --} -- --int vmx_add_msr(u32 msr, int type) --{ -- struct vcpu *curr = current; -- unsigned int idx, *msr_count; -- struct vmx_msr_entry **msr_area, *msr_area_elem; -+ vmx_vmcs_enter(v); - -- if ( type == VMX_GUEST_MSR ) -+ /* Allocate memory on first use. */ -+ if ( unlikely(!*ptr) ) - { -- msr_count = &curr->arch.hvm_vmx.msr_count; -- msr_area = &curr->arch.hvm_vmx.msr_area; -- } -- else -- { -- ASSERT(type == VMX_HOST_MSR); -- msr_count = &curr->arch.hvm_vmx.host_msr_count; -- msr_area = &curr->arch.hvm_vmx.host_msr_area; -- } -+ paddr_t addr; - -- if ( *msr_area == NULL ) -- { -- if ( (*msr_area = alloc_xenheap_page()) == NULL ) -- return -ENOMEM; -+ if ( (*ptr = alloc_xenheap_page()) == NULL ) -+ { -+ rc = -ENOMEM; -+ goto out; -+ } -+ -+ addr = virt_to_maddr(*ptr); - -- if ( type == VMX_GUEST_MSR ) -+ switch ( type ) - { -- __vmwrite(VM_EXIT_MSR_STORE_ADDR, virt_to_maddr(*msr_area)); -- __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, virt_to_maddr(*msr_area)); -+ case VMX_MSR_HOST: -+ __vmwrite(VM_EXIT_MSR_LOAD_ADDR, addr); -+ break; -+ -+ case VMX_MSR_GUEST: -+ case VMX_MSR_GUEST_LOADONLY: -+ __vmwrite(VM_EXIT_MSR_STORE_ADDR, addr); -+ __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, addr); -+ break; - } -- else -- __vmwrite(VM_EXIT_MSR_LOAD_ADDR, virt_to_maddr(*msr_area)); - } - -- for ( idx = 0; idx < *msr_count && (*msr_area)[idx].index <= msr; idx++ ) -- if ( (*msr_area)[idx].index == msr ) -- return 0; -+ start = *ptr; -+ end = start + total; -+ ent = locate_msr_entry(start + substart, start + subend, msr); - -- if ( *msr_count == (PAGE_SIZE / sizeof(struct vmx_msr_entry)) ) -- return -ENOSPC; -+ if ( (ent < end) && (ent->index == msr) ) -+ goto found; - -- memmove(*msr_area + idx + 1, *msr_area + idx, -- sizeof(*msr_area_elem) * (*msr_count - idx)); -+ /* If there isn't an existing entry for msr, insert room for one. */ -+ if ( total == (PAGE_SIZE / sizeof(*ent)) ) -+ { -+ rc = -ENOSPC; -+ goto out; -+ } - -- msr_area_elem = *msr_area + idx; -- msr_area_elem->index = msr; -- msr_area_elem->mbz = 0; -+ memmove(ent + 1, ent, sizeof(*ent) * (end - ent)); - -- ++*msr_count; -+ ent->index = msr; -+ ent->mbz = 0; - -- if ( type == VMX_GUEST_MSR ) -+ switch ( type ) - { -- msr_area_elem->data = 0; -- __vmwrite(VM_EXIT_MSR_STORE_COUNT, *msr_count); -- __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, *msr_count); -- } -- else -- { -- rdmsrl(msr, msr_area_elem->data); -- __vmwrite(VM_EXIT_MSR_LOAD_COUNT, *msr_count); -+ case VMX_MSR_HOST: -+ __vmwrite(VM_EXIT_MSR_LOAD_COUNT, ++vmx->host_msr_count); -+ break; -+ -+ case VMX_MSR_GUEST: -+ __vmwrite(VM_EXIT_MSR_STORE_COUNT, ++vmx->msr_save_count); -+ -+ /* Fallthrough */ -+ case VMX_MSR_GUEST_LOADONLY: -+ __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, ++vmx->msr_load_count); -+ break; - } - -- return 0; -+ /* Set the msr's value. */ -+ found: -+ ent->data = val; -+ rc = 0; -+ -+ out: -+ vmx_vmcs_exit(v); -+ -+ return rc; - } - - void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector) -@@ -1784,10 +1844,7 @@ void vmcs_dump_vcpu(struct vcpu *v) - vmentry_ctl = vmr32(VM_ENTRY_CONTROLS), - vmexit_ctl = vmr32(VM_EXIT_CONTROLS); - cr4 = vmr(GUEST_CR4); -- -- /* EFER.LMA is read as zero, and is loaded from vmentry_ctl on entry. */ -- BUILD_BUG_ON(VM_ENTRY_IA32E_MODE << 1 != EFER_LMA); -- efer = vmr(GUEST_EFER) | ((vmentry_ctl & VM_ENTRY_IA32E_MODE) << 1); -+ efer = vmr(GUEST_EFER); - - printk("*** Guest State ***\n"); - printk("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", -@@ -1797,7 +1854,7 @@ void vmcs_dump_vcpu(struct vcpu *v) - printk("CR3 = 0x%016lx\n", vmr(GUEST_CR3)); - if ( (v->arch.hvm_vmx.secondary_exec_control & - SECONDARY_EXEC_ENABLE_EPT) && -- (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA) ) -+ (cr4 & X86_CR4_PAE) && !(vmentry_ctl & VM_ENTRY_IA32E_MODE) ) - { - printk("PDPTE0 = 0x%016lx PDPTE1 = 0x%016lx\n", - vmr(GUEST_PDPTE(0)), vmr(GUEST_PDPTE(1))); -diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c -index fc505c8cf7..508882e8d2 100644 ---- a/xen/arch/x86/hvm/vmx/vmx.c -+++ b/xen/arch/x86/hvm/vmx/vmx.c -@@ -682,6 +682,12 @@ static void vmx_cpuid_policy_changed(struct vcpu *v) - vmx_clear_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW); - else - vmx_set_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW); -+ -+ /* MSR_FLUSH_CMD is safe to pass through if the guest knows about it. */ -+ if ( cp->feat.l1d_flush ) -+ vmx_clear_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW); -+ else -+ vmx_set_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW); - } - - int vmx_guest_x86_mode(struct vcpu *v) -@@ -1035,7 +1041,8 @@ static void vmx_ctxt_switch_from(struct vcpu *v) - vmx_vmcs_reload(v); - } - -- vmx_fpu_leave(v); -+ if ( !v->arch.fully_eager_fpu ) -+ vmx_fpu_leave(v); - vmx_save_guest_msrs(v); - vmx_restore_host_msrs(); - vmx_save_dr(v); -@@ -1597,7 +1604,10 @@ static void vmx_update_guest_cr(struct vcpu *v, unsigned int cr) - if ( !(v->arch.hvm_vcpu.guest_cr[0] & X86_CR0_TS) ) - { - if ( v != current ) -- hw_cr0_mask |= X86_CR0_TS; -+ { -+ if ( !v->arch.fully_eager_fpu ) -+ hw_cr0_mask |= X86_CR0_TS; -+ } - else if ( v->arch.hvm_vcpu.hw_cr[0] & X86_CR0_TS ) - vmx_fpu_enter(v); - } -@@ -2835,8 +2845,10 @@ enum - - #define LBR_FROM_SIGNEXT_2MSB ((1ULL << 59) | (1ULL << 60)) - --#define FIXUP_LBR_TSX (1u << 0) --#define FIXUP_BDW_ERRATUM_BDF14 (1u << 1) -+#define LBR_MSRS_INSERTED (1u << 0) -+#define LBR_FIXUP_TSX (1u << 1) -+#define LBR_FIXUP_BDF14 (1u << 2) -+#define LBR_FIXUP_MASK (LBR_FIXUP_TSX | LBR_FIXUP_BDF14) - - static bool __read_mostly lbr_tsx_fixup_needed; - static bool __read_mostly bdw_erratum_bdf14_fixup_needed; -@@ -2899,7 +2911,7 @@ static int is_last_branch_msr(u32 ecx) - - static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - { -- const struct vcpu *curr = current; -+ struct vcpu *curr = current; - - HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x", msr); - -@@ -2958,7 +2970,7 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - goto done; - } - -- if ( vmx_read_guest_msr(msr, msr_content) == 0 ) -+ if ( vmx_read_guest_msr(curr, msr, msr_content) == 0 ) - break; - - if ( is_last_branch_msr(msr) ) -@@ -3093,11 +3105,14 @@ void vmx_vlapic_msr_changed(struct vcpu *v) - static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - { - struct vcpu *v = current; -+ const struct cpuid_policy *cp = v->domain->arch.cpuid; - - HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x, msr_value=%#"PRIx64, msr, msr_content); - - switch ( msr ) - { -+ uint64_t rsvd; -+ - case MSR_IA32_SYSENTER_CS: - __vmwrite(GUEST_SYSENTER_CS, msr_content); - break; -@@ -3111,45 +3126,85 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - goto gp_fault; - __vmwrite(GUEST_SYSENTER_EIP, msr_content); - break; -- case MSR_IA32_DEBUGCTLMSR: { -- int i, rc = 0; -- uint64_t supported = IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF; -+ case MSR_IA32_DEBUGCTLMSR: -+ rsvd = ~(IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF); - -- if ( boot_cpu_has(X86_FEATURE_RTM) ) -- supported |= IA32_DEBUGCTLMSR_RTM; -- if ( msr_content & ~supported ) -+ /* TODO: Wire vPMU settings properly through the CPUID policy */ -+ if ( vpmu_is_set(vcpu_vpmu(v), VPMU_CPU_HAS_BTS) ) - { -- /* Perhaps some other bits are supported in vpmu. */ -- if ( vpmu_do_wrmsr(msr, msr_content, supported) ) -- break; -+ rsvd &= ~(IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS | -+ IA32_DEBUGCTLMSR_BTINT); -+ -+ if ( cpu_has(¤t_cpu_data, X86_FEATURE_DSCPL) ) -+ rsvd &= ~(IA32_DEBUGCTLMSR_BTS_OFF_OS | -+ IA32_DEBUGCTLMSR_BTS_OFF_USR); - } -- if ( msr_content & IA32_DEBUGCTLMSR_LBR ) -+ -+ if ( cp->feat.rtm ) -+ rsvd &= ~IA32_DEBUGCTLMSR_RTM; -+ -+ if ( msr_content & rsvd ) -+ goto gp_fault; -+ -+ /* -+ * When a guest first enables LBR, arrange to save and restore the LBR -+ * MSRs and allow the guest direct access. -+ * -+ * MSR_DEBUGCTL and LBR has existed almost as long as MSRs have -+ * existed, and there is no architectural way to hide the feature, or -+ * fail the attempt to enable LBR. -+ * -+ * Unknown host LBR MSRs or hitting -ENOSPC with the guest load/save -+ * list are definitely hypervisor bugs, whereas -ENOMEM for allocating -+ * the load/save list is simply unlucky (and shouldn't occur with -+ * sensible management by the toolstack). -+ * -+ * Either way, there is nothing we can do right now to recover, and -+ * the guest won't execute correctly either. Simply crash the domain -+ * to make the failure obvious. -+ */ -+ if ( !(v->arch.hvm_vmx.lbr_flags & LBR_MSRS_INSERTED) && -+ (msr_content & IA32_DEBUGCTLMSR_LBR) ) - { - const struct lbr_info *lbr = last_branch_msr_get(); -- if ( lbr == NULL ) -- break; - -- for ( ; (rc == 0) && lbr->count; lbr++ ) -- for ( i = 0; (rc == 0) && (i < lbr->count); i++ ) -- if ( (rc = vmx_add_guest_msr(lbr->base + i)) == 0 ) -+ if ( unlikely(!lbr) ) -+ { -+ gprintk(XENLOG_ERR, "Unknown Host LBR MSRs\n"); -+ domain_crash(v->domain); -+ return X86EMUL_OKAY; -+ } -+ -+ for ( ; lbr->count; lbr++ ) -+ { -+ unsigned int i; -+ -+ for ( i = 0; i < lbr->count; i++ ) -+ { -+ int rc = vmx_add_guest_msr(v, lbr->base + i, 0); -+ -+ if ( unlikely(rc) ) - { -- vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW); -- if ( lbr_tsx_fixup_needed ) -- v->arch.hvm_vmx.lbr_fixup_enabled |= FIXUP_LBR_TSX; -- if ( bdw_erratum_bdf14_fixup_needed ) -- v->arch.hvm_vmx.lbr_fixup_enabled |= -- FIXUP_BDW_ERRATUM_BDF14; -+ gprintk(XENLOG_ERR, -+ "Guest load/save list error %d\n", rc); -+ domain_crash(v->domain); -+ return X86EMUL_OKAY; - } -- } - -- if ( (rc < 0) || -- (msr_content && (vmx_add_host_load_msr(msr) < 0)) ) -- hvm_inject_hw_exception(TRAP_machine_check, X86_EVENT_NO_EC); -- else -- __vmwrite(GUEST_IA32_DEBUGCTL, msr_content); -+ vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW); -+ } -+ } -+ -+ v->arch.hvm_vmx.lbr_flags |= LBR_MSRS_INSERTED; -+ if ( lbr_tsx_fixup_needed ) -+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_TSX; -+ if ( bdw_erratum_bdf14_fixup_needed ) -+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_BDF14; -+ } - -+ __vmwrite(GUEST_IA32_DEBUGCTL, msr_content); - break; -- } -+ - case MSR_IA32_FEATURE_CONTROL: - case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: - /* None of these MSRs are writeable. */ -@@ -3175,7 +3230,7 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - switch ( long_mode_do_msr_write(msr, msr_content) ) - { - case HNDL_unhandled: -- if ( (vmx_write_guest_msr(msr, msr_content) != 0) && -+ if ( (vmx_write_guest_msr(v, msr, msr_content) != 0) && - !is_last_branch_msr(msr) ) - switch ( wrmsr_hypervisor_regs(msr, msr_content) ) - { -@@ -3717,6 +3772,7 @@ void vmx_vmexit_handler(struct cpu_user_regs *regs) - */ - __vmread(EXIT_QUALIFICATION, &exit_qualification); - HVMTRACE_1D(TRAP_DEBUG, exit_qualification); -+ __restore_debug_registers(v); - write_debugreg(6, exit_qualification | DR_STATUS_RESERVED_ONE); - if ( !v->domain->debugger_attached ) - { -@@ -4186,11 +4242,11 @@ out: - static void lbr_tsx_fixup(void) - { - struct vcpu *curr = current; -- unsigned int msr_count = curr->arch.hvm_vmx.msr_count; -+ unsigned int msr_count = curr->arch.hvm_vmx.msr_save_count; - struct vmx_msr_entry *msr_area = curr->arch.hvm_vmx.msr_area; - struct vmx_msr_entry *msr; - -- if ( (msr = vmx_find_msr(lbr_from_start, VMX_GUEST_MSR)) != NULL ) -+ if ( (msr = vmx_find_msr(curr, lbr_from_start, VMX_MSR_GUEST)) != NULL ) - { - /* - * Sign extend into bits 61:62 while preserving bit 63 -@@ -4200,15 +4256,15 @@ static void lbr_tsx_fixup(void) - msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2); - } - -- if ( (msr = vmx_find_msr(lbr_lastint_from, VMX_GUEST_MSR)) != NULL ) -+ if ( (msr = vmx_find_msr(curr, lbr_lastint_from, VMX_MSR_GUEST)) != NULL ) - msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2); - } - --static void sign_extend_msr(u32 msr, int type) -+static void sign_extend_msr(struct vcpu *v, u32 msr, int type) - { - struct vmx_msr_entry *entry; - -- if ( (entry = vmx_find_msr(msr, type)) != NULL ) -+ if ( (entry = vmx_find_msr(v, msr, type)) != NULL ) - { - if ( entry->data & VADDR_TOP_BIT ) - entry->data |= CANONICAL_MASK; -@@ -4219,6 +4275,8 @@ static void sign_extend_msr(u32 msr, int type) - - static void bdw_erratum_bdf14_fixup(void) - { -+ struct vcpu *curr = current; -+ - /* - * Occasionally, on certain Broadwell CPUs MSR_IA32_LASTINTTOIP has - * been observed to have the top three bits corrupted as though the -@@ -4228,17 +4286,17 @@ static void bdw_erratum_bdf14_fixup(void) - * erratum BDF14. Fix up MSR_IA32_LASTINT{FROM,TO}IP by - * sign-extending into bits 48:63. - */ -- sign_extend_msr(MSR_IA32_LASTINTFROMIP, VMX_GUEST_MSR); -- sign_extend_msr(MSR_IA32_LASTINTTOIP, VMX_GUEST_MSR); -+ sign_extend_msr(curr, MSR_IA32_LASTINTFROMIP, VMX_MSR_GUEST); -+ sign_extend_msr(curr, MSR_IA32_LASTINTTOIP, VMX_MSR_GUEST); - } - - static void lbr_fixup(void) - { - struct vcpu *curr = current; - -- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_LBR_TSX ) -+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_TSX ) - lbr_tsx_fixup(); -- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_BDW_ERRATUM_BDF14 ) -+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_BDF14 ) - bdw_erratum_bdf14_fixup(); - } - -@@ -4306,7 +4364,7 @@ bool vmx_vmenter_helper(const struct cpu_user_regs *regs) - } - - out: -- if ( unlikely(curr->arch.hvm_vmx.lbr_fixup_enabled) ) -+ if ( unlikely(curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_MASK) ) - lbr_fixup(); - - HVMTRACE_ND(VMENTRY, 0, 1/*cycles*/, 0, 0, 0, 0, 0, 0, 0); -diff --git a/xen/arch/x86/hvm/vpt.c b/xen/arch/x86/hvm/vpt.c -index 181f4cb631..04e3c2e15b 100644 ---- a/xen/arch/x86/hvm/vpt.c -+++ b/xen/arch/x86/hvm/vpt.c -@@ -107,31 +107,49 @@ static int pt_irq_vector(struct periodic_time *pt, enum hvm_intsrc src) - static int pt_irq_masked(struct periodic_time *pt) - { - struct vcpu *v = pt->vcpu; -- unsigned int gsi, isa_irq; -- int mask; -- uint8_t pic_imr; -+ unsigned int gsi = pt->irq; - -- if ( pt->source == PTSRC_lapic ) -+ switch ( pt->source ) -+ { -+ case PTSRC_lapic: - { - struct vlapic *vlapic = vcpu_vlapic(v); -+ - return (!vlapic_enabled(vlapic) || - (vlapic_get_reg(vlapic, APIC_LVTT) & APIC_LVT_MASKED)); - } - -- isa_irq = pt->irq; -- gsi = hvm_isa_irq_to_gsi(isa_irq); -- pic_imr = v->domain->arch.hvm_domain.vpic[isa_irq >> 3].imr; -- mask = vioapic_get_mask(v->domain, gsi); -- if ( mask < 0 ) -+ case PTSRC_isa: - { -- dprintk(XENLOG_WARNING, "d%u: invalid GSI (%u) for platform timer\n", -- v->domain->domain_id, gsi); -- domain_crash(v->domain); -- return -1; -+ uint8_t pic_imr = v->domain->arch.hvm_domain.vpic[pt->irq >> 3].imr; -+ -+ /* Check if the interrupt is unmasked in the PIC. */ -+ if ( !(pic_imr & (1 << (pt->irq & 7))) && vlapic_accept_pic_intr(v) ) -+ return 0; -+ -+ gsi = hvm_isa_irq_to_gsi(pt->irq); -+ } -+ -+ /* Fallthrough to check if the interrupt is masked on the IO APIC. */ -+ case PTSRC_ioapic: -+ { -+ int mask = vioapic_get_mask(v->domain, gsi); -+ -+ if ( mask < 0 ) -+ { -+ dprintk(XENLOG_WARNING, -+ "d%d: invalid GSI (%u) for platform timer\n", -+ v->domain->domain_id, gsi); -+ domain_crash(v->domain); -+ return -1; -+ } -+ -+ return mask; -+ } - } - -- return (((pic_imr & (1 << (isa_irq & 7))) || !vlapic_accept_pic_intr(v)) && -- mask); -+ ASSERT_UNREACHABLE(); -+ return 1; - } - - static void pt_lock(struct periodic_time *pt) -@@ -252,7 +270,7 @@ int pt_update_irq(struct vcpu *v) - struct list_head *head = &v->arch.hvm_vcpu.tm_list; - struct periodic_time *pt, *temp, *earliest_pt; - uint64_t max_lag; -- int irq, is_lapic, pt_vector; -+ int irq, pt_vector = -1; - - spin_lock(&v->arch.hvm_vcpu.tm_lock); - -@@ -288,29 +306,26 @@ int pt_update_irq(struct vcpu *v) - - earliest_pt->irq_issued = 1; - irq = earliest_pt->irq; -- is_lapic = (earliest_pt->source == PTSRC_lapic); - - spin_unlock(&v->arch.hvm_vcpu.tm_lock); - -- /* -- * If periodic timer interrut is handled by lapic, its vector in -- * IRR is returned and used to set eoi_exit_bitmap for virtual -- * interrupt delivery case. Otherwise return -1 to do nothing. -- */ -- if ( is_lapic ) -+ switch ( earliest_pt->source ) - { -+ case PTSRC_lapic: -+ /* -+ * If periodic timer interrupt is handled by lapic, its vector in -+ * IRR is returned and used to set eoi_exit_bitmap for virtual -+ * interrupt delivery case. Otherwise return -1 to do nothing. -+ */ - vlapic_set_irq(vcpu_vlapic(v), irq, 0); - pt_vector = irq; -- } -- else -- { -+ break; -+ -+ case PTSRC_isa: - hvm_isa_irq_deassert(v->domain, irq); - if ( platform_legacy_irq(irq) && vlapic_accept_pic_intr(v) && - v->domain->arch.hvm_domain.vpic[irq >> 3].int_output ) -- { - hvm_isa_irq_assert(v->domain, irq, NULL); -- pt_vector = -1; -- } - else - { - pt_vector = hvm_isa_irq_assert(v->domain, irq, vioapic_get_vector); -@@ -321,6 +336,17 @@ int pt_update_irq(struct vcpu *v) - if ( pt_vector < 0 || !vlapic_test_irq(vcpu_vlapic(v), pt_vector) ) - pt_vector = -1; - } -+ break; -+ -+ case PTSRC_ioapic: -+ /* -+ * NB: At the moment IO-APIC routed interrupts generated by vpt devices -+ * (HPET) are edge-triggered. -+ */ -+ pt_vector = hvm_ioapic_assert(v->domain, irq, false); -+ if ( pt_vector < 0 || !vlapic_test_irq(vcpu_vlapic(v), pt_vector) ) -+ pt_vector = -1; -+ break; - } - - return pt_vector; -@@ -418,7 +444,14 @@ void create_periodic_time( - struct vcpu *v, struct periodic_time *pt, uint64_t delta, - uint64_t period, uint8_t irq, time_cb *cb, void *data) - { -- ASSERT(pt->source != 0); -+ if ( !pt->source || -+ (pt->irq >= NR_ISAIRQS && pt->source == PTSRC_isa) || -+ (pt->irq >= hvm_domain_irq(v->domain)->nr_gsis && -+ pt->source == PTSRC_ioapic) ) -+ { -+ ASSERT_UNREACHABLE(); -+ return; -+ } - - destroy_periodic_time(pt); - -@@ -498,7 +531,7 @@ static void pt_adjust_vcpu(struct periodic_time *pt, struct vcpu *v) - { - int on_list; - -- ASSERT(pt->source == PTSRC_isa); -+ ASSERT(pt->source == PTSRC_isa || pt->source == PTSRC_ioapic); - - if ( pt->vcpu == NULL ) - return; -diff --git a/xen/arch/x86/i387.c b/xen/arch/x86/i387.c -index 88452522ad..a1d128dd94 100644 ---- a/xen/arch/x86/i387.c -+++ b/xen/arch/x86/i387.c -@@ -15,6 +15,7 @@ - #include <asm/i387.h> - #include <asm/xstate.h> - #include <asm/asm_defns.h> -+#include <asm/spec_ctrl.h> - - /*******************************/ - /* FPU Restore Functions */ -@@ -205,13 +206,13 @@ static inline void fpu_fxsave(struct vcpu *v) - /* VCPU FPU Functions */ - /*******************************/ - /* Restore FPU state whenever VCPU is schduled in. */ --void vcpu_restore_fpu_eager(struct vcpu *v) -+void vcpu_restore_fpu_nonlazy(struct vcpu *v, bool need_stts) - { -- ASSERT(!is_idle_vcpu(v)); -- - /* Restore nonlazy extended state (i.e. parts not tracked by CR0.TS). */ -- if ( !v->arch.nonlazy_xstate_used ) -- return; -+ if ( !v->arch.fully_eager_fpu && !v->arch.nonlazy_xstate_used ) -+ goto maybe_stts; -+ -+ ASSERT(!is_idle_vcpu(v)); - - /* Avoid recursion */ - clts(); -@@ -221,17 +222,28 @@ void vcpu_restore_fpu_eager(struct vcpu *v) - * above) we also need to restore full state, to prevent subsequently - * saving state belonging to another vCPU. - */ -- if ( xstate_all(v) ) -+ if ( v->arch.fully_eager_fpu || (v->arch.xsave_area && xstate_all(v)) ) - { -- fpu_xrstor(v, XSTATE_ALL); -+ if ( cpu_has_xsave ) -+ fpu_xrstor(v, XSTATE_ALL); -+ else -+ fpu_fxrstor(v); -+ - v->fpu_initialised = 1; - v->fpu_dirtied = 1; -+ -+ /* Xen doesn't need TS set, but the guest might. */ -+ need_stts = is_pv_vcpu(v) && (v->arch.pv_vcpu.ctrlreg[0] & X86_CR0_TS); - } - else - { - fpu_xrstor(v, XSTATE_NONLAZY); -- stts(); -+ need_stts = true; - } -+ -+ maybe_stts: -+ if ( need_stts ) -+ stts(); - } - - /* -@@ -247,6 +259,8 @@ void vcpu_restore_fpu_lazy(struct vcpu *v) - if ( v->fpu_dirtied ) - return; - -+ ASSERT(!v->arch.fully_eager_fpu); -+ - if ( cpu_has_xsave ) - fpu_xrstor(v, XSTATE_LAZY); - else -@@ -297,6 +311,8 @@ int vcpu_init_fpu(struct vcpu *v) - { - int rc; - -+ v->arch.fully_eager_fpu = opt_eager_fpu; -+ - if ( (rc = xstate_alloc_save_area(v)) != 0 ) - return rc; - -diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c -index bb924e9225..f3dfe35785 100644 ---- a/xen/arch/x86/mm.c -+++ b/xen/arch/x86/mm.c -@@ -125,6 +125,7 @@ - #include <asm/guest.h> - - #include <asm/hvm/grant_table.h> -+#include <asm/pv/domain.h> - #include <asm/pv/grant_table.h> - - #include "pv/mm.h" -@@ -503,12 +504,60 @@ void free_shared_domheap_page(struct page_info *page) - - void make_cr3(struct vcpu *v, mfn_t mfn) - { -+ struct domain *d = v->domain; -+ - v->arch.cr3 = mfn_x(mfn) << PAGE_SHIFT; -+ if ( is_pv_domain(d) && d->arch.pv_domain.pcid ) -+ v->arch.cr3 |= get_pcid_bits(v, false); -+} -+ -+unsigned long pv_guest_cr4_to_real_cr4(const struct vcpu *v) -+{ -+ const struct domain *d = v->domain; -+ unsigned long cr4; -+ -+ cr4 = v->arch.pv_vcpu.ctrlreg[4] & ~X86_CR4_DE; -+ cr4 |= mmu_cr4_features & (X86_CR4_PSE | X86_CR4_SMEP | X86_CR4_SMAP | -+ X86_CR4_OSXSAVE | X86_CR4_FSGSBASE); -+ -+ if ( d->arch.pv_domain.pcid ) -+ cr4 |= X86_CR4_PCIDE; -+ else if ( !d->arch.pv_domain.xpti ) -+ cr4 |= X86_CR4_PGE; -+ -+ cr4 |= d->arch.vtsc ? X86_CR4_TSD : 0; -+ -+ return cr4; - } - - void write_ptbase(struct vcpu *v) - { -- write_cr3(v->arch.cr3); -+ struct cpu_info *cpu_info = get_cpu_info(); -+ unsigned long new_cr4; -+ -+ new_cr4 = (is_pv_vcpu(v) && !is_idle_vcpu(v)) -+ ? pv_guest_cr4_to_real_cr4(v) -+ : ((read_cr4() & ~(X86_CR4_PCIDE | X86_CR4_TSD)) | X86_CR4_PGE); -+ -+ if ( is_pv_vcpu(v) && v->domain->arch.pv_domain.xpti ) -+ { -+ cpu_info->root_pgt_changed = true; -+ cpu_info->pv_cr3 = __pa(this_cpu(root_pgt)); -+ if ( new_cr4 & X86_CR4_PCIDE ) -+ cpu_info->pv_cr3 |= get_pcid_bits(v, true); -+ switch_cr3_cr4(v->arch.cr3, new_cr4); -+ } -+ else -+ { -+ /* Make sure to clear use_pv_cr3 and xen_cr3 before pv_cr3. */ -+ cpu_info->use_pv_cr3 = false; -+ cpu_info->xen_cr3 = 0; -+ /* switch_cr3_cr4() serializes. */ -+ switch_cr3_cr4(v->arch.cr3, new_cr4); -+ cpu_info->pv_cr3 = 0; -+ } -+ -+ ASSERT(is_pv_vcpu(v) || read_cr4() == mmu_cr4_features); - } - - /* -@@ -605,6 +654,9 @@ static int alloc_segdesc_page(struct page_info *page) - return i == 512 ? 0 : -EINVAL; - } - -+static int __get_page_type(struct page_info *page, unsigned long type, -+ int preemptible); -+ - static int get_page_and_type_from_mfn( - mfn_t mfn, unsigned long type, struct domain *d, - int partial, int preemptible) -@@ -616,9 +668,7 @@ static int get_page_and_type_from_mfn( - unlikely(!get_page_from_mfn(mfn, d)) ) - return -EINVAL; - -- rc = (preemptible ? -- get_page_type_preemptible(page, type) : -- (get_page_type(page, type) ? 0 : -EINVAL)); -+ rc = __get_page_type(page, type, preemptible); - - if ( unlikely(rc) && partial >= 0 && - (!preemptible || page != current->arch.old_guest_table) ) -@@ -1108,7 +1158,7 @@ get_page_from_l2e( - int rc; - - if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l2e(d, l2e) ? -ERESTART : 1; - - if ( unlikely((l2e_get_flags(l2e) & L2_DISALLOW_MASK)) ) - { -@@ -1143,7 +1193,7 @@ get_page_from_l3e( - int rc; - - if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l3e(d, l3e) ? -ERESTART : 1; - - if ( unlikely((l3e_get_flags(l3e) & l3_disallow_mask(d))) ) - { -@@ -1176,7 +1226,7 @@ get_page_from_l4e( - int rc; - - if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l4e(d, l4e) ? -ERESTART : 1; - - if ( unlikely((l4e_get_flags(l4e) & L4_DISALLOW_MASK)) ) - { -@@ -1371,6 +1421,13 @@ static int alloc_l1_table(struct page_info *page) - - for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ ) - { -+ if ( !(l1e_get_flags(pl1e[i]) & _PAGE_PRESENT) ) -+ { -+ ret = pv_l1tf_check_l1e(d, pl1e[i]) ? -ERESTART : 0; -+ if ( ret ) -+ goto out; -+ } -+ - switch ( ret = get_page_from_l1e(pl1e[i], d, d) ) - { - default: -@@ -1391,6 +1448,7 @@ static int alloc_l1_table(struct page_info *page) - - fail: - gdprintk(XENLOG_WARNING, "Failure in alloc_l1_table: slot %#x\n", i); -+ out: - while ( i-- > 0 ) - put_page_from_l1e(pl1e[i], d); - -@@ -1438,8 +1496,7 @@ static int create_pae_xen_mappings(struct domain *d, l3_pgentry_t *pl3e) - return 1; - } - --static int alloc_l2_table(struct page_info *page, unsigned long type, -- int preemptible) -+static int alloc_l2_table(struct page_info *page, unsigned long type) - { - struct domain *d = page_get_owner(page); - unsigned long pfn = mfn_x(page_to_mfn(page)); -@@ -1451,8 +1508,7 @@ static int alloc_l2_table(struct page_info *page, unsigned long type, - - for ( i = page->nr_validated_ptes; i < L2_PAGETABLE_ENTRIES; i++ ) - { -- if ( preemptible && i > page->nr_validated_ptes -- && hypercall_preempt_check() ) -+ if ( i > page->nr_validated_ptes && hypercall_preempt_check() ) - { - page->nr_validated_ptes = i; - rc = -ERESTART; -@@ -1463,6 +1519,12 @@ static int alloc_l2_table(struct page_info *page, unsigned long type, - (rc = get_page_from_l2e(pl2e[i], pfn, d)) > 0 ) - continue; - -+ if ( unlikely(rc == -ERESTART) ) -+ { -+ page->nr_validated_ptes = i; -+ break; -+ } -+ - if ( rc < 0 ) - { - gdprintk(XENLOG_WARNING, "Failure in alloc_l2_table: slot %#x\n", i); -@@ -1745,7 +1807,7 @@ static void free_l1_table(struct page_info *page) - } - - --static int free_l2_table(struct page_info *page, int preemptible) -+static int free_l2_table(struct page_info *page) - { - struct domain *d = page_get_owner(page); - unsigned long pfn = mfn_x(page_to_mfn(page)); -@@ -1759,7 +1821,7 @@ static int free_l2_table(struct page_info *page, int preemptible) - do { - if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) && - put_page_from_l2e(pl2e[i], pfn) == 0 && -- preemptible && i && hypercall_preempt_check() ) -+ i && hypercall_preempt_check() ) - { - page->nr_validated_ptes = i; - err = -ERESTART; -@@ -1983,6 +2045,8 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e, - rc = -EBUSY; - } - } -+ else if ( pv_l1tf_check_l1e(pt_dom, nl1e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu, - preserve_ad)) ) - { -@@ -2046,6 +2110,8 @@ static int mod_l2_entry(l2_pgentry_t *pl2e, - rc = -EBUSY; - } - } -+ else if ( pv_l1tf_check_l2e(d, nl2e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2107,6 +2173,8 @@ static int mod_l3_entry(l3_pgentry_t *pl3e, - rc = -EFAULT; - } - } -+ else if ( pv_l1tf_check_l3e(d, nl3e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2172,6 +2240,8 @@ static int mod_l4_entry(l4_pgentry_t *pl4e, - rc = -EFAULT; - } - } -+ else if ( pv_l1tf_check_l4e(d, nl4e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2301,7 +2371,8 @@ static int alloc_page_type(struct page_info *page, unsigned long type, - rc = alloc_l1_table(page); - break; - case PGT_l2_page_table: -- rc = alloc_l2_table(page, type, preemptible); -+ ASSERT(preemptible); -+ rc = alloc_l2_table(page, type); - break; - case PGT_l3_page_table: - ASSERT(preemptible); -@@ -2393,7 +2464,8 @@ int free_page_type(struct page_info *page, unsigned long type, - rc = 0; - break; - case PGT_l2_page_table: -- rc = free_l2_table(page, preemptible); -+ ASSERT(preemptible); -+ rc = free_l2_table(page); - break; - case PGT_l3_page_table: - ASSERT(preemptible); -@@ -2477,7 +2549,7 @@ static int _put_page_type(struct page_info *page, bool preemptible, - nx = x & ~(PGT_validated|PGT_partial); - if ( unlikely((y = cmpxchg(&page->u.inuse.type_info, - x, nx)) != x) ) -- continue; -+ goto maybe_preempt; - /* We cleared the 'valid bit' so we do the clean up. */ - rc = _put_final_page_type(page, x, preemptible, ptpg); - ptpg = NULL; -@@ -2509,12 +2581,13 @@ static int _put_page_type(struct page_info *page, bool preemptible, - */ - cpu_relax(); - y = page->u.inuse.type_info; -- continue; -+ goto maybe_preempt; - } - - if ( likely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) == x) ) - break; - -+ maybe_preempt: - if ( preemptible && hypercall_preempt_check() ) - return -EINTR; - } -@@ -2627,12 +2700,11 @@ static int __get_page_type(struct page_info *page, unsigned long type, - if ( !(x & PGT_partial) ) - { - /* Someone else is updating validation of this page. Wait... */ -- while ( (y = page->u.inuse.type_info) == x ) -- { -+ do { - if ( preemptible && hypercall_preempt_check() ) - return -EINTR; - cpu_relax(); -- } -+ } while ( (y = page->u.inuse.type_info) == x ); - continue; - } - /* Type ref count was left at 1 when PGT_partial got set. */ -@@ -3469,12 +3541,9 @@ long do_mmuext_op( - } - - if ( rc == -ERESTART ) -- { -- ASSERT(i < count); - rc = hypercall_create_continuation( - __HYPERVISOR_mmuext_op, "hihi", - uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom); -- } - else if ( curr->arch.old_guest_table ) - { - XEN_GUEST_HANDLE_PARAM(void) null; -@@ -3674,18 +3743,27 @@ long do_mmu_update( - case PGT_l4_page_table: - rc = mod_l4_entry(va, l4e_from_intpte(req.val), mfn, - cmd == MMU_PT_UPDATE_PRESERVE_AD, v); -- /* -- * No need to sync if all uses of the page can be accounted -- * to the page lock we hold, its pinned status, and uses on -- * this (v)CPU. -- */ -- if ( !rc && !cpu_has_no_xpti && -- ((page->u.inuse.type_info & PGT_count_mask) > -- (1 + !!(page->u.inuse.type_info & PGT_pinned) + -- (pagetable_get_pfn(curr->arch.guest_table) == mfn) + -- (pagetable_get_pfn(curr->arch.guest_table_user) == -- mfn))) ) -- sync_guest = true; -+ if ( !rc && !cpu_has_no_xpti ) -+ { -+ bool local_in_use = false; -+ -+ if ( pagetable_get_pfn(curr->arch.guest_table) == mfn ) -+ { -+ local_in_use = true; -+ get_cpu_info()->root_pgt_changed = true; -+ } -+ -+ /* -+ * No need to sync if all uses of the page can be -+ * accounted to the page lock we hold, its pinned -+ * status, and uses on this (v)CPU. -+ */ -+ if ( (page->u.inuse.type_info & PGT_count_mask) > -+ (1 + !!(page->u.inuse.type_info & PGT_pinned) + -+ (pagetable_get_pfn(curr->arch.guest_table_user) == -+ mfn) + local_in_use) ) -+ sync_guest = true; -+ } - break; - case PGT_writable_page: - perfc_incr(writable_mmu_updates); -@@ -3761,12 +3839,9 @@ long do_mmu_update( - } - - if ( rc == -ERESTART ) -- { -- ASSERT(i < count); - rc = hypercall_create_continuation( - __HYPERVISOR_mmu_update, "hihi", - ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom); -- } - else if ( curr->arch.old_guest_table ) - { - XEN_GUEST_HANDLE_PARAM(void) null; -@@ -3799,7 +3874,7 @@ long do_mmu_update( - - cpumask_andnot(mask, pt_owner->domain_dirty_cpumask, cpumask_of(cpu)); - if ( !cpumask_empty(mask) ) -- flush_mask(mask, FLUSH_TLB_GLOBAL); -+ flush_mask(mask, FLUSH_TLB_GLOBAL | FLUSH_ROOT_PGTBL); - } - - perfc_add(num_page_updates, i); -@@ -4020,7 +4095,13 @@ static int __do_update_va_mapping( - long do_update_va_mapping(unsigned long va, u64 val64, - unsigned long flags) - { -- return __do_update_va_mapping(va, val64, flags, current->domain); -+ int rc = __do_update_va_mapping(va, val64, flags, current->domain); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping, "lll", va, val64, flags); -+ -+ return rc; - } - - long do_update_va_mapping_otherdomain(unsigned long va, u64 val64, -@@ -4037,6 +4118,46 @@ long do_update_va_mapping_otherdomain(unsigned long va, u64 val64, - - put_pg_owner(pg_owner); - -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping_otherdomain, -+ "llli", va, val64, flags, domid); -+ -+ return rc; -+} -+ -+int compat_update_va_mapping(unsigned int va, uint32_t lo, uint32_t hi, -+ unsigned int flags) -+{ -+ int rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, -+ flags, current->domain); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping, "iiii", va, lo, hi, flags); -+ -+ return rc; -+} -+ -+int compat_update_va_mapping_otherdomain(unsigned int va, -+ uint32_t lo, uint32_t hi, -+ unsigned int flags, domid_t domid) -+{ -+ struct domain *pg_owner; -+ int rc; -+ -+ if ( (pg_owner = get_pg_owner(domid)) == NULL ) -+ return -ESRCH; -+ -+ rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, flags, pg_owner); -+ -+ put_pg_owner(pg_owner); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping_otherdomain, -+ "iiiii", va, lo, hi, flags, domid); -+ - return rc; - } - -diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c -index 755a8f83ca..3954e74d43 100644 ---- a/xen/arch/x86/mm/paging.c -+++ b/xen/arch/x86/mm/paging.c -@@ -879,6 +879,8 @@ void paging_dump_domain_info(struct domain *d) - printk(" paging assistance: "); - if ( paging_mode_shadow(d) ) - printk("shadow "); -+ if ( paging_mode_sh_forced(d) ) -+ printk("forced "); - if ( paging_mode_hap(d) ) - printk("hap "); - if ( paging_mode_refcounts(d) ) -diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c -index 3a33e0b148..199dbbc929 100644 ---- a/xen/arch/x86/mm/shadow/common.c -+++ b/xen/arch/x86/mm/shadow/common.c -@@ -3057,6 +3057,15 @@ static void sh_new_mode(struct domain *d, u32 new_mode) - ASSERT(paging_locked_by_me(d)); - ASSERT(d != current->domain); - -+ /* -+ * If PG_SH_forced has previously been activated because of writing an -+ * L1TF-vulnerable PTE, it must remain active for the remaining lifetime -+ * of the domain, even if the logdirty mode needs to be controlled for -+ * migration purposes. -+ */ -+ if ( paging_mode_sh_forced(d) ) -+ new_mode |= PG_SH_forced | PG_SH_enable; -+ - d->arch.paging.mode = new_mode; - for_each_vcpu(d, v) - sh_update_paging_modes(v); -@@ -3935,6 +3944,33 @@ void shadow_audit_tables(struct vcpu *v) - - #endif /* Shadow audit */ - -+#ifdef CONFIG_PV -+ -+void pv_l1tf_tasklet(unsigned long data) -+{ -+ struct domain *d = (void *)data; -+ -+ domain_pause(d); -+ paging_lock(d); -+ -+ if ( !paging_mode_sh_forced(d) && !d->is_dying ) -+ { -+ int ret = shadow_one_bit_enable(d, PG_SH_forced); -+ -+ if ( ret ) -+ { -+ printk(XENLOG_G_ERR "d%d Failed to enable PG_SH_forced: %d\n", -+ d->domain_id, ret); -+ domain_crash(d); -+ } -+ } -+ -+ paging_unlock(d); -+ domain_unpause(d); -+} -+ -+#endif /* CONFIG_PV */ -+ - /* - * Local variables: - * mode: C -diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c -index e93e3b36b1..8c03ba0158 100644 ---- a/xen/arch/x86/mm/shadow/multi.c -+++ b/xen/arch/x86/mm/shadow/multi.c -@@ -952,6 +952,8 @@ static int shadow_set_l4e(struct domain *d, - - /* Write the new entry */ - shadow_write_entries(sl4e, &new_sl4e, 1, sl4mfn); -+ flush_root_pgtbl_domain(d); -+ - flags |= SHADOW_SET_CHANGED; - - if ( shadow_l4e_get_flags(old_sl4e) & _PAGE_PRESENT ) -@@ -966,6 +968,7 @@ static int shadow_set_l4e(struct domain *d, - } - sh_put_ref(d, osl3mfn, paddr); - } -+ - return flags; - } - -diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c -index 49140e46f0..f3f6d48668 100644 ---- a/xen/arch/x86/mpparse.c -+++ b/xen/arch/x86/mpparse.c -@@ -68,19 +68,26 @@ physid_mask_t phys_cpu_present_map; - - void __init set_nr_cpu_ids(unsigned int max_cpus) - { -+ unsigned int tot_cpus = num_processors + disabled_cpus; -+ - if (!max_cpus) -- max_cpus = num_processors + disabled_cpus; -+ max_cpus = tot_cpus; - if (max_cpus > NR_CPUS) - max_cpus = NR_CPUS; - else if (!max_cpus) - max_cpus = 1; - printk(XENLOG_INFO "SMP: Allowing %u CPUs (%d hotplug CPUs)\n", - max_cpus, max_t(int, max_cpus - num_processors, 0)); -- nr_cpu_ids = max_cpus; -+ -+ if (!park_offline_cpus) -+ tot_cpus = max_cpus; -+ nr_cpu_ids = min(tot_cpus, NR_CPUS + 0u); -+ if (park_offline_cpus && nr_cpu_ids < num_processors) -+ printk(XENLOG_WARNING "SMP: Cannot bring up %u further CPUs\n", -+ num_processors - nr_cpu_ids); - - #ifndef nr_cpumask_bits -- nr_cpumask_bits = (max_cpus + (BITS_PER_LONG - 1)) & -- ~(BITS_PER_LONG - 1); -+ nr_cpumask_bits = ROUNDUP(nr_cpu_ids, BITS_PER_LONG); - printk(XENLOG_DEBUG "NR_CPUS:%u nr_cpumask_bits:%u\n", - NR_CPUS, nr_cpumask_bits); - #endif -diff --git a/xen/arch/x86/msr.c b/xen/arch/x86/msr.c -index 48d061d7da..8e39d5f271 100644 ---- a/xen/arch/x86/msr.c -+++ b/xen/arch/x86/msr.c -@@ -131,6 +131,7 @@ int guest_rdmsr(const struct vcpu *v, uint32_t msr, uint64_t *val) - case MSR_AMD_PATCHLOADER: - case MSR_IA32_UCODE_WRITE: - case MSR_PRED_CMD: -+ case MSR_FLUSH_CMD: - /* Write-only */ - goto gp_fault; - -@@ -178,6 +179,8 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val) - - switch ( msr ) - { -+ uint64_t rsvd; -+ - case MSR_INTEL_PLATFORM_INFO: - case MSR_ARCH_CAPABILITIES: - /* Read-only */ -@@ -213,8 +216,10 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val) - * Note: SPEC_CTRL_STIBP is specified as safe to use (i.e. ignored) - * when STIBP isn't enumerated in hardware. - */ -+ rsvd = ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | -+ (cp->feat.ssbd ? SPEC_CTRL_SSBD : 0)); - -- if ( val & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP) ) -+ if ( val & rsvd ) - goto gp_fault; /* Rsvd bit set? */ - - vp->spec_ctrl.raw = val; -@@ -231,14 +236,25 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val) - wrmsrl(MSR_PRED_CMD, val); - break; - -+ case MSR_FLUSH_CMD: -+ if ( !cp->feat.l1d_flush ) -+ goto gp_fault; /* MSR available? */ -+ -+ if ( val & ~FLUSH_CMD_L1D ) -+ goto gp_fault; /* Rsvd bit set? */ -+ -+ if ( v == curr ) -+ wrmsrl(MSR_FLUSH_CMD, val); -+ break; -+ - case MSR_INTEL_MISC_FEATURES_ENABLES: - { -- uint64_t rsvd = ~0ull; - bool old_cpuid_faulting = vp->misc_features_enables.cpuid_faulting; - - if ( !vp->misc_features_enables.available ) - goto gp_fault; - -+ rsvd = ~0ull; - if ( dp->plaform_info.cpuid_faulting ) - rsvd &= ~MSR_MISC_FEATURES_CPUID_FAULTING; - -diff --git a/xen/arch/x86/oprofile/nmi_int.c b/xen/arch/x86/oprofile/nmi_int.c -index d8f5230906..3dfb8fef93 100644 ---- a/xen/arch/x86/oprofile/nmi_int.c -+++ b/xen/arch/x86/oprofile/nmi_int.c -@@ -182,7 +182,7 @@ int nmi_reserve_counters(void) - if (!allocate_msrs()) - return -ENOMEM; - -- /* We walk a thin line between law and rape here. -+ /* - * We need to be careful to install our NMI handler - * without actually triggering any NMIs as this will - * break the core code horrifically. -diff --git a/xen/arch/x86/percpu.c b/xen/arch/x86/percpu.c -index c9997b7937..8be4ebddf4 100644 ---- a/xen/arch/x86/percpu.c -+++ b/xen/arch/x86/percpu.c -@@ -28,7 +28,7 @@ static int init_percpu_area(unsigned int cpu) - char *p; - - if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA ) -- return -EBUSY; -+ return 0; - - if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL ) - return -ENOMEM; -@@ -76,9 +76,12 @@ static int cpu_percpu_callback( - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -- free_percpu_area(cpu); -+ if ( !park_offline_cpus ) -+ free_percpu_area(cpu); - break; -- default: -+ case CPU_REMOVE: -+ if ( park_offline_cpus ) -+ free_percpu_area(cpu); - break; - } - -diff --git a/xen/arch/x86/pv/dom0_build.c b/xen/arch/x86/pv/dom0_build.c -index 5d8909fa13..1a8142f89b 100644 ---- a/xen/arch/x86/pv/dom0_build.c -+++ b/xen/arch/x86/pv/dom0_build.c -@@ -388,6 +388,8 @@ int __init dom0_construct_pv(struct domain *d, - if ( compat32 ) - { - d->arch.is_32bit_pv = d->arch.has_32bit_shinfo = 1; -+ d->arch.pv_domain.xpti = false; -+ d->arch.pv_domain.pcid = false; - v->vcpu_info = (void *)&d->shared_info->compat.vcpu_info[0]; - if ( setup_compat_arg_xlat(v) != 0 ) - BUG(); -@@ -721,7 +723,7 @@ int __init dom0_construct_pv(struct domain *d, - update_cr3(v); - - /* We run on dom0's page tables for the final part of the build process. */ -- write_ptbase(v); -+ switch_cr3_cr4(cr3_pa(v->arch.cr3), read_cr4()); - mapcache_override_current(v); - - /* Copy the OS image and free temporary buffer. */ -@@ -742,7 +744,7 @@ int __init dom0_construct_pv(struct domain *d, - (parms.virt_hypercall >= v_end) ) - { - mapcache_override_current(NULL); -- write_ptbase(current); -+ switch_cr3_cr4(current->arch.cr3, read_cr4()); - printk("Invalid HYPERCALL_PAGE field in ELF notes.\n"); - rc = -1; - goto out; -@@ -875,7 +877,7 @@ int __init dom0_construct_pv(struct domain *d, - - /* Return to idle domain's page tables. */ - mapcache_override_current(NULL); -- write_ptbase(current); -+ switch_cr3_cr4(current->arch.cr3, read_cr4()); - - update_domain_wallclock_time(d); - -diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c -index 74e9e667d2..bdcbd38f15 100644 ---- a/xen/arch/x86/pv/domain.c -+++ b/xen/arch/x86/pv/domain.c -@@ -9,7 +9,54 @@ - #include <xen/lib.h> - #include <xen/sched.h> - -+#include <asm/cpufeature.h> -+#include <asm/invpcid.h> -+#include <asm/spec_ctrl.h> - #include <asm/pv/domain.h> -+#include <asm/shadow.h> -+ -+static __read_mostly enum { -+ PCID_OFF, -+ PCID_ALL, -+ PCID_XPTI, -+ PCID_NOXPTI -+} opt_pcid = PCID_XPTI; -+ -+static __init int parse_pcid(const char *s) -+{ -+ int rc = 0; -+ -+ switch ( parse_bool(s, NULL) ) -+ { -+ case 0: -+ opt_pcid = PCID_OFF; -+ break; -+ -+ case 1: -+ opt_pcid = PCID_ALL; -+ break; -+ -+ default: -+ switch ( parse_boolean("xpti", s, NULL) ) -+ { -+ case 0: -+ opt_pcid = PCID_NOXPTI; -+ break; -+ -+ case 1: -+ opt_pcid = PCID_XPTI; -+ break; -+ -+ default: -+ rc = -EINVAL; -+ break; -+ } -+ break; -+ } -+ -+ return rc; -+} -+custom_runtime_param("pcid", parse_pcid); - - /* Override macros from asm/page.h to make them work with mfn_t */ - #undef mfn_to_page -@@ -81,6 +128,9 @@ int switch_compat(struct domain *d) - - d->arch.x87_fip_width = 4; - -+ d->arch.pv_domain.xpti = false; -+ d->arch.pv_domain.pcid = false; -+ - return 0; - - undo_and_fail: -@@ -166,6 +216,8 @@ int pv_vcpu_initialise(struct vcpu *v) - - void pv_domain_destroy(struct domain *d) - { -+ pv_l1tf_domain_destroy(d); -+ - destroy_perdomain_mapping(d, GDT_LDT_VIRT_START, - GDT_LDT_MBYTES << (20 - PAGE_SHIFT)); - -@@ -187,6 +239,8 @@ int pv_domain_initialise(struct domain *d, unsigned int domcr_flags, - }; - int rc = -ENOMEM; - -+ pv_l1tf_domain_init(d); -+ - d->arch.pv_domain.gdt_ldt_l1tab = - alloc_xenheap_pages(0, MEMF_node(domain_to_node(d))); - if ( !d->arch.pv_domain.gdt_ldt_l1tab ) -@@ -212,6 +266,32 @@ int pv_domain_initialise(struct domain *d, unsigned int domcr_flags, - /* 64-bit PV guest by default. */ - d->arch.is_32bit_pv = d->arch.has_32bit_shinfo = 0; - -+ d->arch.pv_domain.xpti = opt_xpti & (is_hardware_domain(d) -+ ? OPT_XPTI_DOM0 : OPT_XPTI_DOMU); -+ -+ if ( !is_pv_32bit_domain(d) && use_invpcid && cpu_has_pcid ) -+ switch ( opt_pcid ) -+ { -+ case PCID_OFF: -+ break; -+ -+ case PCID_ALL: -+ d->arch.pv_domain.pcid = true; -+ break; -+ -+ case PCID_XPTI: -+ d->arch.pv_domain.pcid = d->arch.pv_domain.xpti; -+ break; -+ -+ case PCID_NOXPTI: -+ d->arch.pv_domain.pcid = !d->arch.pv_domain.xpti; -+ break; -+ -+ default: -+ ASSERT_UNREACHABLE(); -+ break; -+ } -+ - return 0; - - fail: -@@ -239,13 +319,25 @@ void toggle_guest_mode(struct vcpu *v) - - void toggle_guest_pt(struct vcpu *v) - { -+ const struct domain *d = v->domain; -+ - if ( is_pv_32bit_vcpu(v) ) - return; - - v->arch.flags ^= TF_kernel_mode; - update_cr3(v); -+ if ( d->arch.pv_domain.xpti ) -+ { -+ struct cpu_info *cpu_info = get_cpu_info(); -+ -+ cpu_info->root_pgt_changed = true; -+ cpu_info->pv_cr3 = __pa(this_cpu(root_pgt)) | -+ (d->arch.pv_domain.pcid -+ ? get_pcid_bits(v, true) : 0); -+ } -+ - /* Don't flush user global mappings from the TLB. Don't tick TLB clock. */ -- asm volatile ( "mov %0, %%cr3" : : "r" (v->arch.cr3) : "memory" ); -+ write_cr3(v->arch.cr3); - - if ( !(v->arch.flags & TF_kernel_mode) ) - return; -diff --git a/xen/arch/x86/pv/emul-priv-op.c b/xen/arch/x86/pv/emul-priv-op.c -index 642ca312bf..c281936af0 100644 ---- a/xen/arch/x86/pv/emul-priv-op.c -+++ b/xen/arch/x86/pv/emul-priv-op.c -@@ -813,26 +813,6 @@ static int write_cr(unsigned int reg, unsigned long val, - return X86EMUL_UNHANDLEABLE; - } - --static int read_dr(unsigned int reg, unsigned long *val, -- struct x86_emulate_ctxt *ctxt) --{ -- unsigned long res = do_get_debugreg(reg); -- -- if ( IS_ERR_VALUE(res) ) -- return X86EMUL_UNHANDLEABLE; -- -- *val = res; -- -- return X86EMUL_OKAY; --} -- --static int write_dr(unsigned int reg, unsigned long val, -- struct x86_emulate_ctxt *ctxt) --{ -- return do_set_debugreg(reg, val) == 0 -- ? X86EMUL_OKAY : X86EMUL_UNHANDLEABLE; --} -- - static inline uint64_t guest_misc_enable(uint64_t val) - { - val &= ~(MSR_IA32_MISC_ENABLE_PERF_AVAIL | -@@ -906,9 +886,16 @@ static int read_msr(unsigned int reg, uint64_t *val, - return X86EMUL_OKAY; - - case MSR_EFER: -- *val = read_efer(); -+ /* Hide unknown bits, and unconditionally hide SVME from guests. */ -+ *val = read_efer() & EFER_KNOWN_MASK & ~EFER_SVME; -+ /* -+ * Hide the 64-bit features from 32-bit guests. SCE has -+ * vendor-dependent behaviour. -+ */ - if ( is_pv_32bit_domain(currd) ) -- *val &= ~(EFER_LME | EFER_LMA | EFER_LMSLE); -+ *val &= ~(EFER_LME | EFER_LMA | EFER_LMSLE | -+ (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL -+ ? EFER_SCE : 0)); - return X86EMUL_OKAY; - - case MSR_K7_FID_VID_CTL: -@@ -1326,8 +1313,8 @@ static const struct x86_emulate_ops priv_op_ops = { - .read_segment = read_segment, - .read_cr = read_cr, - .write_cr = write_cr, -- .read_dr = read_dr, -- .write_dr = write_dr, -+ .read_dr = x86emul_read_dr, -+ .write_dr = x86emul_write_dr, - .read_msr = read_msr, - .write_msr = write_msr, - .cpuid = pv_emul_cpuid, -diff --git a/xen/arch/x86/pv/misc-hypercalls.c b/xen/arch/x86/pv/misc-hypercalls.c -index 5862130697..1619be7874 100644 ---- a/xen/arch/x86/pv/misc-hypercalls.c -+++ b/xen/arch/x86/pv/misc-hypercalls.c -@@ -30,22 +30,10 @@ long do_set_debugreg(int reg, unsigned long value) - - unsigned long do_get_debugreg(int reg) - { -- struct vcpu *curr = current; -+ unsigned long val; -+ int res = x86emul_read_dr(reg, &val, NULL); - -- switch ( reg ) -- { -- case 0 ... 3: -- case 6: -- return curr->arch.debugreg[reg]; -- case 7: -- return (curr->arch.debugreg[7] | -- curr->arch.debugreg[5]); -- case 4 ... 5: -- return ((curr->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE) ? -- curr->arch.debugreg[reg + 2] : 0); -- } -- -- return -EINVAL; -+ return res == X86EMUL_OKAY ? val : -ENODEV; - } - - long do_fpu_taskswitch(int set) -diff --git a/xen/arch/x86/pv/ro-page-fault.c b/xen/arch/x86/pv/ro-page-fault.c -index 6b2976d3df..622bb7dff0 100644 ---- a/xen/arch/x86/pv/ro-page-fault.c -+++ b/xen/arch/x86/pv/ro-page-fault.c -@@ -29,6 +29,7 @@ - #include <asm/mm.h> - #include <asm/pci.h> - #include <asm/pv/mm.h> -+#include <asm/shadow.h> - - #include "emulate.h" - #include "mm.h" -@@ -127,6 +128,10 @@ static int ptwr_emulated_update(unsigned long addr, paddr_t old, paddr_t val, - - /* Check the new PTE. */ - nl1e = l1e_from_intpte(val); -+ -+ if ( !(l1e_get_flags(nl1e) & _PAGE_PRESENT) && pv_l1tf_check_l1e(d, nl1e) ) -+ return X86EMUL_RETRY; -+ - switch ( ret = get_page_from_l1e(nl1e, d, d) ) - { - default: -diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c -index 482fe11669..1e9eecae04 100644 ---- a/xen/arch/x86/setup.c -+++ b/xen/arch/x86/setup.c -@@ -62,6 +62,14 @@ boolean_param("nosmp", opt_nosmp); - static unsigned int __initdata max_cpus; - integer_param("maxcpus", max_cpus); - -+int8_t __read_mostly opt_smt = -1; -+boolean_param("smt", opt_smt); -+ -+/* opt_invpcid: If false, don't use INVPCID instruction even if available. */ -+static bool __initdata opt_invpcid = true; -+boolean_param("invpcid", opt_invpcid); -+bool __read_mostly use_invpcid; -+ - unsigned long __read_mostly cr4_pv32_mask; - - /* **** Linux config option: propagated to domain0. */ -@@ -169,9 +177,6 @@ static int __init parse_smap_param(const char *s) - } - custom_param("smap", parse_smap_param); - --static int8_t __initdata opt_xpti = -1; --boolean_param("xpti", opt_xpti); -- - bool __read_mostly acpi_disabled; - bool __initdata acpi_force; - static char __initdata acpi_param[10] = ""; -@@ -663,7 +668,7 @@ void __init noreturn __start_xen(unsigned long mbi_p) - { - char *memmap_type = NULL; - char *cmdline, *kextra, *loader; -- unsigned int initrdidx, domcr_flags = DOMCRF_s3_integrity; -+ unsigned int initrdidx, num_parked = 0, domcr_flags = DOMCRF_s3_integrity; - multiboot_info_t *mbi; - module_t *mod; - unsigned long nr_pages, raw_max_page, modules_headroom, *module_map; -@@ -905,6 +910,18 @@ void __init noreturn __start_xen(unsigned long mbi_p) - /* Sanitise the raw E820 map to produce a final clean version. */ - max_page = raw_max_page = init_e820(memmap_type, &e820_raw); - -+ if ( !efi_enabled(EFI_BOOT) ) -+ { -+ /* -+ * Supplement the heuristics in l1tf_calculations() by assuming that -+ * anything referenced in the E820 may be cacheable. -+ */ -+ l1tf_safe_maddr = -+ max(l1tf_safe_maddr, -+ ROUNDUP(e820_raw.map[e820_raw.nr_map - 1].addr + -+ e820_raw.map[e820_raw.nr_map - 1].size, PAGE_SIZE)); -+ } -+ - /* Create a temporary copy of the E820 map. */ - memcpy(&boot_e820, &e820, sizeof(e820)); - -@@ -1485,7 +1502,8 @@ void __init noreturn __start_xen(unsigned long mbi_p) - else - { - set_nr_cpu_ids(max_cpus); -- max_cpus = nr_cpu_ids; -+ if ( !max_cpus ) -+ max_cpus = nr_cpu_ids; - } - - if ( xen_guest ) -@@ -1539,25 +1557,12 @@ void __init noreturn __start_xen(unsigned long mbi_p) - - cr4_pv32_mask = mmu_cr4_features & XEN_CR4_PV32_BITS; - -- if ( opt_xpti < 0 ) -- { -- uint64_t caps = 0; -- -- if ( boot_cpu_data.x86_vendor == X86_VENDOR_AMD ) -- caps = ARCH_CAPABILITIES_RDCL_NO; -- else if ( boot_cpu_has(X86_FEATURE_ARCH_CAPS) ) -- rdmsrl(MSR_ARCH_CAPABILITIES, caps); -- -- opt_xpti = !(caps & ARCH_CAPABILITIES_RDCL_NO); -- } -- if ( opt_xpti ) -- setup_clear_cpu_cap(X86_FEATURE_NO_XPTI); -- else -- setup_force_cpu_cap(X86_FEATURE_NO_XPTI); -- - if ( cpu_has_fsgsbase ) - set_in_cr4(X86_CR4_FSGSBASE); - -+ if ( opt_invpcid && cpu_has_invpcid ) -+ use_invpcid = true; -+ - init_speculation_mitigations(); - - init_idle_domain(); -@@ -1621,16 +1626,30 @@ void __init noreturn __start_xen(unsigned long mbi_p) - /* Set up node_to_cpumask based on cpu_to_node[]. */ - numa_add_cpu(i); - -- if ( (num_online_cpus() < max_cpus) && !cpu_online(i) ) -+ if ( (park_offline_cpus || num_online_cpus() < max_cpus) && -+ !cpu_online(i) ) - { - int ret = cpu_up(i); - if ( ret != 0 ) - printk("Failed to bring up CPU %u (error %d)\n", i, ret); -+ else if ( num_online_cpus() > max_cpus || -+ (!opt_smt && -+ cpu_data[i].compute_unit_id == INVALID_CUID && -+ cpumask_weight(per_cpu(cpu_sibling_mask, i)) > 1) ) -+ { -+ ret = cpu_down(i); -+ if ( !ret ) -+ ++num_parked; -+ else -+ printk("Could not re-offline CPU%u (%d)\n", i, ret); -+ } - } - } - } - - printk("Brought up %ld CPUs\n", (long)num_online_cpus()); -+ if ( num_parked ) -+ printk(XENLOG_INFO "Parked %u CPUs\n", num_parked); - smp_cpus_done(); - - do_initcalls(); -@@ -1746,6 +1765,13 @@ void __init noreturn __start_xen(unsigned long mbi_p) - - setup_io_bitmap(dom0); - -+ if ( bsp_delay_spec_ctrl ) -+ { -+ get_cpu_info()->spec_ctrl_flags &= ~SCF_use_shadow; -+ barrier(); -+ wrmsrl(MSR_SPEC_CTRL, default_xen_spec_ctrl); -+ } -+ - /* Jump to the 1:1 virtual mappings of cpu0_stack. */ - asm volatile ("mov %[stk], %%rsp; jmp %c[fn]" :: - [stk] "g" (__va(__pa(get_stack_bottom()))), -diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c -index fd6d254812..517ba2f70d 100644 ---- a/xen/arch/x86/smp.c -+++ b/xen/arch/x86/smp.c -@@ -208,7 +208,7 @@ void invalidate_interrupt(struct cpu_user_regs *regs) - ack_APIC_irq(); - perfc_incr(ipis); - if ( __sync_local_execstate() ) -- flags &= ~(FLUSH_TLB | FLUSH_TLB_GLOBAL); -+ flags &= ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_ROOT_PGTBL); - flush_area_local(flush_va, flags); - cpumask_clear_cpu(smp_processor_id(), &flush_cpumask); - } -diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c -index e1d023428c..b0496eb66e 100644 ---- a/xen/arch/x86/smpboot.c -+++ b/xen/arch/x86/smpboot.c -@@ -69,6 +69,8 @@ static cpumask_t scratch_cpu0mask; - cpumask_t cpu_online_map __read_mostly; - EXPORT_SYMBOL(cpu_online_map); - -+bool __read_mostly park_offline_cpus; -+ - unsigned int __read_mostly nr_sockets; - cpumask_t **__read_mostly socket_cpumask; - static cpumask_t *secondary_socket_cpumask; -@@ -228,33 +230,41 @@ static void link_thread_siblings(int cpu1, int cpu2) - cpumask_set_cpu(cpu2, per_cpu(cpu_core_mask, cpu1)); - } - --static void set_cpu_sibling_map(int cpu) -+static void set_cpu_sibling_map(unsigned int cpu) - { -- int i; -+ unsigned int i; - struct cpuinfo_x86 *c = cpu_data; - - cpumask_set_cpu(cpu, &cpu_sibling_setup_map); - - cpumask_set_cpu(cpu, socket_cpumask[cpu_to_socket(cpu)]); -+ cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, cpu)); -+ cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu)); - - if ( c[cpu].x86_num_siblings > 1 ) - { - for_each_cpu ( i, &cpu_sibling_setup_map ) - { -- if ( cpu_has(c, X86_FEATURE_TOPOEXT) ) { -- if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) && -- (c[cpu].compute_unit_id == c[i].compute_unit_id) ) -+ if ( cpu == i || c[cpu].phys_proc_id != c[i].phys_proc_id ) -+ continue; -+ if ( c[cpu].compute_unit_id != INVALID_CUID && -+ c[i].compute_unit_id != INVALID_CUID ) -+ { -+ if ( c[cpu].compute_unit_id == c[i].compute_unit_id ) -+ link_thread_siblings(cpu, i); -+ } -+ else if ( c[cpu].cpu_core_id != XEN_INVALID_CORE_ID && -+ c[i].cpu_core_id != XEN_INVALID_CORE_ID ) -+ { -+ if ( c[cpu].cpu_core_id == c[i].cpu_core_id ) - link_thread_siblings(cpu, i); -- } else if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) && -- (c[cpu].cpu_core_id == c[i].cpu_core_id) ) { -- link_thread_siblings(cpu, i); - } -+ else -+ printk(XENLOG_WARNING -+ "CPU%u: unclear relationship with CPU%u\n", -+ cpu, i); - } - } -- else -- { -- cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu)); -- } - - if ( c[cpu].x86_max_cores == 1 ) - { -@@ -330,8 +340,9 @@ void start_secondary(void *unused) - */ - spin_debug_disable(); - -+ get_cpu_info()->use_pv_cr3 = false; - get_cpu_info()->xen_cr3 = 0; -- get_cpu_info()->pv_cr3 = this_cpu(root_pgt) ? __pa(this_cpu(root_pgt)) : 0; -+ get_cpu_info()->pv_cr3 = 0; - - load_system_tables(); - -@@ -351,6 +362,14 @@ void start_secondary(void *unused) - else - microcode_resume_cpu(cpu); - -+ /* -+ * If MSR_SPEC_CTRL is available, apply Xen's default setting and discard -+ * any firmware settings. Note: MSR_SPEC_CTRL may only become available -+ * after loading microcode. -+ */ -+ if ( boot_cpu_has(X86_FEATURE_IBRSB) ) -+ wrmsrl(MSR_SPEC_CTRL, default_xen_spec_ctrl); -+ - if ( xen_guest ) - hypervisor_ap_setup(); - -@@ -870,11 +889,18 @@ static void cleanup_cpu_root_pgt(unsigned int cpu) - l2_pgentry_t *l2t = l3e_to_l2e(l3t[l3_table_offset(stub_linear)]); - l1_pgentry_t *l1t = l2e_to_l1e(l2t[l2_table_offset(stub_linear)]); - -- l1t[l2_table_offset(stub_linear)] = l1e_empty(); -+ l1t[l1_table_offset(stub_linear)] = l1e_empty(); - } - } - --static void cpu_smpboot_free(unsigned int cpu) -+/* -+ * The 'remove' boolean controls whether a CPU is just getting offlined (and -+ * parked), or outright removed / offlined without parking. Parked CPUs need -+ * things like their stack, GDT, IDT, TSS, and per-CPU data still available. -+ * A few other items, in particular CPU masks, are also retained, as it's -+ * difficult to prove that they're entirely unreferenced from parked CPUs. -+ */ -+static void cpu_smpboot_free(unsigned int cpu, bool remove) - { - unsigned int order, socket = cpu_to_socket(cpu); - struct cpuinfo_x86 *c = cpu_data; -@@ -885,15 +911,19 @@ static void cpu_smpboot_free(unsigned int cpu) - socket_cpumask[socket] = NULL; - } - -- c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID; -- c[cpu].cpu_core_id = XEN_INVALID_CORE_ID; -- c[cpu].compute_unit_id = INVALID_CUID; - cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); - -- free_cpumask_var(per_cpu(cpu_sibling_mask, cpu)); -- free_cpumask_var(per_cpu(cpu_core_mask, cpu)); -- if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask ) -- free_cpumask_var(per_cpu(scratch_cpumask, cpu)); -+ if ( remove ) -+ { -+ c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID; -+ c[cpu].cpu_core_id = XEN_INVALID_CORE_ID; -+ c[cpu].compute_unit_id = INVALID_CUID; -+ -+ FREE_CPUMASK_VAR(per_cpu(cpu_sibling_mask, cpu)); -+ FREE_CPUMASK_VAR(per_cpu(cpu_core_mask, cpu)); -+ if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask ) -+ FREE_CPUMASK_VAR(per_cpu(scratch_cpumask, cpu)); -+ } - - cleanup_cpu_root_pgt(cpu); - -@@ -915,19 +945,21 @@ static void cpu_smpboot_free(unsigned int cpu) - } - - order = get_order_from_pages(NR_RESERVED_GDT_PAGES); -- free_xenheap_pages(per_cpu(gdt_table, cpu), order); -+ if ( remove ) -+ FREE_XENHEAP_PAGES(per_cpu(gdt_table, cpu), order); - - free_xenheap_pages(per_cpu(compat_gdt_table, cpu), order); - -- order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -- free_xenheap_pages(idt_tables[cpu], order); -- idt_tables[cpu] = NULL; -- -- if ( stack_base[cpu] != NULL ) -+ if ( remove ) - { -- memguard_unguard_stack(stack_base[cpu]); -- free_xenheap_pages(stack_base[cpu], STACK_ORDER); -- stack_base[cpu] = NULL; -+ order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -+ FREE_XENHEAP_PAGES(idt_tables[cpu], order); -+ -+ if ( stack_base[cpu] ) -+ { -+ memguard_unguard_stack(stack_base[cpu]); -+ FREE_XENHEAP_PAGES(stack_base[cpu], STACK_ORDER); -+ } - } - } - -@@ -941,15 +973,17 @@ static int cpu_smpboot_alloc(unsigned int cpu) - if ( node != NUMA_NO_NODE ) - memflags = MEMF_node(node); - -- stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags); -+ if ( stack_base[cpu] == NULL ) -+ stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags); - if ( stack_base[cpu] == NULL ) - goto oom; - memguard_guard_stack(stack_base[cpu]); - - order = get_order_from_pages(NR_RESERVED_GDT_PAGES); -- per_cpu(gdt_table, cpu) = gdt = alloc_xenheap_pages(order, memflags); -+ gdt = per_cpu(gdt_table, cpu) ?: alloc_xenheap_pages(order, memflags); - if ( gdt == NULL ) - goto oom; -+ per_cpu(gdt_table, cpu) = gdt; - memcpy(gdt, boot_cpu_gdt_table, NR_RESERVED_GDT_PAGES * PAGE_SIZE); - BUILD_BUG_ON(NR_CPUS > 0x10000); - gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu; -@@ -961,13 +995,15 @@ static int cpu_smpboot_alloc(unsigned int cpu) - gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu; - - order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -- idt_tables[cpu] = alloc_xenheap_pages(order, memflags); -+ if ( idt_tables[cpu] == NULL ) -+ idt_tables[cpu] = alloc_xenheap_pages(order, memflags); - if ( idt_tables[cpu] == NULL ) - goto oom; - memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES * sizeof(idt_entry_t)); - set_ist(&idt_tables[cpu][TRAP_double_fault], IST_NONE); - set_ist(&idt_tables[cpu][TRAP_nmi], IST_NONE); - set_ist(&idt_tables[cpu][TRAP_machine_check], IST_NONE); -+ set_ist(&idt_tables[cpu][TRAP_debug], IST_NONE); - - for ( stub_page = 0, i = cpu & ~(STUBS_PER_PAGE - 1); - i < nr_cpu_ids && i <= (cpu | (STUBS_PER_PAGE - 1)); ++i ) -@@ -989,13 +1025,13 @@ static int cpu_smpboot_alloc(unsigned int cpu) - (secondary_socket_cpumask = xzalloc(cpumask_t)) == NULL ) - goto oom; - -- if ( zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) && -- zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) && -- alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu)) ) -+ if ( cond_zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) && -+ cond_zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) && -+ cond_alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu)) ) - return 0; - - oom: -- cpu_smpboot_free(cpu); -+ cpu_smpboot_free(cpu, true); - return -ENOMEM; - } - -@@ -1012,9 +1048,10 @@ static int cpu_smpboot_callback( - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -- cpu_smpboot_free(cpu); -+ cpu_smpboot_free(cpu, !park_offline_cpus); - break; -- default: -+ case CPU_REMOVE: -+ cpu_smpboot_free(cpu, true); - break; - } - -@@ -1047,7 +1084,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) - panic("Error %d setting up PV root page table\n", rc); - if ( per_cpu(root_pgt, 0) ) - { -- get_cpu_info()->pv_cr3 = __pa(per_cpu(root_pgt, 0)); -+ get_cpu_info()->pv_cr3 = 0; - - /* - * All entry points which may need to switch page tables have to start -@@ -1126,6 +1163,7 @@ void __init smp_prepare_boot_cpu(void) - per_cpu(scratch_cpumask, cpu) = &scratch_cpu0mask; - #endif - -+ get_cpu_info()->use_pv_cr3 = false; - get_cpu_info()->xen_cr3 = 0; - get_cpu_info()->pv_cr3 = 0; - } -diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c -index 3c7447bfe6..14e01faff9 100644 ---- a/xen/arch/x86/spec_ctrl.c -+++ b/xen/arch/x86/spec_ctrl.c -@@ -19,13 +19,23 @@ - #include <xen/errno.h> - #include <xen/init.h> - #include <xen/lib.h> -+#include <xen/warning.h> - - #include <asm/microcode.h> - #include <asm/msr.h> - #include <asm/processor.h> -+#include <asm/pv/shim.h> -+#include <asm/setup.h> - #include <asm/spec_ctrl.h> - #include <asm/spec_ctrl_asm.h> - -+/* Cmdline controls for Xen's alternative blocks. */ -+static bool __initdata opt_msr_sc_pv = true; -+static bool __initdata opt_msr_sc_hvm = true; -+static bool __initdata opt_rsb_pv = true; -+static bool __initdata opt_rsb_hvm = true; -+ -+/* Cmdline controls for Xen's speculative settings. */ - static enum ind_thunk { - THUNK_DEFAULT, /* Decide which thunk to use at boot time. */ - THUNK_NONE, /* Missing compiler support for thunks. */ -@@ -35,10 +45,18 @@ static enum ind_thunk { - THUNK_JMP, - } opt_thunk __initdata = THUNK_DEFAULT; - static int8_t __initdata opt_ibrs = -1; --static bool __initdata opt_rsb_native = true; --static bool __initdata opt_rsb_vmexit = true; - bool __read_mostly opt_ibpb = true; --uint8_t __read_mostly default_bti_ist_info; -+bool __read_mostly opt_ssbd = false; -+int8_t __read_mostly opt_eager_fpu = -1; -+int8_t __read_mostly opt_l1d_flush = -1; -+ -+bool __initdata bsp_delay_spec_ctrl; -+uint8_t __read_mostly default_xen_spec_ctrl; -+uint8_t __read_mostly default_spec_ctrl_flags; -+ -+paddr_t __read_mostly l1tf_addr_mask, __read_mostly l1tf_safe_maddr; -+static bool __initdata cpu_has_bug_l1tf; -+static unsigned int __initdata l1d_maxphysaddr; - - static int __init parse_bti(const char *s) - { -@@ -68,9 +86,9 @@ static int __init parse_bti(const char *s) - else if ( (val = parse_boolean("ibpb", s, ss)) >= 0 ) - opt_ibpb = val; - else if ( (val = parse_boolean("rsb_native", s, ss)) >= 0 ) -- opt_rsb_native = val; -+ opt_rsb_pv = val; - else if ( (val = parse_boolean("rsb_vmexit", s, ss)) >= 0 ) -- opt_rsb_vmexit = val; -+ opt_rsb_hvm = val; - else - rc = -EINVAL; - -@@ -81,50 +99,244 @@ static int __init parse_bti(const char *s) - } - custom_param("bti", parse_bti); - --static void __init print_details(enum ind_thunk thunk) -+static int __init parse_spec_ctrl(const char *s) -+{ -+ const char *ss; -+ int val, rc = 0; -+ -+ do { -+ ss = strchr(s, ','); -+ if ( !ss ) -+ ss = strchr(s, '\0'); -+ -+ /* Global and Xen-wide disable. */ -+ val = parse_bool(s, ss); -+ if ( !val ) -+ { -+ opt_msr_sc_pv = false; -+ opt_msr_sc_hvm = false; -+ -+ opt_eager_fpu = 0; -+ -+ if ( opt_xpti < 0 ) -+ opt_xpti = 0; -+ -+ if ( opt_smt < 0 ) -+ opt_smt = 1; -+ -+ if ( opt_pv_l1tf < 0 ) -+ opt_pv_l1tf = 0; -+ -+ disable_common: -+ opt_rsb_pv = false; -+ opt_rsb_hvm = false; -+ -+ opt_thunk = THUNK_JMP; -+ opt_ibrs = 0; -+ opt_ibpb = false; -+ opt_ssbd = false; -+ opt_l1d_flush = 0; -+ } -+ else if ( val > 0 ) -+ rc = -EINVAL; -+ else if ( (val = parse_boolean("xen", s, ss)) >= 0 ) -+ { -+ if ( !val ) -+ goto disable_common; -+ -+ rc = -EINVAL; -+ } -+ -+ /* Xen's alternative blocks. */ -+ else if ( (val = parse_boolean("pv", s, ss)) >= 0 ) -+ { -+ opt_msr_sc_pv = val; -+ opt_rsb_pv = val; -+ } -+ else if ( (val = parse_boolean("hvm", s, ss)) >= 0 ) -+ { -+ opt_msr_sc_hvm = val; -+ opt_rsb_hvm = val; -+ } -+ else if ( (val = parse_boolean("msr-sc", s, ss)) >= 0 ) -+ { -+ opt_msr_sc_pv = val; -+ opt_msr_sc_hvm = val; -+ } -+ else if ( (val = parse_boolean("rsb", s, ss)) >= 0 ) -+ { -+ opt_rsb_pv = val; -+ opt_rsb_hvm = val; -+ } -+ -+ /* Xen's speculative sidechannel mitigation settings. */ -+ else if ( !strncmp(s, "bti-thunk=", 10) ) -+ { -+ s += 10; -+ -+ if ( !strncmp(s, "retpoline", ss - s) ) -+ opt_thunk = THUNK_RETPOLINE; -+ else if ( !strncmp(s, "lfence", ss - s) ) -+ opt_thunk = THUNK_LFENCE; -+ else if ( !strncmp(s, "jmp", ss - s) ) -+ opt_thunk = THUNK_JMP; -+ else -+ rc = -EINVAL; -+ } -+ else if ( (val = parse_boolean("ibrs", s, ss)) >= 0 ) -+ opt_ibrs = val; -+ else if ( (val = parse_boolean("ibpb", s, ss)) >= 0 ) -+ opt_ibpb = val; -+ else if ( (val = parse_boolean("ssbd", s, ss)) >= 0 ) -+ opt_ssbd = val; -+ else if ( (val = parse_boolean("eager-fpu", s, ss)) >= 0 ) -+ opt_eager_fpu = val; -+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 ) -+ opt_l1d_flush = val; -+ else -+ rc = -EINVAL; -+ -+ s = ss + 1; -+ } while ( *ss ); -+ -+ return rc; -+} -+custom_param("spec-ctrl", parse_spec_ctrl); -+ -+int8_t __read_mostly opt_pv_l1tf = -1; -+ -+static __init int parse_pv_l1tf(const char *s) -+{ -+ const char *ss; -+ int val, rc = 0; -+ -+ /* Inhibit the defaults as an explicit choice has been given. */ -+ if ( opt_pv_l1tf == -1 ) -+ opt_pv_l1tf = 0; -+ -+ /* Interpret 'pv-l1tf' alone in its positive boolean form. */ -+ if ( *s == '\0' ) -+ opt_pv_l1tf = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU; -+ -+ do { -+ ss = strchr(s, ','); -+ if ( !ss ) -+ ss = strchr(s, '\0'); -+ -+ switch ( parse_bool(s, ss) ) -+ { -+ case 0: -+ opt_pv_l1tf = 0; -+ break; -+ -+ case 1: -+ opt_pv_l1tf = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU; -+ break; -+ -+ default: -+ if ( (val = parse_boolean("dom0", s, ss)) >= 0 ) -+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOM0) | -+ (val ? OPT_PV_L1TF_DOM0 : 0)); -+ else if ( (val = parse_boolean("domu", s, ss)) >= 0 ) -+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOMU) | -+ (val ? OPT_PV_L1TF_DOMU : 0)); -+ else -+ rc = -EINVAL; -+ break; -+ } -+ -+ s = ss + 1; -+ } while ( *ss ); -+ -+ return rc; -+} -+custom_param("pv-l1tf", parse_pv_l1tf); -+ -+static void __init print_details(enum ind_thunk thunk, uint64_t caps) - { - unsigned int _7d0 = 0, e8b = 0, tmp; -- uint64_t caps = 0; - - /* Collect diagnostics about available mitigations. */ - if ( boot_cpu_data.cpuid_level >= 7 ) - cpuid_count(7, 0, &tmp, &tmp, &tmp, &_7d0); - if ( boot_cpu_data.extended_cpuid_level >= 0x80000008 ) - cpuid(0x80000008, &tmp, &e8b, &tmp, &tmp); -- if ( _7d0 & cpufeat_mask(X86_FEATURE_ARCH_CAPS) ) -- rdmsrl(MSR_ARCH_CAPABILITIES, caps); - -- printk(XENLOG_DEBUG "Speculative mitigation facilities:\n"); -+ printk("Speculative mitigation facilities:\n"); - - /* Hardware features which pertain to speculative mitigations. */ -- printk(XENLOG_DEBUG " Hardware features:%s%s%s%s%s\n", -+ printk(" Hardware features:%s%s%s%s%s%s%s%s%s%s\n", - (_7d0 & cpufeat_mask(X86_FEATURE_IBRSB)) ? " IBRS/IBPB" : "", - (_7d0 & cpufeat_mask(X86_FEATURE_STIBP)) ? " STIBP" : "", -+ (_7d0 & cpufeat_mask(X86_FEATURE_L1D_FLUSH)) ? " L1D_FLUSH" : "", -+ (_7d0 & cpufeat_mask(X86_FEATURE_SSBD)) ? " SSBD" : "", - (e8b & cpufeat_mask(X86_FEATURE_IBPB)) ? " IBPB" : "", - (caps & ARCH_CAPABILITIES_IBRS_ALL) ? " IBRS_ALL" : "", -- (caps & ARCH_CAPABILITIES_RDCL_NO) ? " RDCL_NO" : ""); -- -- /* Compiled-in support which pertains to BTI mitigations. */ -- if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) ) -- printk(XENLOG_DEBUG " Compiled-in support: INDIRECT_THUNK\n"); -- -- printk("BTI mitigations: Thunk %s, Others:%s%s%s%s\n", -+ (caps & ARCH_CAPABILITIES_RDCL_NO) ? " RDCL_NO" : "", -+ (caps & ARCH_CAPS_RSBA) ? " RSBA" : "", -+ (caps & ARCH_CAPS_SKIP_L1DFL) ? " SKIP_L1DFL": "", -+ (caps & ARCH_CAPS_SSB_NO) ? " SSB_NO" : ""); -+ -+ /* Compiled-in support which pertains to mitigations. */ -+ if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) || IS_ENABLED(CONFIG_SHADOW_PAGING) ) -+ printk(" Compiled-in support:" -+#ifdef CONFIG_INDIRECT_THUNK -+ " INDIRECT_THUNK" -+#endif -+#ifdef CONFIG_SHADOW_PAGING -+ " SHADOW_PAGING" -+#endif -+ "\n"); -+ -+ /* Settings for Xen's protection, irrespective of guests. */ -+ printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s\n", - thunk == THUNK_NONE ? "N/A" : - thunk == THUNK_RETPOLINE ? "RETPOLINE" : - thunk == THUNK_LFENCE ? "LFENCE" : - thunk == THUNK_JMP ? "JMP" : "?", -- boot_cpu_has(X86_FEATURE_XEN_IBRS_SET) ? " IBRS+" : -- boot_cpu_has(X86_FEATURE_XEN_IBRS_CLEAR) ? " IBRS-" : "", -- opt_ibpb ? " IBPB" : "", -- boot_cpu_has(X86_FEATURE_RSB_NATIVE) ? " RSB_NATIVE" : "", -- boot_cpu_has(X86_FEATURE_RSB_VMEXIT) ? " RSB_VMEXIT" : ""); -- -- printk("XPTI: %s\n", -- boot_cpu_has(X86_FEATURE_NO_XPTI) ? "disabled" : "enabled"); -+ !boot_cpu_has(X86_FEATURE_IBRSB) ? "No" : -+ (default_xen_spec_ctrl & SPEC_CTRL_IBRS) ? "IBRS+" : "IBRS-", -+ !boot_cpu_has(X86_FEATURE_SSBD) ? "" : -+ (default_xen_spec_ctrl & SPEC_CTRL_SSBD) ? " SSBD+" : " SSBD-", -+ opt_ibpb ? " IBPB" : "", -+ opt_l1d_flush ? " L1D_FLUSH" : ""); -+ -+ /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */ -+ if ( cpu_has_bug_l1tf || opt_pv_l1tf ) -+ printk(" L1TF: believed%s vulnerable, maxphysaddr L1D %u, CPUID %u" -+ ", Safe address %"PRIx64"\n", -+ cpu_has_bug_l1tf ? "" : " not", -+ l1d_maxphysaddr, paddr_bits, l1tf_safe_maddr); -+ -+ /* -+ * Alternatives blocks for protecting against and/or virtualising -+ * mitigation support for guests. -+ */ -+ printk(" Support for VMs: PV:%s%s%s%s, HVM:%s%s%s%s\n", -+ (boot_cpu_has(X86_FEATURE_SC_MSR_PV) || -+ boot_cpu_has(X86_FEATURE_SC_RSB_PV) || -+ opt_eager_fpu) ? "" : " None", -+ boot_cpu_has(X86_FEATURE_SC_MSR_PV) ? " MSR_SPEC_CTRL" : "", -+ boot_cpu_has(X86_FEATURE_SC_RSB_PV) ? " RSB" : "", -+ opt_eager_fpu ? " EAGER_FPU" : "", -+ (boot_cpu_has(X86_FEATURE_SC_MSR_HVM) || -+ boot_cpu_has(X86_FEATURE_SC_RSB_HVM) || -+ opt_eager_fpu) ? "" : " None", -+ boot_cpu_has(X86_FEATURE_SC_MSR_HVM) ? " MSR_SPEC_CTRL" : "", -+ boot_cpu_has(X86_FEATURE_SC_RSB_HVM) ? " RSB" : "", -+ opt_eager_fpu ? " EAGER_FPU" : ""); -+ -+ printk(" XPTI (64-bit PV only): Dom0 %s, DomU %s\n", -+ opt_xpti & OPT_XPTI_DOM0 ? "enabled" : "disabled", -+ opt_xpti & OPT_XPTI_DOMU ? "enabled" : "disabled"); -+ -+ printk(" PV L1TF shadowing: Dom0 %s, DomU %s\n", -+ opt_pv_l1tf & OPT_PV_L1TF_DOM0 ? "enabled" : "disabled", -+ opt_pv_l1tf & OPT_PV_L1TF_DOMU ? "enabled" : "disabled"); - } - - /* Calculate whether Retpoline is known-safe on this CPU. */ --static bool __init retpoline_safe(void) -+static bool __init retpoline_safe(uint64_t caps) - { - unsigned int ucode_rev = this_cpu(ucode_cpu_info).cpu_sig.rev; - -@@ -135,6 +347,13 @@ static bool __init retpoline_safe(void) - boot_cpu_data.x86 != 6 ) - return false; - -+ /* -+ * RSBA may be set by a hypervisor to indicate that we may move to a -+ * processor which isn't retpoline-safe. -+ */ -+ if ( caps & ARCH_CAPS_RSBA ) -+ return false; -+ - switch ( boot_cpu_data.x86_model ) - { - case 0x17: /* Penryn */ -@@ -161,26 +380,337 @@ static bool __init retpoline_safe(void) - * versions. - */ - case 0x3d: /* Broadwell */ -- return ucode_rev >= 0x28; -+ return ucode_rev >= 0x2a; - case 0x47: /* Broadwell H */ -- return ucode_rev >= 0x1b; -+ return ucode_rev >= 0x1d; - case 0x4f: /* Broadwell EP/EX */ -- return ucode_rev >= 0xb000025; -+ return ucode_rev >= 0xb000021; - case 0x56: /* Broadwell D */ -- return false; /* TBD. */ -+ switch ( boot_cpu_data.x86_mask ) -+ { -+ case 2: return ucode_rev >= 0x15; -+ case 3: return ucode_rev >= 0x7000012; -+ case 4: return ucode_rev >= 0xf000011; -+ case 5: return ucode_rev >= 0xe000009; -+ default: -+ printk("Unrecognised CPU stepping %#x - assuming not reptpoline safe\n", -+ boot_cpu_data.x86_mask); -+ return false; -+ } -+ break; - - /* -- * Skylake and later processors are not retpoline-safe. -+ * Skylake, Kabylake and Cannonlake processors are not retpoline-safe. - */ -+ case 0x4e: -+ case 0x55: -+ case 0x5e: -+ case 0x66: -+ case 0x67: -+ case 0x8e: -+ case 0x9e: -+ return false; -+ - default: -+ printk("Unrecognised CPU model %#x - assuming not reptpoline safe\n", -+ boot_cpu_data.x86_model); - return false; - } - } - -+/* Calculate whether this CPU speculates past #NM */ -+static bool __init should_use_eager_fpu(void) -+{ -+ /* -+ * Assume all unrecognised processors are ok. This is only known to -+ * affect Intel Family 6 processors. -+ */ -+ if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || -+ boot_cpu_data.x86 != 6 ) -+ return false; -+ -+ switch ( boot_cpu_data.x86_model ) -+ { -+ /* -+ * Core processors since at least Nehalem are vulnerable. -+ */ -+ case 0x1e: /* Nehalem */ -+ case 0x1f: /* Auburndale / Havendale */ -+ case 0x1a: /* Nehalem EP */ -+ case 0x2e: /* Nehalem EX */ -+ case 0x25: /* Westmere */ -+ case 0x2c: /* Westmere EP */ -+ case 0x2f: /* Westmere EX */ -+ case 0x2a: /* SandyBridge */ -+ case 0x2d: /* SandyBridge EP/EX */ -+ case 0x3a: /* IvyBridge */ -+ case 0x3e: /* IvyBridge EP/EX */ -+ case 0x3c: /* Haswell */ -+ case 0x3f: /* Haswell EX/EP */ -+ case 0x45: /* Haswell D */ -+ case 0x46: /* Haswell H */ -+ case 0x3d: /* Broadwell */ -+ case 0x47: /* Broadwell H */ -+ case 0x4f: /* Broadwell EP/EX */ -+ case 0x56: /* Broadwell D */ -+ case 0x4e: /* Skylake M */ -+ case 0x55: /* Skylake X */ -+ case 0x5e: /* Skylake D */ -+ case 0x66: /* Cannonlake */ -+ case 0x67: /* Cannonlake? */ -+ case 0x8e: /* Kabylake M */ -+ case 0x9e: /* Kabylake D */ -+ return true; -+ -+ /* -+ * Atom processors are not vulnerable. -+ */ -+ case 0x1c: /* Pineview */ -+ case 0x26: /* Lincroft */ -+ case 0x27: /* Penwell */ -+ case 0x35: /* Cloverview */ -+ case 0x36: /* Cedarview */ -+ case 0x37: /* Baytrail / Valleyview (Silvermont) */ -+ case 0x4d: /* Avaton / Rangely (Silvermont) */ -+ case 0x4c: /* Cherrytrail / Brasswell */ -+ case 0x4a: /* Merrifield */ -+ case 0x5a: /* Moorefield */ -+ case 0x5c: /* Goldmont */ -+ case 0x5f: /* Denverton */ -+ case 0x7a: /* Gemini Lake */ -+ return false; -+ -+ /* -+ * Knights processors are not vulnerable. -+ */ -+ case 0x57: /* Knights Landing */ -+ case 0x85: /* Knights Mill */ -+ return false; -+ -+ default: -+ printk("Unrecognised CPU model %#x - assuming vulnerable to LazyFPU\n", -+ boot_cpu_data.x86_model); -+ return true; -+ } -+} -+ -+/* Calculate whether this CPU is vulnerable to L1TF. */ -+static __init void l1tf_calculations(uint64_t caps) -+{ -+ bool hit_default = false; -+ -+ l1d_maxphysaddr = paddr_bits; -+ -+ /* L1TF is only known to affect Intel Family 6 processors at this time. */ -+ if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && -+ boot_cpu_data.x86 == 6 ) -+ { -+ switch ( boot_cpu_data.x86_model ) -+ { -+ /* -+ * Core processors since at least Penryn are vulnerable. -+ */ -+ case 0x17: /* Penryn */ -+ case 0x1d: /* Dunnington */ -+ cpu_has_bug_l1tf = true; -+ break; -+ -+ case 0x1f: /* Auburndale / Havendale */ -+ case 0x1e: /* Nehalem */ -+ case 0x1a: /* Nehalem EP */ -+ case 0x2e: /* Nehalem EX */ -+ case 0x25: /* Westmere */ -+ case 0x2c: /* Westmere EP */ -+ case 0x2f: /* Westmere EX */ -+ cpu_has_bug_l1tf = true; -+ l1d_maxphysaddr = 44; -+ break; -+ -+ case 0x2a: /* SandyBridge */ -+ case 0x2d: /* SandyBridge EP/EX */ -+ case 0x3a: /* IvyBridge */ -+ case 0x3e: /* IvyBridge EP/EX */ -+ case 0x3c: /* Haswell */ -+ case 0x3f: /* Haswell EX/EP */ -+ case 0x45: /* Haswell D */ -+ case 0x46: /* Haswell H */ -+ case 0x3d: /* Broadwell */ -+ case 0x47: /* Broadwell H */ -+ case 0x4f: /* Broadwell EP/EX */ -+ case 0x56: /* Broadwell D */ -+ case 0x4e: /* Skylake M */ -+ case 0x55: /* Skylake X */ -+ case 0x5e: /* Skylake D */ -+ case 0x66: /* Cannonlake */ -+ case 0x67: /* Cannonlake? */ -+ case 0x8e: /* Kabylake M */ -+ case 0x9e: /* Kabylake D */ -+ cpu_has_bug_l1tf = true; -+ l1d_maxphysaddr = 46; -+ break; -+ -+ /* -+ * Atom processors are not vulnerable. -+ */ -+ case 0x1c: /* Pineview */ -+ case 0x26: /* Lincroft */ -+ case 0x27: /* Penwell */ -+ case 0x35: /* Cloverview */ -+ case 0x36: /* Cedarview */ -+ case 0x37: /* Baytrail / Valleyview (Silvermont) */ -+ case 0x4d: /* Avaton / Rangely (Silvermont) */ -+ case 0x4c: /* Cherrytrail / Brasswell */ -+ case 0x4a: /* Merrifield */ -+ case 0x5a: /* Moorefield */ -+ case 0x5c: /* Goldmont */ -+ case 0x5f: /* Denverton */ -+ case 0x7a: /* Gemini Lake */ -+ break; -+ -+ /* -+ * Knights processors are not vulnerable. -+ */ -+ case 0x57: /* Knights Landing */ -+ case 0x85: /* Knights Mill */ -+ break; -+ -+ default: -+ /* Defer printk() until we've accounted for RDCL_NO. */ -+ hit_default = true; -+ cpu_has_bug_l1tf = true; -+ break; -+ } -+ } -+ -+ /* Any processor advertising RDCL_NO should be not vulnerable to L1TF. */ -+ if ( caps & ARCH_CAPABILITIES_RDCL_NO ) -+ cpu_has_bug_l1tf = false; -+ -+ if ( cpu_has_bug_l1tf && hit_default ) -+ printk("Unrecognised CPU model %#x - assuming vulnerable to L1TF\n", -+ boot_cpu_data.x86_model); -+ -+ /* -+ * L1TF safe address heuristics. These apply to the real hardware we are -+ * running on, and are best-effort-only if Xen is virtualised. -+ * -+ * The address mask which the L1D cache uses, which might be wider than -+ * the CPUID-reported maxphysaddr. -+ */ -+ l1tf_addr_mask = ((1ul << l1d_maxphysaddr) - 1) & PAGE_MASK; -+ -+ /* -+ * To be safe, l1tf_safe_maddr must be above the highest cacheable entity -+ * in system physical address space. However, to preserve space for -+ * paged-out metadata, it should be as low as possible above the highest -+ * cacheable address, so as to require fewer high-order bits being set. -+ * -+ * These heuristics are based on some guesswork to improve the likelihood -+ * of safety in the common case, including Linux's L1TF mitigation of -+ * inverting all address bits in a non-present PTE. -+ * -+ * - If L1D is wider than CPUID (Nehalem and later mobile/desktop/low end -+ * server), setting any address bit beyond CPUID maxphysaddr guarantees -+ * to make the PTE safe. This case doesn't require all the high-order -+ * bits being set, and doesn't require any other source of information -+ * for safety. -+ * -+ * - If L1D is the same as CPUID (Pre-Nehalem, or high end server), we -+ * must sacrifice high order bits from the real address space for -+ * safety. Therefore, make a blind guess that there is nothing -+ * cacheable in the top quarter of physical address space. -+ * -+ * It is exceedingly unlikely for machines to be populated with this -+ * much RAM (likely 512G on pre-Nehalem, 16T on Nehalem/Westmere, 64T on -+ * Sandybridge and later) due to the sheer volume of DIMMs this would -+ * actually take. -+ * -+ * However, it is possible to find machines this large, so the "top -+ * quarter" guess is supplemented to push the limit higher if references -+ * to cacheable mappings (E820/SRAT/EFI/etc) are found above the top -+ * quarter boundary. -+ * -+ * Finally, this top quarter guess gives us a good chance of being safe -+ * when running virtualised (and the CPUID maxphysaddr hasn't been -+ * levelled for heterogeneous migration safety), where the safety -+ * consideration is still in terms of host details, but all E820/etc -+ * information is in terms of guest physical layout. -+ */ -+ l1tf_safe_maddr = max(l1tf_safe_maddr, ((l1d_maxphysaddr > paddr_bits) -+ ? (1ul << paddr_bits) -+ : (3ul << (paddr_bits - 2)))); -+} -+ -+int8_t __read_mostly opt_xpti = -1; -+ -+static __init void xpti_init_default(uint64_t caps) -+{ -+ if ( boot_cpu_data.x86_vendor == X86_VENDOR_AMD ) -+ caps = ARCH_CAPABILITIES_RDCL_NO; -+ -+ if ( caps & ARCH_CAPABILITIES_RDCL_NO ) -+ opt_xpti = 0; -+ else -+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU; -+} -+ -+static __init int parse_xpti(const char *s) -+{ -+ const char *ss; -+ int val, rc = 0; -+ -+ /* Inhibit the defaults as an explicit choice has been given. */ -+ if ( opt_xpti == -1 ) -+ opt_xpti = 0; -+ -+ /* Interpret 'xpti' alone in its positive boolean form. */ -+ if ( *s == '\0' ) -+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU; -+ -+ do { -+ ss = strchr(s, ','); -+ if ( !ss ) -+ ss = strchr(s, '\0'); -+ -+ switch ( parse_bool(s, ss) ) -+ { -+ case 0: -+ opt_xpti = 0; -+ break; -+ -+ case 1: -+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU; -+ break; -+ -+ default: -+ if ( !strcmp(s, "default") ) -+ opt_xpti = -1; -+ else if ( (val = parse_boolean("dom0", s, ss)) >= 0 ) -+ opt_xpti = (opt_xpti & ~OPT_XPTI_DOM0) | -+ (val ? OPT_XPTI_DOM0 : 0); -+ else if ( (val = parse_boolean("domu", s, ss)) >= 0 ) -+ opt_xpti = (opt_xpti & ~OPT_XPTI_DOMU) | -+ (val ? OPT_XPTI_DOMU : 0); -+ else -+ rc = -EINVAL; -+ break; -+ } -+ -+ s = ss + 1; -+ } while ( *ss ); -+ -+ return rc; -+} -+custom_param("xpti", parse_xpti); -+ - void __init init_speculation_mitigations(void) - { - enum ind_thunk thunk = THUNK_DEFAULT; -- bool ibrs = false; -+ bool use_spec_ctrl = false, ibrs = false; -+ uint64_t caps = 0; -+ -+ if ( boot_cpu_has(X86_FEATURE_ARCH_CAPS) ) -+ rdmsrl(MSR_ARCH_CAPABILITIES, caps); - - /* - * Has the user specified any custom BTI mitigations? If so, follow their -@@ -209,7 +739,7 @@ void __init init_speculation_mitigations(void) - * On Intel hardware, we'd like to use retpoline in preference to - * IBRS, but only if it is safe on this hardware. - */ -- else if ( retpoline_safe() ) -+ else if ( retpoline_safe(caps) ) - thunk = THUNK_RETPOLINE; - else if ( boot_cpu_has(X86_FEATURE_IBRSB) ) - ibrs = true; -@@ -246,21 +776,35 @@ void __init init_speculation_mitigations(void) - else if ( thunk == THUNK_JMP ) - setup_force_cpu_cap(X86_FEATURE_IND_THUNK_JMP); - -+ /* -+ * If we are on hardware supporting MSR_SPEC_CTRL, see about setting up -+ * the alternatives blocks so we can virtualise support for guests. -+ */ - if ( boot_cpu_has(X86_FEATURE_IBRSB) ) - { -- /* -- * Even if we've chosen to not have IBRS set in Xen context, we still -- * need the IBRS entry/exit logic to virtualise IBRS support for -- * guests. -- */ -- if ( ibrs ) -- setup_force_cpu_cap(X86_FEATURE_XEN_IBRS_SET); -- else -- setup_force_cpu_cap(X86_FEATURE_XEN_IBRS_CLEAR); -+ if ( opt_msr_sc_pv ) -+ { -+ use_spec_ctrl = true; -+ setup_force_cpu_cap(X86_FEATURE_SC_MSR_PV); -+ } -+ -+ if ( opt_msr_sc_hvm ) -+ { -+ use_spec_ctrl = true; -+ setup_force_cpu_cap(X86_FEATURE_SC_MSR_HVM); -+ } -+ -+ if ( use_spec_ctrl ) -+ default_spec_ctrl_flags |= SCF_ist_wrmsr; - -- default_bti_ist_info |= BTI_IST_WRMSR | ibrs; -+ if ( ibrs ) -+ default_xen_spec_ctrl |= SPEC_CTRL_IBRS; - } - -+ /* If we have SSBD available, see whether we should use it. */ -+ if ( boot_cpu_has(X86_FEATURE_SSBD) && opt_ssbd ) -+ default_xen_spec_ctrl |= SPEC_CTRL_SSBD; -+ - /* - * PV guests can poison the RSB to any virtual address from which - * they can execute a call instruction. This is necessarily outside -@@ -274,33 +818,123 @@ void __init init_speculation_mitigations(void) - * If a processors speculates to 32bit PV guest kernel mappings, it is - * speculating in 64bit supervisor mode, and can leak data. - */ -- if ( opt_rsb_native ) -+ if ( opt_rsb_pv ) - { -- setup_force_cpu_cap(X86_FEATURE_RSB_NATIVE); -- default_bti_ist_info |= BTI_IST_RSB; -+ setup_force_cpu_cap(X86_FEATURE_SC_RSB_PV); -+ default_spec_ctrl_flags |= SCF_ist_rsb; - } - - /* - * HVM guests can always poison the RSB to point at Xen supervisor - * mappings. - */ -- if ( opt_rsb_vmexit ) -- setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); -+ if ( opt_rsb_hvm ) -+ setup_force_cpu_cap(X86_FEATURE_SC_RSB_HVM); - - /* Check we have hardware IBPB support before using it... */ - if ( !boot_cpu_has(X86_FEATURE_IBRSB) && !boot_cpu_has(X86_FEATURE_IBPB) ) - opt_ibpb = false; - -- /* (Re)init BSP state now that default_bti_ist_info has been calculated. */ -+ /* Check whether Eager FPU should be enabled by default. */ -+ if ( opt_eager_fpu == -1 ) -+ opt_eager_fpu = should_use_eager_fpu(); -+ -+ /* (Re)init BSP state now that default_spec_ctrl_flags has been calculated. */ - init_shadow_spec_ctrl_state(); - -- print_details(thunk); -+ /* If Xen is using any MSR_SPEC_CTRL settings, adjust the idle path. */ -+ if ( default_xen_spec_ctrl ) -+ setup_force_cpu_cap(X86_FEATURE_SC_MSR_IDLE); -+ -+ if ( opt_xpti == -1 ) -+ xpti_init_default(caps); -+ -+ if ( opt_xpti == 0 ) -+ setup_force_cpu_cap(X86_FEATURE_NO_XPTI); -+ else -+ setup_clear_cpu_cap(X86_FEATURE_NO_XPTI); -+ -+ l1tf_calculations(caps); -+ -+ /* -+ * By default, enable PV domU L1TF mitigations on all L1TF-vulnerable -+ * hardware, except when running in shim mode. -+ * -+ * In shim mode, SHADOW is expected to be compiled out, and a malicious -+ * guest kernel can only attack the shim Xen, not the host Xen. -+ */ -+ if ( opt_pv_l1tf == -1 ) -+ { -+ if ( pv_shim || !cpu_has_bug_l1tf ) -+ opt_pv_l1tf = 0; -+ else -+ opt_pv_l1tf = OPT_PV_L1TF_DOMU; -+ } -+ -+ /* -+ * By default, enable L1D_FLUSH on L1TF-vulnerable hardware, unless -+ * instructed to skip the flush on vmentry by our outer hypervisor. -+ */ -+ if ( !boot_cpu_has(X86_FEATURE_L1D_FLUSH) ) -+ opt_l1d_flush = 0; -+ else if ( opt_l1d_flush == -1 ) -+ opt_l1d_flush = cpu_has_bug_l1tf && !(caps & ARCH_CAPS_SKIP_L1DFL); -+ -+ /* -+ * We do not disable HT by default on affected hardware. -+ * -+ * Firstly, if the user intends to use exclusively PV, or HVM shadow -+ * guests, HT isn't a concern and should remain fully enabled. Secondly, -+ * safety for HVM HAP guests can be arranged by the toolstack with core -+ * parking, pinning or cpupool configurations, including mixed setups. -+ * -+ * However, if we are on affected hardware, with HT enabled, and the user -+ * hasn't explicitly chosen whether to use HT or not, nag them to do so. -+ */ -+ if ( opt_smt == -1 && cpu_has_bug_l1tf && !pv_shim && -+ boot_cpu_data.x86_num_siblings > 1 ) -+ warning_add( -+ "Booted on L1TF-vulnerable hardware with SMT/Hyperthreading\n" -+ "enabled. Please assess your configuration and choose an\n" -+ "explicit 'smt=<bool>' setting. See XSA-273.\n"); -+ -+ print_details(thunk, caps); -+ -+ /* -+ * If MSR_SPEC_CTRL is available, apply Xen's default setting and discard -+ * any firmware settings. For performance reasons, when safe to do so, we -+ * delay applying non-zero settings until after dom0 has been constructed. -+ * -+ * "when safe to do so" is based on whether we are virtualised. A native -+ * boot won't have any other code running in a position to mount an -+ * attack. -+ */ -+ if ( boot_cpu_has(X86_FEATURE_IBRSB) ) -+ { -+ bsp_delay_spec_ctrl = !cpu_has_hypervisor && default_xen_spec_ctrl; -+ -+ /* -+ * If delaying MSR_SPEC_CTRL setup, use the same mechanism as -+ * spec_ctrl_enter_idle(), by using a shadow value of zero. -+ */ -+ if ( bsp_delay_spec_ctrl ) -+ { -+ struct cpu_info *info = get_cpu_info(); -+ -+ info->shadow_spec_ctrl = 0; -+ barrier(); -+ info->spec_ctrl_flags |= SCF_use_shadow; -+ barrier(); -+ } -+ -+ wrmsrl(MSR_SPEC_CTRL, bsp_delay_spec_ctrl ? 0 : default_xen_spec_ctrl); -+ } - } - - static void __init __maybe_unused build_assertions(void) - { - /* The optimised assembly relies on this alias. */ -- BUILD_BUG_ON(BTI_IST_IBRS != SPEC_CTRL_IBRS); -+ BUILD_BUG_ON(SCF_use_shadow != 1); - } - - /* -diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c -index 166eb44fe2..2d70b45909 100644 ---- a/xen/arch/x86/srat.c -+++ b/xen/arch/x86/srat.c -@@ -20,6 +20,7 @@ - #include <xen/pfn.h> - #include <asm/e820.h> - #include <asm/page.h> -+#include <asm/spec_ctrl.h> - - static struct acpi_table_slit *__read_mostly acpi_slit; - -@@ -284,6 +285,11 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma) - if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) - return; - -+ start = ma->base_address; -+ end = start + ma->length; -+ /* Supplement the heuristics in l1tf_calculations(). */ -+ l1tf_safe_maddr = max(l1tf_safe_maddr, ROUNDUP(end, PAGE_SIZE)); -+ - if (num_node_memblks >= NR_NODE_MEMBLKS) - { - dprintk(XENLOG_WARNING, -@@ -292,8 +298,6 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma) - return; - } - -- start = ma->base_address; -- end = start + ma->length; - pxm = ma->proximity_domain; - if (srat_rev < 2) - pxm &= 0xff; -diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c -index 6ba823ca69..e726eee974 100644 ---- a/xen/arch/x86/sysctl.c -+++ b/xen/arch/x86/sysctl.c -@@ -23,6 +23,7 @@ - #include <asm/hvm/hvm.h> - #include <asm/hvm/support.h> - #include <asm/processor.h> -+#include <asm/setup.h> - #include <asm/smp.h> - #include <asm/numa.h> - #include <xen/nodemask.h> -@@ -48,14 +49,27 @@ static void l3_cache_get(void *arg) - - long cpu_up_helper(void *data) - { -- int cpu = (unsigned long)data; -+ unsigned int cpu = (unsigned long)data; - int ret = cpu_up(cpu); -+ - if ( ret == -EBUSY ) - { - /* On EBUSY, flush RCU work and have one more go. */ - rcu_barrier(); - ret = cpu_up(cpu); - } -+ -+ if ( !ret && !opt_smt && -+ cpu_data[cpu].compute_unit_id == INVALID_CUID && -+ cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) > 1 ) -+ { -+ ret = cpu_down_helper(data); -+ if ( ret ) -+ printk("Could not re-offline CPU%u (%d)\n", cpu, ret); -+ else -+ ret = -EPERM; -+ } -+ - return ret; - } - -diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c -index 906124331b..e8f85f716e 100644 ---- a/xen/arch/x86/traps.c -+++ b/xen/arch/x86/traps.c -@@ -96,8 +96,6 @@ string_param("nmi", opt_nmi); - DEFINE_PER_CPU(u64, efer); - static DEFINE_PER_CPU(unsigned long, last_extable_addr); - --DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr); -- - DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table); - DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table); - -@@ -117,6 +115,9 @@ integer_param("debug_stack_lines", debug_stack_lines); - static bool opt_ler; - boolean_param("ler", opt_ler); - -+/* LastExceptionFromIP on this hardware. Zero if LER is not in use. */ -+unsigned int __read_mostly ler_msr; -+ - #define stack_words_per_line 4 - #define ESP_BEFORE_EXCEPTION(regs) ((unsigned long *)regs->rsp) - -@@ -325,13 +326,13 @@ static void show_guest_stack(struct vcpu *v, const struct cpu_user_regs *regs) - /* - * Notes for get_stack_trace_bottom() and get_stack_dump_bottom() - * -- * Stack pages 0, 1 and 2: -+ * Stack pages 0 - 3: - * These are all 1-page IST stacks. Each of these stacks have an exception - * frame and saved register state at the top. The interesting bound for a - * trace is the word adjacent to this, while the bound for a dump is the - * very top, including the exception frame. - * -- * Stack pages 3, 4 and 5: -+ * Stack pages 4 and 5: - * None of these are particularly interesting. With MEMORY_GUARD, page 5 is - * explicitly not present, so attempting to dump or trace it is - * counterproductive. Without MEMORY_GUARD, it is possible for a call chain -@@ -352,12 +353,12 @@ unsigned long get_stack_trace_bottom(unsigned long sp) - { - switch ( get_stack_page(sp) ) - { -- case 0 ... 2: -+ case 0 ... 3: - return ROUNDUP(sp, PAGE_SIZE) - - offsetof(struct cpu_user_regs, es) - sizeof(unsigned long); - - #ifndef MEMORY_GUARD -- case 3 ... 5: -+ case 4 ... 5: - #endif - case 6 ... 7: - return ROUNDUP(sp, STACK_SIZE) - -@@ -372,11 +373,11 @@ unsigned long get_stack_dump_bottom(unsigned long sp) - { - switch ( get_stack_page(sp) ) - { -- case 0 ... 2: -+ case 0 ... 3: - return ROUNDUP(sp, PAGE_SIZE) - sizeof(unsigned long); - - #ifndef MEMORY_GUARD -- case 3 ... 5: -+ case 4 ... 5: - #endif - case 6 ... 7: - return ROUNDUP(sp, STACK_SIZE) - sizeof(unsigned long); -@@ -1722,7 +1723,21 @@ void do_device_not_available(struct cpu_user_regs *regs) - { - struct vcpu *curr = current; - -- BUG_ON(!guest_mode(regs)); -+ if ( !guest_mode(regs) ) -+ { -+ unsigned long fixup = search_exception_table(regs); -+ -+ gprintk(XENLOG_ERR, "#NM: %p [%ps] -> %p\n", -+ _p(regs->rip), _p(regs->rip), _p(fixup)); -+ /* -+ * We shouldn't be able to reach here, but for release builds have -+ * the recovery logic in place nevertheless. -+ */ -+ ASSERT_UNREACHABLE(); -+ BUG_ON(!fixup); -+ regs->rip = fixup; -+ return; -+ } - - vcpu_restore_fpu_lazy(curr); - -@@ -1748,26 +1763,51 @@ void write_efer(u64 val) - wrmsrl(MSR_EFER, val); - } - --static void ler_enable(void) --{ -- u64 debugctl; -- -- if ( !this_cpu(ler_msr) ) -- return; -- -- rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); -- wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl | IA32_DEBUGCTLMSR_LBR); --} -- - void do_debug(struct cpu_user_regs *regs) - { -+ unsigned long dr6; - struct vcpu *v = current; - -+ /* Stash dr6 as early as possible. */ -+ dr6 = read_debugreg(6); -+ - if ( debugger_trap_entry(TRAP_debug, regs) ) - return; - -+ /* -+ * At the time of writing (March 2018), on the subject of %dr6: -+ * -+ * The Intel manual says: -+ * Certain debug exceptions may clear bits 0-3. The remaining contents -+ * of the DR6 register are never cleared by the processor. To avoid -+ * confusion in identifying debug exceptions, debug handlers should -+ * clear the register (except bit 16, which they should set) before -+ * returning to the interrupted task. -+ * -+ * The AMD manual says: -+ * Bits 15:13 of the DR6 register are not cleared by the processor and -+ * must be cleared by software after the contents have been read. -+ * -+ * Some bits are reserved set, some are reserved clear, and some bits -+ * which were previously reserved set are reused and cleared by hardware. -+ * For future compatibility, reset to the default value, which will allow -+ * us to spot any bit being changed by hardware to its non-default value. -+ */ -+ write_debugreg(6, X86_DR6_DEFAULT); -+ -+ /* #DB automatically disabled LBR. Reinstate it if debugging Xen. */ -+ if ( cpu_has_xen_lbr ) -+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR); -+ - if ( !guest_mode(regs) ) - { -+ /* -+ * !!! WARNING !!! -+ * -+ * %dr6 is mostly guest controlled at this point. Any decsions base -+ * on its value must be crosschecked with non-guest controlled state. -+ */ -+ - if ( regs->eflags & X86_EFLAGS_TF ) - { - /* In SYSENTER entry path we can't zap TF until EFLAGS is saved. */ -@@ -1776,7 +1816,7 @@ void do_debug(struct cpu_user_regs *regs) - { - if ( regs->rip == (unsigned long)sysenter_eflags_saved ) - regs->eflags &= ~X86_EFLAGS_TF; -- goto out; -+ return; - } - if ( !debugger_trap_fatal(TRAP_debug, regs) ) - { -@@ -1784,29 +1824,63 @@ void do_debug(struct cpu_user_regs *regs) - regs->eflags &= ~X86_EFLAGS_TF; - } - } -- else -+ -+ /* -+ * Check for fault conditions. General Detect, and instruction -+ * breakpoints are faults rather than traps, at which point attempting -+ * to ignore and continue will result in a livelock. -+ * -+ * However, on entering the #DB handler, hardware clears %dr7.gd for -+ * us (as confirmed by the earlier %dr6 accesses succeeding), meaning -+ * that a real General Detect exception is restartable. -+ * -+ * PV guests are not permitted to point %dr{0..3} at Xen linear -+ * addresses, and Instruction Breakpoints (being faults) don't get -+ * delayed by a MovSS shadow, so we should never encounter one in -+ * hypervisor context. -+ * -+ * If however we do, safety measures need to be enacted. Use a big -+ * hammer and clear all debug settings. -+ */ -+ if ( dr6 & (DR_TRAP3 | DR_TRAP2 | DR_TRAP1 | DR_TRAP0) ) - { -- /* -- * We ignore watchpoints when they trigger within Xen. This may -- * happen when a buffer is passed to us which previously had a -- * watchpoint set on it. No need to bump EIP; the only faulting -- * trap is an instruction breakpoint, which can't happen to us. -- */ -- WARN_ON(!search_exception_table(regs)); -+ unsigned int bp, dr7 = read_debugreg(7); -+ -+ for ( bp = 0; bp < 4; ++bp ) -+ { -+ if ( (dr6 & (1u << bp)) && /* Breakpoint triggered? */ -+ (dr7 & (3u << (bp * DR_ENABLE_SIZE))) && /* Enabled? */ -+ ((dr7 & (3u << ((bp * DR_CONTROL_SIZE) + /* Insn? */ -+ DR_CONTROL_SHIFT))) == DR_RW_EXECUTE) ) -+ { -+ ASSERT_UNREACHABLE(); -+ -+ printk(XENLOG_ERR -+ "Hit instruction breakpoint in Xen context\n"); -+ write_debugreg(7, 0); -+ break; -+ } -+ } - } -- goto out; -+ -+ /* -+ * Whatever caused this #DB should be restartable by this point. Note -+ * it and continue. Guests can trigger this in certain corner cases, -+ * so ensure the message is ratelimited. -+ */ -+ gprintk(XENLOG_WARNING, -+ "Hit #DB in Xen context: %04x:%p [%ps], stk %04x:%p, dr6 %lx\n", -+ regs->cs, _p(regs->rip), _p(regs->rip), -+ regs->ss, _p(regs->rsp), dr6); -+ -+ return; - } - - /* Save debug status register where guest OS can peek at it */ -- v->arch.debugreg[6] = read_debugreg(6); -+ v->arch.debugreg[6] |= (dr6 & ~X86_DR6_DEFAULT); -+ v->arch.debugreg[6] &= (dr6 | ~X86_DR6_DEFAULT); - -- ler_enable(); - pv_inject_hw_exception(TRAP_debug, X86_EVENT_NO_EC); -- return; -- -- out: -- ler_enable(); -- return; - } - - static void __init noinline __set_intr_gate(unsigned int n, -@@ -1850,38 +1924,46 @@ void load_TR(void) - : "=m" (old_gdt) : "rm" (TSS_ENTRY << 3), "m" (tss_gdt) : "memory" ); - } - --void percpu_traps_init(void) -+static unsigned int calc_ler_msr(void) - { -- subarch_percpu_traps_init(); -- -- if ( !opt_ler ) -- return; -- - switch ( boot_cpu_data.x86_vendor ) - { - case X86_VENDOR_INTEL: - switch ( boot_cpu_data.x86 ) - { - case 6: -- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP; -- break; -+ return MSR_IA32_LASTINTFROMIP; -+ - case 15: -- this_cpu(ler_msr) = MSR_P4_LER_FROM_LIP; -- break; -+ return MSR_P4_LER_FROM_LIP; - } - break; -+ - case X86_VENDOR_AMD: - switch ( boot_cpu_data.x86 ) - { - case 6: - case 0xf ... 0x17: -- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP; -- break; -+ return MSR_IA32_LASTINTFROMIP; - } - break; - } - -- ler_enable(); -+ return 0; -+} -+ -+void percpu_traps_init(void) -+{ -+ subarch_percpu_traps_init(); -+ -+ if ( !opt_ler ) -+ return; -+ -+ if ( !ler_msr && (ler_msr = calc_ler_msr()) ) -+ setup_force_cpu_cap(X86_FEATURE_XEN_LBR); -+ -+ if ( cpu_has_xen_lbr ) -+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR); - } - - void __init init_idt_traps(void) -@@ -1917,6 +1999,7 @@ void __init init_idt_traps(void) - set_ist(&idt_table[TRAP_double_fault], IST_DF); - set_ist(&idt_table[TRAP_nmi], IST_NMI); - set_ist(&idt_table[TRAP_machine_check], IST_MCE); -+ set_ist(&idt_table[TRAP_debug], IST_DB); - - /* CPU0 uses the master IDT. */ - idt_tables[0] = idt_table; -@@ -1984,6 +2067,12 @@ void activate_debugregs(const struct vcpu *curr) - } - } - -+/* -+ * Used by hypercalls and the emulator. -+ * -ENODEV => #UD -+ * -EINVAL => #GP Invalid bit -+ * -EPERM => #GP Valid bit, but not permitted to use -+ */ - long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value) - { - int i; -@@ -2015,7 +2104,17 @@ long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value) - if ( v == curr ) - write_debugreg(3, value); - break; -+ -+ case 4: -+ if ( v->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE ) -+ return -ENODEV; -+ -+ /* Fallthrough */ - case 6: -+ /* The upper 32 bits are strictly reserved. */ -+ if ( value != (uint32_t)value ) -+ return -EINVAL; -+ - /* - * DR6: Bits 4-11,16-31 reserved (set to 1). - * Bit 12 reserved (set to 0). -@@ -2025,7 +2124,17 @@ long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value) - if ( v == curr ) - write_debugreg(6, value); - break; -+ -+ case 5: -+ if ( v->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE ) -+ return -ENODEV; -+ -+ /* Fallthrough */ - case 7: -+ /* The upper 32 bits are strictly reserved. */ -+ if ( value != (uint32_t)value ) -+ return -EINVAL; -+ - /* - * DR7: Bit 10 reserved (set to 1). - * Bits 11-12,14-15 reserved (set to 0). -@@ -2038,6 +2147,7 @@ long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value) - */ - if ( value & DR_GENERAL_DETECT ) - return -EPERM; -+ - /* DR7.{G,L}E = 0 => debugging disabled for this domain. */ - if ( value & DR7_ACTIVE_MASK ) - { -@@ -2066,11 +2176,15 @@ long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value) - !(v->arch.debugreg[7] & DR7_ACTIVE_MASK) ) - activate_debugregs(v); - } -+ else -+ /* Zero the emulated controls if %dr7 isn't active. */ -+ v->arch.debugreg[5] = 0; -+ - if ( v == curr ) - write_debugreg(7, value); - break; - default: -- return -EINVAL; -+ return -ENODEV; - } - - v->arch.debugreg[reg] = value; -diff --git a/xen/arch/x86/x86_64/asm-offsets.c b/xen/arch/x86/x86_64/asm-offsets.c -index 13478d4fc1..10c243a039 100644 ---- a/xen/arch/x86/x86_64/asm-offsets.c -+++ b/xen/arch/x86/x86_64/asm-offsets.c -@@ -142,8 +142,10 @@ void __dummy__(void) - OFFSET(CPUINFO_xen_cr3, struct cpu_info, xen_cr3); - OFFSET(CPUINFO_pv_cr3, struct cpu_info, pv_cr3); - OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl); -- OFFSET(CPUINFO_use_shadow_spec_ctrl, struct cpu_info, use_shadow_spec_ctrl); -- OFFSET(CPUINFO_bti_ist_info, struct cpu_info, bti_ist_info); -+ OFFSET(CPUINFO_xen_spec_ctrl, struct cpu_info, xen_spec_ctrl); -+ OFFSET(CPUINFO_spec_ctrl_flags, struct cpu_info, spec_ctrl_flags); -+ OFFSET(CPUINFO_root_pgt_changed, struct cpu_info, root_pgt_changed); -+ OFFSET(CPUINFO_use_pv_cr3, struct cpu_info, use_pv_cr3); - DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info)); - BLANK(); - -diff --git a/xen/arch/x86/x86_64/compat/entry.S b/xen/arch/x86/x86_64/compat/entry.S -index 75497bc292..1c4cd795d2 100644 ---- a/xen/arch/x86/x86_64/compat/entry.S -+++ b/xen/arch/x86/x86_64/compat/entry.S -@@ -39,6 +39,12 @@ ENTRY(compat_test_all_events) - leaq irq_stat+IRQSTAT_softirq_pending(%rip),%rcx - cmpl $0,(%rcx,%rax,1) - jne compat_process_softirqs -+ -+ /* Inject exception if pending. */ -+ lea VCPU_trap_bounce(%rbx), %rdx -+ testb $TBF_EXCEPTION, TRAPBOUNCE_flags(%rdx) -+ jnz .Lcompat_process_trapbounce -+ - testb $1,VCPU_mce_pending(%rbx) - jnz compat_process_mce - .Lcompat_test_guest_nmi: -@@ -68,15 +74,24 @@ compat_process_softirqs: - call do_softirq - jmp compat_test_all_events - -+ ALIGN -+/* %rbx: struct vcpu, %rdx: struct trap_bounce */ -+.Lcompat_process_trapbounce: -+ sti -+.Lcompat_bounce_exception: -+ call compat_create_bounce_frame -+ movb $0, TRAPBOUNCE_flags(%rdx) -+ jmp compat_test_all_events -+ - ALIGN - /* %rbx: struct vcpu */ - compat_process_mce: - testb $1 << VCPU_TRAP_MCE,VCPU_async_exception_mask(%rbx) - jnz .Lcompat_test_guest_nmi - sti -- movb $0,VCPU_mce_pending(%rbx) -- call set_guest_machinecheck_trapbounce -- testl %eax,%eax -+ movb $0, VCPU_mce_pending(%rbx) -+ call set_guest_machinecheck_trapbounce -+ test %al, %al - jz compat_test_all_events - movzbl VCPU_async_exception_mask(%rbx),%edx # save mask for the - movb %dl,VCPU_mce_old_mask(%rbx) # iret hypercall -@@ -88,11 +103,11 @@ compat_process_mce: - /* %rbx: struct vcpu */ - compat_process_nmi: - testb $1 << VCPU_TRAP_NMI,VCPU_async_exception_mask(%rbx) -- jnz compat_test_guest_events -+ jnz compat_test_guest_events - sti -- movb $0,VCPU_nmi_pending(%rbx) -+ movb $0, VCPU_nmi_pending(%rbx) - call set_guest_nmi_trapbounce -- testl %eax,%eax -+ test %al, %al - jz compat_test_all_events - movzbl VCPU_async_exception_mask(%rbx),%edx # save mask for the - movb %dl,VCPU_nmi_old_mask(%rbx) # iret hypercall -@@ -151,7 +166,7 @@ ENTRY(compat_restore_all_guest) - mov VCPUMSR_spec_ctrl_raw(%rax), %eax - - /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ -- SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ -+ SPEC_CTRL_EXIT_TO_PV /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ - - RESTORE_ALL adj=8 compat=1 - .Lft0: iretq -@@ -189,15 +204,6 @@ ENTRY(cr4_pv32_restore) - xor %eax, %eax - ret - --/* %rdx: trap_bounce, %rbx: struct vcpu */ --ENTRY(compat_post_handle_exception) -- testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%rdx) -- jz compat_test_all_events --.Lcompat_bounce_exception: -- call compat_create_bounce_frame -- movb $0,TRAPBOUNCE_flags(%rdx) -- jmp compat_test_all_events -- - .section .text.entry, "ax", @progbits - - /* See lstar_enter for entry register state. */ -@@ -218,10 +224,9 @@ ENTRY(cstar_enter) - - GET_STACK_END(bx) - mov STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx -- neg %rcx -+ test %rcx, %rcx - jz .Lcstar_cr3_okay -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) -- neg %rcx -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) - mov %rcx, %cr3 - movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) - .Lcstar_cr3_okay: -diff --git a/xen/arch/x86/x86_64/compat/mm.c b/xen/arch/x86/x86_64/compat/mm.c -index c2aa6f2fdb..02bc75b91e 100644 ---- a/xen/arch/x86/x86_64/compat/mm.c -+++ b/xen/arch/x86/x86_64/compat/mm.c -@@ -163,19 +163,6 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) - return rc; - } - --int compat_update_va_mapping(unsigned int va, u32 lo, u32 hi, -- unsigned int flags) --{ -- return do_update_va_mapping(va, lo | ((u64)hi << 32), flags); --} -- --int compat_update_va_mapping_otherdomain(unsigned long va, u32 lo, u32 hi, -- unsigned long flags, -- domid_t domid) --{ -- return do_update_va_mapping_otherdomain(va, lo | ((u64)hi << 32), flags, domid); --} -- - DEFINE_XEN_GUEST_HANDLE(mmuext_op_compat_t); - - int compat_mmuext_op(XEN_GUEST_HANDLE_PARAM(void) arg, -diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S -index bdd33e727f..c163c31a60 100644 ---- a/xen/arch/x86/x86_64/entry.S -+++ b/xen/arch/x86/x86_64/entry.S -@@ -42,6 +42,12 @@ test_all_events: - leaq irq_stat+IRQSTAT_softirq_pending(%rip), %rcx - cmpl $0, (%rcx, %rax, 1) - jne process_softirqs -+ -+ /* Inject exception if pending. */ -+ lea VCPU_trap_bounce(%rbx), %rdx -+ testb $TBF_EXCEPTION, TRAPBOUNCE_flags(%rdx) -+ jnz .Lprocess_trapbounce -+ - cmpb $0, VCPU_mce_pending(%rbx) - jne process_mce - .Ltest_guest_nmi: -@@ -69,6 +75,15 @@ process_softirqs: - call do_softirq - jmp test_all_events - -+ ALIGN -+/* %rbx: struct vcpu, %rdx struct trap_bounce */ -+.Lprocess_trapbounce: -+ sti -+.Lbounce_exception: -+ call create_bounce_frame -+ movb $0, TRAPBOUNCE_flags(%rdx) -+ jmp test_all_events -+ - ALIGN - /* %rbx: struct vcpu */ - process_mce: -@@ -77,7 +92,7 @@ process_mce: - sti - movb $0, VCPU_mce_pending(%rbx) - call set_guest_machinecheck_trapbounce -- test %eax, %eax -+ test %al, %al - jz test_all_events - movzbl VCPU_async_exception_mask(%rbx), %edx # save mask for the - movb %dl, VCPU_mce_old_mask(%rbx) # iret hypercall -@@ -93,7 +108,7 @@ process_nmi: - sti - movb $0, VCPU_nmi_pending(%rbx) - call set_guest_nmi_trapbounce -- test %eax, %eax -+ test %al, %al - jz test_all_events - movzbl VCPU_async_exception_mask(%rbx), %edx # save mask for the - movb %dl, VCPU_nmi_old_mask(%rbx) # iret hypercall -@@ -149,11 +164,15 @@ restore_all_guest: - mov VCPU_cr3(%rbx), %r9 - GET_STACK_END(dx) - mov STACK_CPUINFO_FIELD(pv_cr3)(%rdx), %rdi -+ test %rdi, %rdi -+ jz .Lrag_keep_cr3 -+ mov %rdi, %rax -+ cmpb $0, STACK_CPUINFO_FIELD(root_pgt_changed)(%rdx) -+ je .Lrag_copy_done -+ movb $0, STACK_CPUINFO_FIELD(root_pgt_changed)(%rdx) - movabs $PADDR_MASK & PAGE_MASK, %rsi - movabs $DIRECTMAP_VIRT_START, %rcx -- mov %rdi, %rax - and %rsi, %rdi -- jz .Lrag_keep_cr3 - and %r9, %rsi - add %rcx, %rdi - add %rcx, %rsi -@@ -168,20 +187,17 @@ restore_all_guest: - sub $(ROOT_PAGETABLE_FIRST_XEN_SLOT - \ - ROOT_PAGETABLE_LAST_XEN_SLOT - 1) * 8, %rdi - rep movsq -- mov STACK_CPUINFO_FIELD(cr4)(%rdx), %rdi -+.Lrag_copy_done: - mov %r9, STACK_CPUINFO_FIELD(xen_cr3)(%rdx) -- mov %rdi, %rsi -- and $~X86_CR4_PGE, %rdi -- mov %rdi, %cr4 -+ movb $1, STACK_CPUINFO_FIELD(use_pv_cr3)(%rdx) - mov %rax, %cr3 -- mov %rsi, %cr4 - .Lrag_keep_cr3: - - /* Restore stashed SPEC_CTRL value. */ - mov %r15d, %eax - - /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ -- SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ -+ SPEC_CTRL_EXIT_TO_PV /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */ - - RESTORE_ALL - testw $TRAP_syscall,4(%rsp) -@@ -222,20 +238,10 @@ restore_all_xen: - * case we return to late PV exit code (from an NMI or #MC). - */ - GET_STACK_END(bx) -- mov STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rdx -+ cmpb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) -+UNLIKELY_START(ne, exit_cr3) - mov STACK_CPUINFO_FIELD(pv_cr3)(%rbx), %rax -- test %rdx, %rdx -- /* -- * Ideally the condition would be "nsz", but such doesn't exist, -- * so "g" will have to do. -- */ --UNLIKELY_START(g, exit_cr3) -- mov %cr4, %rdi -- mov %rdi, %rsi -- and $~X86_CR4_PGE, %rdi -- mov %rdi, %cr4 - mov %rax, %cr3 -- mov %rsi, %cr4 - UNLIKELY_END(exit_cr3) - - /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ -@@ -276,10 +282,9 @@ ENTRY(lstar_enter) - - GET_STACK_END(bx) - mov STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx -- neg %rcx -+ test %rcx, %rcx - jz .Llstar_cr3_okay -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) -- neg %rcx -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) - mov %rcx, %cr3 - movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) - .Llstar_cr3_okay: -@@ -313,10 +318,9 @@ GLOBAL(sysenter_eflags_saved) - /* PUSHF above has saved EFLAGS.IF clear (the caller had it set). */ - orl $X86_EFLAGS_IF, UREGS_eflags(%rsp) - mov STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx -- neg %rcx -+ test %rcx, %rcx - jz .Lsyse_cr3_okay -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) -- neg %rcx -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) - mov %rcx, %cr3 - movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) - .Lsyse_cr3_okay: -@@ -363,10 +367,9 @@ ENTRY(int80_direct_trap) - - GET_STACK_END(bx) - mov STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx -- neg %rcx -+ test %rcx, %rcx - jz .Lint80_cr3_okay -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) -- neg %rcx -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) - mov %rcx, %cr3 - movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx) - .Lint80_cr3_okay: -@@ -553,24 +556,24 @@ ENTRY(common_interrupt) - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - - mov STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx -+ mov STACK_CPUINFO_FIELD(use_pv_cr3)(%r14), %bl - mov %rcx, %r15 -- neg %rcx -+ test %rcx, %rcx - jz .Lintr_cr3_okay -- jns .Lintr_cr3_load -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -- neg %rcx --.Lintr_cr3_load: -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - mov %rcx, %cr3 - xor %ecx, %ecx - mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14) - testb $3, UREGS_cs(%rsp) - cmovnz %rcx, %r15 -+ cmovnz %rcx, %rbx - .Lintr_cr3_okay: - - CR4_PV32_RESTORE - movq %rsp,%rdi - callq do_IRQ - mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -+ mov %bl, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - jmp ret_from_intr - - ENTRY(page_fault) -@@ -585,18 +588,17 @@ GLOBAL(handle_exception) - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - - mov STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx -+ mov STACK_CPUINFO_FIELD(use_pv_cr3)(%r14), %r13b - mov %rcx, %r15 -- neg %rcx -+ test %rcx, %rcx - jz .Lxcpt_cr3_okay -- jns .Lxcpt_cr3_load -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -- neg %rcx --.Lxcpt_cr3_load: -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - mov %rcx, %cr3 - xor %ecx, %ecx - mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14) - testb $3, UREGS_cs(%rsp) - cmovnz %rcx, %r15 -+ cmovnz %rcx, %r13 - .Lxcpt_cr3_okay: - - handle_exception_saved: -@@ -665,17 +667,12 @@ handle_exception_saved: - mov (%rdx, %rax, 8), %rdx - INDIRECT_CALL %rdx - mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -+ mov %r13b, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - testb $3,UREGS_cs(%rsp) - jz restore_all_xen -- leaq VCPU_trap_bounce(%rbx),%rdx - movq VCPU_domain(%rbx),%rax - testb $1,DOMAIN_is_32bit_pv(%rax) -- jnz compat_post_handle_exception -- testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%rdx) -- jz test_all_events --.Lbounce_exception: -- call create_bounce_frame -- movb $0,TRAPBOUNCE_flags(%rdx) -+ jnz compat_test_all_events - jmp test_all_events - - /* No special register assumptions. */ -@@ -698,6 +695,7 @@ exception_with_ints_disabled: - 1: movq UREGS_error_code(%rsp),%rax # ec/ev - movq %rax,UREGS_kernel_sizeof(%rsp) - mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -+ mov %r13b, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - jmp restore_all_xen # return to fixup code - - /* No special register assumptions. */ -@@ -730,7 +728,7 @@ ENTRY(device_not_available) - ENTRY(debug) - pushq $0 - movl $TRAP_debug,4(%rsp) -- jmp handle_exception -+ jmp handle_ist_exception - - ENTRY(int3) - pushq $0 -@@ -785,10 +783,9 @@ ENTRY(double_fault) - mov STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rbx - test %rbx, %rbx - jz .Ldblf_cr3_okay -- jns .Ldblf_cr3_load -- neg %rbx --.Ldblf_cr3_load: -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - mov %rbx, %cr3 -+ movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%r14) - .Ldblf_cr3_okay: - - movq %rsp,%rdi -@@ -816,13 +813,11 @@ handle_ist_exception: - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - - mov STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx -+ mov STACK_CPUINFO_FIELD(use_pv_cr3)(%r14), %bl - mov %rcx, %r15 -- neg %rcx -+ test %rcx, %rcx - jz .List_cr3_okay -- jns .List_cr3_load -- mov %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -- neg %rcx --.List_cr3_load: -+ movb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - mov %rcx, %cr3 - movq $0, STACK_CPUINFO_FIELD(xen_cr3)(%r14) - .List_cr3_okay: -@@ -835,6 +830,7 @@ handle_ist_exception: - * and copy the context to stack bottom. - */ - xor %r15, %r15 -+ xor %ebx, %ebx - GET_CPUINFO_FIELD(guest_cpu_user_regs,di) - movq %rsp,%rsi - movl $UREGS_kernel_sizeof/8,%ecx -@@ -846,6 +842,7 @@ handle_ist_exception: - mov (%rdx, %rax, 8), %rdx - INDIRECT_CALL %rdx - mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) -+ mov %bl, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) - cmpb $TRAP_nmi,UREGS_entry_vector(%rsp) - jne ret_from_intr - -diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c -index 3652f5ff21..7e8c5de70d 100644 ---- a/xen/arch/x86/x86_64/traps.c -+++ b/xen/arch/x86/x86_64/traps.c -@@ -144,11 +144,12 @@ void show_registers(const struct cpu_user_regs *regs) - printk("CPU: %d\n", smp_processor_id()); - _show_registers(&fault_regs, fault_crs, context, v); - -- if ( this_cpu(ler_msr) && !guest_mode(regs) ) -+ if ( ler_msr && !guest_mode(regs) ) - { - u64 from, to; -- rdmsrl(this_cpu(ler_msr), from); -- rdmsrl(this_cpu(ler_msr) + 1, to); -+ -+ rdmsrl(ler_msr, from); -+ rdmsrl(ler_msr + 1, to); - printk("ler: %016lx -> %016lx\n", from, to); - } - } -diff --git a/xen/arch/x86/x86_emulate.c b/xen/arch/x86/x86_emulate.c -index c7ba221d11..9125c67c9e 100644 ---- a/xen/arch/x86/x86_emulate.c -+++ b/xen/arch/x86/x86_emulate.c -@@ -14,6 +14,7 @@ - #include <asm/processor.h> /* current_cpu_info */ - #include <asm/xstate.h> - #include <asm/amd.h> /* cpu_has_amd_erratum() */ -+#include <asm/debugreg.h> - - /* Avoid namespace pollution. */ - #undef cmpxchg -@@ -41,3 +42,75 @@ - }) - - #include "x86_emulate/x86_emulate.c" -+ -+/* Called with NULL ctxt in hypercall context. */ -+int x86emul_read_dr(unsigned int reg, unsigned long *val, -+ struct x86_emulate_ctxt *ctxt) -+{ -+ struct vcpu *curr = current; -+ -+ /* HVM support requires a bit more plumbing before it will work. */ -+ ASSERT(is_pv_vcpu(curr)); -+ -+ switch ( reg ) -+ { -+ case 0 ... 3: -+ case 6: -+ *val = curr->arch.debugreg[reg]; -+ break; -+ -+ case 7: -+ *val = (curr->arch.debugreg[7] | -+ curr->arch.debugreg[5]); -+ break; -+ -+ case 4 ... 5: -+ if ( !(curr->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE) ) -+ { -+ *val = curr->arch.debugreg[reg + 2]; -+ break; -+ } -+ -+ /* Fallthrough */ -+ default: -+ if ( ctxt ) -+ x86_emul_hw_exception(TRAP_invalid_op, X86_EVENT_NO_EC, ctxt); -+ -+ return X86EMUL_EXCEPTION; -+ } -+ -+ return X86EMUL_OKAY; -+} -+ -+int x86emul_write_dr(unsigned int reg, unsigned long val, -+ struct x86_emulate_ctxt *ctxt) -+{ -+ struct vcpu *curr = current; -+ -+ /* HVM support requires a bit more plumbing before it will work. */ -+ ASSERT(is_pv_vcpu(curr)); -+ -+ switch ( set_debugreg(curr, reg, val) ) -+ { -+ case 0: -+ return X86EMUL_OKAY; -+ -+ case -ENODEV: -+ x86_emul_hw_exception(TRAP_invalid_op, X86_EVENT_NO_EC, ctxt); -+ return X86EMUL_EXCEPTION; -+ -+ default: -+ x86_emul_hw_exception(TRAP_gp_fault, 0, ctxt); -+ return X86EMUL_EXCEPTION; -+ } -+} -+ -+/* -+ * Local variables: -+ * mode: C -+ * c-file-style: "BSD" -+ * c-basic-offset: 4 -+ * tab-width: 4 -+ * indent-tabs-mode: nil -+ * End: -+ */ -diff --git a/xen/arch/x86/x86_emulate/x86_emulate.h b/xen/arch/x86/x86_emulate/x86_emulate.h -index 0c8c80ad5a..9c2bb8157c 100644 ---- a/xen/arch/x86/x86_emulate/x86_emulate.h -+++ b/xen/arch/x86/x86_emulate/x86_emulate.h -@@ -662,6 +662,11 @@ static inline void x86_emulate_free_state(struct x86_emulate_state *state) {} - void x86_emulate_free_state(struct x86_emulate_state *state); - #endif - -+int x86emul_read_dr(unsigned int reg, unsigned long *val, -+ struct x86_emulate_ctxt *ctxt); -+int x86emul_write_dr(unsigned int reg, unsigned long val, -+ struct x86_emulate_ctxt *ctxt); -+ - #endif - - static inline void x86_emul_hw_exception( -diff --git a/xen/arch/x86/xstate.c b/xen/arch/x86/xstate.c -index 845208c189..ebd464e83a 100644 ---- a/xen/arch/x86/xstate.c -+++ b/xen/arch/x86/xstate.c -@@ -670,12 +670,17 @@ static bool valid_xcr0(u64 xcr0) - return !(xcr0 & XSTATE_BNDREGS) == !(xcr0 & XSTATE_BNDCSR); - } - --int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr) -+int validate_xstate(const struct domain *d, uint64_t xcr0, uint64_t xcr0_accum, -+ const struct xsave_hdr *hdr) - { -+ const struct cpuid_policy *cp = d->arch.cpuid; -+ uint64_t xcr0_max = -+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low; - unsigned int i; - - if ( (hdr->xstate_bv & ~xcr0_accum) || - (xcr0 & ~xcr0_accum) || -+ (xcr0_accum & ~xcr0_max) || - !valid_xcr0(xcr0) || - !valid_xcr0(xcr0_accum) ) - return -EINVAL; -@@ -694,20 +699,40 @@ int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr) - int handle_xsetbv(u32 index, u64 new_bv) - { - struct vcpu *curr = current; -+ const struct cpuid_policy *cp = curr->domain->arch.cpuid; -+ uint64_t xcr0_max = -+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low; - u64 mask; - - if ( index != XCR_XFEATURE_ENABLED_MASK ) - return -EOPNOTSUPP; - -- if ( (new_bv & ~xfeature_mask) || !valid_xcr0(new_bv) ) -+ /* -+ * The CPUID logic shouldn't be able to hand out an XCR0 exceeding Xen's -+ * maximum features, but keep the check for robustness. -+ */ -+ if ( unlikely(xcr0_max & ~xfeature_mask) ) -+ { -+ gprintk(XENLOG_ERR, -+ "xcr0_max %016" PRIx64 " exceeds hardware max %016" PRIx64 "\n", -+ xcr0_max, xfeature_mask); -+ domain_crash(curr->domain); -+ -+ return -EINVAL; -+ } -+ -+ if ( (new_bv & ~xcr0_max) || !valid_xcr0(new_bv) ) - return -EINVAL; - -- /* XCR0.PKRU is disabled on PV mode. */ -- if ( is_pv_vcpu(curr) && (new_bv & XSTATE_PKRU) ) -- return -EOPNOTSUPP; -+ /* By this point, new_bv really should be accepted by hardware. */ -+ if ( unlikely(!set_xcr0(new_bv)) ) -+ { -+ gprintk(XENLOG_ERR, "new_bv %016" PRIx64 " rejected by hardware\n", -+ new_bv); -+ domain_crash(curr->domain); - -- if ( !set_xcr0(new_bv) ) - return -EFAULT; -+ } - - mask = new_bv & ~curr->arch.xcr0_accum; - curr->arch.xcr0 = new_bv; -diff --git a/xen/common/cpu.c b/xen/common/cpu.c -index 6350f150bd..653a56b840 100644 ---- a/xen/common/cpu.c -+++ b/xen/common/cpu.c -@@ -67,12 +67,17 @@ void __init register_cpu_notifier(struct notifier_block *nb) - spin_unlock(&cpu_add_remove_lock); - } - --static int take_cpu_down(void *unused) -+static void _take_cpu_down(void *unused) - { - void *hcpu = (void *)(long)smp_processor_id(); - int notifier_rc = notifier_call_chain(&cpu_chain, CPU_DYING, hcpu, NULL); - BUG_ON(notifier_rc != NOTIFY_DONE); - __cpu_disable(); -+} -+ -+static int take_cpu_down(void *arg) -+{ -+ _take_cpu_down(arg); - return 0; - } - -@@ -98,7 +103,9 @@ int cpu_down(unsigned int cpu) - goto fail; - } - -- if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 ) -+ if ( unlikely(system_state < SYS_STATE_active) ) -+ on_selected_cpus(cpumask_of(cpu), _take_cpu_down, NULL, true); -+ else if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 ) - goto fail; - - __cpu_die(cpu); -diff --git a/xen/common/cpupool.c b/xen/common/cpupool.c -index 999839444e..1e8edcbd57 100644 ---- a/xen/common/cpupool.c -+++ b/xen/common/cpupool.c -@@ -490,7 +490,7 @@ static int cpupool_cpu_add(unsigned int cpu) - cpumask_clear_cpu(cpu, &cpupool_locked_cpus); - cpumask_set_cpu(cpu, &cpupool_free_cpus); - -- if ( system_state == SYS_STATE_resume ) -+ if ( system_state == SYS_STATE_suspend || system_state == SYS_STATE_resume ) - { - struct cpupool **c; - -@@ -522,6 +522,7 @@ static int cpupool_cpu_add(unsigned int cpu) - * (or unplugging would have failed) and that is the default behavior - * anyway. - */ -+ per_cpu(cpupool, cpu) = NULL; - ret = cpupool_assign_cpu_locked(cpupool0, cpu); - } - out: -diff --git a/xen/common/efi/boot.c b/xen/common/efi/boot.c -index 01d33004e0..f1d724bd06 100644 ---- a/xen/common/efi/boot.c -+++ b/xen/common/efi/boot.c -@@ -1304,6 +1304,8 @@ efi_start(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) - - #ifndef CONFIG_ARM /* TODO - runtime service support */ - -+#include <asm/spec_ctrl.h> -+ - static bool __initdata efi_map_uc; - - static int __init parse_efi_param(const char *s) -@@ -1419,6 +1421,16 @@ void __init efi_init_memory(void) - desc->PhysicalStart, desc->PhysicalStart + len - 1, - desc->Type, desc->Attribute); - -+ if ( (desc->Attribute & (EFI_MEMORY_WB | EFI_MEMORY_WT)) || -+ (efi_bs_revision >= EFI_REVISION(2, 5) && -+ (desc->Attribute & EFI_MEMORY_WP)) ) -+ { -+ /* Supplement the heuristics in l1tf_calculations(). */ -+ l1tf_safe_maddr = -+ max(l1tf_safe_maddr, -+ ROUNDUP(desc->PhysicalStart + len, PAGE_SIZE)); -+ } -+ - if ( !efi_enabled(EFI_RS) || - (!(desc->Attribute & EFI_MEMORY_RUNTIME) && - (!map_bs || -diff --git a/xen/common/efi/runtime.c b/xen/common/efi/runtime.c -index c38f00a64b..9aa070e77c 100644 ---- a/xen/common/efi/runtime.c -+++ b/xen/common/efi/runtime.c -@@ -111,21 +111,23 @@ struct efi_rs_state efi_rs_enter(void) - asm volatile ( "lgdt %0" : : "m" (gdt_desc) ); - } - -- write_cr3(virt_to_maddr(efi_l4_pgtable)); -+ switch_cr3_cr4(virt_to_maddr(efi_l4_pgtable), read_cr4()); - - return state; - } - - void efi_rs_leave(struct efi_rs_state *state) - { -+ struct vcpu *curr = current; -+ - if ( !state->cr3 ) - return; -- write_cr3(state->cr3); -- if ( is_pv_vcpu(current) && !is_idle_vcpu(current) ) -+ switch_cr3_cr4(state->cr3, read_cr4()); -+ if ( is_pv_vcpu(curr) && !is_idle_vcpu(curr) ) - { - struct desc_ptr gdt_desc = { - .limit = LAST_RESERVED_GDT_BYTE, -- .base = GDT_VIRT_START(current) -+ .base = GDT_VIRT_START(curr) - }; - - asm volatile ( "lgdt %0" : : "m" (gdt_desc) ); -@@ -133,7 +135,7 @@ void efi_rs_leave(struct efi_rs_state *state) - irq_exit(); - efi_rs_on_cpu = NR_CPUS; - spin_unlock(&efi_rs_lock); -- stts(); -+ vcpu_restore_fpu_nonlazy(curr, true); - } - - bool efi_rs_using_pgtables(void) -diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c -index 7650e6d449..64f58fc815 100644 ---- a/xen/common/grant_table.c -+++ b/xen/common/grant_table.c -@@ -97,6 +97,45 @@ static unsigned int __read_mostly max_maptrack_frames = - DEFAULT_MAX_MAPTRACK_FRAMES; - integer_runtime_param("gnttab_max_maptrack_frames", max_maptrack_frames); - -+#ifndef GNTTAB_MAX_VERSION -+#define GNTTAB_MAX_VERSION 2 -+#endif -+ -+static unsigned int __read_mostly opt_gnttab_max_version = GNTTAB_MAX_VERSION; -+static bool __read_mostly opt_transitive_grants = true; -+ -+static int __init parse_gnttab(const char *s) -+{ -+ const char *ss, *e; -+ int val, rc = 0; -+ -+ do { -+ ss = strchr(s, ','); -+ if ( !ss ) -+ ss = strchr(s, '\0'); -+ -+ if ( !strncmp(s, "max-ver:", 8) || -+ !strncmp(s, "max_ver:", 8) ) /* Alias for original XSA-226 patch */ -+ { -+ long ver = simple_strtol(s + 8, &e, 10); -+ -+ if ( e == ss && ver >= 1 && ver <= 2 ) -+ opt_gnttab_max_version = ver; -+ else -+ rc = -EINVAL; -+ } -+ else if ( (val = parse_boolean("transitive", s, ss)) >= 0 ) -+ opt_transitive_grants = val; -+ else -+ rc = -EINVAL; -+ -+ s = ss + 1; -+ } while ( *ss ); -+ -+ return rc; -+} -+custom_param("gnttab", parse_gnttab); -+ - /* - * Note that the three values below are effectively part of the ABI, even if - * we don't need to make them a formal part of it: A guest suspended for -@@ -2725,7 +2764,8 @@ static int gnttab_copy_claim_buf(const struct gnttab_copy *op, - current->domain->domain_id, - buf->read_only, - &buf->frame, &buf->page, -- &buf->ptr.offset, &buf->len, true); -+ &buf->ptr.offset, &buf->len, -+ opt_transitive_grants); - if ( rc != GNTST_okay ) - goto out; - buf->ptr.u.ref = ptr->u.ref; -@@ -2927,6 +2967,10 @@ gnttab_set_version(XEN_GUEST_HANDLE_PARAM(gnttab_set_version_t) uop) - if ( op.version != 1 && op.version != 2 ) - goto out; - -+ res = -ENOSYS; -+ if ( op.version == 2 && opt_gnttab_max_version == 1 ) -+ goto out; /* Behave as before set_version was introduced. */ -+ - res = 0; - if ( gt->gt_version == op.version ) - goto out; -diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c -index 49b2a91751..6d6f2a0628 100644 ---- a/xen/common/page_alloc.c -+++ b/xen/common/page_alloc.c -@@ -1424,7 +1424,7 @@ static void free_heap_pages( - - page_list_del(predecessor, &heap(node, zone, order)); - -- /* Keep predecessor's first_dirty if it is already set. */ -+ /* Update predecessor's first_dirty if necessary. */ - if ( predecessor->u.free.first_dirty == INVALID_DIRTY_IDX && - pg->u.free.first_dirty != INVALID_DIRTY_IDX ) - predecessor->u.free.first_dirty = (1U << order) + -@@ -1445,6 +1445,12 @@ static void free_heap_pages( - - check_and_stop_scrub(successor); - -+ /* Update pg's first_dirty if necessary. */ -+ if ( pg->u.free.first_dirty == INVALID_DIRTY_IDX && -+ successor->u.free.first_dirty != INVALID_DIRTY_IDX ) -+ pg->u.free.first_dirty = (1U << order) + -+ successor->u.free.first_dirty; -+ - page_list_del(successor, &heap(node, zone, order)); - } - -diff --git a/xen/common/schedule.c b/xen/common/schedule.c -index b7884263f2..f21c3e5a64 100644 ---- a/xen/common/schedule.c -+++ b/xen/common/schedule.c -@@ -436,14 +436,9 @@ void sched_destroy_domain(struct domain *d) - cpupool_rm_domain(d); - } - --void vcpu_sleep_nosync(struct vcpu *v) -+void vcpu_sleep_nosync_locked(struct vcpu *v) - { -- unsigned long flags; -- spinlock_t *lock; -- -- TRACE_2D(TRC_SCHED_SLEEP, v->domain->domain_id, v->vcpu_id); -- -- lock = vcpu_schedule_lock_irqsave(v, &flags); -+ ASSERT(spin_is_locked(per_cpu(schedule_data,v->processor).schedule_lock)); - - if ( likely(!vcpu_runnable(v)) ) - { -@@ -452,6 +447,18 @@ void vcpu_sleep_nosync(struct vcpu *v) - - SCHED_OP(vcpu_scheduler(v), sleep, v); - } -+} -+ -+void vcpu_sleep_nosync(struct vcpu *v) -+{ -+ unsigned long flags; -+ spinlock_t *lock; -+ -+ TRACE_2D(TRC_SCHED_SLEEP, v->domain->domain_id, v->vcpu_id); -+ -+ lock = vcpu_schedule_lock_irqsave(v, &flags); -+ -+ vcpu_sleep_nosync_locked(v); - - vcpu_schedule_unlock_irqrestore(lock, flags, v); - } -@@ -567,13 +574,54 @@ static void vcpu_move_nosched(struct vcpu *v, unsigned int new_cpu) - sched_move_irqs(v); - } - --static void vcpu_migrate(struct vcpu *v) -+/* -+ * Initiating migration -+ * -+ * In order to migrate, we need the vcpu in question to have stopped -+ * running and had SCHED_OP(sleep) called (to take it off any -+ * runqueues, for instance); and if it is currently running, it needs -+ * to be scheduled out. Finally, we need to hold the scheduling locks -+ * for both the processor we're migrating from, and the processor -+ * we're migrating to. -+ * -+ * In order to avoid deadlock while satisfying the final requirement, -+ * we must release any scheduling lock we hold, then try to grab both -+ * locks we want, then double-check to make sure that what we started -+ * to do hasn't been changed in the mean time. -+ * -+ * These steps are encapsulated in the following two functions; they -+ * should be called like this: -+ * -+ * lock = vcpu_schedule_lock_irq(v); -+ * vcpu_migrate_start(v); -+ * vcpu_schedule_unlock_irq(lock, v) -+ * vcpu_migrate_finish(v); -+ * -+ * vcpu_migrate_finish() will do the work now if it can, or simply -+ * return if it can't (because v is still running); in that case -+ * vcpu_migrate_finish() will be called by context_saved(). -+ */ -+void vcpu_migrate_start(struct vcpu *v) -+{ -+ set_bit(_VPF_migrating, &v->pause_flags); -+ vcpu_sleep_nosync_locked(v); -+} -+ -+static void vcpu_migrate_finish(struct vcpu *v) - { - unsigned long flags; - unsigned int old_cpu, new_cpu; - spinlock_t *old_lock, *new_lock; - bool_t pick_called = 0; - -+ /* -+ * If the vcpu is currently running, this will be handled by -+ * context_saved(); and in any case, if the bit is cleared, then -+ * someone else has already done the work so we don't need to. -+ */ -+ if ( v->is_running || !test_bit(_VPF_migrating, &v->pause_flags) ) -+ return; -+ - old_cpu = new_cpu = v->processor; - for ( ; ; ) - { -@@ -653,14 +701,11 @@ void vcpu_force_reschedule(struct vcpu *v) - spinlock_t *lock = vcpu_schedule_lock_irq(v); - - if ( v->is_running ) -- set_bit(_VPF_migrating, &v->pause_flags); -+ vcpu_migrate_start(v); -+ - vcpu_schedule_unlock_irq(lock, v); - -- if ( v->pause_flags & VPF_migrating ) -- { -- vcpu_sleep_nosync(v); -- vcpu_migrate(v); -- } -+ vcpu_migrate_finish(v); - } - - void restore_vcpu_affinity(struct domain *d) -@@ -812,10 +857,10 @@ int cpu_disable_scheduler(unsigned int cpu) - * * the scheduler will always fine a suitable solution, or - * things would have failed before getting in here. - */ -- set_bit(_VPF_migrating, &v->pause_flags); -+ vcpu_migrate_start(v); - vcpu_schedule_unlock_irqrestore(lock, flags, v); -- vcpu_sleep_nosync(v); -- vcpu_migrate(v); -+ -+ vcpu_migrate_finish(v); - - /* - * The only caveat, in this case, is that if a vcpu active in -@@ -849,18 +894,14 @@ static int vcpu_set_affinity( - * Always ask the scheduler to re-evaluate placement - * when changing the affinity. - */ -- set_bit(_VPF_migrating, &v->pause_flags); -+ vcpu_migrate_start(v); - } - - vcpu_schedule_unlock_irq(lock, v); - - domain_update_node_affinity(v->domain); - -- if ( v->pause_flags & VPF_migrating ) -- { -- vcpu_sleep_nosync(v); -- vcpu_migrate(v); -- } -+ vcpu_migrate_finish(v); - - return ret; - } -@@ -1088,7 +1129,6 @@ int vcpu_pin_override(struct vcpu *v, int cpu) - { - cpumask_copy(v->cpu_hard_affinity, v->cpu_hard_affinity_saved); - v->affinity_broken = 0; -- set_bit(_VPF_migrating, &v->pause_flags); - ret = 0; - } - } -@@ -1101,20 +1141,18 @@ int vcpu_pin_override(struct vcpu *v, int cpu) - cpumask_copy(v->cpu_hard_affinity_saved, v->cpu_hard_affinity); - v->affinity_broken = 1; - cpumask_copy(v->cpu_hard_affinity, cpumask_of(cpu)); -- set_bit(_VPF_migrating, &v->pause_flags); - ret = 0; - } - } - -+ if ( ret == 0 ) -+ vcpu_migrate_start(v); -+ - vcpu_schedule_unlock_irq(lock, v); - - domain_update_node_affinity(v->domain); - -- if ( v->pause_flags & VPF_migrating ) -- { -- vcpu_sleep_nosync(v); -- vcpu_migrate(v); -- } -+ vcpu_migrate_finish(v); - - return ret; - } -@@ -1501,8 +1539,7 @@ void context_saved(struct vcpu *prev) - - SCHED_OP(vcpu_scheduler(prev), context_saved, prev); - -- if ( unlikely(prev->pause_flags & VPF_migrating) ) -- vcpu_migrate(prev); -+ vcpu_migrate_finish(prev); - } - - /* The scheduler timer: force a run through the scheduler */ -diff --git a/xen/common/tasklet.c b/xen/common/tasklet.c -index 0f0a6f8365..d4fea3151c 100644 ---- a/xen/common/tasklet.c -+++ b/xen/common/tasklet.c -@@ -156,6 +156,10 @@ void tasklet_kill(struct tasklet *t) - - spin_lock_irqsave(&tasklet_lock, flags); - -+ /* Cope with uninitialised tasklets. */ -+ if ( list_head_is_null(&t->list) ) -+ goto unlock; -+ - if ( !list_empty(&t->list) ) - { - BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0)); -@@ -172,6 +176,7 @@ void tasklet_kill(struct tasklet *t) - spin_lock_irqsave(&tasklet_lock, flags); - } - -+ unlock: - spin_unlock_irqrestore(&tasklet_lock, flags); - } - -diff --git a/xen/include/asm-arm/arm32/system.h b/xen/include/asm-arm/arm32/system.h -index c617b40438..ab57abfbc5 100644 ---- a/xen/include/asm-arm/arm32/system.h -+++ b/xen/include/asm-arm/arm32/system.h -@@ -48,6 +48,24 @@ static inline int local_fiq_is_enabled(void) - return !(flags & PSR_FIQ_MASK); - } - -+#define CSDB ".inst 0xe320f014" -+ -+static inline unsigned long array_index_mask_nospec(unsigned long idx, -+ unsigned long sz) -+{ -+ unsigned long mask; -+ -+ asm volatile( "cmp %1, %2\n" -+ "sbc %0, %1, %1\n" -+ CSDB -+ : "=r" (mask) -+ : "r" (idx), "Ir" (sz) -+ : "cc" ); -+ -+ return mask; -+} -+#define array_index_mask_nospec array_index_mask_nospec -+ - #endif - /* - * Local variables: -diff --git a/xen/include/asm-arm/arm64/system.h b/xen/include/asm-arm/arm64/system.h -index 2e2ee212a1..2e36573ac6 100644 ---- a/xen/include/asm-arm/arm64/system.h -+++ b/xen/include/asm-arm/arm64/system.h -@@ -58,6 +58,28 @@ static inline int local_fiq_is_enabled(void) - return !(flags & PSR_FIQ_MASK); - } - -+#define csdb() asm volatile ( "hint #20" : : : "memory" ) -+ -+/* -+ * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz -+ * and 0 otherwise. -+ */ -+static inline unsigned long array_index_mask_nospec(unsigned long idx, -+ unsigned long sz) -+{ -+ unsigned long mask; -+ -+ asm volatile ( "cmp %1, %2\n" -+ "sbc %0, xzr, xzr\n" -+ : "=r" (mask) -+ : "r" (idx), "Ir" (sz) -+ : "cc" ); -+ csdb(); -+ -+ return mask; -+} -+#define array_index_mask_nospec array_index_mask_nospec -+ - #endif - /* - * Local variables: -diff --git a/xen/include/asm-arm/grant_table.h b/xen/include/asm-arm/grant_table.h -index 5b8994cbd5..619721f121 100644 ---- a/xen/include/asm-arm/grant_table.h -+++ b/xen/include/asm-arm/grant_table.h -@@ -7,6 +7,7 @@ - #include <xen/sched.h> - - #define INITIAL_NR_GRANT_FRAMES 1U -+#define GNTTAB_MAX_VERSION 1 - - struct grant_table_arch { - gfn_t *shared_gfn; -diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h -index 62465b20c7..ff6f969e74 100644 ---- a/xen/include/asm-x86/cpufeature.h -+++ b/xen/include/asm-x86/cpufeature.h -@@ -90,6 +90,7 @@ - #define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2) - #define cpu_has_smep boot_cpu_has(X86_FEATURE_SMEP) - #define cpu_has_bmi2 boot_cpu_has(X86_FEATURE_BMI2) -+#define cpu_has_invpcid boot_cpu_has(X86_FEATURE_INVPCID) - #define cpu_has_rtm boot_cpu_has(X86_FEATURE_RTM) - #define cpu_has_fpu_sel (!boot_cpu_has(X86_FEATURE_NO_FPU_SEL)) - #define cpu_has_mpx boot_cpu_has(X86_FEATURE_MPX) -@@ -106,6 +107,7 @@ - #define cpu_has_aperfmperf boot_cpu_has(X86_FEATURE_APERFMPERF) - #define cpu_has_lfence_dispatch boot_cpu_has(X86_FEATURE_LFENCE_DISPATCH) - #define cpu_has_no_xpti boot_cpu_has(X86_FEATURE_NO_XPTI) -+#define cpu_has_xen_lbr boot_cpu_has(X86_FEATURE_XEN_LBR) - - enum _cache_type { - CACHE_TYPE_NULL = 0, -diff --git a/xen/include/asm-x86/cpufeatures.h b/xen/include/asm-x86/cpufeatures.h -index c9b1a48807..8e5cc53dde 100644 ---- a/xen/include/asm-x86/cpufeatures.h -+++ b/xen/include/asm-x86/cpufeatures.h -@@ -26,8 +26,10 @@ XEN_CPUFEATURE(LFENCE_DISPATCH, (FSCAPINTS+0)*32+12) /* lfence set as Dispatch S - XEN_CPUFEATURE(IND_THUNK_LFENCE,(FSCAPINTS+0)*32+13) /* Use IND_THUNK_LFENCE */ - XEN_CPUFEATURE(IND_THUNK_JMP, (FSCAPINTS+0)*32+14) /* Use IND_THUNK_JMP */ - XEN_CPUFEATURE(XEN_IBPB, (FSCAPINTS+0)*32+15) /* IBRSB || IBPB */ --XEN_CPUFEATURE(XEN_IBRS_SET, (FSCAPINTS+0)*32+16) /* IBRSB && IRBS set in Xen */ --XEN_CPUFEATURE(XEN_IBRS_CLEAR, (FSCAPINTS+0)*32+17) /* IBRSB && IBRS clear in Xen */ --XEN_CPUFEATURE(RSB_NATIVE, (FSCAPINTS+0)*32+18) /* RSB overwrite needed for native */ --XEN_CPUFEATURE(RSB_VMEXIT, (FSCAPINTS+0)*32+19) /* RSB overwrite needed for vmexit */ -+XEN_CPUFEATURE(SC_MSR_PV, (FSCAPINTS+0)*32+16) /* MSR_SPEC_CTRL used by Xen for PV */ -+XEN_CPUFEATURE(SC_MSR_HVM, (FSCAPINTS+0)*32+17) /* MSR_SPEC_CTRL used by Xen for HVM */ -+XEN_CPUFEATURE(SC_RSB_PV, (FSCAPINTS+0)*32+18) /* RSB overwrite needed for PV */ -+XEN_CPUFEATURE(SC_RSB_HVM, (FSCAPINTS+0)*32+19) /* RSB overwrite needed for HVM */ - XEN_CPUFEATURE(NO_XPTI, (FSCAPINTS+0)*32+20) /* XPTI mitigation not in use */ -+XEN_CPUFEATURE(SC_MSR_IDLE, (FSCAPINTS+0)*32+21) /* (SC_MSR_PV || SC_MSR_HVM) && default_xen_spec_ctrl */ -+XEN_CPUFEATURE(XEN_LBR, (FSCAPINTS+0)*32+22) /* Xen uses MSR_DEBUGCTL.LBR */ -diff --git a/xen/include/asm-x86/current.h b/xen/include/asm-x86/current.h -index 4678a0fcf5..9a137a1311 100644 ---- a/xen/include/asm-x86/current.h -+++ b/xen/include/asm-x86/current.h -@@ -44,20 +44,33 @@ struct cpu_info { - /* - * Of the two following fields the latter is being set to the CR3 value - * to be used on the given pCPU for loading whenever 64-bit PV guest -- * context is being entered. The value never changes once set. -+ * context is being entered. A value of zero indicates no setting of CR3 -+ * is to be performed. - * The former is the value to restore when re-entering Xen, if any. IOW -- * its value being zero means there's nothing to restore. However, its -- * value can also be negative, indicating to the exit-to-Xen code that -- * restoring is not necessary, but allowing any nested entry code paths -- * to still know the value to put back into CR3. -+ * its value being zero means there's nothing to restore. - */ - unsigned long xen_cr3; - unsigned long pv_cr3; - - /* See asm-x86/spec_ctrl_asm.h for usage. */ - unsigned int shadow_spec_ctrl; -- bool use_shadow_spec_ctrl; -- uint8_t bti_ist_info; -+ uint8_t xen_spec_ctrl; -+ uint8_t spec_ctrl_flags; -+ -+ /* -+ * The following field controls copying of the L4 page table of 64-bit -+ * PV guests to the per-cpu root page table on entering the guest context. -+ * If set the L4 page table is being copied to the root page table and -+ * the field will be reset. -+ */ -+ bool root_pgt_changed; -+ -+ /* -+ * use_pv_cr3 is set in case the value of pv_cr3 is to be written into -+ * CR3 when returning from an interrupt. The main use is when returning -+ * from a NMI or MCE to hypervisor code where pv_cr3 was active. -+ */ -+ bool use_pv_cr3; - - unsigned long __pad; - /* get_stack_bottom() must be 16-byte aligned */ -diff --git a/xen/include/asm-x86/debugreg.h b/xen/include/asm-x86/debugreg.h -index c57914efc6..b3b10eaf40 100644 ---- a/xen/include/asm-x86/debugreg.h -+++ b/xen/include/asm-x86/debugreg.h -@@ -24,6 +24,8 @@ - #define DR_STATUS_RESERVED_ZERO (~0xffffeffful) /* Reserved, read as zero */ - #define DR_STATUS_RESERVED_ONE 0xffff0ff0ul /* Reserved, read as one */ - -+#define X86_DR6_DEFAULT 0xffff0ff0ul /* Default %dr6 value. */ -+ - /* Now define a bunch of things for manipulating the control register. - The top two bytes of the control register consist of 4 fields of 4 - bits - each field corresponds to one of the four debug registers, -diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h -index 4679d5477d..0fbd36bf48 100644 ---- a/xen/include/asm-x86/domain.h -+++ b/xen/include/asm-x86/domain.h -@@ -121,6 +121,11 @@ struct shadow_domain { - - /* Has this domain ever used HVMOP_pagetable_dying? */ - bool_t pagetable_dying_op; -+ -+#ifdef CONFIG_PV -+ /* PV L1 Terminal Fault mitigation. */ -+ struct tasklet pv_l1tf_tasklet; -+#endif /* CONFIG_PV */ - #endif - }; - -@@ -253,6 +258,13 @@ struct pv_domain - - atomic_t nr_l4_pages; - -+ /* XPTI active? */ -+ bool xpti; -+ /* Use PCID feature? */ -+ bool pcid; -+ /* Mitigate L1TF with shadow/crashing? */ -+ bool check_l1tf; -+ - /* map_domain_page() mapping cache. */ - struct mapcache_domain mapcache; - -@@ -564,6 +576,9 @@ struct arch_vcpu - * and thus should be saved/restored. */ - bool_t nonlazy_xstate_used; - -+ /* Restore all FPU state (lazy and non-lazy state) on context switch? */ -+ bool fully_eager_fpu; -+ - /* - * The SMAP check policy when updating runstate_guest(v) and the - * secondary system time. -@@ -612,18 +627,12 @@ void vcpu_show_registers(const struct vcpu *); - unsigned long pv_guest_cr4_fixup(const struct vcpu *, unsigned long guest_cr4); - - /* Convert between guest-visible and real CR4 values. */ --#define pv_guest_cr4_to_real_cr4(v) \ -- (((v)->arch.pv_vcpu.ctrlreg[4] \ -- | (mmu_cr4_features \ -- & (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_SMEP | \ -- X86_CR4_SMAP | X86_CR4_OSXSAVE | \ -- X86_CR4_FSGSBASE)) \ -- | ((v)->domain->arch.vtsc ? X86_CR4_TSD : 0)) \ -- & ~X86_CR4_DE) -+unsigned long pv_guest_cr4_to_real_cr4(const struct vcpu *v); -+ - #define real_cr4_to_pv_guest_cr4(c) \ - ((c) & ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_TSD | \ - X86_CR4_OSXSAVE | X86_CR4_SMEP | \ -- X86_CR4_FSGSBASE | X86_CR4_SMAP)) -+ X86_CR4_FSGSBASE | X86_CR4_SMAP | X86_CR4_PCIDE)) - - #define domain_max_vcpus(d) (is_hvm_domain(d) ? HVM_MAX_VCPUS : MAX_VIRT_CPUS) - -diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h -index 413db692e1..4a930448da 100644 ---- a/xen/include/asm-x86/flushtlb.h -+++ b/xen/include/asm-x86/flushtlb.h -@@ -84,7 +84,7 @@ static inline unsigned long read_cr3(void) - } - - /* Write pagetable base and implicitly tick the tlbflush clock. */ --void write_cr3(unsigned long cr3); -+void switch_cr3_cr4(unsigned long cr3, unsigned long cr4); - - /* flush_* flag fields: */ - /* -@@ -101,6 +101,8 @@ void write_cr3(unsigned long cr3); - #define FLUSH_CACHE 0x400 - /* VA for the flush has a valid mapping */ - #define FLUSH_VA_VALID 0x800 -+ /* Flush the per-cpu root page table */ -+#define FLUSH_ROOT_PGTBL 0x2000 - - /* Flush local TLBs/caches. */ - unsigned int flush_area_local(const void *va, unsigned int flags); -@@ -132,6 +134,12 @@ void flush_area_mask(const cpumask_t *, const void *va, unsigned int flags); - #define flush_tlb_one_all(v) \ - flush_tlb_one_mask(&cpu_online_map, v) - -+#define flush_root_pgtbl_domain(d) \ -+{ \ -+ if ( is_pv_domain(d) && (d)->arch.pv_domain.xpti ) \ -+ flush_mask((d)->domain_dirty_cpumask, FLUSH_ROOT_PGTBL); \ -+} -+ - static inline void flush_page_to_ram(unsigned long mfn, bool sync_icache) {} - static inline int invalidate_dcache_va_range(const void *p, - unsigned long size) -diff --git a/xen/include/asm-x86/hvm/irq.h b/xen/include/asm-x86/hvm/irq.h -index f756cb5a0d..1a52ec6045 100644 ---- a/xen/include/asm-x86/hvm/irq.h -+++ b/xen/include/asm-x86/hvm/irq.h -@@ -207,6 +207,9 @@ int hvm_set_pci_link_route(struct domain *d, u8 link, u8 isa_irq); - - int hvm_inject_msi(struct domain *d, uint64_t addr, uint32_t data); - -+/* Assert an IO APIC pin. */ -+int hvm_ioapic_assert(struct domain *d, unsigned int gsi, bool level); -+ - void hvm_maybe_deassert_evtchn_irq(void); - void hvm_assert_evtchn_irq(struct vcpu *v); - void hvm_set_callback_via(struct domain *d, uint64_t via); -diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h -index 8fb9e3ceee..e216c4ac35 100644 ---- a/xen/include/asm-x86/hvm/vmx/vmcs.h -+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h -@@ -130,10 +130,18 @@ struct arch_vmx_struct { - uint64_t sfmask; - - struct vmx_msr_bitmap *msr_bitmap; -- unsigned int msr_count; -+ -+ /* -+ * Most accesses to the MSR host/guest load/save lists are in current -+ * context. However, the data can be modified by toolstack/migration -+ * actions. Remote access is only permitted for paused vcpus, and is -+ * protected under the domctl lock. -+ */ - struct vmx_msr_entry *msr_area; -- unsigned int host_msr_count; - struct vmx_msr_entry *host_msr_area; -+ unsigned int msr_load_count; -+ unsigned int msr_save_count; -+ unsigned int host_msr_count; - - unsigned long eoi_exitmap_changed; - DECLARE_BITMAP(eoi_exit_bitmap, NR_VECTORS); -@@ -149,7 +157,7 @@ struct arch_vmx_struct { - /* Are we emulating rather than VMENTERing? */ - uint8_t vmx_emulate; - -- uint8_t lbr_fixup_enabled; -+ uint8_t lbr_flags; - - /* Bitmask of segments that we can't safely use in virtual 8086 mode */ - uint16_t vm86_segment_mask; -@@ -511,9 +519,6 @@ enum vmcs_field { - - #define VMCS_VPID_WIDTH 16 - --#define VMX_GUEST_MSR 0 --#define VMX_HOST_MSR 1 -- - /* VM Instruction error numbers */ - enum vmx_insn_errno - { -@@ -531,6 +536,67 @@ enum vmx_insn_errno - VMX_INSN_FAIL_INVALID = ~0, - }; - -+/* MSR load/save list infrastructure. */ -+enum vmx_msr_list_type { -+ VMX_MSR_HOST, /* MSRs loaded on VMExit. */ -+ VMX_MSR_GUEST, /* MSRs saved on VMExit, loaded on VMEntry. */ -+ VMX_MSR_GUEST_LOADONLY, /* MSRs loaded on VMEntry only. */ -+}; -+ -+/** -+ * Add an MSR to an MSR list (inserting space for the entry if necessary), and -+ * set the MSRs value. -+ * -+ * It is undefined behaviour to try and insert the same MSR into both the -+ * GUEST and GUEST_LOADONLY list. -+ * -+ * May fail if unable to allocate memory for the list, or the total number of -+ * entries exceeds the memory allocated. -+ */ -+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val, -+ enum vmx_msr_list_type type); -+ -+static inline int vmx_add_guest_msr(struct vcpu *v, uint32_t msr, uint64_t val) -+{ -+ return vmx_add_msr(v, msr, val, VMX_MSR_GUEST); -+} -+static inline int vmx_add_host_load_msr(struct vcpu *v, uint32_t msr, -+ uint64_t val) -+{ -+ return vmx_add_msr(v, msr, val, VMX_MSR_HOST); -+} -+ -+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr, -+ enum vmx_msr_list_type type); -+ -+static inline int vmx_read_guest_msr(const struct vcpu *v, uint32_t msr, -+ uint64_t *val) -+{ -+ const struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST); -+ -+ if ( !ent ) -+ return -ESRCH; -+ -+ *val = ent->data; -+ -+ return 0; -+} -+ -+static inline int vmx_write_guest_msr(struct vcpu *v, uint32_t msr, -+ uint64_t val) -+{ -+ struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST); -+ -+ if ( !ent ) -+ return -ESRCH; -+ -+ ent->data = val; -+ -+ return 0; -+} -+ -+ -+/* MSR intercept bitmap infrastructure. */ - enum vmx_msr_intercept_type { - VMX_MSR_R = 1, - VMX_MSR_W = 2, -@@ -541,10 +607,6 @@ void vmx_clear_msr_intercept(struct vcpu *v, unsigned int msr, - enum vmx_msr_intercept_type type); - void vmx_set_msr_intercept(struct vcpu *v, unsigned int msr, - enum vmx_msr_intercept_type type); --int vmx_read_guest_msr(u32 msr, u64 *val); --int vmx_write_guest_msr(u32 msr, u64 val); --struct vmx_msr_entry *vmx_find_msr(u32 msr, int type); --int vmx_add_msr(u32 msr, int type); - void vmx_vmcs_switch(paddr_t from, paddr_t to); - void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector); - void vmx_clear_eoi_exit_bitmap(struct vcpu *v, u8 vector); -@@ -559,15 +621,6 @@ void virtual_vmcs_vmwrite(const struct vcpu *, u32 encoding, u64 val); - enum vmx_insn_errno virtual_vmcs_vmwrite_safe(const struct vcpu *v, - u32 vmcs_encoding, u64 val); - --static inline int vmx_add_guest_msr(u32 msr) --{ -- return vmx_add_msr(msr, VMX_GUEST_MSR); --} --static inline int vmx_add_host_load_msr(u32 msr) --{ -- return vmx_add_msr(msr, VMX_HOST_MSR); --} -- - DECLARE_PER_CPU(bool_t, vmxon); - - bool_t vmx_vcpu_pml_enabled(const struct vcpu *v); -diff --git a/xen/include/asm-x86/hvm/vpt.h b/xen/include/asm-x86/hvm/vpt.h -index 21166edd06..0eb5ff632e 100644 ---- a/xen/include/asm-x86/hvm/vpt.h -+++ b/xen/include/asm-x86/hvm/vpt.h -@@ -44,6 +44,7 @@ struct periodic_time { - bool_t warned_timeout_too_short; - #define PTSRC_isa 1 /* ISA time source */ - #define PTSRC_lapic 2 /* LAPIC time source */ -+#define PTSRC_ioapic 3 /* IOAPIC time source */ - u8 source; /* PTSRC_ */ - u8 irq; - struct vcpu *vcpu; /* vcpu timer interrupt delivers to */ -diff --git a/xen/include/asm-x86/hypercall.h b/xen/include/asm-x86/hypercall.h -index b9f3ecf9a3..0e1092845b 100644 ---- a/xen/include/asm-x86/hypercall.h -+++ b/xen/include/asm-x86/hypercall.h -@@ -165,7 +165,7 @@ extern int compat_update_va_mapping( - unsigned int va, u32 lo, u32 hi, unsigned int flags); - - extern int compat_update_va_mapping_otherdomain( -- unsigned long va, u32 lo, u32 hi, unsigned long flags, domid_t domid); -+ unsigned int va, u32 lo, u32 hi, unsigned int flags, domid_t domid); - - DEFINE_XEN_GUEST_HANDLE(trap_info_compat_t); - extern int compat_set_trap_table(XEN_GUEST_HANDLE(trap_info_compat_t) traps); -diff --git a/xen/include/asm-x86/i387.h b/xen/include/asm-x86/i387.h -index 7cfa215d30..243de672eb 100644 ---- a/xen/include/asm-x86/i387.h -+++ b/xen/include/asm-x86/i387.h -@@ -28,7 +28,7 @@ struct ix87_env { - uint16_t fds, _res6; - }; - --void vcpu_restore_fpu_eager(struct vcpu *v); -+void vcpu_restore_fpu_nonlazy(struct vcpu *v, bool need_stts); - void vcpu_restore_fpu_lazy(struct vcpu *v); - void vcpu_save_fpu(struct vcpu *v); - void save_fpu_enable(void); -diff --git a/xen/include/asm-x86/invpcid.h b/xen/include/asm-x86/invpcid.h -new file mode 100644 -index 0000000000..edd8b68706 ---- /dev/null -+++ b/xen/include/asm-x86/invpcid.h -@@ -0,0 +1,72 @@ -+#ifndef _ASM_X86_INVPCID_H_ -+#define _ASM_X86_INVPCID_H_ -+ -+#include <xen/types.h> -+ -+extern bool use_invpcid; -+ -+#define INVPCID_TYPE_INDIV_ADDR 0 -+#define INVPCID_TYPE_SINGLE_CTXT 1 -+#define INVPCID_TYPE_ALL_INCL_GLOBAL 2 -+#define INVPCID_TYPE_ALL_NON_GLOBAL 3 -+ -+#define INVPCID_OPCODE ".byte 0x66, 0x0f, 0x38, 0x82\n" -+#define MODRM_ECX_01 ".byte 0x01\n" -+ -+static inline void invpcid(unsigned int pcid, unsigned long addr, -+ unsigned int type) -+{ -+ struct { -+ uint64_t pcid:12; -+ uint64_t reserved:52; -+ uint64_t addr; -+ } desc = { .pcid = pcid, .addr = addr }; -+ -+ asm volatile ( -+#ifdef HAVE_AS_INVPCID -+ "invpcid %[desc], %q[type]" -+ : /* No output */ -+ : [desc] "m" (desc), [type] "r" (type) -+#else -+ INVPCID_OPCODE MODRM_ECX_01 -+ : /* No output */ -+ : "a" (type), "c" (&desc) -+#endif -+ : "memory" ); -+} -+ -+/* Flush all mappings for a given PCID and addr, not including globals */ -+static inline void invpcid_flush_one(unsigned int pcid, unsigned long addr) -+{ -+ invpcid(pcid, addr, INVPCID_TYPE_INDIV_ADDR); -+} -+ -+/* Flush all mappings for a given PCID, not including globals */ -+static inline void invpcid_flush_single_context(unsigned int pcid) -+{ -+ invpcid(pcid, 0, INVPCID_TYPE_SINGLE_CTXT); -+} -+ -+/* Flush all mappings, including globals, for all PCIDs */ -+static inline void invpcid_flush_all(void) -+{ -+ invpcid(0, 0, INVPCID_TYPE_ALL_INCL_GLOBAL); -+} -+ -+/* Flush all mappings for all PCIDs, excluding globals */ -+static inline void invpcid_flush_all_nonglobals(void) -+{ -+ invpcid(0, 0, INVPCID_TYPE_ALL_NON_GLOBAL); -+} -+ -+#endif /* _ASM_X86_INVPCID_H_ */ -+ -+/* -+ * Local variables: -+ * mode: C -+ * c-file-style: "BSD" -+ * c-basic-offset: 4 -+ * tab-width: 4 -+ * indent-tabs-mode: nil -+ * End: -+ */ -diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h -index a8ceecf3e2..dfeba2821d 100644 ---- a/xen/include/asm-x86/msr-index.h -+++ b/xen/include/asm-x86/msr-index.h -@@ -31,10 +31,14 @@ - #define EFER_LMSLE (1<<_EFER_LMSLE) - #define EFER_FFXSE (1<<_EFER_FFXSE) - -+#define EFER_KNOWN_MASK (EFER_SCE | EFER_LME | EFER_LMA | EFER_NX | \ -+ EFER_SVME | EFER_LMSLE | EFER_FFXSE) -+ - /* Speculation Controls. */ - #define MSR_SPEC_CTRL 0x00000048 - #define SPEC_CTRL_IBRS (_AC(1, ULL) << 0) - #define SPEC_CTRL_STIBP (_AC(1, ULL) << 1) -+#define SPEC_CTRL_SSBD (_AC(1, ULL) << 2) - - #define MSR_PRED_CMD 0x00000049 - #define PRED_CMD_IBPB (_AC(1, ULL) << 0) -@@ -42,6 +46,12 @@ - #define MSR_ARCH_CAPABILITIES 0x0000010a - #define ARCH_CAPABILITIES_RDCL_NO (_AC(1, ULL) << 0) - #define ARCH_CAPABILITIES_IBRS_ALL (_AC(1, ULL) << 1) -+#define ARCH_CAPS_RSBA (_AC(1, ULL) << 2) -+#define ARCH_CAPS_SKIP_L1DFL (_AC(1, ULL) << 3) -+#define ARCH_CAPS_SSB_NO (_AC(1, ULL) << 4) -+ -+#define MSR_FLUSH_CMD 0x0000010b -+#define FLUSH_CMD_L1D (_AC(1, ULL) << 0) - - /* Intel MSRs. Some also available on other CPUs */ - #define MSR_IA32_PERFCTR0 0x000000c1 -diff --git a/xen/include/asm-x86/msr.h b/xen/include/asm-x86/msr.h -index 2c9277b6d5..22d5b7181e 100644 ---- a/xen/include/asm-x86/msr.h -+++ b/xen/include/asm-x86/msr.h -@@ -198,7 +198,7 @@ DECLARE_PER_CPU(u64, efer); - u64 read_efer(void); - void write_efer(u64 val); - --DECLARE_PER_CPU(u32, ler_msr); -+extern unsigned int ler_msr; - - DECLARE_PER_CPU(uint32_t, tsc_aux); - -diff --git a/xen/include/asm-x86/nops.h b/xen/include/asm-x86/nops.h -index 37f9819e82..0016075616 100644 ---- a/xen/include/asm-x86/nops.h -+++ b/xen/include/asm-x86/nops.h -@@ -61,11 +61,12 @@ - #define ASM_NOP7 _ASM_MK_NOP(K8_NOP7) - #define ASM_NOP8 _ASM_MK_NOP(K8_NOP8) - -+#define ASM_NOP14 ASM_NOP8; ASM_NOP6 - #define ASM_NOP17 ASM_NOP8; ASM_NOP7; ASM_NOP2 --#define ASM_NOP21 ASM_NOP8; ASM_NOP8; ASM_NOP5 - #define ASM_NOP24 ASM_NOP8; ASM_NOP8; ASM_NOP8 --#define ASM_NOP29 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP5 --#define ASM_NOP32 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8 -+#define ASM_NOP25 ASM_NOP8; ASM_NOP8; ASM_NOP7; ASM_NOP2 -+#define ASM_NOP33 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP7; ASM_NOP2 -+#define ASM_NOP36 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP4 - #define ASM_NOP40 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8 - - #define ASM_NOP_MAX 8 -diff --git a/xen/include/asm-x86/paging.h b/xen/include/asm-x86/paging.h -index 5607ab4b1f..fc326fe616 100644 ---- a/xen/include/asm-x86/paging.h -+++ b/xen/include/asm-x86/paging.h -@@ -37,11 +37,14 @@ - - #define PG_SH_shift 20 - #define PG_HAP_shift 21 -+#define PG_SHF_shift 22 - /* We're in one of the shadow modes */ - #ifdef CONFIG_SHADOW_PAGING - #define PG_SH_enable (1U << PG_SH_shift) -+#define PG_SH_forced (1U << PG_SHF_shift) - #else - #define PG_SH_enable 0 -+#define PG_SH_forced 0 - #endif - #define PG_HAP_enable (1U << PG_HAP_shift) - -@@ -62,6 +65,7 @@ - - #define paging_mode_enabled(_d) (!!(_d)->arch.paging.mode) - #define paging_mode_shadow(_d) (!!((_d)->arch.paging.mode & PG_SH_enable)) -+#define paging_mode_sh_forced(_d) (!!((_d)->arch.paging.mode & PG_SH_forced)) - #define paging_mode_hap(_d) (!!((_d)->arch.paging.mode & PG_HAP_enable)) - - #define paging_mode_refcounts(_d) (!!((_d)->arch.paging.mode & PG_refcounts)) -diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h -index 80f8411355..90a2701d26 100644 ---- a/xen/include/asm-x86/processor.h -+++ b/xen/include/asm-x86/processor.h -@@ -293,6 +293,21 @@ static inline unsigned long read_cr2(void) - return cr2; - } - -+static inline void write_cr3(unsigned long val) -+{ -+ asm volatile ( "mov %0, %%cr3" : : "r" (val) : "memory" ); -+} -+ -+static inline unsigned long cr3_pa(unsigned long cr3) -+{ -+ return cr3 & X86_CR3_ADDR_MASK; -+} -+ -+static inline unsigned long cr3_pcid(unsigned long cr3) -+{ -+ return cr3 & X86_CR3_PCID_MASK; -+} -+ - static inline unsigned long read_cr4(void) - { - return get_cpu_info()->cr4; -@@ -300,6 +315,9 @@ static inline unsigned long read_cr4(void) - - static inline void write_cr4(unsigned long val) - { -+ /* No global pages in case of PCIDs enabled! */ -+ ASSERT(!(val & X86_CR4_PGE) || !(val & X86_CR4_PCIDE)); -+ - get_cpu_info()->cr4 = val; - asm volatile ( "mov %0,%%cr4" : : "r" (val) ); - } -@@ -329,12 +347,6 @@ static always_inline void set_in_cr4 (unsigned long mask) - write_cr4(read_cr4() | mask); - } - --static always_inline void clear_in_cr4 (unsigned long mask) --{ -- mmu_cr4_features &= ~mask; -- write_cr4(read_cr4() & ~mask); --} -- - static inline unsigned int read_pkru(void) - { - unsigned int pkru; -@@ -445,7 +457,8 @@ struct __packed __cacheline_aligned tss_struct { - #define IST_DF 1UL - #define IST_NMI 2UL - #define IST_MCE 3UL --#define IST_MAX 3UL -+#define IST_DB 4UL -+#define IST_MAX 4UL - - /* Set the interrupt stack table used by a particular interrupt - * descriptor table entry. */ -diff --git a/xen/include/asm-x86/pv/domain.h b/xen/include/asm-x86/pv/domain.h -index acdf140fbd..6778e1bb75 100644 ---- a/xen/include/asm-x86/pv/domain.h -+++ b/xen/include/asm-x86/pv/domain.h -@@ -21,6 +21,37 @@ - #ifndef __X86_PV_DOMAIN_H__ - #define __X86_PV_DOMAIN_H__ - -+/* -+ * PCID values for the address spaces of 64-bit pv domains: -+ * -+ * We are using 4 PCID values for a 64 bit pv domain subject to XPTI: -+ * - hypervisor active and guest in kernel mode PCID 0 -+ * - hypervisor active and guest in user mode PCID 1 -+ * - guest active and in kernel mode PCID 2 -+ * - guest active and in user mode PCID 3 -+ * -+ * Without XPTI only 2 values are used: -+ * - guest in kernel mode PCID 0 -+ * - guest in user mode PCID 1 -+ */ -+ -+#define PCID_PV_PRIV 0x0000 /* Used for other domains, too. */ -+#define PCID_PV_USER 0x0001 -+#define PCID_PV_XPTI 0x0002 /* To be ORed to above values. */ -+ -+/* -+ * Return additional PCID specific cr3 bits. -+ * -+ * Note that X86_CR3_NOFLUSH will not be readable in cr3. Anyone consuming -+ * v->arch.cr3 should mask away X86_CR3_NOFLUSH and X86_CR3_PCIDMASK in case -+ * the value is used to address the root page table. -+ */ -+static inline unsigned long get_pcid_bits(const struct vcpu *v, bool is_xpti) -+{ -+ return X86_CR3_NOFLUSH | (is_xpti ? PCID_PV_XPTI : 0) | -+ ((v->arch.flags & TF_kernel_mode) ? PCID_PV_PRIV : PCID_PV_USER); -+} -+ - #ifdef CONFIG_PV - - void pv_vcpu_destroy(struct vcpu *v); -diff --git a/xen/include/asm-x86/setup.h b/xen/include/asm-x86/setup.h -index b68ec9de4d..ecfd0c2e7b 100644 ---- a/xen/include/asm-x86/setup.h -+++ b/xen/include/asm-x86/setup.h -@@ -66,6 +66,8 @@ extern uint8_t kbd_shift_flags; - extern unsigned long highmem_start; - #endif - -+extern int8_t opt_smt; -+ - #ifdef CONFIG_SHADOW_PAGING - extern bool opt_dom0_shadow; - #else -diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h -index 94a34fd16a..f40f411871 100644 ---- a/xen/include/asm-x86/shadow.h -+++ b/xen/include/asm-x86/shadow.h -@@ -29,6 +29,7 @@ - #include <asm/flushtlb.h> - #include <asm/paging.h> - #include <asm/p2m.h> -+#include <asm/spec_ctrl.h> - - /***************************************************************************** - * Macros to tell which shadow paging mode a domain is in*/ -@@ -115,6 +116,131 @@ static inline int shadow_domctl(struct domain *d, - - #endif /* CONFIG_SHADOW_PAGING */ - -+/* -+ * Mitigations for L1TF / CVE-2018-3620 for PV guests. -+ * -+ * We cannot alter an architecturally-legitimate PTE which a PV guest has -+ * chosen to write, as traditional paged-out metadata is L1TF-vulnerable. -+ * What we can do is force a PV guest which writes a vulnerable PTE into -+ * shadow mode, so Xen controls the pagetables which are reachable by the CPU -+ * pagewalk. -+ * -+ * The core of the L1TF vulnerability is that the address bits of the PTE -+ * (accounting for PSE and factoring in the level-relevant part of the linear -+ * access) are sent for an L1D lookup (to retrieve the next-level PTE, or -+ * eventual memory address) before the Present or reserved bits (which would -+ * cause a terminal fault) are accounted for. If an L1D hit occurs, the -+ * resulting data is available for potentially dependent instructions. -+ * -+ * For Present PTEs, the PV type-count safety logic ensures that the address -+ * bits always point at a guest-accessible frame, which is safe WRT L1TF from -+ * Xen's point of view. In practice, a PV guest should be unable to set any -+ * reserved bits, so should be unable to create any present L1TF-vulnerable -+ * PTEs at all. -+ * -+ * Therefore, these safety checks apply to Not-Present PTEs only, where -+ * traditionally, Xen would have let the guest write any value it chose. -+ * -+ * The all-zero PTE potentially leaks mfn 0. All software on the system is -+ * expected to cooperate and not put any secrets there. In a Xen system, -+ * neither Xen nor dom0 are expected to touch mfn 0, as it typically contains -+ * the real mode IVT and Bios Data Area. Therefore, mfn 0 is considered safe. -+ * -+ * Any PTE whose address is higher than the maximum cacheable address is safe, -+ * as it won't get an L1D hit. -+ * -+ * Speculative superpages also need accounting for, as PSE is considered -+ * irrespective of Present. We disallow PSE being set, as it allows an -+ * attacker to leak 2M or 1G of data starting from mfn 0. Also, because of -+ * recursive/linear pagetables, we must consider PSE even at L4, as hardware -+ * will interpret an L4e as an L3e during a recursive walk. -+ */ -+ -+static inline bool is_l1tf_safe_maddr(intpte_t pte) -+{ -+ paddr_t maddr = pte & l1tf_addr_mask; -+ -+ return maddr == 0 || maddr >= l1tf_safe_maddr; -+} -+ -+static inline bool pv_l1tf_check_pte(struct domain *d, unsigned int level, -+ intpte_t pte) -+{ -+ ASSERT(is_pv_domain(d)); -+ ASSERT(!(pte & _PAGE_PRESENT)); -+ -+ if ( d->arch.pv_domain.check_l1tf && !paging_mode_sh_forced(d) && -+ (((level > 1) && (pte & _PAGE_PSE)) || !is_l1tf_safe_maddr(pte)) ) -+ { -+#ifdef CONFIG_SHADOW_PAGING -+ struct tasklet *t = &d->arch.paging.shadow.pv_l1tf_tasklet; -+ -+ printk(XENLOG_G_WARNING -+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Shadowing\n", -+ d->domain_id, level, pte); -+ /* -+ * Safety consideration for accessing tasklet.scheduled_on without the -+ * tasklet lock. This is a singleshot tasklet with the side effect of -+ * setting PG_SH_forced (checked just above). Multiple vcpus can race -+ * to schedule the tasklet, but if we observe it scheduled anywhere, -+ * that is good enough. -+ */ -+ smp_rmb(); -+ if ( !tasklet_is_scheduled(t) ) -+ tasklet_schedule(t); -+#else -+ printk(XENLOG_G_ERR -+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Crashing\n", -+ d->domain_id, level, pte); -+ domain_crash(d); -+#endif -+ return true; -+ } -+ -+ return false; -+} -+ -+static inline bool pv_l1tf_check_l1e(struct domain *d, l1_pgentry_t l1e) -+{ -+ return pv_l1tf_check_pte(d, 1, l1e.l1); -+} -+ -+static inline bool pv_l1tf_check_l2e(struct domain *d, l2_pgentry_t l2e) -+{ -+ return pv_l1tf_check_pte(d, 2, l2e.l2); -+} -+ -+static inline bool pv_l1tf_check_l3e(struct domain *d, l3_pgentry_t l3e) -+{ -+ return pv_l1tf_check_pte(d, 3, l3e.l3); -+} -+ -+static inline bool pv_l1tf_check_l4e(struct domain *d, l4_pgentry_t l4e) -+{ -+ return pv_l1tf_check_pte(d, 4, l4e.l4); -+} -+ -+void pv_l1tf_tasklet(unsigned long data); -+ -+static inline void pv_l1tf_domain_init(struct domain *d) -+{ -+ d->arch.pv_domain.check_l1tf = -+ opt_pv_l1tf & (is_hardware_domain(d) -+ ? OPT_PV_L1TF_DOM0 : OPT_PV_L1TF_DOMU); -+ -+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV) -+ tasklet_init(&d->arch.paging.shadow.pv_l1tf_tasklet, -+ pv_l1tf_tasklet, (unsigned long)d); -+#endif -+} -+ -+static inline void pv_l1tf_domain_destroy(struct domain *d) -+{ -+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV) -+ tasklet_kill(&d->arch.paging.shadow.pv_l1tf_tasklet); -+#endif -+} -+ - /* Remove all shadows of the guest mfn. */ - static inline void shadow_remove_all_shadows(struct domain *d, mfn_t gmfn) - { -diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h -index 4e5f673fec..09c55458df 100644 ---- a/xen/include/asm-x86/smp.h -+++ b/xen/include/asm-x86/smp.h -@@ -26,6 +26,8 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); - DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); - DECLARE_PER_CPU(cpumask_var_t, scratch_cpumask); - -+extern bool park_offline_cpus; -+ - void smp_send_nmi_allbutself(void); - - void send_IPI_mask(const cpumask_t *, int vector); -diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h -index 5ab4ff3f68..8f8aad40bb 100644 ---- a/xen/include/asm-x86/spec_ctrl.h -+++ b/xen/include/asm-x86/spec_ctrl.h -@@ -27,14 +27,36 @@ - void init_speculation_mitigations(void); - - extern bool opt_ibpb; --extern uint8_t default_bti_ist_info; -+extern bool opt_ssbd; -+extern int8_t opt_eager_fpu; -+extern int8_t opt_l1d_flush; -+ -+extern bool bsp_delay_spec_ctrl; -+extern uint8_t default_xen_spec_ctrl; -+extern uint8_t default_spec_ctrl_flags; -+ -+extern int8_t opt_xpti; -+#define OPT_XPTI_DOM0 0x01 -+#define OPT_XPTI_DOMU 0x02 -+ -+extern int8_t opt_pv_l1tf; -+#define OPT_PV_L1TF_DOM0 0x01 -+#define OPT_PV_L1TF_DOMU 0x02 -+ -+/* -+ * The L1D address mask, which might be wider than reported in CPUID, and the -+ * system physical address above which there are believed to be no cacheable -+ * memory regions, thus unable to leak data via the L1TF vulnerability. -+ */ -+extern paddr_t l1tf_addr_mask, l1tf_safe_maddr; - - static inline void init_shadow_spec_ctrl_state(void) - { - struct cpu_info *info = get_cpu_info(); - -- info->shadow_spec_ctrl = info->use_shadow_spec_ctrl = 0; -- info->bti_ist_info = default_bti_ist_info; -+ info->shadow_spec_ctrl = 0; -+ info->xen_spec_ctrl = default_xen_spec_ctrl; -+ info->spec_ctrl_flags = default_spec_ctrl_flags; - } - - /* WARNING! `ret`, `call *`, `jmp *` not safe after this call. */ -@@ -48,24 +70,24 @@ static always_inline void spec_ctrl_enter_idle(struct cpu_info *info) - */ - info->shadow_spec_ctrl = val; - barrier(); -- info->use_shadow_spec_ctrl = true; -+ info->spec_ctrl_flags |= SCF_use_shadow; - barrier(); -- asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET) -+ asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_SC_MSR_IDLE) - :: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" ); - } - - /* WARNING! `ret`, `call *`, `jmp *` not safe before this call. */ - static always_inline void spec_ctrl_exit_idle(struct cpu_info *info) - { -- uint32_t val = SPEC_CTRL_IBRS; -+ uint32_t val = info->xen_spec_ctrl; - - /* - * Disable shadowing before updating the MSR. There are no SMP issues - * here; only local processor ordering concerns. - */ -- info->use_shadow_spec_ctrl = false; -+ info->spec_ctrl_flags &= ~SCF_use_shadow; - barrier(); -- asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_XEN_IBRS_SET) -+ asm volatile ( ALTERNATIVE(ASM_NOP3, "wrmsr", X86_FEATURE_SC_MSR_IDLE) - :: "a" (val), "c" (MSR_SPEC_CTRL), "d" (0) : "memory" ); - } - -diff --git a/xen/include/asm-x86/spec_ctrl_asm.h b/xen/include/asm-x86/spec_ctrl_asm.h -index 1f2b6f3552..c659f3f146 100644 ---- a/xen/include/asm-x86/spec_ctrl_asm.h -+++ b/xen/include/asm-x86/spec_ctrl_asm.h -@@ -20,10 +20,10 @@ - #ifndef __X86_SPEC_CTRL_ASM_H__ - #define __X86_SPEC_CTRL_ASM_H__ - --/* Encoding of the bottom bits in cpuinfo.bti_ist_info */ --#define BTI_IST_IBRS (1 << 0) --#define BTI_IST_WRMSR (1 << 1) --#define BTI_IST_RSB (1 << 2) -+/* Encoding of cpuinfo.spec_ctrl_flags */ -+#define SCF_use_shadow (1 << 0) -+#define SCF_ist_wrmsr (1 << 1) -+#define SCF_ist_rsb (1 << 2) - - #ifdef __ASSEMBLY__ - #include <asm/msr-index.h> -@@ -50,20 +50,20 @@ - * after VMEXIT. The VMEXIT-specific code reads MSR_SPEC_CTRL and updates - * current before loading Xen's MSR_SPEC_CTRL setting. - * -- * Factor 2 is harder. We maintain a shadow_spec_ctrl value, and -- * use_shadow_spec_ctrl boolean per cpu. The synchronous use is: -+ * Factor 2 is harder. We maintain a shadow_spec_ctrl value, and a use_shadow -+ * boolean in the per cpu spec_ctrl_flags. The synchronous use is: - * - * 1) Store guest value in shadow_spec_ctrl -- * 2) Set use_shadow_spec_ctrl boolean -+ * 2) Set the use_shadow boolean - * 3) Load guest value into MSR_SPEC_CTRL - * 4) Exit to guest - * 5) Entry from guest -- * 6) Clear use_shadow_spec_ctrl boolean -+ * 6) Clear the use_shadow boolean - * 7) Load Xen's value into MSR_SPEC_CTRL - * - * The asynchronous use for interrupts/exceptions is: - * - Set/clear IBRS on entry to Xen -- * - On exit to Xen, check use_shadow_spec_ctrl -+ * - On exit to Xen, check use_shadow - * - If set, load shadow_spec_ctrl - * - * Therefore, an interrupt/exception which hits the synchronous path between -@@ -72,11 +72,14 @@ - * - * The following ASM fragments implement this algorithm. See their local - * comments for further details. -- * - SPEC_CTRL_ENTRY_FROM_VMEXIT -+ * - SPEC_CTRL_ENTRY_FROM_HVM - * - SPEC_CTRL_ENTRY_FROM_PV - * - SPEC_CTRL_ENTRY_FROM_INTR -+ * - SPEC_CTRL_ENTRY_FROM_INTR_IST -+ * - SPEC_CTRL_EXIT_TO_XEN_IST - * - SPEC_CTRL_EXIT_TO_XEN -- * - SPEC_CTRL_EXIT_TO_GUEST -+ * - SPEC_CTRL_EXIT_TO_PV -+ * - SPEC_CTRL_EXIT_TO_HVM - */ - - .macro DO_OVERWRITE_RSB tmp=rax -@@ -117,7 +120,7 @@ - mov %\tmp, %rsp /* Restore old %rsp */ - .endm - --.macro DO_SPEC_CTRL_ENTRY_FROM_VMEXIT ibrs_val:req -+.macro DO_SPEC_CTRL_ENTRY_FROM_HVM - /* - * Requires %rbx=current, %rsp=regs/cpuinfo - * Clobbers %rax, %rcx, %rdx -@@ -135,14 +138,14 @@ - xor %edx, %edx - - /* Clear SPEC_CTRL shadowing *before* loading Xen's value. */ -- movb %dl, CPUINFO_use_shadow_spec_ctrl(%rsp) -+ andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) - - /* Load Xen's intended value. */ -- mov $\ibrs_val, %eax -+ movzbl CPUINFO_xen_spec_ctrl(%rsp), %eax - wrmsr - .endm - --.macro DO_SPEC_CTRL_ENTRY maybexen:req ibrs_val:req -+.macro DO_SPEC_CTRL_ENTRY maybexen:req - /* - * Requires %rsp=regs (also cpuinfo if !maybexen) - * Requires %r14=stack_end (if maybexen) -@@ -161,16 +164,18 @@ - * block so calculate the position directly. - */ - .if \maybexen -+ xor %eax, %eax - /* Branchless `if ( !xen ) clear_shadowing` */ - testb $3, UREGS_cs(%rsp) -- setz %al -- and %al, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%r14) -+ setnz %al -+ not %eax -+ and %al, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) -+ movzbl STACK_CPUINFO_FIELD(xen_spec_ctrl)(%r14), %eax - .else -- movb %dl, CPUINFO_use_shadow_spec_ctrl(%rsp) -+ andb $~SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) -+ movzbl CPUINFO_xen_spec_ctrl(%rsp), %eax - .endif - -- /* Load Xen's intended value. */ -- mov $\ibrs_val, %eax - wrmsr - .endm - -@@ -185,8 +190,8 @@ - */ - xor %edx, %edx - -- cmpb %dl, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%rbx) -- je .L\@_skip -+ testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) -+ jz .L\@_skip - - mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax - mov $MSR_SPEC_CTRL, %ecx -@@ -207,7 +212,7 @@ - mov %eax, CPUINFO_shadow_spec_ctrl(%rsp) - - /* Set SPEC_CTRL shadowing *before* loading the guest value. */ -- movb $1, CPUINFO_use_shadow_spec_ctrl(%rsp) -+ orb $SCF_use_shadow, CPUINFO_spec_ctrl_flags(%rsp) - - mov $MSR_SPEC_CTRL, %ecx - xor %edx, %edx -@@ -215,52 +220,47 @@ - .endm - - /* Use after a VMEXIT from an HVM guest. */ --#define SPEC_CTRL_ENTRY_FROM_VMEXIT \ -+#define SPEC_CTRL_ENTRY_FROM_HVM \ - ALTERNATIVE __stringify(ASM_NOP40), \ -- DO_OVERWRITE_RSB, X86_FEATURE_RSB_VMEXIT; \ -- ALTERNATIVE_2 __stringify(ASM_NOP32), \ -- __stringify(DO_SPEC_CTRL_ENTRY_FROM_VMEXIT \ -- ibrs_val=SPEC_CTRL_IBRS), \ -- X86_FEATURE_XEN_IBRS_SET, \ -- __stringify(DO_SPEC_CTRL_ENTRY_FROM_VMEXIT \ -- ibrs_val=0), \ -- X86_FEATURE_XEN_IBRS_CLEAR -+ DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_HVM; \ -+ ALTERNATIVE __stringify(ASM_NOP36), \ -+ DO_SPEC_CTRL_ENTRY_FROM_HVM, X86_FEATURE_SC_MSR_HVM - - /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ - #define SPEC_CTRL_ENTRY_FROM_PV \ - ALTERNATIVE __stringify(ASM_NOP40), \ -- DO_OVERWRITE_RSB, X86_FEATURE_RSB_NATIVE; \ -- ALTERNATIVE_2 __stringify(ASM_NOP21), \ -- __stringify(DO_SPEC_CTRL_ENTRY maybexen=0 \ -- ibrs_val=SPEC_CTRL_IBRS), \ -- X86_FEATURE_XEN_IBRS_SET, \ -- __stringify(DO_SPEC_CTRL_ENTRY maybexen=0 ibrs_val=0), \ -- X86_FEATURE_XEN_IBRS_CLEAR -+ DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ -+ ALTERNATIVE __stringify(ASM_NOP25), \ -+ __stringify(DO_SPEC_CTRL_ENTRY maybexen=0), X86_FEATURE_SC_MSR_PV - - /* Use in interrupt/exception context. May interrupt Xen or PV context. */ - #define SPEC_CTRL_ENTRY_FROM_INTR \ - ALTERNATIVE __stringify(ASM_NOP40), \ -- DO_OVERWRITE_RSB, X86_FEATURE_RSB_NATIVE; \ -- ALTERNATIVE_2 __stringify(ASM_NOP29), \ -- __stringify(DO_SPEC_CTRL_ENTRY maybexen=1 \ -- ibrs_val=SPEC_CTRL_IBRS), \ -- X86_FEATURE_XEN_IBRS_SET, \ -- __stringify(DO_SPEC_CTRL_ENTRY maybexen=1 ibrs_val=0), \ -- X86_FEATURE_XEN_IBRS_CLEAR -+ DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ -+ ALTERNATIVE __stringify(ASM_NOP33), \ -+ __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), X86_FEATURE_SC_MSR_PV - - /* Use when exiting to Xen context. */ - #define SPEC_CTRL_EXIT_TO_XEN \ -- ALTERNATIVE_2 __stringify(ASM_NOP17), \ -- DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_XEN_IBRS_SET, \ -- DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_XEN_IBRS_CLEAR -+ ALTERNATIVE __stringify(ASM_NOP17), \ -+ DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_SC_MSR_PV - --/* Use when exiting to guest context. */ --#define SPEC_CTRL_EXIT_TO_GUEST \ -- ALTERNATIVE_2 __stringify(ASM_NOP24), \ -- DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_XEN_IBRS_SET, \ -- DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_XEN_IBRS_CLEAR -+/* Use when exiting to PV guest context. */ -+#define SPEC_CTRL_EXIT_TO_PV \ -+ ALTERNATIVE __stringify(ASM_NOP24), \ -+ DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV - --/* TODO: Drop these when the alternatives infrastructure is NMI/#MC safe. */ -+/* Use when exiting to HVM guest context. */ -+#define SPEC_CTRL_EXIT_TO_HVM \ -+ ALTERNATIVE __stringify(ASM_NOP24), \ -+ DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_HVM -+ -+/* -+ * Use in IST interrupt/exception context. May interrupt Xen or PV context. -+ * Fine grain control of SCF_ist_wrmsr is needed for safety in the S3 resume -+ * path to avoid using MSR_SPEC_CTRL before the microcode introducing it has -+ * been reloaded. -+ */ - .macro SPEC_CTRL_ENTRY_FROM_INTR_IST - /* - * Requires %rsp=regs, %r14=stack_end -@@ -269,29 +269,27 @@ - * This is logical merge of DO_OVERWRITE_RSB and DO_SPEC_CTRL_ENTRY - * maybexen=1, but with conditionals rather than alternatives. - */ -- movzbl STACK_CPUINFO_FIELD(bti_ist_info)(%r14), %eax -+ movzbl STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14), %eax - -- testb $BTI_IST_RSB, %al -+ test $SCF_ist_rsb, %al - jz .L\@_skip_rsb - - DO_OVERWRITE_RSB tmp=rdx /* Clobbers %rcx/%rdx */ - - .L\@_skip_rsb: - -- testb $BTI_IST_WRMSR, %al -+ test $SCF_ist_wrmsr, %al - jz .L\@_skip_wrmsr - - xor %edx, %edx - testb $3, UREGS_cs(%rsp) -- setz %dl -- and %dl, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%r14) -+ setnz %dl -+ not %edx -+ and %dl, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) - -- /* -- * Load Xen's intended value. SPEC_CTRL_IBRS vs 0 is encoded in the -- * bottom bit of bti_ist_info, via a deliberate alias with BTI_IST_IBRS. -- */ -+ /* Load Xen's intended value. */ - mov $MSR_SPEC_CTRL, %ecx -- and $BTI_IST_IBRS, %eax -+ movzbl STACK_CPUINFO_FIELD(xen_spec_ctrl)(%r14), %eax - xor %edx, %edx - wrmsr - -@@ -309,12 +307,13 @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): - UNLIKELY_END(\@_serialise) - .endm - -+/* Use when exiting to Xen in IST context. */ - .macro SPEC_CTRL_EXIT_TO_XEN_IST - /* - * Requires %rbx=stack_end - * Clobbers %rax, %rcx, %rdx - */ -- testb $BTI_IST_WRMSR, STACK_CPUINFO_FIELD(bti_ist_info)(%rbx) -+ testb $SCF_ist_wrmsr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) - jz .L\@_skip - - DO_SPEC_CTRL_EXIT_TO_XEN -diff --git a/xen/include/asm-x86/system.h b/xen/include/asm-x86/system.h -index eb498f5e71..605768be12 100644 ---- a/xen/include/asm-x86/system.h -+++ b/xen/include/asm-x86/system.h -@@ -185,6 +185,30 @@ static always_inline unsigned long __xadd( - #define set_mb(var, value) do { xchg(&var, value); } while (0) - #define set_wmb(var, value) do { var = value; wmb(); } while (0) - -+/** -+ * array_index_mask_nospec() - generate a mask that is ~0UL when the -+ * bounds check succeeds and 0 otherwise -+ * @index: array element index -+ * @size: number of elements in array -+ * -+ * Returns: -+ * 0 - (index < size) -+ */ -+static inline unsigned long array_index_mask_nospec(unsigned long index, -+ unsigned long size) -+{ -+ unsigned long mask; -+ -+ asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];" -+ : [mask] "=r" (mask) -+ : [size] "g" (size), [index] "r" (index) ); -+ -+ return mask; -+} -+ -+/* Override default implementation in nospec.h. */ -+#define array_index_mask_nospec array_index_mask_nospec -+ - #define local_irq_disable() asm volatile ( "cli" : : : "memory" ) - #define local_irq_enable() asm volatile ( "sti" : : : "memory" ) - -diff --git a/xen/include/asm-x86/x86-defns.h b/xen/include/asm-x86/x86-defns.h -index 70453e8dfb..10b366a07d 100644 ---- a/xen/include/asm-x86/x86-defns.h -+++ b/xen/include/asm-x86/x86-defns.h -@@ -42,6 +42,13 @@ - #define X86_CR0_CD 0x40000000 /* Cache Disable (RW) */ - #define X86_CR0_PG 0x80000000 /* Paging (RW) */ - -+/* -+ * Intel CPU flags in CR3 -+ */ -+#define X86_CR3_NOFLUSH (_AC(1, ULL) << 63) -+#define X86_CR3_ADDR_MASK (PAGE_MASK & PADDR_MASK) -+#define X86_CR3_PCID_MASK _AC(0x0fff, ULL) /* Mask for PCID */ -+ - /* - * Intel CPU features in CR4 - */ -diff --git a/xen/include/asm-x86/xstate.h b/xen/include/asm-x86/xstate.h -index d36f422b59..9ba2a04c74 100644 ---- a/xen/include/asm-x86/xstate.h -+++ b/xen/include/asm-x86/xstate.h -@@ -116,8 +116,9 @@ void xsave(struct vcpu *v, uint64_t mask); - void xrstor(struct vcpu *v, uint64_t mask); - void xstate_set_init(uint64_t mask); - bool xsave_enabled(const struct vcpu *v); --int __must_check validate_xstate(u64 xcr0, u64 xcr0_accum, -- const struct xsave_hdr *); -+int __must_check validate_xstate(const struct domain *d, -+ uint64_t xcr0, uint64_t xcr0_accum, -+ const struct xsave_hdr *hdr); - int __must_check handle_xsetbv(u32 index, u64 new_bv); - void expand_xsave_states(struct vcpu *v, void *dest, unsigned int size); - void compress_xsave_states(struct vcpu *v, const void *src, unsigned int size); -diff --git a/xen/include/public/arch-x86/cpufeatureset.h b/xen/include/public/arch-x86/cpufeatureset.h -index 8da5783f7a..6c82816fd3 100644 ---- a/xen/include/public/arch-x86/cpufeatureset.h -+++ b/xen/include/public/arch-x86/cpufeatureset.h -@@ -243,8 +243,10 @@ XEN_CPUFEATURE(IBPB, 8*32+12) /*A IBPB support only (no IBRS, used by - XEN_CPUFEATURE(AVX512_4VNNIW, 9*32+ 2) /*A AVX512 Neural Network Instructions */ - XEN_CPUFEATURE(AVX512_4FMAPS, 9*32+ 3) /*A AVX512 Multiply Accumulation Single Precision */ - XEN_CPUFEATURE(IBRSB, 9*32+26) /*A IBRS and IBPB support (used by Intel) */ --XEN_CPUFEATURE(STIBP, 9*32+27) /*A! STIBP */ -+XEN_CPUFEATURE(STIBP, 9*32+27) /*A STIBP */ -+XEN_CPUFEATURE(L1D_FLUSH, 9*32+28) /*S MSR_FLUSH_CMD and L1D flush. */ - XEN_CPUFEATURE(ARCH_CAPS, 9*32+29) /* IA32_ARCH_CAPABILITIES MSR */ -+XEN_CPUFEATURE(SSBD, 9*32+31) /*A MSR_SPEC_CTRL.SSBD available */ - - #endif /* XEN_CPUFEATURE */ - -diff --git a/xen/include/xen/compiler.h b/xen/include/xen/compiler.h -index 533a8ea0f3..a7e05681c9 100644 ---- a/xen/include/xen/compiler.h -+++ b/xen/include/xen/compiler.h -@@ -81,6 +81,9 @@ - #pragma GCC visibility push(hidden) - #endif - -+/* Make the optimizer believe the variable can be manipulated arbitrarily. */ -+#define OPTIMIZER_HIDE_VAR(var) __asm__ ( "" : "+g" (var) ) -+ - /* This macro obfuscates arithmetic on a variable address so that gcc - shouldn't recognize the original var, and make assumptions about it */ - /* -diff --git a/xen/include/xen/cpu.h b/xen/include/xen/cpu.h -index ffefc09f8e..2fe3ec05d8 100644 ---- a/xen/include/xen/cpu.h -+++ b/xen/include/xen/cpu.h -@@ -47,6 +47,8 @@ void register_cpu_notifier(struct notifier_block *nb); - #define CPU_DYING (0x0007 | NOTIFY_REVERSE) - /* CPU_DEAD: CPU is dead. */ - #define CPU_DEAD (0x0008 | NOTIFY_REVERSE) -+/* CPU_REMOVE: CPU was removed. */ -+#define CPU_REMOVE (0x0009 | NOTIFY_REVERSE) - - /* Perform CPU hotplug. May return -EAGAIN. */ - int cpu_down(unsigned int cpu); -diff --git a/xen/include/xen/cpumask.h b/xen/include/xen/cpumask.h -index 3f340d619a..ee4399865a 100644 ---- a/xen/include/xen/cpumask.h -+++ b/xen/include/xen/cpumask.h -@@ -349,16 +349,35 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask) - return *mask != NULL; - } - -+static inline bool cond_alloc_cpumask_var(cpumask_var_t *mask) -+{ -+ if (*mask == NULL) -+ *mask = _xmalloc(nr_cpumask_bits / 8, sizeof(long)); -+ return *mask != NULL; -+} -+ - static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask) - { - *(void **)mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long)); - return *mask != NULL; - } - -+static inline bool cond_zalloc_cpumask_var(cpumask_var_t *mask) -+{ -+ if (*mask == NULL) -+ *mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long)); -+ else -+ cpumask_clear(*mask); -+ return *mask != NULL; -+} -+ - static inline void free_cpumask_var(cpumask_var_t mask) - { - xfree(mask); - } -+ -+/* Free an allocated mask, and zero the pointer to it. */ -+#define FREE_CPUMASK_VAR(m) XFREE(m) - #else - typedef cpumask_t cpumask_var_t[1]; - -@@ -366,16 +385,20 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask) - { - return 1; - } -+#define cond_alloc_cpumask_var alloc_cpumask_var - - static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask) - { - cpumask_clear(*mask); - return 1; - } -+#define cond_zalloc_cpumask_var zalloc_cpumask_var - - static inline void free_cpumask_var(cpumask_var_t mask) - { - } -+ -+#define FREE_CPUMASK_VAR(m) free_cpumask_var(m) - #endif - - #if NR_CPUS > 1 -diff --git a/xen/include/xen/list.h b/xen/include/xen/list.h -index fa07d720ee..1387abb211 100644 ---- a/xen/include/xen/list.h -+++ b/xen/include/xen/list.h -@@ -51,6 +51,11 @@ static inline void INIT_LIST_HEAD(struct list_head *list) - list->prev = list; - } - -+static inline bool list_head_is_null(const struct list_head *list) -+{ -+ return !list->next && !list->prev; -+} -+ - /* - * Insert a new entry between two known consecutive entries. - * -diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h -index e813c07b22..fdcb90841a 100644 ---- a/xen/include/xen/mm.h -+++ b/xen/include/xen/mm.h -@@ -162,6 +162,14 @@ void free_xenheap_pages(void *v, unsigned int order); - bool scrub_free_pages(void); - #define alloc_xenheap_page() (alloc_xenheap_pages(0,0)) - #define free_xenheap_page(v) (free_xenheap_pages(v,0)) -+ -+/* Free an allocation, and zero the pointer to it. */ -+#define FREE_XENHEAP_PAGES(p, o) do { \ -+ free_xenheap_pages(p, o); \ -+ (p) = NULL; \ -+} while ( false ) -+#define FREE_XENHEAP_PAGE(p) FREE_XENHEAP_PAGES(p, 0) -+ - /* Map machine page range in Xen virtual address space. */ - int map_pages_to_xen( - unsigned long virt, -diff --git a/xen/include/xen/nospec.h b/xen/include/xen/nospec.h -new file mode 100644 -index 0000000000..48793996e8 ---- /dev/null -+++ b/xen/include/xen/nospec.h -@@ -0,0 +1,70 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* Copyright(c) 2018 Linus Torvalds. All rights reserved. */ -+/* Copyright(c) 2018 Alexei Starovoitov. All rights reserved. */ -+/* Copyright(c) 2018 Intel Corporation. All rights reserved. */ -+/* Copyright(c) 2018 Citrix Systems R&D Ltd. All rights reserved. */ -+ -+#ifndef XEN_NOSPEC_H -+#define XEN_NOSPEC_H -+ -+#include <asm/system.h> -+ -+/** -+ * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise -+ * @index: array element index -+ * @size: number of elements in array -+ * -+ * When @index is out of bounds (@index >= @size), the sign bit will be -+ * set. Extend the sign bit to all bits and invert, giving a result of -+ * zero for an out of bounds index, or ~0 if within bounds [0, @size). -+ */ -+#ifndef array_index_mask_nospec -+static inline unsigned long array_index_mask_nospec(unsigned long index, -+ unsigned long size) -+{ -+ /* -+ * Always calculate and emit the mask even if the compiler -+ * thinks the mask is not needed. The compiler does not take -+ * into account the value of @index under speculation. -+ */ -+ OPTIMIZER_HIDE_VAR(index); -+ return ~(long)(index | (size - 1UL - index)) >> (BITS_PER_LONG - 1); -+} -+#endif -+ -+/* -+ * array_index_nospec - sanitize an array index after a bounds check -+ * -+ * For a code sequence like: -+ * -+ * if (index < size) { -+ * index = array_index_nospec(index, size); -+ * val = array[index]; -+ * } -+ * -+ * ...if the CPU speculates past the bounds check then -+ * array_index_nospec() will clamp the index within the range of [0, -+ * size). -+ */ -+#define array_index_nospec(index, size) \ -+({ \ -+ typeof(index) _i = (index); \ -+ typeof(size) _s = (size); \ -+ unsigned long _mask = array_index_mask_nospec(_i, _s); \ -+ \ -+ BUILD_BUG_ON(sizeof(_i) > sizeof(long)); \ -+ BUILD_BUG_ON(sizeof(_s) > sizeof(long)); \ -+ \ -+ (typeof(_i)) (_i & _mask); \ -+}) -+ -+#endif /* XEN_NOSPEC_H */ -+ -+/* -+ * Local variables: -+ * mode: C -+ * c-file-style: "BSD" -+ * c-basic-offset: 4 -+ * indent-tabs-mode: nil -+ * End: -+ */ -diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h -index 2541ecb04f..eaa83dc97e 100644 ---- a/xen/include/xen/sched.h -+++ b/xen/include/xen/sched.h -@@ -796,7 +796,7 @@ static inline struct domain *next_domain_in_cpupool( - #define _VPF_parked 8 - #define VPF_parked (1UL<<_VPF_parked) - --static inline int vcpu_runnable(struct vcpu *v) -+static inline bool vcpu_runnable(const struct vcpu *v) - { - return !(v->pause_flags | - atomic_read(&v->pause_count) | -diff --git a/xen/include/xen/tasklet.h b/xen/include/xen/tasklet.h -index 23d69c738e..bc9ddace6d 100644 ---- a/xen/include/xen/tasklet.h -+++ b/xen/include/xen/tasklet.h -@@ -50,6 +50,11 @@ static inline bool tasklet_work_to_do(unsigned int cpu) - TASKLET_scheduled); - } - -+static inline bool tasklet_is_scheduled(const struct tasklet *t) -+{ -+ return t->scheduled_on != -1; -+} -+ - void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu); - void tasklet_schedule(struct tasklet *t); - void do_tasklet(void); -diff --git a/xen/include/xen/xmalloc.h b/xen/include/xen/xmalloc.h -index cc2673d8ae..9aa5edf593 100644 ---- a/xen/include/xen/xmalloc.h -+++ b/xen/include/xen/xmalloc.h -@@ -26,6 +26,12 @@ - /* Free any of the above. */ - extern void xfree(void *); - -+/* Free an allocation, and zero the pointer to it. */ -+#define XFREE(p) do { \ -+ xfree(p); \ -+ (p) = NULL; \ -+} while ( false ) -+ - /* Underlying functions */ - extern void *_xmalloc(unsigned long size, unsigned long align); - extern void *_xzalloc(unsigned long size, unsigned long align); -diff --git a/xen/tools/gen-cpuid.py b/xen/tools/gen-cpuid.py -index 613b909c3d..65526ff120 100755 ---- a/xen/tools/gen-cpuid.py -+++ b/xen/tools/gen-cpuid.py -@@ -257,10 +257,19 @@ def crunch_numbers(state): - AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW, - AVX512_4FMAPS, AVX512_VPOPCNTDQ], - -- # Single Thread Indirect Branch Predictors enumerates a new bit in the -- # MSR enumerated by Indirect Branch Restricted Speculation/Indirect -- # Branch Prediction Barrier enumeration. -- IBRSB: [STIBP], -+ # The features: -+ # * Single Thread Indirect Branch Predictors -+ # * Speculative Store Bypass Disable -+ # -+ # enumerate new bits in MSR_SPEC_CTRL, which is enumerated by Indirect -+ # Branch Restricted Speculation/Indirect Branch Prediction Barrier. -+ # -+ # In practice, these features also enumerate the presense of -+ # MSR_SPEC_CTRL. However, no real hardware will exist with SSBD but -+ # not IBRSB, and we pass this MSR directly to guests. Treating them -+ # as dependent features simplifies Xen's logic, and prevents the guest -+ # from seeing implausible configurations. -+ IBRSB: [STIBP, SSBD], - } - - deep_features = tuple(sorted(deps.keys())) |