From 456fe99fa67ca9a4b09b3521568205e9cafb197a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 18 Jan 2019 23:53:20 +0100 Subject: [PATCH] Disable scrubbing memory pages during initial balloon down Balloon driver scrub memory page before giving it back to the hypervisor. Normally this is a good thing, to avoid leaking VM's memory data into Xen and other domains. But during initial startup when maxmem is bigger than initial memory, on HVM and PVH, Populate-on-Demand (PoD) is in use. This means every page on initial balloon down needs to be first mapped by Xen into VM's memory (as it wasn't populated before - and in fact didn't have any data), scrubbed by the kernel and then given back to Xen. This is great waste of time. Such operation with default settings (initial memory 400M, maxmem 4000M) can take few seconds, delaying every VM startup (including DispVM). In extreme situation, when running inside nested virtualization, the effect is much worse. Avoid this problem by disabling memory scrubbing during initial boot, and re-enable it as soon as user space kicks in - in initramfs, before mounting root filesystem, to be sure it's enabled before memory contains any kind of secrets. This commit handle only one case - when kernel in managed by the VM itself. It is critical to enable initramfs module whenever xen_scrub_pages=0 kernel option is given, so make them depend on the same condition and ship them in the same package. Fixes QubesOS/qubes-issues#1963 --- Makefile | 2 + debian/qubes-kernel-vm-support.install | 2 + debian/qubes-kernel-vm-support.postinst | 47 +++++++++++++++++++ dracut/simple/init.sh | 5 ++ dracut/xen-balloon-scrub-pages/Makefile | 4 ++ .../xen-balloon-scrub-pages/module-setup.sh | 17 +++++++ dracut/xen-balloon-scrub-pages/scrub_pages.sh | 20 ++++++++ grub/Makefile | 7 +++ grub/grub.qubes-kernel-vm-support | 5 ++ initramfs-tools/Makefile | 2 + initramfs-tools/local-top/scrub_pages.sh | 1 + rpm_spec/qubes-kernel-vm-support.spec.in | 14 ++++++ 12 files changed, 126 insertions(+) create mode 100644 debian/qubes-kernel-vm-support.postinst create mode 100644 dracut/xen-balloon-scrub-pages/Makefile create mode 100644 dracut/xen-balloon-scrub-pages/module-setup.sh create mode 100644 dracut/xen-balloon-scrub-pages/scrub_pages.sh create mode 100644 grub/Makefile create mode 100644 grub/grub.qubes-kernel-vm-support create mode 120000 initramfs-tools/local-top/scrub_pages.sh diff --git a/Makefile b/Makefile index fa0ace5..269ca2b 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,13 @@ install: install-fedora-kernel-support: $(MAKE) -C dracut install $(MAKE) -C kernel-modules install + $(MAKE) -C grub install-fedora install-debian-kernel-support: $(MAKE) -C initramfs-tools install $(MAKE) -C dracut install $(MAKE) -C kernel-modules install + $(MAKE) -C grub install-debian # expand module version rm -f debian/qubes-kernel-vm-support.dkms echo debian/tmp/usr/src/u2mfn-*/dkms.conf > debian/qubes-kernel-vm-support.dkms diff --git a/debian/qubes-kernel-vm-support.install b/debian/qubes-kernel-vm-support.install index 59e4153..12c33e7 100644 --- a/debian/qubes-kernel-vm-support.install +++ b/debian/qubes-kernel-vm-support.install @@ -1,6 +1,8 @@ +usr/share/initramfs-tools/scripts/local-top/scrub_pages usr/share/initramfs-tools/scripts/local-top/qubes_cow_setup usr/share/initramfs-tools/hooks/qubes_vm usr/lib/dracut/modules.d/90qubes-vm/* usr/lib/dracut/modules.d/90qubes-vm-modules/* usr/lib/dracut/modules.d/90qubes-vm-simple/* usr/src/u2mfn-*/* +etc/default/grub.d/30-qubes-kernel-vm-support.cfg diff --git a/debian/qubes-kernel-vm-support.postinst b/debian/qubes-kernel-vm-support.postinst new file mode 100644 index 0000000..89e375a --- /dev/null +++ b/debian/qubes-kernel-vm-support.postinst @@ -0,0 +1,47 @@ +#!/bin/bash +# postinst script for qubes-kernel-vm-support +# +# see: dh_installdeb(1) + +set -e + +# The postinst script may be called in the following ways: +# * 'configure' +# * 'abort-upgrade' +# * 'abort-remove' 'in-favour' +# +# * 'abort-remove' +# * 'abort-deconfigure' 'in-favour' +# 'removing' +# +# +# For details, see http://www.debian.org/doc/debian-policy/ or +# https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html or +# the debian-policy package + + +case "${1}" in + configure) + if [ -x /usr/sbin/update-initramfs ]; then + update-initramfs -u + fi + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + exit 0 + ;; + + *) + echo "postinst called with unknown argument \`${1}'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 + +# vim: set ts=4 sw=4 sts=4 et : diff --git a/dracut/simple/init.sh b/dracut/simple/init.sh index 132e7f9..da06739 100644 --- a/dracut/simple/init.sh +++ b/dracut/simple/init.sh @@ -6,6 +6,11 @@ mount -t proc proc /proc mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev +if [ -w /sys/devices/system/xen_memory/xen_memory0/scrub_pages ]; then + # re-enable xen-balloon pages scrubbing, after initial balloon down + echo 1 > /sys/devices/system/xen_memory/xen_memory0/scrub_pages +fi + if [ -e /dev/mapper/dmroot ] ; then echo "Qubes: FATAL error: /dev/mapper/dmroot already exists?!" fi diff --git a/dracut/xen-balloon-scrub-pages/Makefile b/dracut/xen-balloon-scrub-pages/Makefile new file mode 100644 index 0000000..36bf206 --- /dev/null +++ b/dracut/xen-balloon-scrub-pages/Makefile @@ -0,0 +1,4 @@ +install: + install -d $(DESTDIR)/usr/lib/dracut/modules.d/80xen-scrub-pages + install module-setup.sh scrub_pages.sh \ + $(DESTDIR)/usr/lib/dracut/modules.d/80xen-scrub-pages/ diff --git a/dracut/xen-balloon-scrub-pages/module-setup.sh b/dracut/xen-balloon-scrub-pages/module-setup.sh new file mode 100644 index 0000000..f75dfb6 --- /dev/null +++ b/dracut/xen-balloon-scrub-pages/module-setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +check() { + if [ -r /usr/share/qubes/marker-vm ]; then + return 0 + else + return 255 + fi +} + +depends() { + return 0 +} + +install() { + inst_hook pre-trigger 60 $moddir/scrub_pages.sh +} diff --git a/dracut/xen-balloon-scrub-pages/scrub_pages.sh b/dracut/xen-balloon-scrub-pages/scrub_pages.sh new file mode 100644 index 0000000..18bd43c --- /dev/null +++ b/dracut/xen-balloon-scrub-pages/scrub_pages.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# This file should be placed in pre-trigger directory in dracut's initramfs, or +# scripts/local-top in case of initramfs-tools +# + +# initramfs-tools (Debian) API +PREREQS="" +case "$1" in + prereqs) + # This runs during initramfs creation + echo "$PREREQS" + exit 0 + ;; +esac + +if [ -w /sys/devices/system/xen_memory/xen_memory0/scrub_pages ]; then + # re-enable xen-balloon pages scrubbing, after initial balloon down + echo 1 > /sys/devices/system/xen_memory/xen_memory0/scrub_pages +fi diff --git a/grub/Makefile b/grub/Makefile new file mode 100644 index 0000000..cdce37a --- /dev/null +++ b/grub/Makefile @@ -0,0 +1,7 @@ +install-fedora: + install -D -m 0644 grub.qubes-kernel-vm-support \ + $(DESTDIR)/etc/default/grub.qubes-kernel-vm-support + +install-debian: + install -D -m 0644 grub.qubes-kernel-vm-support \ + $(DESTDIR)/etc/default/grub.d/30-qubes-kernel-vm-support.cfg diff --git a/grub/grub.qubes-kernel-vm-support b/grub/grub.qubes-kernel-vm-support new file mode 100644 index 0000000..58910c2 --- /dev/null +++ b/grub/grub.qubes-kernel-vm-support @@ -0,0 +1,5 @@ +# add kernel options only in VM +if [ -r /usr/share/qubes/marker-vm ]; then + GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX xen_scrub_pages=0" +fi + diff --git a/initramfs-tools/Makefile b/initramfs-tools/Makefile index b9592a3..65f2004 100644 --- a/initramfs-tools/Makefile +++ b/initramfs-tools/Makefile @@ -1,6 +1,8 @@ install: install -D local-top/qubes_cow_setup.sh \ $(DESTDIR)/usr/share/initramfs-tools/scripts/local-top/qubes_cow_setup + install -D local-top/scrub_pages.sh \ + $(DESTDIR)/usr/share/initramfs-tools/scripts/local-top/scrub_pages install -D qubes_vm \ $(DESTDIR)/usr/share/initramfs-tools/hooks/qubes_vm diff --git a/initramfs-tools/local-top/scrub_pages.sh b/initramfs-tools/local-top/scrub_pages.sh new file mode 120000 index 0000000..d975cb1 --- /dev/null +++ b/initramfs-tools/local-top/scrub_pages.sh @@ -0,0 +1 @@ +../../dracut/xen-balloon-scrub-pages/scrub_pages.sh \ No newline at end of file diff --git a/rpm_spec/qubes-kernel-vm-support.spec.in b/rpm_spec/qubes-kernel-vm-support.spec.in index 8ccad23..8a2d2a9 100644 --- a/rpm_spec/qubes-kernel-vm-support.spec.in +++ b/rpm_spec/qubes-kernel-vm-support.spec.in @@ -54,6 +54,14 @@ make install-fedora-kernel-support DESTDIR=%{buildroot} /usr/lib/dracut/modules.d/90qubes-vm-simple /usr/src/u2mfn-%{version}/ /usr/sbin/qubes-prepare-vm-kernel +%config(noreplace) /etc/default/grub.qubes-kernel-vm-support + +%triggerin -- grub2-tools +if ! grep -q '/etc/default/grub.qubes-kernel-vm-support$' /etc/default/grub 2>/dev/null; then + # do not keep Qubes-related settings directly in user-controlled config, + # include another file + echo '. /etc/default/grub.qubes-kernel-vm-support' >> /etc/default/grub +fi %post dkms add -m u2mfn -v %{version} --rpm_safe_upgrade @@ -61,5 +69,11 @@ dkms add -m u2mfn -v %{version} --rpm_safe_upgrade %preun dkms remove -m u2mfn -v %{version} --all --rpm_safe_upgrade +if [ $1 -eq 0 ]; then + if grep -q '/etc/default/grub.qubes-kernel-vm-support$' /etc/default/grub 2>/dev/null; then + sed -i -e '/grub.qubes-kernel-vm-support$/d' /etc/default/grub + fi +fi + %changelog @CHANGELOG@