From f3ef0562675966b70a55c549133adf9a5bdece9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 12 Sep 2018 02:51:28 +0200 Subject: [PATCH] Add xen_scrub_pages runtime control Fix performance issue in nested virtualization (used for system tests). See patch description for details. Similar patch is already accepted for Linux 4.19. --- ...runtime-control-for-scrubbing-balloo.patch | 167 ++++++++++++++++++ series.conf | 1 + 2 files changed, 168 insertions(+) create mode 100644 patches.xen/xen-balloon-add-runtime-control-for-scrubbing-balloo.patch diff --git a/patches.xen/xen-balloon-add-runtime-control-for-scrubbing-balloo.patch b/patches.xen/xen-balloon-add-runtime-control-for-scrubbing-balloo.patch new file mode 100644 index 0000000..6517cc9 --- /dev/null +++ b/patches.xen/xen-balloon-add-runtime-control-for-scrubbing-balloo.patch @@ -0,0 +1,167 @@ +From 59e44d8c45afe91204e90229bef64a6d2c5ac103 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= + +Date: Thu, 6 Sep 2018 15:09:44 +0200 +Subject: [PATCH] xen/balloon: add runtime control for scrubbing ballooned out + pages + +Scrubbing pages on initial balloon down can take some time, especially +in nested virtualization case (nested EPT is slow). When HVM/PVH guest is +started with memory= significantly lower than maxmem=, all the extra +pages will be scrubbed before returning to Xen. But since most of them +weren't used at all at that point, Xen needs to populate them first +(from populate-on-demand pool). In nested virt case (Xen inside KVM) +this slows down the guest boot by 15-30s with just 1.5GB needed to be +returned to Xen. + +Add runtime parameter to enable/disable it, to allow initially disabling +scrubbing, then enable it back during boot (for example in initramfs). +Such usage relies on assumption that a) most pages ballooned out during +initial boot weren't used at all, and b) even if they were, very few +secrets are in the guest at that time (before any serious userspace +kicks in). +Convert CONFIG_XEN_SCRUB_PAGES to CONFIG_XEN_SCRUB_PAGES_DEFAULT (also +enabled by default), controlling default value for the new runtime +switch. + +Signed-off-by: Marek Marczykowski-Górecki + +--- + .../ABI/stable/sysfs-devices-system-xen_memory | 9 +++++++++ + Documentation/admin-guide/kernel-parameters.txt | 6 ++++++ + drivers/xen/Kconfig | 10 +++++++--- + drivers/xen/balloon.c | 9 ++++++--- + drivers/xen/xen-balloon.c | 2 ++ + include/xen/balloon.h | 1 + + 6 files changed, 31 insertions(+), 6 deletions(-) + +diff --git a/Documentation/ABI/stable/sysfs-devices-system-xen_memory b/Documentation/ABI/stable/sysfs-devices-system-xen_memory +index caa311d59ac1..6d83f95a8a8e 100644 +--- a/Documentation/ABI/stable/sysfs-devices-system-xen_memory ++++ b/Documentation/ABI/stable/sysfs-devices-system-xen_memory +@@ -75,3 +75,12 @@ Contact: Konrad Rzeszutek Wilk + Description: + Amount (in KiB) of low (or normal) memory in the + balloon. ++ ++What: /sys/devices/system/xen_memory/xen_memory0/scrub_pages ++Date: September 2018 ++KernelVersion: 4.20 ++Contact: xen-devel@lists.xenproject.org ++Description: ++ Control scrubbing pages before returning them to Xen for others domains ++ use. Can be set with xen_scrub_pages cmdline ++ parameter. Default value controlled with CONFIG_XEN_SCRUB_PAGES_DEFAULT. +diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt +index 1370b424a453..c62691ddd476 100644 +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -4921,6 +4921,12 @@ + Disables the PV optimizations forcing the HVM guest to + run as generic HVM guest with no PV drivers. + ++ xen_scrub_pages= [XEN] ++ Boolean option to control scrubbing pages before giving them back ++ to Xen, for use by other domains. Can be also changed at runtime ++ with /sys/devices/system/xen_memory/xen_memory0/scrub_pages. ++ Default value controlled with CONFIG_XEN_SCRUB_PAGES_DEFAULT. ++ + xirc2ps_cs= [NET,PCMCIA] + Format: + ,,,,,[,[,[,]]] +diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig +index e5d0c28372ea..2f0353290ce5 100644 +--- a/drivers/xen/Kconfig ++++ b/drivers/xen/Kconfig +@@ -79,15 +79,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT + This value is used to allocate enough space in internal + tables needed for physical memory administration. + +-config XEN_SCRUB_PAGES +- bool "Scrub pages before returning them to system" ++config XEN_SCRUB_PAGES_DEFAULT ++ bool "Scrub pages before returning them to system by default" + depends on XEN_BALLOON + default y + help + Scrub pages before returning them to the system for reuse by + other domains. This makes sure that any confidential data + is not accidentally visible to other domains. Is it more +- secure, but slightly less efficient. ++ secure, but slightly less efficient. This can be controlled with ++ xen_scrub_pages=0 parameter and ++ /sys/devices/system/xen_memory/xen_memory0/scrub_pages. ++ This option only sets the default value. ++ + If in doubt, say yes. + + config XEN_DEV_EVTCHN +diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c +index 065f0b607373..a3e5dfa71796 100644 +--- a/drivers/xen/balloon.c ++++ b/drivers/xen/balloon.c +@@ -56,6 +56,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -74,6 +75,9 @@ + + static int xen_hotplug_unpopulated; + ++bool __read_mostly xen_scrub_pages = IS_ENABLED(CONFIG_XEN_SCRUB_PAGES_DEFAULT); ++core_param(xen_scrub_pages, xen_scrub_pages, bool, 0); ++ + #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG + + static int zero; +@@ -159,9 +163,8 @@ static DECLARE_DELAYED_WORK(balloon_worker, balloon_process); + + static void scrub_page(struct page *page) + { +-#ifdef CONFIG_XEN_SCRUB_PAGES +- clear_highpage(page); +-#endif ++ if (xen_scrub_pages) ++ clear_highpage(page); + } + + /* balloon_append: add the given page to the balloon. */ +diff --git a/drivers/xen/xen-balloon.c b/drivers/xen/xen-balloon.c +index b437fccd4e62..a3dd7c6ec371 100644 +--- a/drivers/xen/xen-balloon.c ++++ b/drivers/xen/xen-balloon.c +@@ -137,6 +137,7 @@ static DEVICE_ULONG_ATTR(schedule_delay, 0444, balloon_stats.schedule_delay); + static DEVICE_ULONG_ATTR(max_schedule_delay, 0644, balloon_stats.max_schedule_delay); + static DEVICE_ULONG_ATTR(retry_count, 0444, balloon_stats.retry_count); + static DEVICE_ULONG_ATTR(max_retry_count, 0644, balloon_stats.max_retry_count); ++static DEVICE_BOOL_ATTR(scrub_pages, 0644, xen_scrub_pages); + + static ssize_t show_target_kb(struct device *dev, struct device_attribute *attr, + char *buf) +@@ -203,6 +204,7 @@ static struct attribute *balloon_attrs[] = { + &dev_attr_max_schedule_delay.attr.attr, + &dev_attr_retry_count.attr.attr, + &dev_attr_max_retry_count.attr.attr, ++ &dev_attr_scrub_pages.attr.attr, + NULL + }; + +diff --git a/include/xen/balloon.h b/include/xen/balloon.h +index 61f410fd74e4..0ec832b9ff55 100644 +--- a/include/xen/balloon.h ++++ b/include/xen/balloon.h +@@ -21,6 +21,7 @@ struct balloon_stats { + }; + + extern struct balloon_stats balloon_stats; ++extern bool xen_scrub_pages; + + void balloon_set_new_target(unsigned long target); + +-- +2.17.1 + diff --git a/series.conf b/series.conf index 5dfb2c1..5d98d94 100644 --- a/series.conf +++ b/series.conf @@ -3,6 +3,7 @@ patches.rpmify/makefile-after_link.patch patches.xen/xen-netfront-detach-crash.patch patches.xen/0001-mce-hide-EBUSY-initialization-error-on-Xen.patch patches.xen/irq-bind-debug-log.patch +patches.xen/xen-balloon-add-runtime-control-for-scrubbing-balloo.patch # Additional features #patches.xen/pvops-0100-usb-xen-pvusb-driver.patch