From 764b0f3f0778c7675ded7aa67aa954917045f20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 12 Sep 2017 04:25:34 +0200 Subject: [PATCH] udev: major cleanup in block devices handling 1. Do not detach device forcefully when it's removed. This breaks libvirt (which thinks the device is still there). After this change, it is possible to detach device using libvirt, even if it was already removed physically from backend domain (unless it is dom0 - in which case it is still broken). So, this is partial fix for QubesOS/qubes-issues#1082. 2. Do not trigger "change" udev event when only QubesDB state needs to be updated - this leads to massive udev events queue, and heavy I/O usage - for example scanning all LVM many times. In some cases it even caused infinite event queue. 3. Do not use QUBES_EXPOSED udev property - it was needed a while back before QubesDB, because concurrent xenstore accesses are expensive (because of transactions). It isn't the problem on QubesDB. 4. Cache information about device-mapper, so it is possible to reconstruct it at device remove - when the actual device cannot be queried anymore. This is specifically about list of lower layer devices used. 5. Allow excluding loop devices pointing at a file in directory marked with ".qubes-exclude-block-devices" file. This is more generic than hardcoding /var/lib/qubes. QubesOS/qubes-issues#3084 Fixes QubesOS/qubes-issues#3073 QubesOS/qubes-issues#1082 --- udev/Makefile | 1 - udev/udev-block-add-change | 52 +++++++++++++++++++++++++++---------- udev/udev-block-cleanup | 8 ------ udev/udev-block-remove | 40 ++++++---------------------- udev/udev-qubes-block.rules | 9 ++----- 5 files changed, 49 insertions(+), 61 deletions(-) delete mode 100755 udev/udev-block-cleanup diff --git a/udev/Makefile b/udev/Makefile index a2cf288..6eff34a 100644 --- a/udev/Makefile +++ b/udev/Makefile @@ -9,6 +9,5 @@ install: mkdir -p $(DESTDIR)$(SCRIPTSDIR) cp udev-block-add-change $(DESTDIR)$(SCRIPTSDIR) cp udev-block-remove $(DESTDIR)$(SCRIPTSDIR) - cp udev-block-cleanup $(DESTDIR)$(SCRIPTSDIR) cp udev-usb-add-change $(DESTDIR)$(SCRIPTSDIR) cp udev-usb-remove $(DESTDIR)$(SCRIPTSDIR) diff --git a/udev/udev-block-add-change b/udev/udev-block-add-change index 30cafcd..d3dce92 100755 --- a/udev/udev-block-add-change +++ b/udev/udev-block-add-change @@ -15,11 +15,10 @@ xs_remove() { return 0 fi - if [ "$QUBES_EXPOSED" == "1" ]; then + if qubesdb-read -q "$QDB_KEY/desc" >/dev/null; then qubesdb-rm "$QDB_KEY/" qubesdb-write /qubes-block-devices '' fi - echo QUBES_EXPOSED=0 } is_used() { @@ -42,6 +41,14 @@ is_used() { return 1 } +refresh_another() { + # launch this script for other device + local devpath=$1 + local launch_env=$(udevadm info -q all -x -p "$devpath" \ + | grep ^E: | cut -d ' ' -f 2-) + env -i PATH=$PATH $launch_env $0 +} + # communicate with xenstored through socket in dom0 # trying to access xenstore before xenstored is started, hang forever (in # non-killable state), so better fail ('-s' in VM if /proc/xen isn't mounted @@ -53,14 +60,16 @@ else fi is_attached() { dev_hex=$(stat -c %t:%T /dev/$(basename $1)) + if [ -z "$dev_hex" -o "$dev_hex" = "0:0" ]; then + return 1 + fi $XENSTORE_LS backend/vbd | grep -q "physical-device = \"$dev_hex\"" } # update info about parent devices, if any: if [ -f /sys$DEVPATH/partition ]; then parent=$(dirname $(readlink -f /sys$DEVPATH)) - udevadm trigger \ - --property-match=DEVPATH=/$(realpath --relative-to=/sys $parent) + refresh_another /$(realpath --relative-to=/sys $parent) # if parent device is already attached, skip its partitions if is_attached $parent; then xs_remove @@ -70,12 +79,23 @@ fi # and underlying devices of device-mapper (if any) for dev in /sys$DEVPATH/slaves/*; do - udevadm trigger \ - --property-match=DEVPATH=/$(realpath --relative-to=/sys $dev) + refresh_another /$(realpath --relative-to=/sys $dev) done +# cache slave devices for remove event +if [ -n "$DM_NAME" ]; then + ls -1 /sys$DEVPATH/slaves/ > /var/run/qubes/block-slave-cache-$NAME +fi + # then take care of this device: +# udev rules already excluded this device: + +if [ "$DM_UDEV_DISABLE_DISK_RULES_FLAG" = "1" ]; then + xs_remove + exit 0 +fi + # device itself is already used if is_used /sys$DEVPATH; then xs_remove @@ -100,12 +120,19 @@ if [ "$MAJOR" -eq 7 -a ! -d /sys/$DEVPATH/loop ]; then exit 0 fi -# ... and temporary devices used during VM startup -if [[ "$NAME" = 'loop'* ]] && \ - [[ "`cat /sys/block/${NAME%p*}/loop/backing_file`" = \ - '/var/lib/qubes/'*'/volatile.img' ]]; then - xs_remove - exit 0 +# ... and loop devices from excluded directories +if [[ "$NAME" = 'loop'* ]]; then + backing_file=$(cat /sys/block/${NAME%p*}/loop/backing_file) + if [ -n "$backing_file" ]; then + dir_to_check=$(dirname "$backing_file") + while [ "$dir_to_check" != "/" -a "$dir_to_check" != "." ]; do + if [ -e "$dir_to_check/.qubes-exclude-block-devices" ]; then + xs_remove + exit 0 + fi + dir_to_check=$(dirname "$dir_to_check") + done + fi fi # Check if device is read-only @@ -138,7 +165,6 @@ qubesdb-write \ "$QDB_KEY/size" "$SIZE" \ "$QDB_KEY/mode" "$MODE" \ /qubes-block-devices '' -echo QUBES_EXPOSED=1 # Make sure that block backend is loaded /sbin/modprobe xen-blkback 2> /dev/null || /sbin/modprobe blkbk diff --git a/udev/udev-block-cleanup b/udev/udev-block-cleanup deleted file mode 100755 index bc4535f..0000000 --- a/udev/udev-block-cleanup +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -DEVID=$[ $MAJOR * 256 + $MINOR ] - -XS_PATH="device/vbd/$DEVID" - -# Double check that DEVID is not empty -[ -n "$DEVID" ] && xenstore-rm $XS_PATH diff --git a/udev/udev-block-remove b/udev/udev-block-remove index ae2902f..0fc8486 100755 --- a/udev/udev-block-remove +++ b/udev/udev-block-remove @@ -2,39 +2,15 @@ NAME=${DEVNAME#/dev/} QDB_KEY="/qubes-block-devices/$NAME" -# Trailing slash is intentional - it will remove whole directory, instead of -# single base entry +# Trailing slash is intentional - it will remove the whole directory, instead of +# a single base entry qubesdb-rm "$QDB_KEY/" qubesdb-write /qubes-block-devices '' -# If device was connected to some VM - detach it -# Notice: this can be run also in VM, so we cannot use xl... - -device_detach() { - xs_path=$1 - - xenstore-write $xs_path/online 0 $xs_path/state 5 - - # Wait for backend to finish dev shutdown - try=30 - # -lt will break loop also when 'state' will be empty - while [ "`xenstore-read $xs_path/state 2> /dev/null`" -lt 6 ]; do - try=$[ $try - 1 ] - [ "$try" -le 0 ] && break - sleep 0.1 +if [ -r /var/run/qubes/block-slave-cache-$NAME ]; then + # update info about underlying devices of device-mapper (if any); + for dev in $(cat /var/run/qubes/block-slave-cache-$NAME); do + udevadm trigger /sys/class/block/$dev done - xenstore-rm $xs_path -} - -# update info about underlying devices of device-mapper (if any) -# at this stage device-mapper is already removed, so can't check what devices -# were used there -udevadm trigger --subsystem-match=block - -for XS_DEV_PATH in `xenstore-ls -f backend/vbd | grep 'backend/vbd/[0-9]*/[0-9]* ' | cut -f 1 -d ' '`; do - CUR_DEVICE=`xenstore-read "$XS_DEV_PATH/params"` - if [ "$CUR_DEVICE" == "$DEVNAME" ]; then - device_detach "$XS_DEV_PATH" - exit 0 - fi -done + rm -f /var/run/qubes/block-slave-cache-$NAME +fi diff --git a/udev/udev-qubes-block.rules b/udev/udev-qubes-block.rules index 2c25049..8d3a3af 100644 --- a/udev/udev-qubes-block.rules +++ b/udev/udev-qubes-block.rules @@ -14,14 +14,9 @@ ENV{MAJOR}=="202", GOTO="qubes_block_end" KERNEL=="dm-*", ENV{DM_NAME}=="snapshot-*", GOTO="qubes_block_end" KERNEL=="dm-*", ENV{DM_NAME}=="origin-*", GOTO="qubes_block_end" KERNEL=="dm-*", ENV{DM_NAME}=="", GOTO="qubes_block_end" -ENV{DM_UDEV_DISABLE_DISK_RULES_FLAG}=="1", GOTO="qubes_block_end" -IMPORT{db}="QUBES_EXPOSED" -ACTION=="add", IMPORT{program}="/usr/lib/qubes/udev-block-add-change" -ACTION=="change", IMPORT{program}="/usr/lib/qubes/udev-block-add-change" +ACTION=="add", RUN+="/usr/lib/qubes/udev-block-add-change" +ACTION=="change", RUN+="/usr/lib/qubes/udev-block-add-change" ACTION=="remove", RUN+="/usr/lib/qubes/udev-block-remove" LABEL="qubes_block_end" - -# Cleanup disconnected frontend from xenstore -ACTION=="remove", SUBSYSTEM=="block", ENV{MAJOR}=="202", RUN+="/usr/lib/qubes/udev-block-cleanup"