udev: major cleanup in block devices handling

1. Do not detach device forcefully when it's removed. This breaks
libvirt (which thinks the device is still there). After this change, it
is possible to detach device using libvirt, even if it was already
removed physically from backend domain (unless it is dom0 - in which
case it is still broken). So, this is partial fix for
QubesOS/qubes-issues#1082.

2. Do not trigger "change" udev event when only QubesDB state needs to
be updated - this leads to massive udev events queue, and heavy I/O
usage - for example scanning all LVM many times. In some cases it even
caused infinite event queue.

3. Do not use QUBES_EXPOSED udev property - it was needed a while back
before QubesDB, because concurrent xenstore accesses are expensive
(because of transactions). It isn't the problem on QubesDB.

4. Cache information about device-mapper, so it is possible to
reconstruct it at device remove - when the actual device cannot be
queried anymore. This is specifically about list of lower layer devices
used.

5. Allow excluding loop devices pointing at a file in directory marked
with ".qubes-exclude-block-devices" file. This is more generic than
hardcoding /var/lib/qubes.

QubesOS/qubes-issues#3084
Fixes QubesOS/qubes-issues#3073
QubesOS/qubes-issues#1082

udev/Makefile

@@ -9,6 +9,5 @@ install:
 	mkdir -p $(DESTDIR)$(SCRIPTSDIR)
 	cp udev-block-add-change $(DESTDIR)$(SCRIPTSDIR)
 	cp udev-block-remove $(DESTDIR)$(SCRIPTSDIR)
-	cp udev-block-cleanup $(DESTDIR)$(SCRIPTSDIR)
 	cp udev-usb-add-change $(DESTDIR)$(SCRIPTSDIR)
 	cp udev-usb-remove $(DESTDIR)$(SCRIPTSDIR)

udev/udev-block-add-change

@@ -15,11 +15,10 @@ xs_remove() {
         return 0
     fi
 
-    if [ "$QUBES_EXPOSED" == "1" ]; then
+    if qubesdb-read -q "$QDB_KEY/desc" >/dev/null; then
         qubesdb-rm "$QDB_KEY/"
         qubesdb-write /qubes-block-devices ''
     fi
-    echo QUBES_EXPOSED=0
 }
 
 is_used() {
@@ -42,6 +41,14 @@ is_used() {
     return 1
 }
 
+refresh_another() {
+    # launch this script for other device
+    local devpath=$1
+    local launch_env=$(udevadm info -q all -x -p "$devpath" \
+            | grep ^E: | cut -d ' ' -f 2-)
+    env -i PATH=$PATH $launch_env $0
+}
+
 # communicate with xenstored through socket in dom0
 # trying to access xenstore before xenstored is started, hang forever (in
 # non-killable state), so better fail ('-s' in VM if /proc/xen isn't mounted
@@ -53,14 +60,16 @@ else
 fi
 is_attached() {
     dev_hex=$(stat -c %t:%T /dev/$(basename $1))
+    if [ -z "$dev_hex" -o "$dev_hex" = "0:0" ]; then
+        return 1
+    fi
     $XENSTORE_LS backend/vbd | grep -q "physical-device = \"$dev_hex\""
 }
 
 # update info about parent devices, if any:
 if [ -f /sys$DEVPATH/partition ]; then
     parent=$(dirname $(readlink -f /sys$DEVPATH))
-    udevadm trigger \
-        --property-match=DEVPATH=/$(realpath --relative-to=/sys $parent)
+    refresh_another /$(realpath --relative-to=/sys $parent)
     # if parent device is already attached, skip its partitions
     if is_attached $parent; then
         xs_remove
@@ -70,12 +79,23 @@ fi
 
 # and underlying devices of device-mapper (if any)
 for dev in /sys$DEVPATH/slaves/*; do
-    udevadm trigger \
-        --property-match=DEVPATH=/$(realpath --relative-to=/sys $dev)
+    refresh_another /$(realpath --relative-to=/sys $dev)
 done
 
+# cache slave devices for remove event
+if [ -n "$DM_NAME" ]; then
+    ls -1 /sys$DEVPATH/slaves/ > /var/run/qubes/block-slave-cache-$NAME
+fi
+
 # then take care of this device:
 
+# udev rules already excluded this device:
+
+if [ "$DM_UDEV_DISABLE_DISK_RULES_FLAG" = "1" ]; then
+    xs_remove
+    exit 0
+fi
+
 # device itself is already used
 if is_used /sys$DEVPATH; then
     xs_remove
@@ -100,12 +120,19 @@ if [ "$MAJOR" -eq 7 -a ! -d /sys/$DEVPATH/loop ]; then
     exit 0
 fi
 
-# ... and temporary devices used during VM startup
-if [[ "$NAME" = 'loop'* ]] && \
-       [[ "`cat /sys/block/${NAME%p*}/loop/backing_file`" = \
-            '/var/lib/qubes/'*'/volatile.img' ]]; then
-    xs_remove
-    exit 0
+# ... and loop devices from excluded directories
+if [[ "$NAME" = 'loop'* ]]; then
+    backing_file=$(cat /sys/block/${NAME%p*}/loop/backing_file)
+    if [ -n "$backing_file" ]; then
+        dir_to_check=$(dirname "$backing_file")
+        while [ "$dir_to_check" != "/" -a "$dir_to_check" != "." ]; do
+            if [ -e "$dir_to_check/.qubes-exclude-block-devices" ]; then
+                xs_remove
+                exit 0
+            fi
+            dir_to_check=$(dirname "$dir_to_check")
+        done
+    fi
 fi
 
 # Check if device is read-only
@@ -138,7 +165,6 @@ qubesdb-write \
     "$QDB_KEY/size" "$SIZE" \
     "$QDB_KEY/mode" "$MODE" \
     /qubes-block-devices ''
-echo QUBES_EXPOSED=1
 
 # Make sure that block backend is loaded
 /sbin/modprobe xen-blkback 2> /dev/null || /sbin/modprobe blkbk

udev/udev-block-cleanup

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-DEVID=$[ $MAJOR * 256 + $MINOR ]
-
-XS_PATH="device/vbd/$DEVID"
-
-# Double check that DEVID is not empty
-[ -n "$DEVID" ] && xenstore-rm $XS_PATH

udev/udev-block-remove

@@ -2,39 +2,15 @@
 
 NAME=${DEVNAME#/dev/}
 QDB_KEY="/qubes-block-devices/$NAME"
-# Trailing slash is intentional - it will remove whole directory, instead of
-# single base entry
+# Trailing slash is intentional - it will remove the whole directory, instead of
+# a single base entry
 qubesdb-rm "$QDB_KEY/"
 qubesdb-write /qubes-block-devices ''
 
-# If device was connected to some VM - detach it
-# Notice: this can be run also in VM, so we cannot use xl...
-
-device_detach() {
-    xs_path=$1
-
-    xenstore-write $xs_path/online 0 $xs_path/state 5
-
-    # Wait for backend to finish dev shutdown
-    try=30
-    # -lt will break loop also when 'state' will be empty
-    while [ "`xenstore-read $xs_path/state 2> /dev/null`" -lt 6 ]; do
-        try=$[ $try - 1 ]
-        [ "$try" -le 0 ] && break
-        sleep 0.1
+if [ -r /var/run/qubes/block-slave-cache-$NAME ]; then
+    # update info about underlying devices of device-mapper (if any);
+    for dev in $(cat /var/run/qubes/block-slave-cache-$NAME); do
+        udevadm trigger /sys/class/block/$dev
     done
-    xenstore-rm $xs_path
-}
-
-# update info about underlying devices of device-mapper (if any)
-# at this stage device-mapper is already removed, so can't check what devices
-# were used there
-udevadm trigger --subsystem-match=block
-
-for XS_DEV_PATH in `xenstore-ls -f backend/vbd | grep 'backend/vbd/[0-9]*/[0-9]* ' | cut -f 1 -d ' '`; do
-    CUR_DEVICE=`xenstore-read "$XS_DEV_PATH/params"`
-    if [ "$CUR_DEVICE" == "$DEVNAME" ]; then
-        device_detach "$XS_DEV_PATH"
-        exit 0
-    fi
-done
+    rm -f /var/run/qubes/block-slave-cache-$NAME
+fi

udev/udev-qubes-block.rules

@@ -14,14 +14,9 @@ ENV{MAJOR}=="202", GOTO="qubes_block_end"
 KERNEL=="dm-*", ENV{DM_NAME}=="snapshot-*", GOTO="qubes_block_end"
 KERNEL=="dm-*", ENV{DM_NAME}=="origin-*", GOTO="qubes_block_end"
 KERNEL=="dm-*", ENV{DM_NAME}=="", GOTO="qubes_block_end"
-ENV{DM_UDEV_DISABLE_DISK_RULES_FLAG}=="1", GOTO="qubes_block_end"
 
-IMPORT{db}="QUBES_EXPOSED"
-ACTION=="add", IMPORT{program}="/usr/lib/qubes/udev-block-add-change"
-ACTION=="change", IMPORT{program}="/usr/lib/qubes/udev-block-add-change"
+ACTION=="add", RUN+="/usr/lib/qubes/udev-block-add-change"
+ACTION=="change", RUN+="/usr/lib/qubes/udev-block-add-change"
 ACTION=="remove", RUN+="/usr/lib/qubes/udev-block-remove"
 
 LABEL="qubes_block_end"
-
-# Cleanup disconnected frontend from xenstore
-ACTION=="remove", SUBSYSTEM=="block", ENV{MAJOR}=="202", RUN+="/usr/lib/qubes/udev-block-cleanup"