TAILORED: build 4k sector sized templates only

https://github.com/QubesOS/qubes-issues/issues/4974#issuecomment-575955927

Makefile

@@ -20,10 +20,12 @@ TEMPLATE_ENV_WHITELIST += \
 	TEMPLATE_ROOT_WITH_PARTITIONS TEMPLATE_ROOT_SIZE \
 	USE_QUBES_REPO_VERSION USE_QUBES_REPO_TESTING \
 	BUILDER_TURBO_MODE REPO_PROXY FEDORA_MIRROR \
-	CENTOS_MIRROR EPEL_MIRROR
+	CENTOS_MIRROR EPEL_MIRROR QUBES_MIRROR
 
 # Make sure names are < 32 characters, process aliases
-fix_up := $(shell TEMPLATE_NAME=$(TEMPLATE_NAME) ./builder_fix_filenames)
+fix_up := $(shell TEMPLATE_NAME=$(TEMPLATE_NAME) \
+		TEMPLATE_LABEL="$(TEMPLATE_LABEL)" \
+		./builder_fix_filenames)
 TEMPLATE_NAME := $(word 1,$(fix_up))
 
 export TEMPLATE_NAME
@@ -31,7 +33,10 @@ export TEMPLATE_SCRIPTS
 export DISTRIBUTION
 
 VERSION := $(shell cat version)
-TIMESTAMP := $(shell date -u +%Y%m%d%H%M)
+TEMPLATE_TIMESTAMP ?= $(shell date -u +%Y%m%d%H%M)
+
+.PHONY: help template-name prepare package rpms rootimg-build
+.PHONY: update-repo-templates-itl update-repo-templates-community
 
 help:
 	@echo "make rpms                  -- generate template rpm"
@@ -43,7 +48,7 @@ template-name:
 
 prepare:
 	@echo "Building template: $(TEMPLATE_NAME)"
-	@echo $(TIMESTAMP) > build_timestamp_$(TEMPLATE_NAME)
+	@echo $(TEMPLATE_TIMESTAMP) > build_timestamp_$(TEMPLATE_NAME)
 
 package:
 	./build_template_rpm $(TEMPLATE_NAME)
@@ -72,6 +77,9 @@ update-repo-installer:
 	[ -z "$$UPDATE_REPO" ] && UPDATE_REPO=../installer/yum/qubes-dom0;\
 	ln -f rpm/noarch/qubes-template-$(TEMPLATE_NAME)-$(VERSION)-$(shell cat build_timestamp_$(TEMPLATE_NAME))*.noarch.rpm $$UPDATE_REPO/rpm
 
+sign:
+	setsid -w rpmsign $$RPMSIGN_OPTS --addsign rpm/noarch/qubes-template-$(TEMPLATE_NAME)-$(VERSION)-$(shell cat build_timestamp_$(TEMPLATE_NAME))*.noarch.rpm
+
 prepare-repo-template:
 	rm -rf pkgs-for-template/$(DIST)
 	mkdir -p pkgs-for-template/$(DIST)

Makefile.builder

@@ -0,0 +1,5 @@
+# pretend that normal package is built from this repo, to reuse update-repo-* 
+ifeq ($(PACKAGE_SET),vm)
+OUTPUT_DIR = rpm
+RPM_SPEC_FILES = templates.spec
+endif

build_template_rpm

@@ -26,3 +26,7 @@ rpmbuild --target noarch \
          --define "_topdir $PWD/rpmbuild" \
          --define "_tmppath $PWD/rpmbuild/tmp" \
          -bb templates.spec
+
+if [ "0$DISCARD_PREPARED_IMAGE" -eq "1" ]; then
+    rm -rf "qubeized_images/$NAME"
+fi

functions-name.sh

@@ -71,7 +71,7 @@ templateNameDist() {
     template_name="$(templateName)" && dist_name="${template_name}"
 
     # Automaticly correct name length if it's greater than 32 chars
-    dist_name="$(templateNameFixLength ${template_name})"
+    dist_name="$(templateNameFixLength ${dist_name})"
 
     # Remove and '+' characters from name since they are invalid for name
     dist_name="${dist_name//+/-}"
@@ -79,7 +79,7 @@ templateNameDist() {
 }
 
 templateName() {
-    local template_flavor=${1-${TEMPLATE_FLAVOR}}
+    local template_flavor=${1:-${TEMPLATE_FLAVOR}}
     retval=1 # Default is 1; mean no replace happened
 
     # Only apply options if $1 was not passed

mount_root.sh

@@ -18,7 +18,7 @@ mkdir -p mnt
 
 MNTDIR=$(pwd)/mnt
 
-LOOP=$(/sbin/losetup -f --show -o $OFFSET $ROOTIMG)
+LOOP=$(/sbin/losetup -b 4096 -f --show -o $OFFSET $ROOTIMG)
 
 if [ x$LOOP = x ] ; then
 echo "Cannot setup loopback device for the $ROOTIMG file -- perhaps a permissions problem?"

prepare_image

@@ -50,10 +50,13 @@ echo "-> Preparing instalation of ${DIST} template..."
 if [ -f "${IMG}" ]; then
     echo "-> Image file already exists, assuming *update*..."
     if [ "0$TEMPLATE_ROOT_WITH_PARTITIONS" -eq 1 ]; then
-        IMG_LOOP=$(/sbin/losetup -P -f --show "$IMG")
+        IMG_LOOP=$(/sbin/losetup -b 4096 -P -f --show "$IMG")
+        # BUG: losetup's partscan (-P) does nothing when trying 4096 sectors formatted images on a 512 sectors backing device.
+        #      Hence it needs an additional partprobe run.
+        /sbin/partprobe $IMG_LOOP
         IMG_DEV=${IMG_LOOP}p3
     else
-        IMG_LOOP=$(/sbin/losetup -f --show "$IMG")
+        IMG_LOOP=$(/sbin/losetup -b 4096 -f --show "$IMG")
         IMG_DEV=${IMG_LOOP}
     fi
 else
@@ -62,8 +65,12 @@ else
 
     if [ "0$TEMPLATE_ROOT_WITH_PARTITIONS" -eq 1 ]; then
         echo "-> Creating partition table"
-        # have static UUIDs to make partition table reproducible
-        sfdisk "$IMG" <<EOF || exit 1
+        IMG_LOOP=$(/sbin/losetup -b 4096 -P -f --show "$IMG")
+        # When a 4096 sectors formatted image gets created with the sfdisk, the partitions are automatically visible
+        # to the system. Hence no need to run additional partprobe.
+
+        # Have static UUIDs to make partition table reproducible
+        sfdisk "$IMG_LOOP" <<EOF || exit 1
 label: gpt
 label-id: f4796a2a-e377-45bd-b539-d6d49e569055
 
@@ -72,17 +79,21 @@ size=2MiB, type=21686148-6449-6E6F-744E-656564454649, uuid=1e6c9db4-1e91-46c4-84
 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, uuid=693244e6-3e07-47bf-ad79-acade4293fe7, name="Root filesystem"
 EOF
 
-        IMG_LOOP=$(/sbin/losetup -P -f --show "$IMG")
         IMG_DEV=${IMG_LOOP}p3
     else
-        IMG_LOOP=$(/sbin/losetup -f --show "$IMG")
+        IMG_LOOP=$(/sbin/losetup -b 4096 -f --show "$IMG")
         IMG_DEV=${IMG_LOOP}
     fi
 
+    # make sure the IMG_DEV is present
+    c=0; while [ ! -b $IMG_DEV ]; do sleep 1; c=$((c+1)); [ $c -ge 5 ] && break; done; unset c
+
     echo "-> Creating filesystem..."
-    mkfs.ext4 -q -F "${IMG_DEV}" || exit 1
+    /sbin/mkfs.ext4 -q -F "${IMG_DEV}" || exit 1
 fi
 
+# make sure the IMG_DEV is present
+c=0; while [ ! -b $IMG_DEV ]; do sleep 1; c=$((c+1)); [ $c -ge 5 ] && break; done; unset c
 mount "${IMG_DEV}" "${INSTALLDIR}" || exit 1
 trap "umount_kill $(readlink -m ${INSTALLDIR})" EXIT
 "${SCRIPTSDIR}/01_install_core.sh"

qubeize_image

@@ -72,12 +72,17 @@ fi
 echo "--> Mounting $IMG"
 mkdir -p mnt
 if [ "0$TEMPLATE_ROOT_WITH_PARTITIONS" -eq 1 ]; then
-    IMG_LOOP=$(/sbin/losetup -P -f --show "$IMG")
+    IMG_LOOP=$(/sbin/losetup -b 4096 -P -f --show "$IMG")
+    # BUG: losetup's partscan (-P) does nothing when trying 4096 sectors formatted images on a 512 sectors backing device.
+    #      Hence it needs an additional partprobe run.
+    /sbin/partprobe $IMG_LOOP
     IMG_DEV=${IMG_LOOP}p3
 else
-    IMG_LOOP=$(/sbin/losetup -f --show "$IMG")
+    IMG_LOOP=$(/sbin/losetup -b 4096 -f --show "$IMG")
     IMG_DEV=${IMG_LOOP}
 fi
+# make sure the IMG_DEV is present
+c=0; while [ ! -b $IMG_DEV ]; do sleep 1; c=$((c+1)); [ $c -ge 5 ] && break; done; unset c
 mount "$IMG_DEV" mnt || exit 1
 export INSTALLDIR=mnt
 
@@ -94,8 +99,12 @@ _appmenus_dir="${APPMENUS_DIR:-${SCRIPTSDIR}}"
 rm -f appmenus
 if [ -d "${_appmenus_dir}/appmenus_${DIST}_${TEMPLATE_FLAVOR}" ]; then
     ln -s "${_appmenus_dir}/appmenus_${DIST}_${TEMPLATE_FLAVOR}" appmenus
+elif [ -d "${_appmenus_dir}/appmenus_${DIST//[0-9]*}_${TEMPLATE_FLAVOR}" ]; then
+    ln -s "${_appmenus_dir}/appmenus_${DIST//[0-9]*}_${TEMPLATE_FLAVOR}" appmenus
 elif [ -d "${_appmenus_dir}/appmenus_$DIST" ]; then
     ln -s "${_appmenus_dir}/appmenus_$DIST" appmenus
+elif [ -d "${_appmenus_dir}/appmenus_${DIST//[0-9]*}" ]; then
+    ln -s "${_appmenus_dir}/appmenus_${DIST//[0-9]*}" appmenus
 elif [ -d "${_appmenus_dir}/appmenus" ]; then
     ln -s "${_appmenus_dir}/appmenus" appmenus
 else
@@ -111,7 +120,7 @@ mkdir mnt/home
 
 echo "--> Linking /usr/local to /rw/usrlocal..."
 mv mnt/usr/local mnt/usr/local.orig
-ln -sf /rw/usrlocal mnt/usr/local
+mkdir mnt/usr/local
 
 echo "Reducing image size (calling cleanup_image)..."
 ls -als $IMG

templates.spec

@@ -3,8 +3,9 @@
 # This includes the VM's root image, patched with all qubes rpms, etc
 #
 
-%{!?version: %define version %(cat version)}
-%{!?rel: %define rel %(cat build_timestamp_%{template_name})}
+%{!?template_name: %global template_name %{getenv:TEMPLATE_NAME}}
+%{!?version: %global version %(cat version)}
+%{!?rel: %global rel %(cat build_timestamp_%{template_name} || echo unavailable)}
 
 Name:		qubes-template-%{template_name}
 Version:	%{version}
@@ -17,6 +18,7 @@ Source:		.
 
 Requires:	xdg-utils
 Requires(post):	tar
+BuildArch:  noarch
 Provides:	qubes-template
 Obsoletes:  %{name} > %{version}-%{release}
 
@@ -76,6 +78,7 @@ if command -v qvm-template-postprocess >/dev/null 2>&1; then
 fi
 
 echo "--> Processing the root.img... (this might take a while)"
+rm -f %{dest_dir}/root.img
 cat %{dest_dir}/root.img.part.* | tar --sparse -xf - -C %{dest_dir}
 rm -f %{dest_dir}/root.img.part.*
 chown root.qubes %{dest_dir}/root.img

version

@@ -1 +1 @@
-4.0.0
+4.0.1