From f0979fae03bcc4767ac4df1cb8fad9277e952637 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 26 Feb 2024 14:39:12 +0100 Subject: debvm-create: switch from genext2fs to mkfs.ext4 Why? genext2fs is slow. It has O(n^2) behaviour, which becomes annoying with larger filesystems. It also creates an ext2 image and we have to upgrade it to ext4 in multiple steps. Even then, the image has an inode size of 128, which is incompatible with 2038 and does not allow storing sub-second precision time stamps. How? We use mkfs.ext4 from e2fsprogs (which is required anyway) instead. Running this from within a --customize-hook does not work, because the user namespace used for the hook often has no privileges to write to the output image. Instead, we now ask mmdebstrap to output to a directory and create the filesystem separately. We construct a special namespace that has both access to the tree and to the output image and perform the mkfs.ext4 there. As a consequence, we no longer support any mode but unshare and since mkfs.ext4 consumes the size parameter, the size syntax is reduced a bit. --- bin/debvm-create | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) (limited to 'bin/debvm-create') diff --git a/bin/debvm-create b/bin/debvm-create index bfc02e3..b93fa23 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -74,8 +74,8 @@ This is specific to using B, B, B or B as init syst =item B -Internally, B creates a tar archive first and converts that to ext2, which is then upgraded to ext4. -This option causes the conversion to ext2 and further steps to be skipped and the output image will be a tar archive instead. +Normally, B bootstraps to a temporary directory and we create the output ext4 image from that directory. +This option causes the creation of the ext4 image to be skipped and the output file becomes a tar archive of the temporary directory instead. Such a tar archive is not suitable for being booted by B. =item B @@ -116,7 +116,7 @@ Without the hook, dependencies will pull the B package as needed, whic =item B<-z> I, B<--size>=I Specify the minimum image size as an integer and optional unit (example: 10K is 10*1024). -Units are K,M,G,T,P,E,Z,Y (powers of 1024) or KB,MB,... (powers of 1000). +Units are K,M,G,T (powers of 1024). The resulting image will be grown as a sparse file to this size if necessary. The default is 1 GB. @@ -294,7 +294,7 @@ if ! check_skip kernel; then set -- "--customize-hook=$SHARE_DIR/customize-kernel.sh" "$@" fi -MMFORMAT=ext2 +MMFORMAT=directory # output a tarball if the ext4 step is skipped if check_skip ext4; then MMFORMAT=tar @@ -303,6 +303,7 @@ fi # construct mmdebstrap options as $@: set -- \ --verbose \ + --mode=unshare \ --variant=apt \ "--format=$MMFORMAT" \ '--customize-hook=echo "LABEL=debvm / ext4 defaults 0 0" >"$1/etc/fstab"' \ @@ -393,20 +394,34 @@ if ! check_skip autologin; then set -- "--customize-hook=$SHARE_DIR/customize-autologin.sh" "$@" fi -set -- "$SUITE" "$IMAGE" "$@" +run_inside_userns() { + unshare --user --map-auto --map-user=65536 --map-group=65536 --keep-caps -- "$@" +} -set -ex +TEMPROOT= +cleanup() { + if test -n "$TEMPROOT"; then + run_inside_userns rm -Rf "$TEMPROOT" + fi +} -mmdebstrap "$@" +if check_skip ext4; then + set -- "$IMAGE" "$@" +else + trap cleanup EXIT + trap 'exit 1' HUP INT QUIT TERM + TEMPROOT="$(mktemp -d)" -{ set +x; } 2>/dev/null -check_skip ext4 && exit + set -- "$TEMPROOT" "$@" +fi + +set -- "$SUITE" "$@" -set -x +set -ex + +mmdebstrap "$@" -truncate -s ">$SIZE" "$IMAGE" -/sbin/resize2fs "$IMAGE" -/sbin/tune2fs -L debvm -c 0 -i 0 -O dir_index,dir_nlink,extents,extra_isize,flex_bg,has_journal,huge_file "$IMAGE" -/sbin/resize2fs -b "$IMAGE" -# Must fsck after tune2fs: https://ext4.wiki.kernel.org/index.php/UpgradeToExt4 -/sbin/fsck.ext4 -fDp "$IMAGE" +# Create and truncate the file with "normal" permission. +: >"$IMAGE" +check_skip ext4 || + run_inside_userns /sbin/mkfs.ext4 -L debvm -d "$TEMPROOT" "$IMAGE" "$SIZE" -- cgit v1.2.3 From ae025c7aba7e4cbbe794cb48de4464d0e136dfee Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 26 Feb 2024 16:25:59 +0100 Subject: debvm-create: allow running without unshare again In a CI-environment such as salsa-ci, we tend to run as root and want --mode=root rather than --mode=unshare. --- bin/debvm-create | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'bin/debvm-create') diff --git a/bin/debvm-create b/bin/debvm-create index b93fa23..a8d5727 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -303,12 +303,17 @@ fi # construct mmdebstrap options as $@: set -- \ --verbose \ - --mode=unshare \ --variant=apt \ "--format=$MMFORMAT" \ '--customize-hook=echo "LABEL=debvm / ext4 defaults 0 0" >"$1/etc/fstab"' \ "$@" +if test "$(id -u)" = 0; then + set -- --mode=root "$@" +else + set -- --mode=unshare "$@" +fi + if test -n "$ARCHITECTURE"; then set -- "--architecture=$ARCHITECTURE" "$@" fi @@ -394,9 +399,15 @@ if ! check_skip autologin; then set -- "--customize-hook=$SHARE_DIR/customize-autologin.sh" "$@" fi -run_inside_userns() { - unshare --user --map-auto --map-user=65536 --map-group=65536 --keep-caps -- "$@" -} +if test "$(id -u)" = 0; then + run_inside_userns() { + "$@" + } +else + run_inside_userns() { + unshare --user --map-auto --map-user=65536 --map-group=65536 --keep-caps -- "$@" + } +fi TEMPROOT= cleanup() { -- cgit v1.2.3 From 12d193bb8d108f5718dc12053ccf78a1b6a876ce Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 27 Feb 2024 07:50:50 +0100 Subject: debvm-create: fix --skip ext4 We were truncating the output file after generating it. Instead, split the logic and create the file prior to running mmdebstrap to make permission failures fail early. --- bin/debvm-create | 28 ++++++++++++++-------------- debian/control | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'bin/debvm-create') diff --git a/bin/debvm-create b/bin/debvm-create index a8d5727..0e1a969 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -409,6 +409,12 @@ else } fi +if check_skip ext4; then + set -- "$SUITE" "$IMAGE" "$@" + set -x + exec mmdebstrap "$@" +fi + TEMPROOT= cleanup() { if test -n "$TEMPROOT"; then @@ -416,23 +422,17 @@ cleanup() { fi } -if check_skip ext4; then - set -- "$IMAGE" "$@" -else - trap cleanup EXIT - trap 'exit 1' HUP INT QUIT TERM - TEMPROOT="$(mktemp -d)" - - set -- "$TEMPROOT" "$@" -fi +trap cleanup EXIT +trap 'exit 1' HUP INT QUIT TERM +TEMPROOT="$(mktemp -d)" -set -- "$SUITE" "$@" +set -- "$SUITE" "$TEMPROOT" "$@" set -ex -mmdebstrap "$@" - # Create and truncate the file with "normal" permission. : >"$IMAGE" -check_skip ext4 || - run_inside_userns /sbin/mkfs.ext4 -L debvm -d "$TEMPROOT" "$IMAGE" "$SIZE" + +mmdebstrap "$@" + +run_inside_userns /sbin/mkfs.ext4 -L debvm -d "$TEMPROOT" "$IMAGE" "$SIZE" diff --git a/debian/control b/debian/control index 0474049..341e8a7 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,6 @@ Depends: qemu-kvm (>> 1:8.0) | qemu-system-any, # Required for -device virtio-net-pci,netdev=net0 ipxe-qemu, - uidmap, Recommends: arch-test, file, @@ -29,6 +28,7 @@ Recommends: qemu-system, qemu-user-static, seabios, + uidmap, Suggests: qemu-system-gui, Description: create and run virtual machines for various Debian releases and architectures -- cgit v1.2.3 From 93436f32e595e216de538d693a1e579aa23de9a8 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 27 Feb 2024 08:45:55 +0100 Subject: debvm-create: emit an empty /dev With the change from genext2fs to mkfs.ext4, we no longer create an intermediate tar archive and thus no longer create device nodes if using unshare. Thus our output is slightly unreproducible. At the same time, we never need devices, because /dev is mounted as devtmpfs. Hence, we unconditionally skip them and thus improve reproducibility. Reported-by: Johannes Schauer Marin Rodrigues --- bin/debvm-create | 1 + 1 file changed, 1 insertion(+) (limited to 'bin/debvm-create') diff --git a/bin/debvm-create b/bin/debvm-create index 0e1a969..c148ed2 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -305,6 +305,7 @@ set -- \ --verbose \ --variant=apt \ "--format=$MMFORMAT" \ + --skip=output/dev \ '--customize-hook=echo "LABEL=debvm / ext4 defaults 0 0" >"$1/etc/fstab"' \ "$@" -- cgit v1.2.3 From fd97cb52975a11ba2e019f12c1aecd5123bd2204 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 27 Feb 2024 11:17:27 +0100 Subject: debvm-create: explain the trap 'exit 1' Reported-by: Johannes Schauer Marin Rodrigues --- bin/debvm-create | 1 + 1 file changed, 1 insertion(+) (limited to 'bin/debvm-create') diff --git a/bin/debvm-create b/bin/debvm-create index c148ed2..1c59a83 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -424,6 +424,7 @@ cleanup() { } trap cleanup EXIT +# The default action for these signals does not invoke the EXIT trap. trap 'exit 1' HUP INT QUIT TERM TEMPROOT="$(mktemp -d)" -- cgit v1.2.3