From aece2dad3e16feb6073e17a3b61dfd8d69007ecb Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 11 Jan 2023 19:25:08 +0100 Subject: move the debvm-* tools to bin The purpose of this change is adding support files to be referenced and called from these tools. Those support files shall be located in ../share and this way of locating them shall work both in-source and when installed. --- .gitlab-ci.yml | 8 +- README.md | 2 +- bin/debvm-create | 342 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/debvm-run | 272 +++++++++++++++++++++++++++++++++++++++++++ bin/debvm-waitssh | 178 ++++++++++++++++++++++++++++ debvm-create | 342 ------------------------------------------------------ debvm-run | 272 ------------------------------------------- debvm-waitssh | 178 ---------------------------- 8 files changed, 797 insertions(+), 797 deletions(-) create mode 100755 bin/debvm-create create mode 100755 bin/debvm-run create mode 100755 bin/debvm-waitssh delete mode 100755 debvm-create delete mode 100755 debvm-run delete mode 100755 debvm-waitssh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 456e8d9..0da847f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,14 +5,14 @@ shellcheck: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install shellcheck - - shellcheck -P tests debvm-* tests/*.sh + - shellcheck -P tests bin/* tests/*.sh codespell: script: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install codespell - - codespell debvm-* tests/*.sh + - codespell bin/* tests/*.sh release_test: parallel: @@ -28,7 +28,7 @@ release_test: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install e2fsprogs genext2fs mmdebstrap openssh-client qemu-kvm - - PATH=.:$PATH ./tests/create-and-run.sh $(dpkg --print-architecture) "$RELEASE" + - PATH=./bin:$PATH ./tests/create-and-run.sh $(dpkg --print-architecture) "$RELEASE" arch_test: parallel: @@ -46,4 +46,4 @@ arch_test: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install e2fsprogs genext2fs mmdebstrap openssh-client qemu-system binfmt-support arch-test qemu-user-static - - PATH=.:$PATH ./tests/create-and-run.sh "$ARCHITECTURE" sid + - PATH=./bin:$PATH ./tests/create-and-run.sh "$ARCHITECTURE" sid diff --git a/README.md b/README.md index 5ccbf12..6f042d7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ invocation. The following two invocations will give you a shell inside a qemu virtual machine of the native architecture, leaving all the settings at their defaults: - ./debvm-create && ./debvm-run + ./bin/debvm-create && ./bin/debvm-run What do I need? =============== diff --git a/bin/debvm-create b/bin/debvm-create new file mode 100755 index 0000000..0dc619b --- /dev/null +++ b/bin/debvm-create @@ -0,0 +1,342 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT + +# shellcheck disable=SC2016 # Intentional quoting technique + +: <<'POD2MAN' +=head1 NAME + +debvm-create - Create a VM image for various Debian releases and architectures + +=head1 SYNOPSIS + +B [B<-a> I] [B<-h> I] [B<-k> F] [B<-m> I] [B<-o> F] [B<-p> I] [B<-r> I] [B<-z> I] [B<--> I] + +=head1 DESCRIPTION + +B is essentially a thin wrapper around B for creating a raw ext4 filesystem image for booting with B. +The purpose of these images primarily is testing the different releases and architectures without access to a physical machine of that architecture. +Beyond essential packages, the image will contain B, an init system and a suitable kernel package. +Notably absent is a bootloader and a partition table. +In order to boot such an image, one is supposed to extract the kernel and initrd from the image and pass it to a suitable bootloader. +No user account is created and root can login without specifying a password. + +=head1 OPTIONS + +=over 8 + +=item B<-a> I, B<--architecture>=I + +Specify a Debian architecture name. +By default, the native architecture is being used. +A suitable kernel image is automatically selected and installed into the image. + +=item B<-h> I, B<--hostname>=I + +Set the hostname of the virtual machine. +By default, the hostname is B. + +=item B<-k> F, B<--sshkey>=F + +Install the given ssh public key file into the virtual machine image for the root user. +This option also causes the ssh server to be installed. +By default, no key or server is installed. +To connect to the vm, pass a port number to B with the B<-s> option. + +=item B<-m> I, B<--mirror>=I + +Specify the Debian mirror to be used for downloading packages and to be configured inside the virtual machine image. +By default, L is being used. + +=item B<-o> F, B<--output>=F + +Specify the file name of the resulting virtual machine image. +By default, it is written to F. + +=item B<-p> I, B<--package>=I + +Request additional packages to be installed into the virtual machine image. +This option can be specified multiple times and packages can be separated by a comma. +Package recommendations are not honoured. +If a linux-image is passed here, it will replace the one selected by default. + +=item B<-r> I, B<--release>=I + +Use the given Debian release. +By default, B is being used. + +=item B<-z> I, B<--size>=I + +Specify the minimum image size in giga bytes. +The resulting image will be grown as a sparse file to this size if necessary. +The default is 1 GB. + +=item B<--> I + +All options beyond a double dash are passed to B before the suite, target and mirror specification. +This can be used to provide additional hooks for image customization. + +=back + +=head1 SEE ALSO + + debvm-run(1) mmdebstrap(1) + +=cut +POD2MAN + +set -u + +ARCHITECTURE=$(dpkg --print-architecture) +IMAGE=rootfs.ext4 +INCLUDE_PACKAGES=init +MIRROR="http://deb.debian.org/debian" +SIZE=$((1024*1024*1024)) +SSHKEY= +SUITE=unstable +VMNAME=testvm + +nth_arg() { + shift "$1" + printf "%s" "$1" +} + +die() { + echo "$*" 1>&2 + exit 1 +} +usage() { + die "usage: $0 [-a architecture] [-h hostname] [-k sshkey] [-m mirror] [-o output] [-p packages] [-r release] [-z size_in_GB] [-- mmdebstrap options]" +} +usage_error() { + echo "error: $*" 1>&2 + usage +} + +opt_architecture() { + ARCHITECTURE=$1 +} +opt_hostname() { + VMNAME=$1 +} +opt_mirror() { + MIRROR=$1 +} +opt_sshkey() { + SSHKEY=$1 +} +opt_output() { + IMAGE=$1 +} +opt_package() { + INCLUDE_PACKAGES="$INCLUDE_PACKAGES,$1" +} +opt_release() { + SUITE=$1 +} +opt_size() { + SIZE=$(($1*1024*1024*1024)) +} + +while getopts :a:h:k:m:o:p:r:z:-: OPTCHAR; do + case "$OPTCHAR" in + a) opt_architecture "$OPTARG" ;; + h) opt_hostname "$OPTARG" ;; + k) opt_sshkey "$OPTARG" ;; + m) opt_mirror "$OPTARG" ;; + o) opt_output "$OPTARG" ;; + p) opt_package "$OPTARG" ;; + r) opt_release "$OPTARG" ;; + z) opt_size "$OPTARG" ;; + -) + case "$OPTARG" in + help) + usage + ;; + architecture|hostname|mirror|output|package|release|size|sshkey) + test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" + "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" + OPTIND=$((OPTIND+1)) + ;; + architecture=*|hostname=*|mirror=*|output=*|package=*|release=*|size=*|sshkey=*) + "opt_${OPTARG%%=*}" "${OPTARG#*=}" + ;; + *) + usage_error "unrecognized option --$OPTARG" + ;; + esac + ;; + :) + usage_error "missing argument for -$OPTARG" + ;; + '?') + usage_error "unrecognized option -$OPTARG" + ;; + *) + die "internal error while parsing command options, please report a bug" + ;; + esac +done +shift "$((OPTIND - 1))" + +if test -n "$SSHKEY" && ! test -f "$SSHKEY"; then + die "error: ssh keyfile '$SSHKEY' not found" +fi + +case "$SUITE" in + jessie) + DEBVER=8 + ;; + stretch) + DEBVER=9 + ;; + buster) + DEBVER=10 + ;; + bullseye|stable) + DEBVER=11 + ;; + bookworm|testing) + DEBVER=12 + ;; + trixie) + DEBVER=13 + ;; + forky) + DEBVER=14 + ;; + sid|unstable) + DEBVER=999 + ;; + *) + die "unrecognized Debian release: $SUITE" + ;; +esac + +KERNEL_SUFFIX=-$ARCHITECTURE +case "$ARCHITECTURE" in + amd64|arm64) + KERNEL_SUFFIX="-cloud-$ARCHITECTURE" + if test "$DEBVER" -le 9; then + KERNEL_SUFFIX="-$ARCHITECTURE" + fi + ;; + armhf) + KERNEL_SUFFIX=-armmp + ;; + i386) + KERNEL_SUFFIX=-686-pae + ;; + mips64el) + KERNEL_SUFFIX=-5kc-malta + ;; + mipsel) + KERNEL_SUFFIX=-4kc-malta + ;; + ppc64el) + KERNEL_SUFFIX=-powerpc64le + ;; +esac + +case ",$INCLUDE_PACKAGES," in + *,linux-image-*) + ;; + *) + INCLUDE_PACKAGES="$INCLUDE_PACKAGES,linux-image$KERNEL_SUFFIX" + ;; +esac + +if test -n "$SSHKEY"; then + INCLUDE_PACKAGES="$INCLUDE_PACKAGES,openssh-server" +fi + +# add a DNS resolver +if test "$DEBVER" -ge 9; then + INCLUDE_PACKAGES="$INCLUDE_PACKAGES,libnss-resolve" +fi +if test "$DEBVER" -le 11; then + set -- '--customize-hook=chroot "$1" systemctl enable systemd-resolved.service' "$@" +fi +if test "$DEBVER" -le 9; then + set -- '--customize-hook=ln -fs ../run/systemd/resolve/resolv.conf "$1/etc/resolv.conf"' "$@" +elif test "$DEBVER" -le 11; then + set -- '--customize-hook=ln -fs ../run/systemd/resolve/stub-resolv.conf "$1/etc/resolv.conf"' "$@" +fi + +# construct mmdebstrap options as $@: +set -- \ + --verbose \ + --variant=apt \ + --format=ext2 \ + "--architecture=$ARCHITECTURE" \ + "--include=$INCLUDE_PACKAGES" \ + '--customize-hook=echo "LABEL=debvm / ext4 defaults 0 1" >"$1/etc/fstab"' \ + "$@" + + +# set up a hostname +set -- \ + "--customize-hook=echo $VMNAME >"'"$1/etc/hostname"' \ + "--customize-hook=echo 127.0.0.1 localhost $VMNAME >"'"$1/etc/hosts"' \ + "$@" + +# allow password-less root login +set -- '--customize-hook=chroot "$1" passwd --delete root' "$@" + +# dhcp on all network interfaces +SYSD_NET_MATCH='Name=en*\n' +test "$DEBVER" -le 8 && SYSD_NET_MATCH="${SYSD_NET_MATCH}Name=eth*\\n" +SYSD_NET_NET='DHCP=yes\n' +# This anchor is included by default since bullseye. Fails DNSSEC validation when missing. +test "$DEBVER" -le 11 && SYSD_NET_NET="${SYSD_NET_NET}DNSSECNegativeTrustAnchors=home.arpa\\n" +set -- \ + '--customize-hook=chroot "$1" systemctl enable systemd-networkd.service' \ + "--customize-hook=printf \"[Match]\\n$SYSD_NET_MATCH\\n[Network]\\n$SYSD_NET_NET"'\n[DHCP]\nUseDomains=yes\n" > "$1/etc/systemd/network/20-wired.network"' \ + "$@" + +# add ssh key for root +if test -n "$SSHKEY"; then + set -- \ + '--customize-hook=mkdir -m700 -p "$1/root/.ssh"' \ + "--customize-hook=upload $SSHKEY /root/.ssh/authorized_keys" \ + "$@" +fi + +set -- --skip=cleanup/apt/lists "$@" + +# We need /var/lib/dpkg/available for dpkg --set-selections to work. +set -- '--customize-hook=chroot "$1" apt-cache dumpavail | chroot "$1" dpkg --update-avail' "$@" + +if test "$DEBVER" -le 8; then + # Use obsolete and expired keys. + set -- '--keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg' "$@" + set -- --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"' "$@" + set -- --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older "$@" +fi + +if test "$DEBVER" -ge 12; then + # Avoid the usrmerge package + set -- --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr "$@" +fi + +set -- \ + '--customize-hook=mkdir "$1/etc/systemd/system/serial-getty@.service.d"' \ + "--customize-hook=sed -n -e '1i[Service]' -e '1iExecStart=' -e 's,^ExecStart=-/sbin/agetty ,&-a root ,p'"' "$1/lib/systemd/system/serial-getty@.service" > "$1/etc/systemd/system/serial-getty@.service.d/autologin.conf"' \ + "$@" + +# suite target mirror +set -- "$@" "$SUITE" "$IMAGE" "deb $MIRROR $SUITE main" + +set -ex + +mmdebstrap "$@" + +IMAGESIZE=$(stat -c %s "$IMAGE") +if test "$IMAGESIZE" -lt "$SIZE"; then + truncate -s "$SIZE" "$IMAGE" + /sbin/resize2fs "$IMAGE" +fi +/sbin/tune2fs -L debvm -i 0 -O extents,uninit_bg,dir_index,has_journal "$IMAGE" +# Must fsck after tune2fs: https://ext4.wiki.kernel.org/index.php/UpgradeToExt4 +/sbin/fsck.ext4 -fDp "$IMAGE" diff --git a/bin/debvm-run b/bin/debvm-run new file mode 100755 index 0000000..c2a3f21 --- /dev/null +++ b/bin/debvm-run @@ -0,0 +1,272 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT + +: <<'POD2MAN' +=head1 NAME + +debvm-run - Run a VM image created by debvm-create + +=head1 SYNOPSIS + +B [B<-g>] [B<-i> F] [B<-s> I] [B<--> I] + +=head1 DESCRIPTION + +B is essentially a thin wrapper around B for running a virtual machine image created by B or something compatible. +The virtual machine image is expected to be a raw ext4 image with file system label B. +The architecture of the machine is detected from the contained F. +It must contain a symbolic link pointing to a kernel image at F or F depending on the architecture and a symbolic link pointing to an initrd image at F. +Both are extracted and passed to B. +A net interface configured for user mode is added automatically. + +=head1 OPTIONS + +=over 8 + +=item B<-g>, B<--graphical> + +By default, the option B<-nographic> is passed to B and one interacts with the serial console of the machine. +This configuration is skipped in the presence of this option. + +=item B<-i> F, B<--image>=F + +This option specifies the location of the virtual machine image file. +By default F in the working directory is used. + +=item B<-s> I, B<--sshport>=I + +If given, B is configured to pass connections to I<127.0.0.1:sshport> to port 22 of the virtual machine. +You can connect to your virtual machine without updating your known hosts like this: + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p $sshport root@127.0.0.1 + +=item B<--> I + +All options beyond a double dash are passed to B. +This can be used to configure additional hardware components. +One possible use of this method is passing B<-snapshot> to avoid modifying the virtual machine image. + +=back + +=head1 EXAMPLES + +Run a virtual machine stored in the image F (the default) with +local port 8022 routed to port 22 of the virtual machine. The B<-snapshot> +argument is passed to QEMU and prevents any permanent changes to +F, resulting in an ephemeral run. + + debvm-run -s 8022 -i rootfs.ext4 -- -snapshot + +=head1 LIMITATIONS + +Due to the way kernel and bootloader are being extracted before running B, one cannot upgrade a kernel and then just reboot. +Attempting to do so, will still use the old kernel. +Instead, B must be terminated and B should be launched again to pick up the new kernel. +In order to avoid accidental reboots, one may pass B<-no-reboot> to B. + +=head1 SEE ALSO + + debvm-create(1) qemu(1) + +=cut +POD2MAN + +set -u + +IMAGE=rootfs.ext4 +SSHPORT= +GRAPHICAL= + +nth_arg() { + shift "$1" + printf "%s" "$1" +} + +die() { + echo "$*" 1>&2 + exit 1 +} +usage() { + die "usage: $0 [-g] [-i image] [-s sshport] [-- qemu options]" +} +usage_error() { + echo "error: $*" 1>&2 + usage +} + +opt_graphical() { + GRAPHICAL=1 +} +opt_image() { + IMAGE=$1 +} +opt_sshport() { + SSHPORT=$1 +} + +while getopts :gi:s:-: OPTCHAR; do + case "$OPTCHAR" in + g) opt_graphical ;; + i) opt_image "$OPTARG" ;; + s) opt_sshport "$OPTARG" ;; + -) + case "$OPTARG" in + help) + usage + ;; + graphical|image|sshport) + test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" + "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" + OPTIND=$((OPTIND+1)) + ;; + image=*|sshport=*) + "opt_${OPTARG%%=*}" "${OPTARG#*=}" + ;; + *) + usage_error "unrecognized option --$OPTARG" + ;; + esac + ;; + :) + usage_error "missing argument for -$OPTARG" + ;; + '?') + usage_erro "unrecognized option -$OPTARG" + ;; + *) + die "internal error while parsing command options, please report a bug" + ;; + esac +done +shift "$((OPTIND - 1))" + +test -f "$IMAGE" || die "image '$IMAGE' not found" +test -s "$IMAGE" || die "image '$IMAGE' is empty" + +if ! printf '\123\357' | cmp --bytes=2 "$IMAGE" - 1080; then + die "image '$IMAGE' is not in ext4 format" +fi + +if ! printf 'debvm\000' | cmp --bytes=6 "$IMAGE" - 1144; then + die "image '$IMAGE' was not created by debvm-create (wrong disk label)" +fi + +cleanup() { + set +x + test -n "$KERNELTMP" && rm -f "$KERNELTMP" + test -n "$INITRDTMP" && rm -f "$INITRDTMP" +} + +trap cleanup EXIT INT TERM QUIT + +KERNELTMP=$(mktemp) +INITRDTMP=$(mktemp) + +ARCHITECTURE=$(dpkg --print-architecture) +VMARCH=$ARCHITECTURE +if command -v elf-arch >/dev/null 2>&1; then + /sbin/debugfs "$IMAGE" -R "cat /bin/true" > "$KERNELTMP" + VMARCH=$(elf-arch "$KERNELTMP") +fi +case "$VMARCH" in + mips*|ppc64el) + KERNELLINK=vmlinux + ;; + *) + KERNELLINK=vmlinuz + ;; +esac + +KERNELNAME=$(/sbin/debugfs "$IMAGE" -R "stat $KERNELLINK" | sed 's/Fast link dest: "\(.*\)"/\1/;t;d') +INITRDNAME=$(/sbin/debugfs "$IMAGE" -R "stat initrd.img" | sed 's/Fast link dest: "\(.*\)"/\1/;t;d') + +test -n "$KERNELNAME" || die "failed to discover kernel image" +test -n "$INITRDNAME" || die "failed to discover initrd image" + +KERNEL_CMDLINE="root=LABEL=debvm rw" +NETDEV="user,id=net0" + +set -- \ + -no-user-config \ + -name "debvm-run $IMAGE" \ + -m 1G \ + -kernel "$KERNELTMP" \ + -initrd "$INITRDTMP" \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "media=disk,format=raw,discard=unmap,file=$IMAGE,if=virtio,cache=unsafe" \ + -device "virtio-net-pci,netdev=net0" \ + "$@" + +MAX_SMP= +if test "$ARCHITECTURE" = "$VMARCH"; then + QEMU=kvm + # While kvm will fall back gracefully, the following options can only + # be passed when kvm really is available. + if test -w /dev/kvm; then + set -- -enable-kvm -cpu host "$@" + fi + case "$VMARCH" in + arm64) + set -- -machine type=virt,gic-version=host "$@" + ;; + esac +else + QEMU="qemu-system-$VMARCH" + case "$VMARCH" in + arm64) + QEMU=qemu-system-aarch64 + set -- -machine virt -cpu max "$@" + ;; + arm|armel|armhf) + QEMU=qemu-system-arm + set -- -machine virt -cpu max "$@" + ;; + ppc64el) + QEMU=qemu-system-ppc64 + ;; + mips64el) + MAX_SMP=1 + set -- -cpu 5KEc "$@" + ;; + mipsel) + MAX_SMP=1 + ;; + riscv64) + set -- -machine virt "$@" + ;; + esac +fi +if test -z "$MAX_SMP" || test "$MAX_SMP" -gt 1; then + NPROC=$(nproc) + test -n "$MAX_SMP" && test "$NPROC" -gt "$MAX_SMP" && NPROC=$MAX_SMP + set -- -smp "$NPROC" "$@" +fi + +if test -z "$GRAPHICAL"; then + set -- -nographic "$@" + case "$VMARCH" in + amd64|i386) + KERNEL_CMDLINE="$KERNEL_CMDLINE console=ttyS0" + ;; + esac +fi + +if test -n "$SSHPORT"; then + NETDEV="$NETDEV,hostfwd=tcp:127.0.0.1:$SSHPORT-:22" +fi +DNSSEARCH=$(dnsdomainname) +if test -n "$DNSSEARCH"; then + NETDEV="$NETDEV,domainname=$DNSSEARCH" +fi +set -- \ + -append "$KERNEL_CMDLINE" \ + -netdev "$NETDEV" \ + "$@" + +set -ex + +/sbin/debugfs "$IMAGE" -R "cat $KERNELNAME" > "$KERNELTMP" +/sbin/debugfs "$IMAGE" -R "cat $INITRDNAME" > "$INITRDTMP" + +"$QEMU" "$@" diff --git a/bin/debvm-waitssh b/bin/debvm-waitssh new file mode 100755 index 0000000..82eb14d --- /dev/null +++ b/bin/debvm-waitssh @@ -0,0 +1,178 @@ +#!/bin/sh +# Copyright 2023 Helmut Grohne +# SPDX-License-Identifier: MIT + +: <<'POD2MAN' +=head1 NAME + +debvm-waitssh - Wait for a ssh server to be reachable + +=head1 SYNOPSIS + +B [B<-q>] [B<-t> I] [I:]I + +=head1 DESCRIPTION + +B can be used to wait for a virtual machine with exposed ssh port to be reachable on that port. +If no hostname is given, B<127.0.0.1> is assumed. No authentication is attempted by B, so neither +a username nor a key have to be supplied. + +=head1 OPTIONS + +=over 8 + +=item B<-t> I, B<--timeout>=I + +Set the maximum duration for waiting in seconds. +Defaults to one minute. + +=item B<-q>, B<--quiet> + +Be quiet. +Do not output a message when the timeout has been reached without success. + +=back + +=head1 EXIT VALUES + +=over 8 + +=item B<0> + +The server is reachable. + +=item B<1> + +A timeout was reached before the server answered. + +=item B<2> + +Usage error. + +=back + +=head1 SEE ALSO + + debvm-run(1) + +=cut +POD2MAN + +set -u + +TOTALTIMEOUT=60 +SCANTIMEOUT=10 +SCANDELAY=1 +VERBOSITY=1 + +nth_arg() { + shift "$1" + printf "%s" "$1" +} + +die() { + echo "$*" >&2 + exit 2 +} +usage() { + die "usage: $0 [-q] [-t ] [:]" +} +usage_error() { + echo "error: $*" >&2 + usage +} + +opt_help() { + # shellcheck disable=SC2317 # not dead, called as "opt_$OPTARG" + usage +} +opt_quiet() { + VERBOSITY=0 +} +opt_timeout() { + TOTALTIMEOUT=$1 +} + +while getopts :qt:-: OPTCHAR; do + case "$OPTCHAR" in + q) opt_quiet ;; + t) opt_timeout "$OPTARG" ;; + -) + case "$OPTARG" in + help|quiet) + "opt_$OPTARG" + ;; + timeout) + test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" + "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" + OPTIND=$((OPTIND+1)) + ;; + timeout=) + "opt_${OPTARG%%=*}" "${OPTARG#*=}" + ;; + *) + usage_error "unrecognized option --$OPTARG" + ;; + esac + ;; + :) + usage_error "missing argument for -$OPTARG" + ;; + '?') + usage_error "unrecognized option -$OPTARG" + ;; + *) + die "internal error while parsing command options, please report a bug" + ;; + esac +done +shift "$((OPTIND - 1))" + +test "$#" = 1 || usage + +case "$1" in + "") + usage + ;; + *:*) + HOST=${1%:*} + PORT=${1##*:} + ;; + *) + HOST=127.0.0.1 + PORT=$1 + ;; +esac + +case "$HOST" in *@*) + die "$0: hostname '$HOST' must not contain the '@' character. No username is required." +;; esac + +# Guard against strings containing anything but digits, strings starting with +# zero and empty strings as the port number. +# +# We cannot use [!0-9] because that matches on any character (or possibly +# multi-character collation element) that sorts in between 0 and 9. +case "$PORT" in *[!0123456789]*|0?*|""|??????*) + die "$0: port '$PORT' is not an integer between 1 and 65535" +;; esac +if test "$PORT" -lt 1 -o "$PORT" -gt 65535; then + die "$0: port '$PORT' is not an integer between 1 and 65535" +fi + +now=$(date +%s) +deadline=$((now + TOTALTIMEOUT)) +while test "$now" -lt "$deadline"; do + start=$now + ssh-keyscan -t rsa -T "$SCANTIMEOUT" -p "$PORT" "$HOST" >/dev/null 2>&1 && exit 0 + now=$(date +%s) + if test "$((now - start))" -lt "$SCANTIMEOUT"; then + sleep "$SCANDELAY" + now=$(date +%s) + fi +done +if [ "$VERBOSITY" -ge 1 ]; then + echo "$0: timeout reached trying to contact $HOST:$PORT after waiting $TOTALTIMEOUT seconds." >&2 +fi +exit 1 + diff --git a/debvm-create b/debvm-create deleted file mode 100755 index 0dc619b..0000000 --- a/debvm-create +++ /dev/null @@ -1,342 +0,0 @@ -#!/bin/sh -# Copyright 2022 Helmut Grohne -# SPDX-License-Identifier: MIT - -# shellcheck disable=SC2016 # Intentional quoting technique - -: <<'POD2MAN' -=head1 NAME - -debvm-create - Create a VM image for various Debian releases and architectures - -=head1 SYNOPSIS - -B [B<-a> I] [B<-h> I] [B<-k> F] [B<-m> I] [B<-o> F] [B<-p> I] [B<-r> I] [B<-z> I] [B<--> I] - -=head1 DESCRIPTION - -B is essentially a thin wrapper around B for creating a raw ext4 filesystem image for booting with B. -The purpose of these images primarily is testing the different releases and architectures without access to a physical machine of that architecture. -Beyond essential packages, the image will contain B, an init system and a suitable kernel package. -Notably absent is a bootloader and a partition table. -In order to boot such an image, one is supposed to extract the kernel and initrd from the image and pass it to a suitable bootloader. -No user account is created and root can login without specifying a password. - -=head1 OPTIONS - -=over 8 - -=item B<-a> I, B<--architecture>=I - -Specify a Debian architecture name. -By default, the native architecture is being used. -A suitable kernel image is automatically selected and installed into the image. - -=item B<-h> I, B<--hostname>=I - -Set the hostname of the virtual machine. -By default, the hostname is B. - -=item B<-k> F, B<--sshkey>=F - -Install the given ssh public key file into the virtual machine image for the root user. -This option also causes the ssh server to be installed. -By default, no key or server is installed. -To connect to the vm, pass a port number to B with the B<-s> option. - -=item B<-m> I, B<--mirror>=I - -Specify the Debian mirror to be used for downloading packages and to be configured inside the virtual machine image. -By default, L is being used. - -=item B<-o> F, B<--output>=F - -Specify the file name of the resulting virtual machine image. -By default, it is written to F. - -=item B<-p> I, B<--package>=I - -Request additional packages to be installed into the virtual machine image. -This option can be specified multiple times and packages can be separated by a comma. -Package recommendations are not honoured. -If a linux-image is passed here, it will replace the one selected by default. - -=item B<-r> I, B<--release>=I - -Use the given Debian release. -By default, B is being used. - -=item B<-z> I, B<--size>=I - -Specify the minimum image size in giga bytes. -The resulting image will be grown as a sparse file to this size if necessary. -The default is 1 GB. - -=item B<--> I - -All options beyond a double dash are passed to B before the suite, target and mirror specification. -This can be used to provide additional hooks for image customization. - -=back - -=head1 SEE ALSO - - debvm-run(1) mmdebstrap(1) - -=cut -POD2MAN - -set -u - -ARCHITECTURE=$(dpkg --print-architecture) -IMAGE=rootfs.ext4 -INCLUDE_PACKAGES=init -MIRROR="http://deb.debian.org/debian" -SIZE=$((1024*1024*1024)) -SSHKEY= -SUITE=unstable -VMNAME=testvm - -nth_arg() { - shift "$1" - printf "%s" "$1" -} - -die() { - echo "$*" 1>&2 - exit 1 -} -usage() { - die "usage: $0 [-a architecture] [-h hostname] [-k sshkey] [-m mirror] [-o output] [-p packages] [-r release] [-z size_in_GB] [-- mmdebstrap options]" -} -usage_error() { - echo "error: $*" 1>&2 - usage -} - -opt_architecture() { - ARCHITECTURE=$1 -} -opt_hostname() { - VMNAME=$1 -} -opt_mirror() { - MIRROR=$1 -} -opt_sshkey() { - SSHKEY=$1 -} -opt_output() { - IMAGE=$1 -} -opt_package() { - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,$1" -} -opt_release() { - SUITE=$1 -} -opt_size() { - SIZE=$(($1*1024*1024*1024)) -} - -while getopts :a:h:k:m:o:p:r:z:-: OPTCHAR; do - case "$OPTCHAR" in - a) opt_architecture "$OPTARG" ;; - h) opt_hostname "$OPTARG" ;; - k) opt_sshkey "$OPTARG" ;; - m) opt_mirror "$OPTARG" ;; - o) opt_output "$OPTARG" ;; - p) opt_package "$OPTARG" ;; - r) opt_release "$OPTARG" ;; - z) opt_size "$OPTARG" ;; - -) - case "$OPTARG" in - help) - usage - ;; - architecture|hostname|mirror|output|package|release|size|sshkey) - test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" - "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" - OPTIND=$((OPTIND+1)) - ;; - architecture=*|hostname=*|mirror=*|output=*|package=*|release=*|size=*|sshkey=*) - "opt_${OPTARG%%=*}" "${OPTARG#*=}" - ;; - *) - usage_error "unrecognized option --$OPTARG" - ;; - esac - ;; - :) - usage_error "missing argument for -$OPTARG" - ;; - '?') - usage_error "unrecognized option -$OPTARG" - ;; - *) - die "internal error while parsing command options, please report a bug" - ;; - esac -done -shift "$((OPTIND - 1))" - -if test -n "$SSHKEY" && ! test -f "$SSHKEY"; then - die "error: ssh keyfile '$SSHKEY' not found" -fi - -case "$SUITE" in - jessie) - DEBVER=8 - ;; - stretch) - DEBVER=9 - ;; - buster) - DEBVER=10 - ;; - bullseye|stable) - DEBVER=11 - ;; - bookworm|testing) - DEBVER=12 - ;; - trixie) - DEBVER=13 - ;; - forky) - DEBVER=14 - ;; - sid|unstable) - DEBVER=999 - ;; - *) - die "unrecognized Debian release: $SUITE" - ;; -esac - -KERNEL_SUFFIX=-$ARCHITECTURE -case "$ARCHITECTURE" in - amd64|arm64) - KERNEL_SUFFIX="-cloud-$ARCHITECTURE" - if test "$DEBVER" -le 9; then - KERNEL_SUFFIX="-$ARCHITECTURE" - fi - ;; - armhf) - KERNEL_SUFFIX=-armmp - ;; - i386) - KERNEL_SUFFIX=-686-pae - ;; - mips64el) - KERNEL_SUFFIX=-5kc-malta - ;; - mipsel) - KERNEL_SUFFIX=-4kc-malta - ;; - ppc64el) - KERNEL_SUFFIX=-powerpc64le - ;; -esac - -case ",$INCLUDE_PACKAGES," in - *,linux-image-*) - ;; - *) - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,linux-image$KERNEL_SUFFIX" - ;; -esac - -if test -n "$SSHKEY"; then - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,openssh-server" -fi - -# add a DNS resolver -if test "$DEBVER" -ge 9; then - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,libnss-resolve" -fi -if test "$DEBVER" -le 11; then - set -- '--customize-hook=chroot "$1" systemctl enable systemd-resolved.service' "$@" -fi -if test "$DEBVER" -le 9; then - set -- '--customize-hook=ln -fs ../run/systemd/resolve/resolv.conf "$1/etc/resolv.conf"' "$@" -elif test "$DEBVER" -le 11; then - set -- '--customize-hook=ln -fs ../run/systemd/resolve/stub-resolv.conf "$1/etc/resolv.conf"' "$@" -fi - -# construct mmdebstrap options as $@: -set -- \ - --verbose \ - --variant=apt \ - --format=ext2 \ - "--architecture=$ARCHITECTURE" \ - "--include=$INCLUDE_PACKAGES" \ - '--customize-hook=echo "LABEL=debvm / ext4 defaults 0 1" >"$1/etc/fstab"' \ - "$@" - - -# set up a hostname -set -- \ - "--customize-hook=echo $VMNAME >"'"$1/etc/hostname"' \ - "--customize-hook=echo 127.0.0.1 localhost $VMNAME >"'"$1/etc/hosts"' \ - "$@" - -# allow password-less root login -set -- '--customize-hook=chroot "$1" passwd --delete root' "$@" - -# dhcp on all network interfaces -SYSD_NET_MATCH='Name=en*\n' -test "$DEBVER" -le 8 && SYSD_NET_MATCH="${SYSD_NET_MATCH}Name=eth*\\n" -SYSD_NET_NET='DHCP=yes\n' -# This anchor is included by default since bullseye. Fails DNSSEC validation when missing. -test "$DEBVER" -le 11 && SYSD_NET_NET="${SYSD_NET_NET}DNSSECNegativeTrustAnchors=home.arpa\\n" -set -- \ - '--customize-hook=chroot "$1" systemctl enable systemd-networkd.service' \ - "--customize-hook=printf \"[Match]\\n$SYSD_NET_MATCH\\n[Network]\\n$SYSD_NET_NET"'\n[DHCP]\nUseDomains=yes\n" > "$1/etc/systemd/network/20-wired.network"' \ - "$@" - -# add ssh key for root -if test -n "$SSHKEY"; then - set -- \ - '--customize-hook=mkdir -m700 -p "$1/root/.ssh"' \ - "--customize-hook=upload $SSHKEY /root/.ssh/authorized_keys" \ - "$@" -fi - -set -- --skip=cleanup/apt/lists "$@" - -# We need /var/lib/dpkg/available for dpkg --set-selections to work. -set -- '--customize-hook=chroot "$1" apt-cache dumpavail | chroot "$1" dpkg --update-avail' "$@" - -if test "$DEBVER" -le 8; then - # Use obsolete and expired keys. - set -- '--keyring=/usr/share/keyrings/debian-archive-removed-keys.gpg' "$@" - set -- --aptopt='Apt::Key::gpgvcommand "/usr/libexec/mmdebstrap/gpgvnoexpkeysig"' "$@" - set -- --hook-dir=/usr/share/mmdebstrap/hooks/jessie-or-older "$@" -fi - -if test "$DEBVER" -ge 12; then - # Avoid the usrmerge package - set -- --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr "$@" -fi - -set -- \ - '--customize-hook=mkdir "$1/etc/systemd/system/serial-getty@.service.d"' \ - "--customize-hook=sed -n -e '1i[Service]' -e '1iExecStart=' -e 's,^ExecStart=-/sbin/agetty ,&-a root ,p'"' "$1/lib/systemd/system/serial-getty@.service" > "$1/etc/systemd/system/serial-getty@.service.d/autologin.conf"' \ - "$@" - -# suite target mirror -set -- "$@" "$SUITE" "$IMAGE" "deb $MIRROR $SUITE main" - -set -ex - -mmdebstrap "$@" - -IMAGESIZE=$(stat -c %s "$IMAGE") -if test "$IMAGESIZE" -lt "$SIZE"; then - truncate -s "$SIZE" "$IMAGE" - /sbin/resize2fs "$IMAGE" -fi -/sbin/tune2fs -L debvm -i 0 -O extents,uninit_bg,dir_index,has_journal "$IMAGE" -# Must fsck after tune2fs: https://ext4.wiki.kernel.org/index.php/UpgradeToExt4 -/sbin/fsck.ext4 -fDp "$IMAGE" diff --git a/debvm-run b/debvm-run deleted file mode 100755 index c2a3f21..0000000 --- a/debvm-run +++ /dev/null @@ -1,272 +0,0 @@ -#!/bin/sh -# Copyright 2022 Helmut Grohne -# SPDX-License-Identifier: MIT - -: <<'POD2MAN' -=head1 NAME - -debvm-run - Run a VM image created by debvm-create - -=head1 SYNOPSIS - -B [B<-g>] [B<-i> F] [B<-s> I] [B<--> I] - -=head1 DESCRIPTION - -B is essentially a thin wrapper around B for running a virtual machine image created by B or something compatible. -The virtual machine image is expected to be a raw ext4 image with file system label B. -The architecture of the machine is detected from the contained F. -It must contain a symbolic link pointing to a kernel image at F or F depending on the architecture and a symbolic link pointing to an initrd image at F. -Both are extracted and passed to B. -A net interface configured for user mode is added automatically. - -=head1 OPTIONS - -=over 8 - -=item B<-g>, B<--graphical> - -By default, the option B<-nographic> is passed to B and one interacts with the serial console of the machine. -This configuration is skipped in the presence of this option. - -=item B<-i> F, B<--image>=F - -This option specifies the location of the virtual machine image file. -By default F in the working directory is used. - -=item B<-s> I, B<--sshport>=I - -If given, B is configured to pass connections to I<127.0.0.1:sshport> to port 22 of the virtual machine. -You can connect to your virtual machine without updating your known hosts like this: - - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p $sshport root@127.0.0.1 - -=item B<--> I - -All options beyond a double dash are passed to B. -This can be used to configure additional hardware components. -One possible use of this method is passing B<-snapshot> to avoid modifying the virtual machine image. - -=back - -=head1 EXAMPLES - -Run a virtual machine stored in the image F (the default) with -local port 8022 routed to port 22 of the virtual machine. The B<-snapshot> -argument is passed to QEMU and prevents any permanent changes to -F, resulting in an ephemeral run. - - debvm-run -s 8022 -i rootfs.ext4 -- -snapshot - -=head1 LIMITATIONS - -Due to the way kernel and bootloader are being extracted before running B, one cannot upgrade a kernel and then just reboot. -Attempting to do so, will still use the old kernel. -Instead, B must be terminated and B should be launched again to pick up the new kernel. -In order to avoid accidental reboots, one may pass B<-no-reboot> to B. - -=head1 SEE ALSO - - debvm-create(1) qemu(1) - -=cut -POD2MAN - -set -u - -IMAGE=rootfs.ext4 -SSHPORT= -GRAPHICAL= - -nth_arg() { - shift "$1" - printf "%s" "$1" -} - -die() { - echo "$*" 1>&2 - exit 1 -} -usage() { - die "usage: $0 [-g] [-i image] [-s sshport] [-- qemu options]" -} -usage_error() { - echo "error: $*" 1>&2 - usage -} - -opt_graphical() { - GRAPHICAL=1 -} -opt_image() { - IMAGE=$1 -} -opt_sshport() { - SSHPORT=$1 -} - -while getopts :gi:s:-: OPTCHAR; do - case "$OPTCHAR" in - g) opt_graphical ;; - i) opt_image "$OPTARG" ;; - s) opt_sshport "$OPTARG" ;; - -) - case "$OPTARG" in - help) - usage - ;; - graphical|image|sshport) - test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" - "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" - OPTIND=$((OPTIND+1)) - ;; - image=*|sshport=*) - "opt_${OPTARG%%=*}" "${OPTARG#*=}" - ;; - *) - usage_error "unrecognized option --$OPTARG" - ;; - esac - ;; - :) - usage_error "missing argument for -$OPTARG" - ;; - '?') - usage_erro "unrecognized option -$OPTARG" - ;; - *) - die "internal error while parsing command options, please report a bug" - ;; - esac -done -shift "$((OPTIND - 1))" - -test -f "$IMAGE" || die "image '$IMAGE' not found" -test -s "$IMAGE" || die "image '$IMAGE' is empty" - -if ! printf '\123\357' | cmp --bytes=2 "$IMAGE" - 1080; then - die "image '$IMAGE' is not in ext4 format" -fi - -if ! printf 'debvm\000' | cmp --bytes=6 "$IMAGE" - 1144; then - die "image '$IMAGE' was not created by debvm-create (wrong disk label)" -fi - -cleanup() { - set +x - test -n "$KERNELTMP" && rm -f "$KERNELTMP" - test -n "$INITRDTMP" && rm -f "$INITRDTMP" -} - -trap cleanup EXIT INT TERM QUIT - -KERNELTMP=$(mktemp) -INITRDTMP=$(mktemp) - -ARCHITECTURE=$(dpkg --print-architecture) -VMARCH=$ARCHITECTURE -if command -v elf-arch >/dev/null 2>&1; then - /sbin/debugfs "$IMAGE" -R "cat /bin/true" > "$KERNELTMP" - VMARCH=$(elf-arch "$KERNELTMP") -fi -case "$VMARCH" in - mips*|ppc64el) - KERNELLINK=vmlinux - ;; - *) - KERNELLINK=vmlinuz - ;; -esac - -KERNELNAME=$(/sbin/debugfs "$IMAGE" -R "stat $KERNELLINK" | sed 's/Fast link dest: "\(.*\)"/\1/;t;d') -INITRDNAME=$(/sbin/debugfs "$IMAGE" -R "stat initrd.img" | sed 's/Fast link dest: "\(.*\)"/\1/;t;d') - -test -n "$KERNELNAME" || die "failed to discover kernel image" -test -n "$INITRDNAME" || die "failed to discover initrd image" - -KERNEL_CMDLINE="root=LABEL=debvm rw" -NETDEV="user,id=net0" - -set -- \ - -no-user-config \ - -name "debvm-run $IMAGE" \ - -m 1G \ - -kernel "$KERNELTMP" \ - -initrd "$INITRDTMP" \ - -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ - -drive "media=disk,format=raw,discard=unmap,file=$IMAGE,if=virtio,cache=unsafe" \ - -device "virtio-net-pci,netdev=net0" \ - "$@" - -MAX_SMP= -if test "$ARCHITECTURE" = "$VMARCH"; then - QEMU=kvm - # While kvm will fall back gracefully, the following options can only - # be passed when kvm really is available. - if test -w /dev/kvm; then - set -- -enable-kvm -cpu host "$@" - fi - case "$VMARCH" in - arm64) - set -- -machine type=virt,gic-version=host "$@" - ;; - esac -else - QEMU="qemu-system-$VMARCH" - case "$VMARCH" in - arm64) - QEMU=qemu-system-aarch64 - set -- -machine virt -cpu max "$@" - ;; - arm|armel|armhf) - QEMU=qemu-system-arm - set -- -machine virt -cpu max "$@" - ;; - ppc64el) - QEMU=qemu-system-ppc64 - ;; - mips64el) - MAX_SMP=1 - set -- -cpu 5KEc "$@" - ;; - mipsel) - MAX_SMP=1 - ;; - riscv64) - set -- -machine virt "$@" - ;; - esac -fi -if test -z "$MAX_SMP" || test "$MAX_SMP" -gt 1; then - NPROC=$(nproc) - test -n "$MAX_SMP" && test "$NPROC" -gt "$MAX_SMP" && NPROC=$MAX_SMP - set -- -smp "$NPROC" "$@" -fi - -if test -z "$GRAPHICAL"; then - set -- -nographic "$@" - case "$VMARCH" in - amd64|i386) - KERNEL_CMDLINE="$KERNEL_CMDLINE console=ttyS0" - ;; - esac -fi - -if test -n "$SSHPORT"; then - NETDEV="$NETDEV,hostfwd=tcp:127.0.0.1:$SSHPORT-:22" -fi -DNSSEARCH=$(dnsdomainname) -if test -n "$DNSSEARCH"; then - NETDEV="$NETDEV,domainname=$DNSSEARCH" -fi -set -- \ - -append "$KERNEL_CMDLINE" \ - -netdev "$NETDEV" \ - "$@" - -set -ex - -/sbin/debugfs "$IMAGE" -R "cat $KERNELNAME" > "$KERNELTMP" -/sbin/debugfs "$IMAGE" -R "cat $INITRDNAME" > "$INITRDTMP" - -"$QEMU" "$@" diff --git a/debvm-waitssh b/debvm-waitssh deleted file mode 100755 index 82eb14d..0000000 --- a/debvm-waitssh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/sh -# Copyright 2023 Helmut Grohne -# SPDX-License-Identifier: MIT - -: <<'POD2MAN' -=head1 NAME - -debvm-waitssh - Wait for a ssh server to be reachable - -=head1 SYNOPSIS - -B [B<-q>] [B<-t> I] [I:]I - -=head1 DESCRIPTION - -B can be used to wait for a virtual machine with exposed ssh port to be reachable on that port. -If no hostname is given, B<127.0.0.1> is assumed. No authentication is attempted by B, so neither -a username nor a key have to be supplied. - -=head1 OPTIONS - -=over 8 - -=item B<-t> I, B<--timeout>=I - -Set the maximum duration for waiting in seconds. -Defaults to one minute. - -=item B<-q>, B<--quiet> - -Be quiet. -Do not output a message when the timeout has been reached without success. - -=back - -=head1 EXIT VALUES - -=over 8 - -=item B<0> - -The server is reachable. - -=item B<1> - -A timeout was reached before the server answered. - -=item B<2> - -Usage error. - -=back - -=head1 SEE ALSO - - debvm-run(1) - -=cut -POD2MAN - -set -u - -TOTALTIMEOUT=60 -SCANTIMEOUT=10 -SCANDELAY=1 -VERBOSITY=1 - -nth_arg() { - shift "$1" - printf "%s" "$1" -} - -die() { - echo "$*" >&2 - exit 2 -} -usage() { - die "usage: $0 [-q] [-t ] [:]" -} -usage_error() { - echo "error: $*" >&2 - usage -} - -opt_help() { - # shellcheck disable=SC2317 # not dead, called as "opt_$OPTARG" - usage -} -opt_quiet() { - VERBOSITY=0 -} -opt_timeout() { - TOTALTIMEOUT=$1 -} - -while getopts :qt:-: OPTCHAR; do - case "$OPTCHAR" in - q) opt_quiet ;; - t) opt_timeout "$OPTARG" ;; - -) - case "$OPTARG" in - help|quiet) - "opt_$OPTARG" - ;; - timeout) - test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" - "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" - OPTIND=$((OPTIND+1)) - ;; - timeout=) - "opt_${OPTARG%%=*}" "${OPTARG#*=}" - ;; - *) - usage_error "unrecognized option --$OPTARG" - ;; - esac - ;; - :) - usage_error "missing argument for -$OPTARG" - ;; - '?') - usage_error "unrecognized option -$OPTARG" - ;; - *) - die "internal error while parsing command options, please report a bug" - ;; - esac -done -shift "$((OPTIND - 1))" - -test "$#" = 1 || usage - -case "$1" in - "") - usage - ;; - *:*) - HOST=${1%:*} - PORT=${1##*:} - ;; - *) - HOST=127.0.0.1 - PORT=$1 - ;; -esac - -case "$HOST" in *@*) - die "$0: hostname '$HOST' must not contain the '@' character. No username is required." -;; esac - -# Guard against strings containing anything but digits, strings starting with -# zero and empty strings as the port number. -# -# We cannot use [!0-9] because that matches on any character (or possibly -# multi-character collation element) that sorts in between 0 and 9. -case "$PORT" in *[!0123456789]*|0?*|""|??????*) - die "$0: port '$PORT' is not an integer between 1 and 65535" -;; esac -if test "$PORT" -lt 1 -o "$PORT" -gt 65535; then - die "$0: port '$PORT' is not an integer between 1 and 65535" -fi - -now=$(date +%s) -deadline=$((now + TOTALTIMEOUT)) -while test "$now" -lt "$deadline"; do - start=$now - ssh-keyscan -t rsa -T "$SCANTIMEOUT" -p "$PORT" "$HOST" >/dev/null 2>&1 && exit 0 - now=$(date +%s) - if test "$((now - start))" -lt "$SCANTIMEOUT"; then - sleep "$SCANDELAY" - now=$(date +%s) - fi -done -if [ "$VERBOSITY" -ge 1 ]; then - echo "$0: timeout reached trying to contact $HOST:$PORT after waiting $TOTALTIMEOUT seconds." >&2 -fi -exit 1 - -- cgit v1.2.3 From 0fb18186ca3ed2ac16efeb34ea3840e17aafaa3d Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 11 Jan 2023 20:34:12 +0100 Subject: CI: attempt to fix by not using a relative path in $PATH --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0da847f..d7f2aee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ release_test: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install e2fsprogs genext2fs mmdebstrap openssh-client qemu-kvm - - PATH=./bin:$PATH ./tests/create-and-run.sh $(dpkg --print-architecture) "$RELEASE" + - PATH=$(pwd)/bin:$PATH ./tests/create-and-run.sh $(dpkg --print-architecture) "$RELEASE" arch_test: parallel: @@ -46,4 +46,4 @@ arch_test: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install e2fsprogs genext2fs mmdebstrap openssh-client qemu-system binfmt-support arch-test qemu-user-static - - PATH=./bin:$PATH ./tests/create-and-run.sh "$ARCHITECTURE" sid + - PATH=$(pwd)/bin:$PATH ./tests/create-and-run.sh "$ARCHITECTURE" sid -- cgit v1.2.3 From c751e229a1f8118787954d26bab4c300d114300e Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 11 Jan 2023 19:29:46 +0100 Subject: debvm-create: move functionality into support files The benefit of this change is that we don't have to escape that much. As a consequence, it becomes easier to put more code into the customization hooks, which have access to installed package versions. Thus we can reduce the use of DEBVER and thus improve working with snapshot.d.o. --- .gitlab-ci.yml | 4 ++-- bin/debvm-create | 26 +++++--------------------- share/customize-autologin.sh | 20 ++++++++++++++++++++ share/customize-networkd.sh | 37 +++++++++++++++++++++++++++++++++++++ share/customize-resolved.sh | 26 ++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 23 deletions(-) create mode 100755 share/customize-autologin.sh create mode 100755 share/customize-networkd.sh create mode 100755 share/customize-resolved.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7f2aee..29e0783 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,14 +5,14 @@ shellcheck: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install shellcheck - - shellcheck -P tests bin/* tests/*.sh + - shellcheck -P tests bin/* share/*.sh tests/*.sh codespell: script: - apt-get update - apt-get dist-upgrade --yes - apt-get --no-install-recommends --yes install codespell - - codespell bin/* tests/*.sh + - codespell bin/* share/*.sh tests/*.sh release_test: parallel: diff --git a/bin/debvm-create b/bin/debvm-create index 0dc619b..9d637b1 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -97,6 +97,8 @@ SSHKEY= SUITE=unstable VMNAME=testvm +SHARE_DIR="${0%/*}/../share" + nth_arg() { shift "$1" printf "%s" "$1" @@ -255,14 +257,7 @@ fi if test "$DEBVER" -ge 9; then INCLUDE_PACKAGES="$INCLUDE_PACKAGES,libnss-resolve" fi -if test "$DEBVER" -le 11; then - set -- '--customize-hook=chroot "$1" systemctl enable systemd-resolved.service' "$@" -fi -if test "$DEBVER" -le 9; then - set -- '--customize-hook=ln -fs ../run/systemd/resolve/resolv.conf "$1/etc/resolv.conf"' "$@" -elif test "$DEBVER" -le 11; then - set -- '--customize-hook=ln -fs ../run/systemd/resolve/stub-resolv.conf "$1/etc/resolv.conf"' "$@" -fi +set -- "--customize-hook=$SHARE_DIR/customize-resolved.sh" "$@" # construct mmdebstrap options as $@: set -- \ @@ -285,15 +280,7 @@ set -- \ set -- '--customize-hook=chroot "$1" passwd --delete root' "$@" # dhcp on all network interfaces -SYSD_NET_MATCH='Name=en*\n' -test "$DEBVER" -le 8 && SYSD_NET_MATCH="${SYSD_NET_MATCH}Name=eth*\\n" -SYSD_NET_NET='DHCP=yes\n' -# This anchor is included by default since bullseye. Fails DNSSEC validation when missing. -test "$DEBVER" -le 11 && SYSD_NET_NET="${SYSD_NET_NET}DNSSECNegativeTrustAnchors=home.arpa\\n" -set -- \ - '--customize-hook=chroot "$1" systemctl enable systemd-networkd.service' \ - "--customize-hook=printf \"[Match]\\n$SYSD_NET_MATCH\\n[Network]\\n$SYSD_NET_NET"'\n[DHCP]\nUseDomains=yes\n" > "$1/etc/systemd/network/20-wired.network"' \ - "$@" +set -- "--customize-hook=$SHARE_DIR/customize-networkd.sh" "$@" # add ssh key for root if test -n "$SSHKEY"; then @@ -320,10 +307,7 @@ if test "$DEBVER" -ge 12; then set -- --hook-dir=/usr/share/mmdebstrap/hooks/merged-usr "$@" fi -set -- \ - '--customize-hook=mkdir "$1/etc/systemd/system/serial-getty@.service.d"' \ - "--customize-hook=sed -n -e '1i[Service]' -e '1iExecStart=' -e 's,^ExecStart=-/sbin/agetty ,&-a root ,p'"' "$1/lib/systemd/system/serial-getty@.service" > "$1/etc/systemd/system/serial-getty@.service.d/autologin.conf"' \ - "$@" +set -- "--customize-hook=$SHARE_DIR/customize-autologin.sh" "$@" # suite target mirror set -- "$@" "$SUITE" "$IMAGE" "deb $MIRROR $SUITE main" diff --git a/share/customize-autologin.sh b/share/customize-autologin.sh new file mode 100755 index 0000000..5592a03 --- /dev/null +++ b/share/customize-autologin.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT +# +# This is a mmdebstrap customize hook that configures automatic root login on a +# serial console. + +set -eu + +TARGET=$1 + +UNIT=serial-getty@.service + +mkdir "$TARGET/etc/systemd/system/$UNIT.d" + +( + echo '[Service]' + echo 'ExecStart=' + sed -n 's,^ExecStart=-/sbin/agetty ,&-a root ,p' "$TARGET/lib/systemd/system/$UNIT" +) > "$TARGET/etc/systemd/system/$UNIT.d/autologin.conf" diff --git a/share/customize-networkd.sh b/share/customize-networkd.sh new file mode 100755 index 0000000..c89aae2 --- /dev/null +++ b/share/customize-networkd.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT +# +# This is a mmdebstrap customize hook that enables and configures +# systemd-networkd on various Debian releases. + +set -eu + +TARGET=$1 + +SYSTEMD_VERSION=$(dpkg-query --root "$TARGET" -f '${Version}' -W systemd) + +if test "${MMDEBSTRAP_MODE:-}" = chrootless; then + systemctl --root "$TARGET" enable systemd-networkd.service +else + chroot "$TARGET" systemctl enable systemd-networkd.service +fi + +{ + echo '[Match]' + echo 'Name=en*' + if dpkg --compare-versions "$SYSTEMD_VERSION" lt 220-7~; then + echo 'Name=eth*' + fi + + echo '[Network]' + echo 'DHCP=yes' + + if dpkg --compare-versions "$SYSTEMD_VERSION" lt 249; then + # This anchor is included by default since bullseye. Fails DNSSEC + # validation when missing. + echo 'DNSSECNegativeTrustAnchors=home.arpa' + fi + echo '[DHCP]' + echo 'UseDomains=yes' +} >"$TARGET/etc/systemd/network/20-wired.network" diff --git a/share/customize-resolved.sh b/share/customize-resolved.sh new file mode 100755 index 0000000..e8fe248 --- /dev/null +++ b/share/customize-resolved.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT +# +# This is a mmdebstrap customize hook that enables systemd-resolved on various +# Debian releases. + +set -eu + +TARGET=$1 + +LIBNSS_RESOLVE_VERSION=$(dpkg-query --root "$TARGET" -f '${Version}' -W libnss-resolve 2>/dev/null) || : + +if dpkg --compare-versions "$LIBNSS_RESOLVE_VERSION" lt 251.3-2~exp1; then + if test "${MMDEBSTRAP_MODE:-}" = chrootless; then + systemctl --root "$TARGET" enable systemd-resolved.service + else + chroot "$TARGET" systemctl enable systemd-resolved.service + fi + + if test -z "$LIBNSS_RESOLVE_VERSION"; then + ln -fs ../run/systemd/resolve/resolv.conf "$TARGET/etc/resolv.conf" + else + ln -fs ../run/systemd/resolve/stub-resolv.conf "$TARGET/etc/resolv.conf" + fi +fi -- cgit v1.2.3 From 3fde9f313c991aa19fbcedc38a4531de9e838f8d Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 12 Jan 2023 08:05:32 +0100 Subject: debvm-create: also move dpkg's available database to a hook file --- bin/debvm-create | 4 +--- share/customize-dpkgavailable.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100755 share/customize-dpkgavailable.sh diff --git a/bin/debvm-create b/bin/debvm-create index 9d637b1..981b450 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -291,9 +291,7 @@ if test -n "$SSHKEY"; then fi set -- --skip=cleanup/apt/lists "$@" - -# We need /var/lib/dpkg/available for dpkg --set-selections to work. -set -- '--customize-hook=chroot "$1" apt-cache dumpavail | chroot "$1" dpkg --update-avail' "$@" +set -- "--customize-hook=$SHARE_DIR/customize-dpkgavailable.sh" "$@" if test "$DEBVER" -le 8; then # Use obsolete and expired keys. diff --git a/share/customize-dpkgavailable.sh b/share/customize-dpkgavailable.sh new file mode 100755 index 0000000..f35b81f --- /dev/null +++ b/share/customize-dpkgavailable.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Copyright 2022 Helmut Grohne +# SPDX-License-Identifier: MIT +# +# This is a mmdebstrap customize hook that initializes dpkg's available +# database from an updated apt package list cache. +# +# Without the available database, dpkg --set-selections won't work. + +set -eu + +APT_CONFIG=$MMDEBSTRAP_APT_CONFIG apt-cache dumpavail | dpkg --root "$1" --update-avail -- cgit v1.2.3 From 52202bfc705e433a170e78529558556b0976ae71 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Fri, 13 Jan 2023 07:04:28 +0100 Subject: automatically set up agetty TERM when possible * debvm-create will now parse a new kernel cmdline debvm.term and if present will pass its value to agetty as TERM. * debvm-run will now detect whether it is running in a terminal in non-graphic mode and pass its environment TERM variable as debvm.term to the kernel cmdline thus closing the loop. --- bin/debvm-run | 3 +++ share/customize-autologin.sh | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/debvm-run b/bin/debvm-run index c2a3f21..4f7ff44 100755 --- a/bin/debvm-run +++ b/bin/debvm-run @@ -250,6 +250,9 @@ if test -z "$GRAPHICAL"; then KERNEL_CMDLINE="$KERNEL_CMDLINE console=ttyS0" ;; esac + if test -t 0 && test -t 1 && test -n "$TERM"; then + KERNEL_CMDLINE="$KERNEL_CMDLINE debvm.term=$TERM" + fi fi if test -n "$SSHPORT"; then diff --git a/share/customize-autologin.sh b/share/customize-autologin.sh index 5592a03..d7df555 100755 --- a/share/customize-autologin.sh +++ b/share/customize-autologin.sh @@ -3,7 +3,8 @@ # SPDX-License-Identifier: MIT # # This is a mmdebstrap customize hook that configures automatic root login on a -# serial console. +# serial console. It also parses the debvm.term kernel cmdline and passes it as +# TERM to agetty. set -eu @@ -15,6 +16,8 @@ mkdir "$TARGET/etc/systemd/system/$UNIT.d" ( echo '[Service]' + printf '%s\n' 'ExecStartPre=/bin/sed -n -e "s/.*\\(^\\| \\)debvm\\.term=\\([^ ]*\\).*/TERM=\\2/w/run/debvmterm" /proc/cmdline' + echo 'EnvironmentFile=-/run/debvmterm' echo 'ExecStart=' sed -n 's,^ExecStart=-/sbin/agetty ,&-a root ,p' "$TARGET/lib/systemd/system/$UNIT" ) > "$TARGET/etc/systemd/system/$UNIT.d/autologin.conf" -- cgit v1.2.3 From 8d13bcead097b2052ad300e594d1bfb9183d0164 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Fri, 13 Jan 2023 07:43:46 +0100 Subject: debvm-create: defer the decision of installing libnss-resolve to apt Rather than check the DEBVER, let apt figure out whether it knows about a package called libnss-resolve and install it when available. --- bin/debvm-create | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/debvm-create b/bin/debvm-create index 981b450..d4a291b 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -254,9 +254,7 @@ if test -n "$SSHKEY"; then fi # add a DNS resolver -if test "$DEBVER" -ge 9; then - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,libnss-resolve" -fi +INCLUDE_PACKAGES="$INCLUDE_PACKAGES,?exact-name(libnss-resolve)" set -- "--customize-hook=$SHARE_DIR/customize-resolved.sh" "$@" # construct mmdebstrap options as $@: -- cgit v1.2.3 From 636fdb3b8fe582d2ba5da2d8b04276335e8451ab Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Fri, 13 Jan 2023 09:38:40 +0100 Subject: debvm-create: remove --package option This option was meant to make it convenient to add packages, but mmdebstrap's --include provides the same convenience at more flexibility. You can provide apt patterns there provided that you use multiple --include options. So rather than duplicate this functionality under a different name, just refer to mmdebstrap thus reinforcing the notion of being a thin wrapper. This also removes the convenience of overriding the kernel image. You need to use --skip=kernel when passing --include=linux-image-something to mmdebstrap now. This implements "explicit is better than implicit". --- bin/debvm-create | 25 ++++++------------------- useraddhook/customize.sh | 2 +- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/bin/debvm-create b/bin/debvm-create index 59811e8..b34399b 100755 --- a/bin/debvm-create +++ b/bin/debvm-create @@ -11,7 +11,7 @@ debvm-create - Create a VM image for various Debian releases and architectures =head1 SYNOPSIS -B [B<-a> I] [B<-h> I] [B<-k> F] [B<-m> I] [B<-o> F] [B<-p> I] [B<-r> I] [B<-s> ] [B<-z> I] [B<--> I] +B [B<-a> I] [B<-h> I] [B<-k> F] [B<-m> I] [B<-o> F] [B<-r> I] [B<-s> ] [B<-z> I] [B<--> I] =head1 DESCRIPTION @@ -54,13 +54,6 @@ By default, L is being used. Specify the file name of the resulting virtual machine image. By default, it is written to F. -=item B<-p> I, B<--package>=I - -Request additional packages to be installed into the virtual machine image. -This option can be specified multiple times and packages can be separated by a comma. -Package recommendations are not honoured. -If a linux-image is passed here, it will also C<--skip=kernel>. - =item B<-r> I, B<--release>=I Use the given Debian release. @@ -108,6 +101,7 @@ The default is 1 GB. All options beyond a double dash are passed to B before the suite, target and mirror specification. This can be used to provide additional hooks for image customization. +You can also request additional packages to be installed into the image using B's B<--include> option. =back @@ -142,7 +136,7 @@ die() { exit 1 } usage() { - die "usage: $0 [-a architecture] [-h hostname] [-k sshkey] [-m mirror] [-o output] [-p packages] [-r release] [-z size_in_GB] [-- mmdebstrap options]" + die "usage: $0 [-a architecture] [-h hostname] [-k sshkey] [-m mirror] [-o output] [-r release] [-s task] [-z size_in_GB] [-- mmdebstrap options]" } usage_error() { echo "error: $*" 1>&2 @@ -167,12 +161,6 @@ opt_sshkey() { opt_output() { IMAGE=$1 } -opt_package() { - INCLUDE_PACKAGES="$INCLUDE_PACKAGES,$1" - case "$1" in linux-image*) - opt_skip kernel - ;; esac -} opt_release() { SUITE=$1 } @@ -180,14 +168,13 @@ opt_size() { SIZE=$(($1*1024*1024*1024)) } -while getopts :a:h:k:m:o:p:r:s:z:-: OPTCHAR; do +while getopts :a:h:k:m:o:r:s:z:-: OPTCHAR; do case "$OPTCHAR" in a) opt_architecture "$OPTARG" ;; h) opt_hostname "$OPTARG" ;; k) opt_sshkey "$OPTARG" ;; m) opt_mirror "$OPTARG" ;; o) opt_output "$OPTARG" ;; - p) opt_package "$OPTARG" ;; r) opt_release "$OPTARG" ;; s) opt_skip "$OPTARG" ;; z) opt_size "$OPTARG" ;; @@ -196,12 +183,12 @@ while getopts :a:h:k:m:o:p:r:s:z:-: OPTCHAR; do help) usage ;; - architecture|hostname|mirror|output|package|release|size|skip|sshkey) + architecture|hostname|mirror|output|release|size|skip|sshkey) test "$OPTIND" -gt "$#" && usage_error "missing argument for --$OPTARG" "opt_$OPTARG" "$(nth_arg "$OPTIND" "$@")" OPTIND=$((OPTIND+1)) ;; - architecture=*|hostname=*|mirror=*|output=*|package=*|release=*|size=*|skip=*|sshkey=*) + architecture=*|hostname=*|mirror=*|output=*|release=*|size=*|skip=*|sshkey=*) "opt_${OPTARG%%=*}" "${OPTARG#*=}" ;; *) diff --git a/useraddhook/customize.sh b/useraddhook/customize.sh index a4390bd..3bba263 100755 --- a/useraddhook/customize.sh +++ b/useraddhook/customize.sh @@ -13,7 +13,7 @@ # # Example usage: # -# $ debvm-create -p sudo -k ~/.ssh/id_rsa.pub -- --hook-dir=.../useraddhook +# $ debvm-create -k ~/.ssh/id_rsa.pub -- --hook-dir=.../useraddhook --include sudo # $ debvm-run -s 8022 # $ ssh -l user -p 8022 127.0.0.1 whoami # user -- cgit v1.2.3 From ca2cbf6d84dcc96845fcc6521aebe3b5076cdb32 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Fri, 13 Jan 2023 12:55:10 +0100 Subject: change cmdline variable debvm.term to TERM Jochen Sprickerhof found that systemd interprets the TERM variable according to our intended meaning and applies it to pid 1. It doesn't apply it to serial-getty, though so we keep that code for now. --- bin/debvm-run | 2 +- share/customize-autologin.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/debvm-run b/bin/debvm-run index 839653c..6ed47c4 100755 --- a/bin/debvm-run +++ b/bin/debvm-run @@ -268,7 +268,7 @@ if test -z "$GRAPHICAL"; then ;; esac if test -t 0 && test -t 1 && test -n "$TERM"; then - KERNEL_CMDLINE="$KERNEL_CMDLINE debvm.term=$TERM" + KERNEL_CMDLINE="$KERNEL_CMDLINE TERM=$TERM" fi fi diff --git a/share/customize-autologin.sh b/share/customize-autologin.sh index d7df555..b9a4a8b 100755 --- a/share/customize-autologin.sh +++ b/share/customize-autologin.sh @@ -16,7 +16,7 @@ mkdir "$TARGET/etc/systemd/system/$UNIT.d" ( echo '[Service]' - printf '%s\n' 'ExecStartPre=/bin/sed -n -e "s/.*\\(^\\| \\)debvm\\.term=\\([^ ]*\\).*/TERM=\\2/w/run/debvmterm" /proc/cmdline' + printf '%s\n' 'ExecStartPre=/bin/sed -n -e "s/^\\(.* \\)\\?\\(TERM=[^ ]*\\).*/\\2/w/run/debvmterm" /proc/cmdline' echo 'EnvironmentFile=-/run/debvmterm' echo 'ExecStart=' sed -n 's,^ExecStart=-/sbin/agetty ,&-a root ,p' "$TARGET/lib/systemd/system/$UNIT" -- cgit v1.2.3