initial checkin of mdbp

Proof-of-concept status. Some things work.

6 files changed, 530 insertions, 0 deletions

diff --git a/mdbp/__init__.py b/mdbp/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mdbp/__init__.py
diff --git a/mdbp/build_schema.json b/mdbp/build_schema.json
new file mode 100644
index 0000000..b8d905d
--- /dev/null
+++ b/mdbp/build_schema.json
@@ -0,0 +1,129 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "type": "object",
+  "required": [ "input", "distribution", "output" ],
+  "properties": {
+    "input": {
+      "type": "object",
+      "oneOf": [ {
+        "required": [ "dscpath" ],
+        "additionalProperties": false,
+        "properties": {
+          "dscpath": {
+            "type": "string",
+            "description": "path to the .dsc file that is to be built, can be relative to the location of this json file"
+          }
+        }
+      }, {
+        "required": [ "dscuri" ],
+        "additionalProperties": false,
+        "properties": {
+          "dscuri": {
+            "type": "string",
+            "format": "uri",
+            "description": "uri for downloading the .dsc file"
+          },
+          "checksums": {
+            "type": "object",
+            "patternProperties": { ".*": { "type": "string" } },
+            "default": {},
+            "description": "a mapping of checksum algorithms to the expected values"
+          }
+        }
+      } ]
+    },
+    "distribution": {
+      "type": "string",
+      "pattern": "^[a-z0-9-]+$",
+      "description": "selects the base chroot used for building"
+    },
+    "extrarepositories": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "pattern": "^(deb|deb-src) "
+      },
+      "default": [],
+      "description": "extra repository specifications to be added to sources.list"
+    },
+    "type": {
+      "type": "string",
+      "enum": [ "any", "all", "binary" ],
+      "default": "binary",
+      "description": "select an arch-only, indep-only or full build"
+    },
+    "buildarch": {
+      "type": "string",
+      "minLength": 2,
+      "pattern": "^[a-z0-9-]+$",
+      "description": "build architecture, defaults to the native architecure"
+    },
+    "hostarch": {
+      "type": "string",
+      "minLength": 2,
+      "pattern": "^[a-z0-9-]+$",
+      "description": "host architecture, defaults to the build architecture"
+    },
+    "profiles": {
+      "type": "array",
+      "items": { "type": "string", "pattern": "^[a-z0-9.-]+$" },
+      "uniqueItems": true,
+      "default": [],
+      "description": "select build profiles to enabled"
+    },
+    "options": {
+      "type": "array",
+      "items": { "type": "string", "pattern": "^[a-z0-9.=_-]+$" },
+      "uniqueItems": true,
+      "default": [],
+      "description": "values of DEB_BUILD_OPTIONS"
+    },
+    "environment": {
+      "type": "object",
+      "propertyNames": { "pattern": "^[^=-][^=]*$" },
+      "patternProperties": { ".*": { "type": "string" } },
+      "default": [],
+      "description": "extra environment variables"
+    },
+    "buildpath": {
+      "type": "string",
+      "description": "the path inside the chroot to peform the build"
+    },
+    "lintian": {
+      "type": "object",
+      "properties": {
+        "run": {
+          "type": "boolean",
+          "default": false,
+          "description": "whether to run lintian after the build"
+        },
+        "options": {
+          "type": "array",
+          "items": { "type": "string" },
+          "default": [],
+          "description": "extra options to pass to lintian"
+        }
+      }
+    },
+    "bd-uninstallable-explainer": {
+      "enum": [ null, "apt", "dose3" ],
+      "default": null,
+      "description": "when installing Build-Depends fails, an explainer can be used to give details"
+    },
+    "network": {
+      "enum": [ "enable", "disable", "try-disable", "try-enable", "undefined" ],
+      "default": "undefined",
+      "description": "whether the build should be able to access the internet"
+    },
+    "output": {
+      "type": "object",
+      "required": [ "directory" ],
+      "properties": {
+        "directory": {
+          "type": "string",
+          "description": "target directory to place output artifacts"
+        }
+      }
+    }
+  }
+}
diff --git a/mdbp/common.py b/mdbp/common.py
new file mode 100644
index 0000000..d0d2f2d
--- /dev/null
+++ b/mdbp/common.py
@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: MIT
+"""Common functions used by multiple backends"""
+
+from __future__ import annotations
+import argparse
+import contextlib
+import hashlib
+import importlib.resources
+import json
+import pathlib
+import tempfile
+import typing
+import urllib.parse
+
+import debian.deb822
+import requests
+
+try:
+    import jsonschema
+except ImportError:
+    jsonschema = None
+
+def json_load(filecontextmanager:
+              typing.ContextManager[typing.IO[typing.AnyStr]]) -> typing.Any:
+    """Load the json context from a file context manager."""
+    with filecontextmanager as fileobj:
+        return json.load(fileobj)
+
+JsonObject = typing.Dict[str, typing.Any]
+
+def buildjson(filename: str) -> JsonObject:
+    """Type constructor for argparse validating a build json file path and
+    returning the parsed json object."""
+    buildobj = json_load(argparse.FileType("r")(filename))
+    if jsonschema:
+        jsonschema.validate(
+            buildobj,
+            json_load(
+                importlib.resources.open_text("mdbp", "build_schema.json")))
+    assert isinstance(buildobj, dict)
+    return buildobj
+
+def compute_env(build: JsonObject) -> typing.Dict[str, str]:
+    """Compute the process environment from the build object."""
+    env = dict(PATH="/usr/bin:/bin")
+    env.update(build.get("environment", {}))
+    if build.get("options"):
+        env["DEB_BUILD_OPTIONS"] = " ".join(build["options"])
+    return env
+
+class HashSumMismatch(Exception):
+    """Raised from `hash_check` when validation fails."""
+
+# pylint does not grok from __future__ import annotations yet
+# pylint: disable=E1101,W0212
+def hash_check(iterable: typing.Iterable[bytes], hashobj: hashlib._Hash,
+               expected_digest: str) -> \
+        typing.Iterator[bytes]:
+    """Wraps an iterable that yields bytes. It doesn't modify the sequence,
+    but on the final element it verifies that the concatenation of bytes
+    yields an expected digest value. Upon failure, the final next() results in
+    a HashSumMismatch rather than StopIteration.
+    """
+    for data in iterable:
+        hashobj.update(data)
+        yield data
+    if hashobj.hexdigest() != expected_digest:
+        raise HashSumMismatch()
+
+def download(uri: str, checksums: typing.Dict[str, str],
+             dest: pathlib.Path) -> None:
+    """Download the given uri and save it as the given dest path provided that
+    the given checksums match. When checksums do not match, raise a
+    HashSumMismatch.
+    """
+    with requests.get(uri, stream=True) as resp:
+        resp.raise_for_status()
+        iterable = resp.iter_content(None)
+        for algo, csum in checksums.items():
+            iterable = hash_check(iterable, hashlib.new(algo), csum)
+        try:
+            with dest.open("wb") as out:
+                for chunk in iterable:
+                    out.write(chunk)
+        except HashSumMismatch:
+            dest.unlink()
+            raise
+
+@contextlib.contextmanager
+def get_dsc(build: JsonObject) -> typing.Iterator[pathlib.Path]:
+    """A context manager that provides a path pointing at the .dsc file for the
+    duration of the context. If the .dsc is supplied as a path, it simply is
+    returned. If it is supplied as a uri, it and the referred components are
+    downloaded to a temporary location.
+    """
+    try:
+        dscpath = build["input"]["dscpath"]
+    except KeyError:
+        dscuri = build["input"]["dscuri"]
+        with tempfile.TemporaryDirectory() as tdirname:
+            tdir = pathlib.Path(tdirname)
+            dscpath = tdir / dscuri.split("/")[-1]
+            download(dscuri, build["input"].get("checksums", {}), dscpath)
+            files: typing.Dict[str, typing.Dict[str, str]] = {}
+            with dscpath.open("r") as dscf:
+                for key, value in debian.deb822.Dsc(dscf).items():
+                    if key.lower().startswith("checksums-"):
+                        for entry in value:
+                            algo = key[10:].lower()
+                            files.setdefault(entry["name"], dict())[algo] = \
+                                    entry[algo]
+            for name, checksums in files.items():
+                download(urllib.parse.urljoin(dscuri, name), checksums,
+                         tdir / name)
+            yield dscpath
+    else:
+        yield pathlib.Path(dscpath)
+
+def get_dsc_files(dscpath: pathlib.Path) -> typing.List[pathlib.Path]:
+    """Get the component names referenced by the .dsc file."""
+    with dscpath.open("r") as dscf:
+        dsc = debian.deb822.Dsc(dscf)
+    return [dscpath.parent / item["name"] for item in dsc["Files"]]
+
+def make_option(optname: str, value: typing.Optional[str]) -> typing.List[str]:
+    """Construct a valued option if a value is given."""
+    if not value:
+        return []
+    if optname.endswith("="):
+        return [optname + value]
+    return [optname, value]
+
+def profile_option(build: JsonObject, optname: str) -> typing.List[str]:
+    """Construct the option for specifying build profiles if required."""
+    return make_option(optname, ",".join(build.get("profiles", ())))
diff --git a/mdbp/mmdebstrap.py b/mdbp/mmdebstrap.py
new file mode 100644
index 0000000..df96e47
--- /dev/null
+++ b/mdbp/mmdebstrap.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+"""mdbp backend using mmdebstrap"""
+
+import argparse
+import pathlib
+import shlex
+import subprocess
+import tempfile
+import typing
+
+from .common import buildjson, compute_env, get_dsc, get_dsc_files, \
+        profile_option
+
+# pylint: disable=W0102 # as we do not modify env
+def priv_drop(cmd: typing.List[str], *, chroot: bool = False,
+              chdir: typing.Optional[str] = None,
+              setuid: typing.Optional[str] = None,
+              privnet: bool = False,
+              env: typing.Dict[str, str] = {}) -> str:
+    """Returns shlext.join(cmd) with certain privilege dropping operations
+    applied:
+     * When chroot is True, chroot to "$1".
+     * When chdir is set, chdir to it after chrooting. Should be absolute.
+     * When setuid is set, change the user id and group id. Names allowed.
+     * When privnet is True, create a new network namespace with an active
+       loopback interface.
+     * Environment variables from env will be set.
+    """
+    if chdir or env:
+        envcmd = ["env"]
+        if chdir:
+            envcmd.extend(["--chdir", chdir])
+        envcmd.extend(map("%s=%s".__mod__, env.items()))
+        cmd = envcmd + cmd
+    cmdstring = ""
+    if setuid or chroot:
+        cmdstring = "chroot "
+        if setuid:
+            cmdstring += "--userspec %s " % \
+                    shlex.quote("%s:%s" % (setuid, setuid))
+        cmdstring += '"$1" ' if chroot else "/ "
+    cmdstring += shlex.join(cmd)
+    if privnet:
+        cmd = ["unshare", "--net",
+               "sh", "-c", "ip link set dev lo up && " + cmdstring]
+        cmdstring = shlex.join(cmd)
+        if chroot:
+            cmdstring += ' exec "$1"'
+    return cmdstring
+
+def main() -> None:
+    """Entry point for mdbp-mmdebstrap backend"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--mirror", type=str, action="store",
+                        help="mirror url to fetch packages from")
+    parser.add_argument("buildjson", type=buildjson)
+    args = parser.parse_args()
+    build = args.buildjson
+
+    if build.get("lintian", {}).get("run"):
+        raise ValueError("running lintian not supported")
+    if build.get("bd-uninstallable-explainer") not in (None, "apt"):
+        raise ValueError("bd-uinstallable-explainer %r not supported" %
+                         build.get("bd-uinstallable-explainer"))
+
+    buildarch = build.get("buildarch") or \
+        subprocess.check_output(["dpkg", "--print-architecture"],
+                                encoding="ascii").strip()
+    hostarch = build.get("hostarch") or buildarch
+
+    if buildarch == hostarch:
+        buildessential = set(("build-essential",))
+    else:
+        buildessential = set(("crossbuild-essential-" + hostarch,
+                              "libc-dev:" + hostarch,
+                              "libstdc++-dev:" + hostarch))
+    buildpath = pathlib.PurePath(build.get("buildpath", "/build/build"))
+
+    with get_dsc(build) as dscpath, \
+         tempfile.NamedTemporaryFile("w+") as script:
+        dscfiles = [dscpath] + get_dsc_files(dscpath)
+
+        script.write("#!/bin/sh\nset -u\nset -e\n")
+        script.write("%s\n" % priv_drop(["chown", "-R", "build:build",
+                                         str(buildpath.parent)],
+                                        chroot=True))
+        cmd = ["apt-get", "build-dep", "--yes",
+               "--host-architecture", hostarch]
+        cmd.extend(dict(any=["--arch-only"],
+                        all=["--indep-only"]).get(build.get("type"), []))
+        cmd.extend(profile_option(build, "--build-profiles"))
+        cmd.append(str(buildpath.parent / dscpath.name))
+        if build.get("bd-uninstallable-explainer") == "apt":
+            script.write("if ! %s\nthen\n" % priv_drop(cmd, chroot=True))
+            cmd[-1:-1] = ['-oDebug::pkgProblemResolver=true',
+                          '-oDebug::pkgDepCache::Marker=1',
+                          '-oDebug::pkgDepCache::AutoInstall=1',
+                          '-oDebug::BuildDeps=1']
+            script.write("%s\nfi\n" % priv_drop(cmd, chroot=True))
+        else:
+            script.write("%s\n" % priv_drop(cmd, chroot=True))
+        script.write("%s\n" % priv_drop(
+            ["dpkg-source", "--no-check", "--extract", dscpath.name,
+             buildpath.name],
+            setuid="build", chroot=True, chdir=str(buildpath.parent)))
+        script.write("%s\n" % priv_drop(["rm"] + [f.name for f in dscfiles],
+                                        chroot=True,
+                                        chdir=str(buildpath.parent)))
+        cmd = ["dpkg-buildpackage", "-uc", "--host-arch=" + hostarch,
+               "--build=" + build.get("type", "binary")]
+        cmd.extend(profile_option(build, "--build-profiles="))
+        script.write("%s\n" % priv_drop(
+            cmd, chroot=True, setuid="build",
+            privnet=not build.get("network") in ("enable", "try-enable"),
+            chdir=str(buildpath), env=compute_env(build)))
+        script.write("%s\n" % priv_drop(["rm", "-R", str(buildpath)],
+                                        chroot=True))
+        # Only close the file object, script.close would delete it.
+        # Unfortunatly, mypy doesn't grok the indirection and thinks that
+        # script already is the file.
+        script.file.close() # type: ignore[attr-defined]
+        # World readable as the hook may be run by a different uid.
+        pathlib.Path(script.name).chmod(0o755)
+        cmd = [
+            "mmdebstrap",
+            "--verbose",
+            "--mode=unshare",
+            "--variant=apt",
+            "--architectures=" +
+                ",".join(dict.fromkeys((buildarch, hostarch))),
+            "--include=" + ",".join(buildessential),
+            '--customize-hook=chroot "$1" useradd --user-group --create-home '
+                '--home-dir %s build --skel /nonexistent' %
+                shlex.quote(str(buildpath.parent)),
+            "--customize-hook=copy-in %s %s" %
+                (shlex.join(map(str, dscfiles)),
+                 shlex.quote(str(buildpath.parent))),
+            "--customize-hook=" + script.name,
+            "--customize-hook=sync-out " +
+                shlex.join([str(buildpath.parent),
+                            build["output"]["directory"]]),
+            build["distribution"],
+            "/dev/null",
+            args.mirror or "http://deb.debian.org/debian",
+        ]
+        cmd.extend(build.get("extrarepositories", ()))
+        proc = subprocess.Popen(cmd)
+        proc.wait()
+
+if __name__ == "__main__":
+    main()
diff --git a/mdbp/pbuilder.py b/mdbp/pbuilder.py
new file mode 100644
index 0000000..51111e3
--- /dev/null
+++ b/mdbp/pbuilder.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+"""mdbp backend using pbuilder"""
+
+import argparse
+import os
+import pathlib
+import subprocess
+
+from .common import buildjson, compute_env, get_dsc, make_option, \
+        profile_option
+
+def main() -> None:
+    """Entry point for mdbp-pbuilder backend"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("buildjson", type=buildjson)
+    args = parser.parse_args()
+    build = args.buildjson
+
+    if build.get("lintian", {}).get("run"):
+        raise ValueError("running lintian not supported")
+    if build.get("bd-uinstallable-explainer"):
+        raise ValueError("bd-uninstallable-explainer %r not supported" %
+                         build.get("bd-uinstallable-explainer"))
+    if build.get("buildpath"):
+        raise ValueError("buildpath not supported")
+    if build["distribution"] in ("sid", "unstable"):
+        basetgz = None
+    else:
+        for pat in ("/var/cache/pbuilder/%s-base.tgz",
+                    "/var/cache/pbuidler/%s.tgz"):
+            basetgz = pat % build["distribution"]
+            if pathlib.Path(basetgz).is_file():
+                break
+        else:
+            raise ValueError("unsupported distribution %s" %
+                             build["distribution"])
+
+    cmd = []
+    if os.getuid() != 0:
+        cmd.extend(["sudo", "-E", "--"])
+    cmd.extend(["/usr/sbin/pbuilder", "build"])
+    cmd.extend(make_option("--basetgz", basetgz))
+    cmd.extend(make_option("--architecture", build.get("buildarch")))
+    cmd.extend(make_option("--host-arch", build.get("hostarch")))
+    cmd.extend(make_option("--othermirror",
+                           "|".join(build.get("extrarepositories", ()))))
+    cmd.extend(make_option("--use-network",
+        {"enable": "yes", "try-enable": "yes", "disable": "no",
+         "try-disable": "no"}.get(build.get("network"))))
+    cmd.extend(dict(any=["--binary-arch"],
+                    all=["--binary-indep"],
+                    binary=["--debbuildopts", "-b"])[
+                        build.get("type", "binary")])
+    cmd.extend(profile_option(build, "--profiles"))
+    cmd.extend(["--buildresult", build["output"]["directory"]])
+    with get_dsc(build) as dscpath:
+        cmd.append(str(dscpath))
+        proc = subprocess.Popen(cmd, env=compute_env(build))
+        proc.wait()
+
+if __name__ == "__main__":
+    main()
diff --git a/mdbp/sbuild.py b/mdbp/sbuild.py
new file mode 100644
index 0000000..6f1d75a
--- /dev/null
+++ b/mdbp/sbuild.py
@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+"""mdbp backend using sbuild"""
+
+import argparse
+import subprocess
+
+from .common import buildjson, compute_env, get_dsc, make_option, \
+        profile_option
+
+def main() -> None:
+    """Entry point for mdbp-sbuild backend"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("buildjson", type=buildjson)
+    args = parser.parse_args()
+    build = args.buildjson
+
+    if build.get("network") == "disable":
+        raise ValueError("disabling network not supported with sbuild")
+
+    cmd = [
+        "sbuild",
+        "--dist=" + build["distribution"],
+        "--no-arch-any" if build.get("type") == "all" else "--arch-any",
+        "--no-arch-all" if build.get("type") == "any" else "--arch-all",
+        "--bd-uninstallable-explainer=" +
+            (build.get("bd-uninstallable-explainer") or ""),
+        "--run-lintian" if build.get("lintian", {}).get("run") else
+            "--no-run-lintian",
+    ]
+    cmd.extend(make_option("--build=", build.get("buildarch")))
+    cmd.extend(make_option("--host=", build.get("hostarch")))
+    cmd.extend(map("--extra-repository=".__add__,
+                   build.get("extrarepositories", ())))
+    cmd.extend(profile_option(build, "--profiles="))
+    cmd.extend(make_option("--build-path=", build.get("buildpath")))
+    if build.get("network") == "try-disable":
+        cmd.extend([
+            "--starting-build-commands="
+                "mv /etc/resolv.conf /etc/resolv.conf.disabled",
+            "--finished-build-commands="
+                "mv /etc/resolv.conf.disabled /etc/resolv.conf",
+        ])
+    with get_dsc(build) as dscpath:
+        cmd.append(str(dscpath.absolute()))
+        proc = subprocess.Popen(cmd, env=compute_env(build),
+                                cwd=build["output"]["directory"])
+        proc.wait()
+
+if __name__ == "__main__":
+    main()