diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | LICENSE | 19 | ||||
| -rw-r--r-- | README.md | 65 | ||||
| -rw-r--r-- | TODO.md | 5 | ||||
| -rwxr-xr-x | checks.sh | 3 | ||||
| -rw-r--r-- | mdbp/__init__.py | 0 | ||||
| -rw-r--r-- | mdbp/build_schema.json | 129 | ||||
| -rw-r--r-- | mdbp/common.py | 135 | ||||
| -rw-r--r-- | mdbp/mmdebstrap.py | 152 | ||||
| -rw-r--r-- | mdbp/pbuilder.py | 63 | ||||
| -rw-r--r-- | mdbp/sbuild.py | 51 | ||||
| -rwxr-xr-x | setup.py | 27 |
12 files changed, 650 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ @@ -0,0 +1,19 @@ +Copyright (c) 2021 Helmut Grohne <helmut@subdivi.de> + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..603d157 --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +mdbp +==== + +A machine-friendly abstraction of build environments such as `sbuild` or +`pbuilder`. Primarily, it is an API. To make it useful, it also carries a +couple of implementations henceforth called backends. Each backend is supplied +by its own command. + +Usage +----- + +To use it, create a JSON file describing the build you intend to perform. There +is a schema describing the available parameters in `mdbp/build_schema.json`. An +example build request could be: + + { + "input": {"dscpath": "./hello_2.10-2.dsc"}, + "distribution": "unstable", + "output": {"directory": "out"} + } + +The output directory `out` is supposed to be an empty directory. Then run one +of the backends `mdbp-BACKEND [BACKENDOPTIONS] REQUEST.json`. The process +blocks until the build has finished and its return code signifies success or +failure of the operation. Usage of unsupported parameters of the particular +backend result in a failure of the operation as a whole. + +When to use mdbp? +----------------- + +The target audience performs many builds in a mechanical way. Typically, a +higher level application needs to perform Debian package builds. Multiple users +of such an application may prefer different backends. + +When not to use mdbp? +--------------------- + +Detailed customization and interaction are non-goals. If you need to spawn an +interactive shell during a failed build, using the underlying implementation is +better suited. + +mdbp-sbuild +----------- + +This backend uses `sbuild` to perform the actual build. It expects that +`sbuild` is set up and the user running the build has permission to do so. It +has no backend-specific options. + +mdbp-pbuilder +------------- + +This backend uses `pbuilder` to perform the actual build. Unless run as root, +it performs the build via `sudo`. It assumes that suitable tarballs exist for. +For distributions `unstable` and `sid`, it'll use the predefined one and for +any other it'll look for a `<distribution>-base.tgz` or `<distribution>.tgz` in +`/var/cache/pbuilder`. It has no backend-specific options. + +mdbp-mmdebstrap +--------------- + +This backend bootstraps a temporary chroot in a user namespace and performs the +build within. As such it requires working user namespaces including a subuid +allocation and the suid tool `newuidmap`. Unlike other backends, it does not +need a chroot or base to be set up before. Given that there is no external +state, it allows supplying the mirror to be used via the `--mirror` option. @@ -0,0 +1,5 @@ +* A build log should be supplied in a machine-consumable way. +* It should be requestable which build artifacts are to be retained. +* There should be a remote backend performing builds via ssh. +* There should be a backend supporting a container thingy (e.g. `debspawn`, + `debocker`, `whalebuilder`). diff --git a/checks.sh b/checks.sh new file mode 100755 index 0000000..3ea4b81 --- /dev/null +++ b/checks.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mypy --strict --ignore-missing-imports mdbp +pylint mdbp diff --git a/mdbp/__init__.py b/mdbp/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/mdbp/__init__.py diff --git a/mdbp/build_schema.json b/mdbp/build_schema.json new file mode 100644 index 0000000..b8d905d --- /dev/null +++ b/mdbp/build_schema.json @@ -0,0 +1,129 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "required": [ "input", "distribution", "output" ], + "properties": { + "input": { + "type": "object", + "oneOf": [ { + "required": [ "dscpath" ], + "additionalProperties": false, + "properties": { + "dscpath": { + "type": "string", + "description": "path to the .dsc file that is to be built, can be relative to the location of this json file" + } + } + }, { + "required": [ "dscuri" ], + "additionalProperties": false, + "properties": { + "dscuri": { + "type": "string", + "format": "uri", + "description": "uri for downloading the .dsc file" + }, + "checksums": { + "type": "object", + "patternProperties": { ".*": { "type": "string" } }, + "default": {}, + "description": "a mapping of checksum algorithms to the expected values" + } + } + } ] + }, + "distribution": { + "type": "string", + "pattern": "^[a-z0-9-]+$", + "description": "selects the base chroot used for building" + }, + "extrarepositories": { + "type": "array", + "items": { + "type": "string", + "pattern": "^(deb|deb-src) " + }, + "default": [], + "description": "extra repository specifications to be added to sources.list" + }, + "type": { + "type": "string", + "enum": [ "any", "all", "binary" ], + "default": "binary", + "description": "select an arch-only, indep-only or full build" + }, + "buildarch": { + "type": "string", + "minLength": 2, + "pattern": "^[a-z0-9-]+$", + "description": "build architecture, defaults to the native architecure" + }, + "hostarch": { + "type": "string", + "minLength": 2, + "pattern": "^[a-z0-9-]+$", + "description": "host architecture, defaults to the build architecture" + }, + "profiles": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-z0-9.-]+$" }, + "uniqueItems": true, + "default": [], + "description": "select build profiles to enabled" + }, + "options": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-z0-9.=_-]+$" }, + "uniqueItems": true, + "default": [], + "description": "values of DEB_BUILD_OPTIONS" + }, + "environment": { + "type": "object", + "propertyNames": { "pattern": "^[^=-][^=]*$" }, + "patternProperties": { ".*": { "type": "string" } }, + "default": [], + "description": "extra environment variables" + }, + "buildpath": { + "type": "string", + "description": "the path inside the chroot to peform the build" + }, + "lintian": { + "type": "object", + "properties": { + "run": { + "type": "boolean", + "default": false, + "description": "whether to run lintian after the build" + }, + "options": { + "type": "array", + "items": { "type": "string" }, + "default": [], + "description": "extra options to pass to lintian" + } + } + }, + "bd-uninstallable-explainer": { + "enum": [ null, "apt", "dose3" ], + "default": null, + "description": "when installing Build-Depends fails, an explainer can be used to give details" + }, + "network": { + "enum": [ "enable", "disable", "try-disable", "try-enable", "undefined" ], + "default": "undefined", + "description": "whether the build should be able to access the internet" + }, + "output": { + "type": "object", + "required": [ "directory" ], + "properties": { + "directory": { + "type": "string", + "description": "target directory to place output artifacts" + } + } + } + } +} diff --git a/mdbp/common.py b/mdbp/common.py new file mode 100644 index 0000000..d0d2f2d --- /dev/null +++ b/mdbp/common.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: MIT +"""Common functions used by multiple backends""" + +from __future__ import annotations +import argparse +import contextlib +import hashlib +import importlib.resources +import json +import pathlib +import tempfile +import typing +import urllib.parse + +import debian.deb822 +import requests + +try: + import jsonschema +except ImportError: + jsonschema = None + +def json_load(filecontextmanager: + typing.ContextManager[typing.IO[typing.AnyStr]]) -> typing.Any: + """Load the json context from a file context manager.""" + with filecontextmanager as fileobj: + return json.load(fileobj) + +JsonObject = typing.Dict[str, typing.Any] + +def buildjson(filename: str) -> JsonObject: + """Type constructor for argparse validating a build json file path and + returning the parsed json object.""" + buildobj = json_load(argparse.FileType("r")(filename)) + if jsonschema: + jsonschema.validate( + buildobj, + json_load( + importlib.resources.open_text("mdbp", "build_schema.json"))) + assert isinstance(buildobj, dict) + return buildobj + +def compute_env(build: JsonObject) -> typing.Dict[str, str]: + """Compute the process environment from the build object.""" + env = dict(PATH="/usr/bin:/bin") + env.update(build.get("environment", {})) + if build.get("options"): + env["DEB_BUILD_OPTIONS"] = " ".join(build["options"]) + return env + +class HashSumMismatch(Exception): + """Raised from `hash_check` when validation fails.""" + +# pylint does not grok from __future__ import annotations yet +# pylint: disable=E1101,W0212 +def hash_check(iterable: typing.Iterable[bytes], hashobj: hashlib._Hash, + expected_digest: str) -> \ + typing.Iterator[bytes]: + """Wraps an iterable that yields bytes. It doesn't modify the sequence, + but on the final element it verifies that the concatenation of bytes + yields an expected digest value. Upon failure, the final next() results in + a HashSumMismatch rather than StopIteration. + """ + for data in iterable: + hashobj.update(data) + yield data + if hashobj.hexdigest() != expected_digest: + raise HashSumMismatch() + +def download(uri: str, checksums: typing.Dict[str, str], + dest: pathlib.Path) -> None: + """Download the given uri and save it as the given dest path provided that + the given checksums match. When checksums do not match, raise a + HashSumMismatch. + """ + with requests.get(uri, stream=True) as resp: + resp.raise_for_status() + iterable = resp.iter_content(None) + for algo, csum in checksums.items(): + iterable = hash_check(iterable, hashlib.new(algo), csum) + try: + with dest.open("wb") as out: + for chunk in iterable: + out.write(chunk) + except HashSumMismatch: + dest.unlink() + raise + +@contextlib.contextmanager +def get_dsc(build: JsonObject) -> typing.Iterator[pathlib.Path]: + """A context manager that provides a path pointing at the .dsc file for the + duration of the context. If the .dsc is supplied as a path, it simply is + returned. If it is supplied as a uri, it and the referred components are + downloaded to a temporary location. + """ + try: + dscpath = build["input"]["dscpath"] + except KeyError: + dscuri = build["input"]["dscuri"] + with tempfile.TemporaryDirectory() as tdirname: + tdir = pathlib.Path(tdirname) + dscpath = tdir / dscuri.split("/")[-1] + download(dscuri, build["input"].get("checksums", {}), dscpath) + files: typing.Dict[str, typing.Dict[str, str]] = {} + with dscpath.open("r") as dscf: + for key, value in debian.deb822.Dsc(dscf).items(): + if key.lower().startswith("checksums-"): + for entry in value: + algo = key[10:].lower() + files.setdefault(entry["name"], dict())[algo] = \ + entry[algo] + for name, checksums in files.items(): + download(urllib.parse.urljoin(dscuri, name), checksums, + tdir / name) + yield dscpath + else: + yield pathlib.Path(dscpath) + +def get_dsc_files(dscpath: pathlib.Path) -> typing.List[pathlib.Path]: + """Get the component names referenced by the .dsc file.""" + with dscpath.open("r") as dscf: + dsc = debian.deb822.Dsc(dscf) + return [dscpath.parent / item["name"] for item in dsc["Files"]] + +def make_option(optname: str, value: typing.Optional[str]) -> typing.List[str]: + """Construct a valued option if a value is given.""" + if not value: + return [] + if optname.endswith("="): + return [optname + value] + return [optname, value] + +def profile_option(build: JsonObject, optname: str) -> typing.List[str]: + """Construct the option for specifying build profiles if required.""" + return make_option(optname, ",".join(build.get("profiles", ()))) diff --git a/mdbp/mmdebstrap.py b/mdbp/mmdebstrap.py new file mode 100644 index 0000000..df96e47 --- /dev/null +++ b/mdbp/mmdebstrap.py @@ -0,0 +1,152 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT +"""mdbp backend using mmdebstrap""" + +import argparse +import pathlib +import shlex +import subprocess +import tempfile +import typing + +from .common import buildjson, compute_env, get_dsc, get_dsc_files, \ + profile_option + +# pylint: disable=W0102 # as we do not modify env +def priv_drop(cmd: typing.List[str], *, chroot: bool = False, + chdir: typing.Optional[str] = None, + setuid: typing.Optional[str] = None, + privnet: bool = False, + env: typing.Dict[str, str] = {}) -> str: + """Returns shlext.join(cmd) with certain privilege dropping operations + applied: + * When chroot is True, chroot to "$1". + * When chdir is set, chdir to it after chrooting. Should be absolute. + * When setuid is set, change the user id and group id. Names allowed. + * When privnet is True, create a new network namespace with an active + loopback interface. + * Environment variables from env will be set. + """ + if chdir or env: + envcmd = ["env"] + if chdir: + envcmd.extend(["--chdir", chdir]) + envcmd.extend(map("%s=%s".__mod__, env.items())) + cmd = envcmd + cmd + cmdstring = "" + if setuid or chroot: + cmdstring = "chroot " + if setuid: + cmdstring += "--userspec %s " % \ + shlex.quote("%s:%s" % (setuid, setuid)) + cmdstring += '"$1" ' if chroot else "/ " + cmdstring += shlex.join(cmd) + if privnet: + cmd = ["unshare", "--net", + "sh", "-c", "ip link set dev lo up && " + cmdstring] + cmdstring = shlex.join(cmd) + if chroot: + cmdstring += ' exec "$1"' + return cmdstring + +def main() -> None: + """Entry point for mdbp-mmdebstrap backend""" + parser = argparse.ArgumentParser() + parser.add_argument("--mirror", type=str, action="store", + help="mirror url to fetch packages from") + parser.add_argument("buildjson", type=buildjson) + args = parser.parse_args() + build = args.buildjson + + if build.get("lintian", {}).get("run"): + raise ValueError("running lintian not supported") + if build.get("bd-uninstallable-explainer") not in (None, "apt"): + raise ValueError("bd-uinstallable-explainer %r not supported" % + build.get("bd-uinstallable-explainer")) + + buildarch = build.get("buildarch") or \ + subprocess.check_output(["dpkg", "--print-architecture"], + encoding="ascii").strip() + hostarch = build.get("hostarch") or buildarch + + if buildarch == hostarch: + buildessential = set(("build-essential",)) + else: + buildessential = set(("crossbuild-essential-" + hostarch, + "libc-dev:" + hostarch, + "libstdc++-dev:" + hostarch)) + buildpath = pathlib.PurePath(build.get("buildpath", "/build/build")) + + with get_dsc(build) as dscpath, \ + tempfile.NamedTemporaryFile("w+") as script: + dscfiles = [dscpath] + get_dsc_files(dscpath) + + script.write("#!/bin/sh\nset -u\nset -e\n") + script.write("%s\n" % priv_drop(["chown", "-R", "build:build", + str(buildpath.parent)], + chroot=True)) + cmd = ["apt-get", "build-dep", "--yes", + "--host-architecture", hostarch] + cmd.extend(dict(any=["--arch-only"], + all=["--indep-only"]).get(build.get("type"), [])) + cmd.extend(profile_option(build, "--build-profiles")) + cmd.append(str(buildpath.parent / dscpath.name)) + if build.get("bd-uninstallable-explainer") == "apt": + script.write("if ! %s\nthen\n" % priv_drop(cmd, chroot=True)) + cmd[-1:-1] = ['-oDebug::pkgProblemResolver=true', + '-oDebug::pkgDepCache::Marker=1', + '-oDebug::pkgDepCache::AutoInstall=1', + '-oDebug::BuildDeps=1'] + script.write("%s\nfi\n" % priv_drop(cmd, chroot=True)) + else: + script.write("%s\n" % priv_drop(cmd, chroot=True)) + script.write("%s\n" % priv_drop( + ["dpkg-source", "--no-check", "--extract", dscpath.name, + buildpath.name], + setuid="build", chroot=True, chdir=str(buildpath.parent))) + script.write("%s\n" % priv_drop(["rm"] + [f.name for f in dscfiles], + chroot=True, + chdir=str(buildpath.parent))) + cmd = ["dpkg-buildpackage", "-uc", "--host-arch=" + hostarch, + "--build=" + build.get("type", "binary")] + cmd.extend(profile_option(build, "--build-profiles=")) + script.write("%s\n" % priv_drop( + cmd, chroot=True, setuid="build", + privnet=not build.get("network") in ("enable", "try-enable"), + chdir=str(buildpath), env=compute_env(build))) + script.write("%s\n" % priv_drop(["rm", "-R", str(buildpath)], + chroot=True)) + # Only close the file object, script.close would delete it. + # Unfortunatly, mypy doesn't grok the indirection and thinks that + # script already is the file. + script.file.close() # type: ignore[attr-defined] + # World readable as the hook may be run by a different uid. + pathlib.Path(script.name).chmod(0o755) + cmd = [ + "mmdebstrap", + "--verbose", + "--mode=unshare", + "--variant=apt", + "--architectures=" + + ",".join(dict.fromkeys((buildarch, hostarch))), + "--include=" + ",".join(buildessential), + '--customize-hook=chroot "$1" useradd --user-group --create-home ' + '--home-dir %s build --skel /nonexistent' % + shlex.quote(str(buildpath.parent)), + "--customize-hook=copy-in %s %s" % + (shlex.join(map(str, dscfiles)), + shlex.quote(str(buildpath.parent))), + "--customize-hook=" + script.name, + "--customize-hook=sync-out " + + shlex.join([str(buildpath.parent), + build["output"]["directory"]]), + build["distribution"], + "/dev/null", + args.mirror or "http://deb.debian.org/debian", + ] + cmd.extend(build.get("extrarepositories", ())) + proc = subprocess.Popen(cmd) + proc.wait() + +if __name__ == "__main__": + main() diff --git a/mdbp/pbuilder.py b/mdbp/pbuilder.py new file mode 100644 index 0000000..51111e3 --- /dev/null +++ b/mdbp/pbuilder.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT +"""mdbp backend using pbuilder""" + +import argparse +import os +import pathlib +import subprocess + +from .common import buildjson, compute_env, get_dsc, make_option, \ + profile_option + +def main() -> None: + """Entry point for mdbp-pbuilder backend""" + parser = argparse.ArgumentParser() + parser.add_argument("buildjson", type=buildjson) + args = parser.parse_args() + build = args.buildjson + + if build.get("lintian", {}).get("run"): + raise ValueError("running lintian not supported") + if build.get("bd-uinstallable-explainer"): + raise ValueError("bd-uninstallable-explainer %r not supported" % + build.get("bd-uinstallable-explainer")) + if build.get("buildpath"): + raise ValueError("buildpath not supported") + if build["distribution"] in ("sid", "unstable"): + basetgz = None + else: + for pat in ("/var/cache/pbuilder/%s-base.tgz", + "/var/cache/pbuidler/%s.tgz"): + basetgz = pat % build["distribution"] + if pathlib.Path(basetgz).is_file(): + break + else: + raise ValueError("unsupported distribution %s" % + build["distribution"]) + + cmd = [] + if os.getuid() != 0: + cmd.extend(["sudo", "-E", "--"]) + cmd.extend(["/usr/sbin/pbuilder", "build"]) + cmd.extend(make_option("--basetgz", basetgz)) + cmd.extend(make_option("--architecture", build.get("buildarch"))) + cmd.extend(make_option("--host-arch", build.get("hostarch"))) + cmd.extend(make_option("--othermirror", + "|".join(build.get("extrarepositories", ())))) + cmd.extend(make_option("--use-network", + {"enable": "yes", "try-enable": "yes", "disable": "no", + "try-disable": "no"}.get(build.get("network")))) + cmd.extend(dict(any=["--binary-arch"], + all=["--binary-indep"], + binary=["--debbuildopts", "-b"])[ + build.get("type", "binary")]) + cmd.extend(profile_option(build, "--profiles")) + cmd.extend(["--buildresult", build["output"]["directory"]]) + with get_dsc(build) as dscpath: + cmd.append(str(dscpath)) + proc = subprocess.Popen(cmd, env=compute_env(build)) + proc.wait() + +if __name__ == "__main__": + main() diff --git a/mdbp/sbuild.py b/mdbp/sbuild.py new file mode 100644 index 0000000..6f1d75a --- /dev/null +++ b/mdbp/sbuild.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT +"""mdbp backend using sbuild""" + +import argparse +import subprocess + +from .common import buildjson, compute_env, get_dsc, make_option, \ + profile_option + +def main() -> None: + """Entry point for mdbp-sbuild backend""" + parser = argparse.ArgumentParser() + parser.add_argument("buildjson", type=buildjson) + args = parser.parse_args() + build = args.buildjson + + if build.get("network") == "disable": + raise ValueError("disabling network not supported with sbuild") + + cmd = [ + "sbuild", + "--dist=" + build["distribution"], + "--no-arch-any" if build.get("type") == "all" else "--arch-any", + "--no-arch-all" if build.get("type") == "any" else "--arch-all", + "--bd-uninstallable-explainer=" + + (build.get("bd-uninstallable-explainer") or ""), + "--run-lintian" if build.get("lintian", {}).get("run") else + "--no-run-lintian", + ] + cmd.extend(make_option("--build=", build.get("buildarch"))) + cmd.extend(make_option("--host=", build.get("hostarch"))) + cmd.extend(map("--extra-repository=".__add__, + build.get("extrarepositories", ()))) + cmd.extend(profile_option(build, "--profiles=")) + cmd.extend(make_option("--build-path=", build.get("buildpath"))) + if build.get("network") == "try-disable": + cmd.extend([ + "--starting-build-commands=" + "mv /etc/resolv.conf /etc/resolv.conf.disabled", + "--finished-build-commands=" + "mv /etc/resolv.conf.disabled /etc/resolv.conf", + ]) + with get_dsc(build) as dscpath: + cmd.append(str(dscpath.absolute())) + proc = subprocess.Popen(cmd, env=compute_env(build), + cwd=build["output"]["directory"]) + proc.wait() + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..265ecf2 --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT + +from setuptools import setup + +setup(name="mdbp", + description="dpkg-buildpackage wrapper", + packages="mdbp", + classifiers=[ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX :: Linux", + "Topic :: Software Development :: Build Tools", + ], + install_requires=[ + "debian", + "jsonschema", + "requests", + ], + entry_points=dict( + console_scripts=[ + "mdbp-mmdebstrap=mdbp.mmdebstrap:main", + "mdbp-pbuilder=mdbp.pbuilder:main", + "mdbp-sbuild=mdbp.sbuild:main", + ] + ) +) |