From 9faa39b400a155c9dddeea4fe712301462093d4f Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 11 May 2021 14:52:29 +0200 Subject: add ssh remote backend and streamapi wrapper The ssh backend calls into another host and passes all information on stdin/stdout/stderr. To do so, it uses a streamapi wrapper on the remote side that itself calls into a regular backend. --- TODO.md | 1 - mdbp/common.py | 36 ++++++++++++++++++++++----- mdbp/ssh.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ mdbp/streamapi.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 ++ 5 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 mdbp/ssh.py create mode 100644 mdbp/streamapi.py diff --git a/TODO.md b/TODO.md index fbcd688..1407dc8 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,3 @@ * It should be requestable which build artifacts are to be retained. -* There should be a remote backend performing builds via ssh. * There should be a backend supporting a container thingy (e.g. `debspawn`, `debocker`, `whalebuilder`). diff --git a/mdbp/common.py b/mdbp/common.py index 097c7a3..830aad7 100644 --- a/mdbp/common.py +++ b/mdbp/common.py @@ -9,6 +9,7 @@ import importlib.resources import json import multiprocessing import pathlib +import tarfile import tempfile import typing import urllib.parse @@ -29,15 +30,21 @@ def json_load(filecontextmanager: JsonObject = typing.Dict[str, typing.Any] -def buildjson(filename: str) -> JsonObject: - """Type constructor for argparse validating a build json file path and - returning the parsed json object.""" - buildobj = json_load(argparse.FileType("r")(filename)) +def buildjson_validate(buildobj: JsonObject) -> None: + """Validate the given build json object against the schema.""" if jsonschema: jsonschema.validate( buildobj, json_load( importlib.resources.open_text("mdbp", "build_schema.json"))) + +def buildjson_patch_relative(buildobj: JsonObject, + basedir: pathlib.PurePath) -> None: + """Resolve relative paths used in the buildobj using the given basedir: + * .input.dscpath + * .output.directory + The operation is performed in-place and modifes the given buildobj. + """ for attrs in (("input", "dscpath"), ("output", "directory")): obj = buildobj for attr in attrs[:-1]: @@ -47,10 +54,16 @@ def buildjson(filename: str) -> JsonObject: break else: try: - obj[attrs[-1]] = str(pathlib.Path(filename).parent / - pathlib.Path(obj[attrs[-1]])) + obj[attrs[-1]] = str(basedir / pathlib.Path(obj[attrs[-1]])) except KeyError: pass + +def buildjson(filename: str) -> JsonObject: + """Type constructor for argparse validating a build json file path and + returning the parsed json object.""" + buildobj = json_load(argparse.FileType("r")(filename)) + buildjson_validate(buildobj) + buildjson_patch_relative(buildobj, pathlib.Path(filename).parent) assert isinstance(buildobj, dict) return buildobj @@ -153,3 +166,14 @@ def make_option(optname: str, value: typing.Optional[str]) -> typing.List[str]: def profile_option(build: JsonObject, optname: str) -> typing.List[str]: """Construct the option for specifying build profiles if required.""" return make_option(optname, ",".join(build.get("profiles", ()))) + +def tar_add(tarobj: tarfile.TarFile, path: pathlib.Path) -> None: + """Add the given file as its basename to the tarobj retaining its + modification time, but no mode or ownership information. + """ + info = tarfile.TarInfo(path.name) + statres = path.stat() + info.size = statres.st_size + info.mtime = int(statres.st_mtime) + with path.open("rb") as fobj: + tarobj.addfile(info, fobj) diff --git a/mdbp/ssh.py b/mdbp/ssh.py new file mode 100644 index 0000000..8af6bde --- /dev/null +++ b/mdbp/ssh.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT +"""mdbp backend wrapper via ssh""" + +import argparse +import io +import json +import pathlib +import subprocess +import sys +import tarfile +import typing + +from .common import JsonObject, buildjson, get_dsc_files, tar_add + +def produce_request_tar(buildjsonobj: JsonObject, + fileobj: typing.IO[bytes]) -> None: + """Write a tar file suitable for mdbp-streamapi into the given `fileobj` + based on the given `buildjsonobj`. + * An .output.directory is discarded. + * A referenced .dsc file and its components is included. + """ + sendjsonobj = buildjsonobj.copy() + sendjsonobj["output"] = sendjsonobj["output"].copy() + del sendjsonobj["output"]["directory"] + dscpath: typing.Optional[pathlib.Path] + try: + dscpath = pathlib.Path(buildjsonobj["input"]["dscpath"]) + except KeyError: + dscpath = None + else: + sendjsonobj["input"] = sendjsonobj["input"].copy() + sendjsonobj["input"]["dscpath"] = dscpath.name + tar = tarfile.open(mode="w|", fileobj=fileobj) + info = tarfile.TarInfo("build.json") + sendjsonfile = io.BytesIO() + for chunk in json.JSONEncoder().iterencode(sendjsonobj): + sendjsonfile.write(chunk.encode("utf8")) + info.size = sendjsonfile.tell() + sendjsonfile.seek(0) + tar.addfile(info, sendjsonfile) + if dscpath: + for path in [dscpath] + get_dsc_files(dscpath): + tar_add(tar, path) + +def main() -> None: + """Entry point for mdbp-ssh backend""" + parser = argparse.ArgumentParser() + parser.add_argument("host", type=str) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if len(args.command) < 2: + parser.error("missing command or json file") + build = buildjson(args.command.pop()) + + cmd = ["ssh", args.host, "mdbp-streamapi", *args.command] + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=sys.stdout + if build["output"].get("log", True) + else subprocess.DEVNULL) + assert proc.stdin is not None + produce_request_tar(build, proc.stdin) + proc.stdin.close() + with tarfile.open(fileobj=proc.stdout, mode="r|") as outtar: + for member in outtar: + if "/" in member.name or not member.isfile(): + raise ValueError("expected flat tar as output") + outtar.extract(member, build["output"]["directory"], + set_attrs=False) + sys.exit(proc.wait()) + +if __name__ == "__main__": + main() diff --git a/mdbp/streamapi.py b/mdbp/streamapi.py new file mode 100644 index 0000000..8f67366 --- /dev/null +++ b/mdbp/streamapi.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: MIT +"""mdbp frontend without filesystem interaction. It basically wraps another +backend and is used as frontend from the ssh backend on the remote side. +Differences to the regular backend API: + * It expects an uncompressed tar file on stdin. The first member must be named + "build.json". Any other members must be regular files. The build.json file + should lack the .output.directory and if an .input.dscpath is given, it + should not contain any slashes. + * The build log is issued on stderr instead of stdout. + * All the requested artifacts are emitted as a tar stream on stdout. +""" + +import argparse +import json +import pathlib +import subprocess +import sys +import tarfile +import tempfile + +from .common import json_load, buildjson_validate, buildjson_patch_relative, \ + tar_add + +def main() -> None: + """Entry point for mdbp-streamapi wrapper""" + parser = argparse.ArgumentParser() + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if not args.command: + parser.error("missing command") + + with tempfile.TemporaryDirectory() as tdirname: + indir = pathlib.Path(tdirname) / "input" + outdir = pathlib.Path(tdirname) / "output" + indir.mkdir() + outdir.mkdir() + with tarfile.open(fileobj=sys.stdin.buffer, mode="r|") as intar: + seenjson = False + for member in intar: + if "/" in member.name or not member.isfile(): + raise ValueError("expected flat tar as input") + if seenjson: + intar.extract(member, indir, set_attrs=False) + continue + if member.name != "build.json": + raise ValueError("first input member must be build.json") + jsonfileobj = intar.extractfile(member) + # We already checked .isfile(), but mypy doesn't know. + assert jsonfileobj is not None + build = json_load(jsonfileobj) + build["output"]["directory"] = str(outdir) + buildjson_validate(build) + buildjson_patch_relative(build, indir) + (indir / "build.json").write_text(json.dumps(build)) + seenjson = True + if not seenjson: + raise ValueError("input is an empty tar archive") + proc = subprocess.Popen([*args.command, str(indir / "build.json")], + stdout=sys.stderr + if build["output"].get("log", True) + else subprocess.DEVNULL, + stderr=None if build["output"].get("log", True) + else subprocess.DEVNULL) + code = proc.wait() + if code != 0: + sys.exit(code) + with tarfile.open(fileobj=sys.stdout.buffer, mode="w|") as outtar: + for elem in outdir.iterdir(): + tar_add(outtar, elem) + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 265ecf2..eac91ef 100755 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ setup(name="mdbp", "mdbp-mmdebstrap=mdbp.mmdebstrap:main", "mdbp-pbuilder=mdbp.pbuilder:main", "mdbp-sbuild=mdbp.sbuild:main", + "mdbp-ssh=mdbp.ssh:main", + "mdbp-streamapi=mdbp.streamapi:main", ] ) ) -- cgit v1.2.3