add ssh remote backend and streamapi wrapper

The ssh backend calls into another host and passes all information on
stdin/stdout/stderr. To do so, it uses a streamapi wrapper on the remote
side that itself calls into a regular backend.

5 files changed, 178 insertions, 7 deletions

diff --git a/TODO.md b/TODO.md
index fbcd688..1407dc8 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,4 +1,3 @@
 * It should be requestable which build artifacts are to be retained.
-* There should be a remote backend performing builds via ssh.
 * There should be a backend supporting a container thingy (e.g. `debspawn`,
   `debocker`, `whalebuilder`).
diff --git a/mdbp/common.py b/mdbp/common.py
index 097c7a3..830aad7 100644
--- a/mdbp/common.py
+++ b/mdbp/common.py
@@ -9,6 +9,7 @@ import importlib.resources
 import json
 import multiprocessing
 import pathlib
+import tarfile
 import tempfile
 import typing
 import urllib.parse
@@ -29,15 +30,21 @@ def json_load(filecontextmanager:
 
 JsonObject = typing.Dict[str, typing.Any]
 
-def buildjson(filename: str) -> JsonObject:
-    """Type constructor for argparse validating a build json file path and
-    returning the parsed json object."""
-    buildobj = json_load(argparse.FileType("r")(filename))
+def buildjson_validate(buildobj: JsonObject) -> None:
+    """Validate the given build json object against the schema."""
     if jsonschema:
         jsonschema.validate(
             buildobj,
             json_load(
                 importlib.resources.open_text("mdbp", "build_schema.json")))
+
+def buildjson_patch_relative(buildobj: JsonObject,
+                             basedir: pathlib.PurePath) -> None:
+    """Resolve relative paths used in the buildobj using the given basedir:
+     * .input.dscpath
+     * .output.directory
+    The operation is performed in-place and modifes the given buildobj.
+    """
     for attrs in (("input", "dscpath"), ("output", "directory")):
         obj = buildobj
         for attr in attrs[:-1]:
@@ -47,10 +54,16 @@ def buildjson(filename: str) -> JsonObject:
                 break
         else:
             try:
-                obj[attrs[-1]] = str(pathlib.Path(filename).parent /
-                                     pathlib.Path(obj[attrs[-1]]))
+                obj[attrs[-1]] = str(basedir / pathlib.Path(obj[attrs[-1]]))
             except KeyError:
                 pass
+
+def buildjson(filename: str) -> JsonObject:
+    """Type constructor for argparse validating a build json file path and
+    returning the parsed json object."""
+    buildobj = json_load(argparse.FileType("r")(filename))
+    buildjson_validate(buildobj)
+    buildjson_patch_relative(buildobj, pathlib.Path(filename).parent)
     assert isinstance(buildobj, dict)
     return buildobj
 
@@ -153,3 +166,14 @@ def make_option(optname: str, value: typing.Optional[str]) -> typing.List[str]:
 def profile_option(build: JsonObject, optname: str) -> typing.List[str]:
     """Construct the option for specifying build profiles if required."""
     return make_option(optname, ",".join(build.get("profiles", ())))
+
+def tar_add(tarobj: tarfile.TarFile, path: pathlib.Path) -> None:
+    """Add the given file as its basename to the tarobj retaining its
+    modification time, but no mode or ownership information.
+    """
+    info = tarfile.TarInfo(path.name)
+    statres = path.stat()
+    info.size = statres.st_size
+    info.mtime = int(statres.st_mtime)
+    with path.open("rb") as fobj:
+        tarobj.addfile(info, fobj)
diff --git a/mdbp/ssh.py b/mdbp/ssh.py
new file mode 100644
index 0000000..8af6bde
--- /dev/null
+++ b/mdbp/ssh.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+"""mdbp backend wrapper via ssh"""
+
+import argparse
+import io
+import json
+import pathlib
+import subprocess
+import sys
+import tarfile
+import typing
+
+from .common import JsonObject, buildjson, get_dsc_files, tar_add
+
+def produce_request_tar(buildjsonobj: JsonObject,
+                        fileobj: typing.IO[bytes]) -> None:
+    """Write a tar file suitable for mdbp-streamapi into the given `fileobj`
+    based on the given `buildjsonobj`.
+     * An .output.directory is discarded.
+     * A referenced .dsc file and its components is included.
+    """
+    sendjsonobj = buildjsonobj.copy()
+    sendjsonobj["output"] = sendjsonobj["output"].copy()
+    del sendjsonobj["output"]["directory"]
+    dscpath: typing.Optional[pathlib.Path]
+    try:
+        dscpath = pathlib.Path(buildjsonobj["input"]["dscpath"])
+    except KeyError:
+        dscpath = None
+    else:
+        sendjsonobj["input"] = sendjsonobj["input"].copy()
+        sendjsonobj["input"]["dscpath"] = dscpath.name
+    tar = tarfile.open(mode="w|", fileobj=fileobj)
+    info = tarfile.TarInfo("build.json")
+    sendjsonfile = io.BytesIO()
+    for chunk in json.JSONEncoder().iterencode(sendjsonobj):
+        sendjsonfile.write(chunk.encode("utf8"))
+    info.size = sendjsonfile.tell()
+    sendjsonfile.seek(0)
+    tar.addfile(info, sendjsonfile)
+    if dscpath:
+        for path in [dscpath] + get_dsc_files(dscpath):
+            tar_add(tar, path)
+
+def main() -> None:
+    """Entry point for mdbp-ssh backend"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("host", type=str)
+    parser.add_argument("command", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    if len(args.command) < 2:
+        parser.error("missing command or json file")
+    build = buildjson(args.command.pop())
+
+    cmd = ["ssh", args.host, "mdbp-streamapi", *args.command]
+    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                            stderr=sys.stdout
+                                   if build["output"].get("log", True)
+                                   else subprocess.DEVNULL)
+    assert proc.stdin is not None
+    produce_request_tar(build, proc.stdin)
+    proc.stdin.close()
+    with tarfile.open(fileobj=proc.stdout, mode="r|") as outtar:
+        for member in outtar:
+            if "/" in member.name or not member.isfile():
+                raise ValueError("expected flat tar as output")
+            outtar.extract(member, build["output"]["directory"],
+                           set_attrs=False)
+    sys.exit(proc.wait())
+
+if __name__ == "__main__":
+    main()
diff --git a/mdbp/streamapi.py b/mdbp/streamapi.py
new file mode 100644
index 0000000..8f67366
--- /dev/null
+++ b/mdbp/streamapi.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: MIT
+"""mdbp frontend without filesystem interaction. It basically wraps another
+backend and is used as frontend from the ssh backend on the remote side.
+Differences to the regular backend API:
+ * It expects an uncompressed tar file on stdin. The first member must be named
+   "build.json". Any other members must be regular files. The build.json file
+   should lack the .output.directory and if an .input.dscpath is given, it
+   should not contain any slashes.
+ * The build log is issued on stderr instead of stdout.
+ * All the requested artifacts are emitted as a tar stream on stdout.
+"""
+
+import argparse
+import json
+import pathlib
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+from .common import json_load, buildjson_validate, buildjson_patch_relative, \
+        tar_add
+
+def main() -> None:
+    """Entry point for mdbp-streamapi wrapper"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("command", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    if not args.command:
+        parser.error("missing command")
+
+    with tempfile.TemporaryDirectory() as tdirname:
+        indir = pathlib.Path(tdirname) / "input"
+        outdir = pathlib.Path(tdirname) / "output"
+        indir.mkdir()
+        outdir.mkdir()
+        with tarfile.open(fileobj=sys.stdin.buffer, mode="r|") as intar:
+            seenjson = False
+            for member in intar:
+                if "/" in member.name or not member.isfile():
+                    raise ValueError("expected flat tar as input")
+                if seenjson:
+                    intar.extract(member, indir, set_attrs=False)
+                    continue
+                if member.name != "build.json":
+                    raise ValueError("first input member must be build.json")
+                jsonfileobj = intar.extractfile(member)
+                # We already checked .isfile(), but mypy doesn't know.
+                assert jsonfileobj is not None
+                build = json_load(jsonfileobj)
+                build["output"]["directory"] = str(outdir)
+                buildjson_validate(build)
+                buildjson_patch_relative(build, indir)
+                (indir / "build.json").write_text(json.dumps(build))
+                seenjson = True
+            if not seenjson:
+                raise ValueError("input is an empty tar archive")
+        proc = subprocess.Popen([*args.command, str(indir / "build.json")],
+                                stdout=sys.stderr
+                                       if build["output"].get("log", True)
+                                       else subprocess.DEVNULL,
+                                stderr=None if build["output"].get("log", True)
+                                       else subprocess.DEVNULL)
+        code = proc.wait()
+        if code != 0:
+            sys.exit(code)
+        with tarfile.open(fileobj=sys.stdout.buffer, mode="w|") as outtar:
+            for elem in outdir.iterdir():
+                tar_add(outtar, elem)
+
+if __name__ == "__main__":
+    main()
diff --git a/setup.py b/setup.py
index 265ecf2..eac91ef 100755
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,8 @@ setup(name="mdbp",
               "mdbp-mmdebstrap=mdbp.mmdebstrap:main",
               "mdbp-pbuilder=mdbp.pbuilder:main",
               "mdbp-sbuild=mdbp.sbuild:main",
+              "mdbp-ssh=mdbp.ssh:main",
+              "mdbp-streamapi=mdbp.streamapi:main",
           ]
       )
 )