diff --git a/NEWS b/NEWS index a314699..433ab15 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,15 @@ IMPROVEMENTS * Stop installing tests. (Jelmer Vernooij) + * Add ``subunit-combine`` script that runs multiple subunit-producing + commands and merges their streams, optionally prefixing each command's + test ids. (Jelmer Vernooij, #2150097) + + * ``subunit-combine`` supports testr-style ``$LISTOPT``, ``$IDOPTION``, + ``$IDFILE`` and ``$IDLIST`` placeholders in command argv, along with + ``--list``, ``--load-list`` and positional test ids for per-command + filtering. (Jelmer Vernooij) + 1.4.5 (2025-11-10) --------------------- diff --git a/pyproject.toml b/pyproject.toml index c6f3423..b9295c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Programming Language :: Python", "Topic :: Software Development :: Testing", ] -dependencies = ["iso8601", "testtools>=2.7"] +dependencies = ["iso8601", "PyYAML", "testtools>=2.7"] description = "Python implementation of subunit test streaming protocol" dynamic = ["version"] keywords = ["python", "streaming", "test"] @@ -54,6 +54,7 @@ where = ["python"] [project.scripts] "subunit-1to2" = "subunit.filter_scripts.subunit_1to2:main" "subunit-2to1" = "subunit.filter_scripts.subunit_2to1:main" +"subunit-combine" = "subunit.filter_scripts.subunit_combine:main" "subunit-filter" = "subunit.filter_scripts.subunit_filter:main" "subunit-ls" = "subunit.filter_scripts.subunit_ls:main" "subunit-notify" = "subunit.filter_scripts.subunit_notify:main" diff --git a/python/subunit/filter_scripts/subunit_combine.py b/python/subunit/filter_scripts/subunit_combine.py new file mode 100644 index 0000000..04f18cb --- /dev/null +++ b/python/subunit/filter_scripts/subunit_combine.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +# subunit: extensions to python unittest to get test results from subprocesses. +# Copyright (C) 2026 Jelmer Vernooij +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + +"""Run multiple commands producing subunit v2 output and merge their streams. + +The commands to run are described in a YAML configuration file. Each command +may have a test id ``prefix`` prepended to all test ids it emits, which makes +it possible to combine output from, for example, a Python and a Rust test +suite without test id collisions. + +Example configuration:: + + commands: + - prefix: "python/" + argv: ["python", "-m", "subunit.run", "discover", ".", "$LISTOPT", "$IDOPTION"] + list_option: "--list" + id_option: "--load-list $IDFILE" + - prefix: "rust/" + argv: ["cargo", "test", "--", "--format=subunit"] + cwd: "rust" + +The combined subunit v2 stream is written to stdout. The exit code is 0 if +all commands exited 0, 1 otherwise. + +testr-style substitutions +------------------------- + +Each command's ``argv``, ``list_option`` and ``id_option`` entries may contain +the following placeholders, which are expanded per invocation: + +* ``$LISTOPT`` -- expands to ``list_option`` when ``subunit-combine --list`` + is used, empty otherwise. +* ``$IDLIST`` -- space-separated list of (prefix-stripped) test ids to run + for this command. Empty if no ids were requested or none match. +* ``$IDFILE`` -- path to a temporary file containing one test id per line. +* ``$IDOPTION`` -- expands to ``id_option`` (with ``$IDFILE`` further + substituted) when ids are being supplied, empty otherwise. + +Test ids can be passed as positional arguments after the config file or via +``--load-list FILE``. Ids that start with a command's ``prefix`` are routed +(with the prefix stripped) to that command; ids that don't match any prefix +are ignored for that command. +""" + +import os +import re +import subprocess +import sys +import tempfile +from argparse import ArgumentParser +from typing import Optional + +import yaml + +from subunit import ByteStreamToStreamResult, StreamResultToBytes + + +_VARIABLE_RE = re.compile(r"\$(IDOPTION|IDFILE|IDLIST|LISTOPT)") + + +class _PrefixingStreamResult: + """Forward StreamResult events, prepending ``prefix`` to every test_id.""" + + def __init__(self, target, prefix: str): + self._target = target + self._prefix = prefix + + def startTestRun(self): + self._target.startTestRun() + + def stopTestRun(self): + self._target.stopTestRun() + + def status(self, test_id=None, **kwargs): + if test_id is not None: + test_id = self._prefix + test_id + self._target.status(test_id=test_id, **kwargs) + + +def load_config(path: str) -> list[dict]: + """Load and validate a combine configuration file. + + Returns a list of command dictionaries. + """ + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise ValueError(f"{path}: top-level configuration must be a mapping") + commands = data.get("commands") + if not isinstance(commands, list) or not commands: + raise ValueError(f"{path}: 'commands' must be a non-empty list") + for i, cmd in enumerate(commands): + if not isinstance(cmd, dict): + raise ValueError(f"{path}: commands[{i}] must be a mapping") + argv = cmd.get("argv") + if not isinstance(argv, list) or not argv or not all(isinstance(a, str) for a in argv): + raise ValueError(f"{path}: commands[{i}].argv must be a non-empty list of strings") + prefix = cmd.get("prefix", "") + if not isinstance(prefix, str): + raise ValueError(f"{path}: commands[{i}].prefix must be a string") + for key in ("list_option", "id_option"): + value = cmd.get(key) + if value is not None and not isinstance(value, str): + raise ValueError(f"{path}: commands[{i}].{key} must be a string") + return commands + + +def _substitute(template: str, variables: dict[str, str]) -> list[str]: + """Substitute $VAR placeholders in ``template`` and return a shell-split list. + + Empty values are expanded to the empty string; the resulting string is then + split on whitespace so that e.g. an empty ``$LISTOPT`` disappears rather + than leaving an empty argument behind. + """ + + def repl(match: re.Match) -> str: + return variables.get(match.group(1), "") + + return re.sub(_VARIABLE_RE, repl, template).split() + + +def _expand_argv( + cmd: dict, + *, + list_mode: bool, + test_ids: Optional[list[str]], + idfile_path: Optional[str], +) -> list[str]: + """Expand testr-style placeholders in ``cmd['argv']``.""" + list_option = cmd.get("list_option", "") if list_mode else "" + id_option_template = cmd.get("id_option", "") + if test_ids is None or not id_option_template: + id_option = "" + else: + id_option = re.sub( + _VARIABLE_RE, + lambda m: {"IDFILE": idfile_path or "", "IDLIST": " ".join(test_ids)}.get(m.group(1), ""), + id_option_template, + ) + variables = { + "LISTOPT": list_option, + "IDOPTION": id_option, + "IDFILE": idfile_path or "", + "IDLIST": " ".join(test_ids) if test_ids else "", + } + expanded: list[str] = [] + for piece in cmd["argv"]: + if _VARIABLE_RE.search(piece): + expanded.extend(_substitute(piece, variables)) + else: + expanded.append(piece) + return expanded + + +def _select_ids_for_command(cmd: dict, test_ids: Optional[list[str]]) -> Optional[list[str]]: + """Return the ids that belong to ``cmd`` (with the prefix stripped). + + Returns None when no filtering should be applied (no ids were requested + globally). + """ + if test_ids is None: + return None + prefix = cmd.get("prefix", "") + if not prefix: + return list(test_ids) + return [tid[len(prefix) :] for tid in test_ids if tid.startswith(prefix)] + + +def _write_idfile(test_ids: list[str]) -> str: + fd, path = tempfile.mkstemp(prefix="subunit-combine-", suffix=".list") + with os.fdopen(fd, "w") as f: + for tid in test_ids: + f.write(tid + "\n") + return path + + +def run_command( + cmd: dict, + output, + *, + list_mode: bool = False, + test_ids: Optional[list[str]] = None, +) -> int: + """Run a single command and forward its subunit v2 output. + + The child's stdout is parsed as subunit v2 and re-emitted to ``output`` + (a :class:`StreamResultToBytes`) with each test_id prefixed with + ``cmd['prefix']`` (if any). + + :param list_mode: If True, ``$LISTOPT`` is substituted with the command's + ``list_option`` so the child lists tests rather than running them. + :param test_ids: If not None, the list of (prefix-stripped) test ids to + supply to the child via ``$IDLIST`` / ``$IDOPTION`` / ``$IDFILE``. + :return: The exit code of the child process. + """ + prefix = cmd.get("prefix", "") + cwd = cmd.get("cwd") + env = os.environ.copy() + extra_env = cmd.get("env") + if extra_env: + env.update(extra_env) + + idfile_path: Optional[str] = None + if test_ids is not None and test_ids: + idfile_path = _write_idfile(test_ids) + + try: + argv = _expand_argv( + cmd, + list_mode=list_mode, + test_ids=test_ids, + idfile_path=idfile_path, + ) + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, cwd=cwd, env=env) + try: + assert proc.stdout is not None + result = _PrefixingStreamResult(output, prefix) if prefix else output + ByteStreamToStreamResult(proc.stdout, non_subunit_name="stdout").run(result) + finally: + returncode = proc.wait() + finally: + if idfile_path is not None: + try: + os.unlink(idfile_path) + except OSError: + pass + return returncode + + +def combine( + commands: list[dict], + output_stream, + *, + list_mode: bool = False, + test_ids: Optional[list[str]] = None, +) -> int: + """Run ``commands`` and merge their subunit streams into ``output_stream``. + + :param list_mode: Run each command in listing mode (``$LISTOPT`` expanded). + :param test_ids: Optional list of test ids to restrict execution to. + Each command only sees ids whose prefix matches; commands with no + matching ids are skipped entirely. + :return: 0 if every command exited 0, 1 otherwise. + """ + output = StreamResultToBytes(output_stream) + output.startTestRun() + failed = False + try: + for cmd in commands: + cmd_ids = _select_ids_for_command(cmd, test_ids) + if test_ids is not None and not cmd_ids: + # Ids were requested, but none match this command. + continue + rc = run_command(cmd, output, list_mode=list_mode, test_ids=cmd_ids) + if rc != 0: + failed = True + finally: + output.stopTestRun() + return 1 if failed else 0 + + +def _read_id_list(path: str) -> list[str]: + """Read test ids from a file, one per line; blank lines and # comments are skipped.""" + ids = [] + with open(path) as f: + for line in f: + line = line.split("#", 1)[0].strip() + if line: + ids.append(line) + return ids + + +def make_parser() -> ArgumentParser: + parser = ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument( + "config", + help="Path to a YAML configuration file describing the commands to run.", + ) + parser.add_argument( + "test_ids", + nargs="*", + help="Optional test ids to restrict execution to. Ids whose prefix " + "matches a command are routed to that command.", + ) + parser.add_argument( + "--list", + dest="list_mode", + action="store_true", + help="List tests that would be run instead of running them. Each " + "command is invoked with its configured list_option substituted for " + "$LISTOPT.", + ) + parser.add_argument( + "--load-list", + dest="load_list", + metavar="FILE", + help="Read test ids (one per line) from FILE to restrict execution.", + ) + return parser + + +def main(argv: Optional[list[str]] = None) -> None: + parser = make_parser() + options = parser.parse_args(argv) + commands = load_config(options.config) + + test_ids: Optional[list[str]] = None + if options.load_list: + test_ids = _read_id_list(options.load_list) + if options.test_ids: + test_ids = (test_ids or []) + options.test_ids + + sys.exit( + combine( + commands, + sys.stdout, + list_mode=options.list_mode, + test_ids=test_ids, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/python/tests/test_subunit_combine.py b/python/tests/test_subunit_combine.py new file mode 100644 index 0000000..761811e --- /dev/null +++ b/python/tests/test_subunit_combine.py @@ -0,0 +1,475 @@ +# +# subunit: extensions to Python unittest to get test results from subprocesses. +# Copyright (C) 2026 Jelmer Vernooij +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. +# + +"""Tests for subunit.filter_scripts.subunit_combine.""" + +import os +import subprocess +import sys +import tempfile +from io import BytesIO + +from testtools import TestCase +from testtools.testresult.doubles import StreamResult + +from subunit import ByteStreamToStreamResult, StreamResultToBytes +from subunit.filter_scripts.subunit_combine import ( + _PrefixingStreamResult, + _expand_argv, + _read_id_list, + _select_ids_for_command, + combine, + load_config, +) + + +def _stream_with_tests(test_ids): + """Build a subunit v2 byte stream with an inprogress+success pair per id.""" + buf = BytesIO() + out = StreamResultToBytes(buf) + for tid in test_ids: + out.status(test_id=tid, test_status="inprogress") + out.status(test_id=tid, test_status="success") + return buf.getvalue() + + +def _parse(data): + """Parse a subunit v2 byte stream into a list of (test_id, test_status).""" + events = StreamResult() + ByteStreamToStreamResult(BytesIO(data)).run(events) + return [(ev[1], ev[2]) for ev in events._events if ev[0] == "status"] + + +class TestPrefixingStreamResult(TestCase): + def test_prefixes_test_id(self): + target = StreamResult() + result = _PrefixingStreamResult(target, "py/") + result.status(test_id="foo", test_status="success") + self.assertEqual([("status", "py/foo", "success")], [ev[:3] for ev in target._events]) + + def test_passes_through_none_test_id(self): + target = StreamResult() + result = _PrefixingStreamResult(target, "py/") + result.status(file_name="stdout", file_bytes=b"hi") + self.assertEqual([("status", None, None)], [ev[:3] for ev in target._events]) + + +class TestLoadConfig(TestCase): + def _write(self, text): + fd, path = tempfile.mkstemp(suffix=".yaml") + self.addCleanup(os.unlink, path) + with os.fdopen(fd, "w") as f: + f.write(text) + return path + + def test_basic(self): + path = self._write("commands:\n - prefix: 'a/'\n argv: ['echo', 'hi']\n - argv: ['true']\n") + self.assertEqual( + [{"prefix": "a/", "argv": ["echo", "hi"]}, {"argv": ["true"]}], + load_config(path), + ) + + def test_rejects_non_mapping(self): + path = self._write("- 1\n- 2\n") + self.assertRaises(ValueError, load_config, path) + + def test_rejects_empty_commands(self): + path = self._write("commands: []\n") + self.assertRaises(ValueError, load_config, path) + + def test_rejects_missing_argv(self): + path = self._write("commands:\n - prefix: 'a/'\n") + self.assertRaises(ValueError, load_config, path) + + def test_rejects_non_string_prefix(self): + path = self._write("commands:\n - argv: ['x']\n prefix: 42\n") + self.assertRaises(ValueError, load_config, path) + + +class TestCombine(TestCase): + def _cat_cmd(self, data): + """A command that writes the given bytes to stdout and exits 0. + + We use python -c so this works on all platforms and without relying on + shell quoting. + """ + src = "import sys; sys.stdout.buffer.write({!r})".format(data) + return [sys.executable, "-c", src] + + def test_merges_streams_with_prefixes(self): + stream_a = _stream_with_tests(["one", "two"]) + stream_b = _stream_with_tests(["alpha"]) + commands = [ + {"prefix": "py/", "argv": self._cat_cmd(stream_a)}, + {"prefix": "rs/", "argv": self._cat_cmd(stream_b)}, + ] + output = BytesIO() + rc = combine(commands, output) + self.assertEqual(0, rc) + self.assertEqual( + [ + ("py/one", "inprogress"), + ("py/one", "success"), + ("py/two", "inprogress"), + ("py/two", "success"), + ("rs/alpha", "inprogress"), + ("rs/alpha", "success"), + ], + _parse(output.getvalue()), + ) + + def test_no_prefix(self): + stream = _stream_with_tests(["solo"]) + commands = [{"argv": self._cat_cmd(stream)}] + output = BytesIO() + rc = combine(commands, output) + self.assertEqual(0, rc) + self.assertEqual( + [("solo", "inprogress"), ("solo", "success")], + _parse(output.getvalue()), + ) + + def test_nonzero_exit_propagates(self): + stream = _stream_with_tests(["only"]) + src = "import sys; sys.stdout.buffer.write({!r}); sys.exit(3)".format(stream) + commands = [{"argv": [sys.executable, "-c", src]}] + output = BytesIO() + rc = combine(commands, output) + self.assertEqual(1, rc) + + +class TestCombineCommand(TestCase): + def test_end_to_end(self): + stream_a = _stream_with_tests(["a"]) + stream_b = _stream_with_tests(["b"]) + + def cmd_for(data): + src = "import sys; sys.stdout.buffer.write({!r})".format(data) + return [sys.executable, "-c", src] + + import yaml + + config = { + "commands": [ + {"prefix": "py/", "argv": cmd_for(stream_a)}, + {"prefix": "rs/", "argv": cmd_for(stream_b)}, + ] + } + fd, path = tempfile.mkstemp(suffix=".yaml") + self.addCleanup(os.unlink, path) + with os.fdopen(fd, "w") as f: + yaml.safe_dump(config, f) + + ps = subprocess.Popen( + [sys.executable, "-m", "subunit.filter_scripts.subunit_combine", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = ps.communicate() + self.assertEqual(0, ps.returncode, err) + self.assertEqual( + [ + ("py/a", "inprogress"), + ("py/a", "success"), + ("rs/b", "inprogress"), + ("rs/b", "success"), + ], + _parse(out), + ) + + +class TestExpandArgv(TestCase): + def test_listopt_expanded_in_list_mode(self): + cmd = {"argv": ["runner", "$LISTOPT", "$IDOPTION"], "list_option": "--list"} + self.assertEqual( + ["runner", "--list"], + _expand_argv(cmd, list_mode=True, test_ids=None, idfile_path=None), + ) + + def test_listopt_empty_when_not_listing(self): + cmd = {"argv": ["runner", "$LISTOPT", "$IDOPTION"], "list_option": "--list"} + self.assertEqual( + ["runner"], + _expand_argv(cmd, list_mode=False, test_ids=None, idfile_path=None), + ) + + def test_idoption_expanded_with_idfile(self): + cmd = { + "argv": ["runner", "$IDOPTION"], + "id_option": "--load-list $IDFILE", + } + self.assertEqual( + ["runner", "--load-list", "/tmp/ids"], + _expand_argv(cmd, list_mode=False, test_ids=["a", "b"], idfile_path="/tmp/ids"), + ) + + def test_idlist_expanded(self): + cmd = {"argv": ["runner", "$IDLIST"]} + self.assertEqual( + ["runner", "a", "b", "c"], + _expand_argv(cmd, list_mode=False, test_ids=["a", "b", "c"], idfile_path=None), + ) + + def test_idoption_empty_without_ids(self): + cmd = { + "argv": ["runner", "$IDOPTION"], + "id_option": "--load-list $IDFILE", + } + self.assertEqual( + ["runner"], + _expand_argv(cmd, list_mode=False, test_ids=None, idfile_path=None), + ) + + def test_literal_argv_entries_untouched(self): + cmd = {"argv": ["runner", "--tag=$literal", "$IDLIST"]} + self.assertEqual( + ["runner", "--tag=$literal", "x"], + _expand_argv(cmd, list_mode=False, test_ids=["x"], idfile_path=None), + ) + + +class TestSelectIdsForCommand(TestCase): + def test_none_returns_none(self): + self.assertIsNone(_select_ids_for_command({"prefix": "py/"}, None)) + + def test_filters_by_prefix_and_strips(self): + ids = ["py/one", "rs/two", "py/three"] + self.assertEqual(["one", "three"], _select_ids_for_command({"prefix": "py/"}, ids)) + + def test_no_prefix_returns_all(self): + ids = ["one", "two"] + self.assertEqual(ids, _select_ids_for_command({}, ids)) + self.assertIsNot(ids, _select_ids_for_command({}, ids)) # must copy + + def test_no_matches_returns_empty(self): + self.assertEqual([], _select_ids_for_command({"prefix": "py/"}, ["rs/a"])) + + +class TestReadIdList(TestCase): + def test_skips_blank_and_comments(self): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + with os.fdopen(fd, "w") as f: + f.write("one\n\n# comment\ntwo # trailing\nthree\n") + self.assertEqual(["one", "two", "three"], _read_id_list(path)) + + +class TestCombineListAndFilter(TestCase): + def _cmd_for(self, data): + src = "import sys; sys.stdout.buffer.write({!r})".format(data) + return [sys.executable, "-c", src] + + def _exists_stream(self, test_ids): + buf = BytesIO() + out = StreamResultToBytes(buf) + for tid in test_ids: + out.status(test_id=tid, test_status="exists") + return buf.getvalue() + + def test_list_mode_substitutes_listopt(self): + # Child emits ids only when --list is present: we simulate this by + # embedding the argument directly in the command and asserting it's + # what gets executed. + list_stream = self._exists_stream(["foo", "bar"]) + src = ( + "import sys; args = sys.argv[1:]; assert args == ['--list'], args; sys.stdout.buffer.write({!r})" + ).format(list_stream) + commands = [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$LISTOPT"], + "list_option": "--list", + } + ] + output = BytesIO() + rc = combine(commands, output, list_mode=True) + self.assertEqual(0, rc) + self.assertEqual( + [("py/foo", "exists"), ("py/bar", "exists")], + _parse(output.getvalue()), + ) + + def test_listopt_absent_when_not_listing(self): + # When list_mode is False, $LISTOPT should expand to nothing. + run_stream = _stream_with_tests(["x"]) + src = ("import sys; args = sys.argv[1:]; assert args == [], args; sys.stdout.buffer.write({!r})").format( + run_stream + ) + commands = [ + { + "argv": [sys.executable, "-c", src, "$LISTOPT"], + "list_option": "--list", + } + ] + output = BytesIO() + rc = combine(commands, output) + self.assertEqual(0, rc) + + def test_idoption_and_idfile_substituted(self): + # The child asserts it received --load-list with two ids in it. + run_stream = _stream_with_tests(["one", "two"]) + src = ( + "import sys, pathlib; " + "args = sys.argv[1:]; " + "assert args[0] == '--load-list', args; " + "contents = pathlib.Path(args[1]).read_text().splitlines(); " + "assert contents == ['one', 'two'], contents; " + "sys.stdout.buffer.write({!r})" + ).format(run_stream) + commands = [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$IDOPTION"], + "id_option": "--load-list $IDFILE", + } + ] + output = BytesIO() + rc = combine(commands, output, test_ids=["py/one", "py/two"]) + self.assertEqual(0, rc) + + def test_idlist_substituted(self): + run_stream = _stream_with_tests(["a"]) + src = ( + "import sys; args = sys.argv[1:]; assert args == ['a', 'b'], args; sys.stdout.buffer.write({!r})" + ).format(run_stream) + commands = [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$IDLIST"], + } + ] + output = BytesIO() + rc = combine(commands, output, test_ids=["py/a", "py/b"]) + self.assertEqual(0, rc) + + def test_command_with_no_matching_ids_is_skipped(self): + # 'py/' receives ids; 'rs/' has no matching ids and must not run. + py_stream = _stream_with_tests(["one"]) + py_src = "import sys; sys.stdout.buffer.write({!r})".format(py_stream) + rs_src = "import sys; sys.exit('must not run')" + commands = [ + {"prefix": "py/", "argv": [sys.executable, "-c", py_src]}, + {"prefix": "rs/", "argv": [sys.executable, "-c", rs_src]}, + ] + output = BytesIO() + rc = combine(commands, output, test_ids=["py/one"]) + self.assertEqual(0, rc) + self.assertEqual( + [("py/one", "inprogress"), ("py/one", "success")], + _parse(output.getvalue()), + ) + + def test_cli_list_flag(self): + list_stream = self._exists_stream(["foo"]) + src = ( + "import sys; args = sys.argv[1:]; assert args == ['--list'], args; sys.stdout.buffer.write({!r})" + ).format(list_stream) + config = { + "commands": [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$LISTOPT"], + "list_option": "--list", + } + ] + } + fd, path = tempfile.mkstemp(suffix=".yaml") + self.addCleanup(os.unlink, path) + with os.fdopen(fd, "w") as f: + import yaml + + yaml.safe_dump(config, f) + ps = subprocess.Popen( + [sys.executable, "-m", "subunit.filter_scripts.subunit_combine", "--list", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = ps.communicate() + self.assertEqual(0, ps.returncode, err) + self.assertEqual([("py/foo", "exists")], _parse(out)) + + def test_cli_positional_ids(self): + run_stream = _stream_with_tests(["a"]) + src = ("import sys; args = sys.argv[1:]; assert args == ['a'], args; sys.stdout.buffer.write({!r})").format( + run_stream + ) + config = { + "commands": [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$IDLIST"], + } + ] + } + fd, path = tempfile.mkstemp(suffix=".yaml") + self.addCleanup(os.unlink, path) + with os.fdopen(fd, "w") as f: + import yaml + + yaml.safe_dump(config, f) + ps = subprocess.Popen( + [sys.executable, "-m", "subunit.filter_scripts.subunit_combine", path, "py/a"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = ps.communicate() + self.assertEqual(0, ps.returncode, err) + self.assertEqual( + [("py/a", "inprogress"), ("py/a", "success")], + _parse(out), + ) + + def test_cli_load_list(self): + run_stream = _stream_with_tests(["one"]) + src = ("import sys; args = sys.argv[1:]; assert args == ['one'], args; sys.stdout.buffer.write({!r})").format( + run_stream + ) + config = { + "commands": [ + { + "prefix": "py/", + "argv": [sys.executable, "-c", src, "$IDLIST"], + } + ] + } + cfg_fd, cfg_path = tempfile.mkstemp(suffix=".yaml") + self.addCleanup(os.unlink, cfg_path) + with os.fdopen(cfg_fd, "w") as f: + import yaml + + yaml.safe_dump(config, f) + list_fd, list_path = tempfile.mkstemp(suffix=".list") + self.addCleanup(os.unlink, list_path) + with os.fdopen(list_fd, "w") as f: + f.write("py/one\n") + ps = subprocess.Popen( + [ + sys.executable, + "-m", + "subunit.filter_scripts.subunit_combine", + "--load-list", + list_path, + cfg_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = ps.communicate() + self.assertEqual(0, ps.returncode, err) + self.assertEqual( + [("py/one", "inprogress"), ("py/one", "success")], + _parse(out), + )