Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions tmt-recipe-tool/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,16 @@ tmt-recipe-tool [OPTIONS] COMMAND [ARGS]...

#### `filter-tests`

Filter recipe tests by their result outcome, keeping only those that match.
Filter recipe tests by their result outcome or name, keeping only those that match.

```
tmt-recipe-tool -i RECIPE filter-tests [--use-reportportal] [--result RESULT]...
tmt-recipe-tool -i RECIPE filter-tests [--use-reportportal] [--result RESULT]... [--name NAME]...
```

| Option | Default | Description |
|--------|---------|-------------|
| `--result RESULT` | `fail`, `error`, `warn` | Keep tests with this outcome (repeatable) |
| `--name NAME` | | Keep tests whose name matches this regex pattern (repeatable). Combined with `--result` using OR logic. Uses [search mode](https://tmt.readthedocs.io/en/stable/overview.html#regular-expressions) for matching. |
| `--use-reportportal` | `false` | Fetch test results from ReportPortal instead of a local results file |

When `--use-reportportal` is used, results are fetched from any report phase in the recipe that has `how: reportportal`. The phase must include `launch-uuid` and `test-uuids` (populated automatically by the tmt [ReportPortal](https://tmt.readthedocs.io/en/stable/plugins/report/reportportal.html) plugin after a run finishes). The `launch-uuid` and `test-uuids` fields are stripped from the output recipe so that a subsequent run creates a new ReportPortal launch. Plans that do not have a `reportportal` report phase always fall back to their local `results.yaml` file, even when `--use-reportportal` is passed.
Expand Down Expand Up @@ -113,3 +114,15 @@ Fetch failures from ReportPortal and rerun them immediately:
```bash
tmt-recipe-tool -i recipe.yaml --run filter-tests --use-reportportal --result fail
```

Keep only tests whose name matches a regex pattern, regardless of their result:

```bash
tmt-recipe-tool -i recipe.yaml -o subset.yaml filter-tests --name '/smoke'
```

Keep only tests whose name matches a pattern or whose result is `fail`:

```bash
tmt-recipe-tool -i recipe.yaml -o subset.yaml filter-tests --result fail --name '/smoke'
```
21 changes: 18 additions & 3 deletions tmt-recipe-tool/tmt_recipe_tool/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ def main(
"Can be specified multiple times."
),
)
@click.option(
"--name",
"names",
metavar="NAME",
multiple=True,
default=(),
help=(
"Keep tests whose name matches the given regex pattern. Can be specified multiple times."
),
)
@click.option(
"--use-reportportal",
is_flag=True,
Expand All @@ -89,14 +99,19 @@ def main(
)
@click.pass_context
def filter_tests(
ctx: click.Context, results: list[str], use_reportportal: bool, **kwargs: Any
ctx: click.Context,
results: list[str],
names: list[str],

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sequence would be better, or tuple[str, ...], because Click will deliver multiple as a tuple. Also, you set default=() in the option declaration above, so that's another conflict (I think we can drop default, Click will initialize it correctly thanks to multiple=True).

use_reportportal: bool,
**kwargs: Any,
) -> None:
"""Filter recipe tests by their result outcome."""
"""Filter recipe tests by their result outcome or name."""
assert ctx.parent is not None

ctx.obj["recipe"] = filter_recipe(
ctx.parent.params["input"],
results,
filter_results=results,
filter_names=names,
run_workdir=ctx.parent.params.get("run_workdir"),
use_reportportal=use_reportportal,
)
Expand Down
16 changes: 12 additions & 4 deletions tmt-recipe-tool/tmt_recipe_tool/recipe.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import re
from collections.abc import Iterable
from pathlib import Path
from typing import Optional, cast
Expand Down Expand Up @@ -71,9 +72,13 @@ def _filter_tests(
tests: list[tmt.recipe._RecipeTest],
results: list[Result],
filter_results: list[str],
filter_names: list[str],
) -> Iterable[tmt.recipe._RecipeTest]:
"""Return only the tests whose result outcome matches the filter."""
"""Return only the tests whose result outcome or name matches the filter."""
for test in tests:
if any(re.search(name, test.name) for name in filter_names):
yield test
continue
for result in results:
if (
test.name == result.name
Expand All @@ -87,10 +92,11 @@ def _filter_tests(
def filter_recipe(
input_path: Path,
filter_results: list[str],
filter_names: list[str],
run_workdir: Optional[Path] = None,
use_reportportal: bool = False,
) -> tmt.recipe.Recipe:
"""Load a recipe and keep only tests matching the specified result outcomes."""
"""Load a recipe and keep only tests matching the specified result outcomes or names."""
recipe = _load_recipe(input_path)

filtered_plans = []
Expand All @@ -101,14 +107,16 @@ def filter_recipe(
rp_phases = get_rp_phases(plan)
if rp_phases and use_reportportal:
plan.discover.tests = list(
filter_tests_from_rp(plan.discover.tests, rp_phases, filter_results)
filter_tests_from_rp(plan.discover.tests, rp_phases, filter_results, filter_names)
)
plan.report.phases = edit_rp_phases(plan.report.phases)
else:
results = _load_results(
_resolve_results_path(plan, input_path, run_workdir), plan.name
)
plan.discover.tests = list(_filter_tests(plan.discover.tests, results, filter_results))
plan.discover.tests = list(
_filter_tests(plan.discover.tests, results, filter_results, filter_names)
)
filtered_plans.append(plan)

recipe.plans = filtered_plans
Expand Down
7 changes: 7 additions & 0 deletions tmt-recipe-tool/tmt_recipe_tool/reportportal.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import re
from collections.abc import Iterable
from typing import Union, cast

Expand Down Expand Up @@ -121,6 +122,7 @@ def filter_tests_from_rp(
tests: list[_RecipeTest],
rp_phases: list[ReportPortalPhase],
filter_results: list[str],
filter_names: list[str],
) -> Iterable[_RecipeTest]:
"""Filter the tests from the ReportPortal results"""
filter_results = list(
Expand All @@ -129,6 +131,11 @@ def filter_tests_from_rp(

filtered_tests: dict[int, _RecipeTest] = {}

for test in tests:
if any(re.search(name, test.name) for name in filter_names):
filtered_tests[test.serial_number] = test
continue

for phase in rp_phases:
with retry_session(
status_forcelist=STATUS_FORCELIST,
Expand Down