diff --git a/AGENTS.md b/AGENTS.md index 4da2b37404..00b0851643 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,7 +190,7 @@ To run a blueprint directly from Python: autoconnect(module_a(), module_b(), module_c()).build().loop() ``` -Expose as a module-level variable for `dimos run` to find it. Add to the registry by running `pytest dimos/robot/test_all_blueprints_generation.py`. +For in-repo DimOS blueprints, expose a module-level variable for `dimos run` to find it and add it to the built-in registry by running `pytest dimos/robot/test_all_blueprints_generation.py`. External packages should not edit `all_blueprints.py`; expose runnable blueprints through installed Python package entry points in the `dimos.blueprints` group. ### GlobalConfig @@ -366,13 +366,13 @@ Code style rules: ## `all_blueprints.py` is auto-generated -`dimos/robot/all_blueprints.py` is generated by `test_all_blueprints_generation.py`. After adding or renaming blueprints: +`dimos/robot/all_blueprints.py` is generated by `test_all_blueprints_generation.py` for in-repo built-in blueprints and modules. After adding or renaming built-in blueprints: ```bash pytest dimos/robot/test_all_blueprints_generation.py ``` -CI asserts the file is current — if it's stale, CI fails. +CI asserts the file is current — if it's stale, CI fails. Externally packaged blueprints are discovered from installed `dimos.blueprints` entry points and do not require regenerating this file. --- diff --git a/dimos/core/coordination/test_module_coordinator.py b/dimos/core/coordination/test_module_coordinator.py index c1baad17b2..8a14d5448c 100644 --- a/dimos/core/coordination/test_module_coordinator.py +++ b/dimos/core/coordination/test_module_coordinator.py @@ -39,6 +39,7 @@ from dimos.core.module import Module from dimos.core.stream import In, Out from dimos.msgs.sensor_msgs.Image import Image +import dimos.robot.get_all_blueprints as resolver from dimos.spec.utils import Spec # Disable Rerun for tests (prevents viewer spawn and gRPC flush errors) @@ -94,6 +95,10 @@ class TargetModule(Module): remapped_data: In[Data1] +class ExternalNameLoadModule(Module): + pass + + # ModuleRef / RPC tests class CalculatorSpec(Spec, Protocol): @rpc @@ -790,3 +795,21 @@ def test_list_module_names(dynamic_coordinator) -> None: dynamic_coordinator.load_module(ModuleA) dynamic_coordinator.load_module(ModuleC) assert set(dynamic_coordinator.list_module_names()) == {"ModuleA", "ModuleC"} + + +def test_load_blueprint_by_name_uses_shared_resolver( + monkeypatch: pytest.MonkeyPatch, mocker +) -> None: + expected_blueprint = ExternalNameLoadModule.blueprint() + + def fake_get_by_name(name: str): + assert name == "my-test-stack.demo" + return expected_blueprint + + coordinator = ModuleCoordinator() + load_blueprint = mocker.patch.object(ModuleCoordinator, "load_blueprint") + monkeypatch.setattr(resolver, "get_by_name", fake_get_by_name) + + coordinator.load_blueprint_by_name("my-test-stack.demo") + + load_blueprint.assert_called_once_with(expected_blueprint) diff --git a/dimos/core/module.py b/dimos/core/module.py index 26a2b6f893..09d71f24a3 100644 --- a/dimos/core/module.py +++ b/dimos/core/module.py @@ -25,6 +25,7 @@ ClassVar, Literal, Protocol, + TypeGuard, get_args, get_origin, get_type_hints, @@ -813,7 +814,7 @@ def connect_stream(self, input_name: str, remote_stream: RemoteOut[T]): # type: ModuleSpec = tuple[type[ModuleBase], GlobalConfig, dict[str, Any]] -def is_module_type(value: Any) -> bool: +def is_module_type(value: object) -> TypeGuard[type[Module]]: try: return inspect.isclass(value) and issubclass(value, Module) except Exception: diff --git a/dimos/porcelain/test_dimos.py b/dimos/porcelain/test_dimos.py index b9ab867195..27e157f3fa 100644 --- a/dimos/porcelain/test_dimos.py +++ b/dimos/porcelain/test_dimos.py @@ -61,6 +61,18 @@ def test_resolve_string_name(): assert bp is not None +def test_resolve_external_string_name_uses_shared_resolver(monkeypatch): + expected = StressTestModule.blueprint() + + def fake_get_by_name(name: str): + assert name == "my-test-stack.demo" + return expected + + monkeypatch.setattr("dimos.porcelain.dimos.get_by_name", fake_get_by_name) + + assert _resolve_target("my-test-stack.demo") is expected + + def test_resolve_unknown_string(): with pytest.raises(ValueError, match="Unknown"): _resolve_target("nonexistent-blueprint-xyz") diff --git a/dimos/robot/cli/dimos.py b/dimos/robot/cli/dimos.py index 3f94a6be4e..31d8d28a3c 100644 --- a/dimos/robot/cli/dimos.py +++ b/dimos/robot/cli/dimos.py @@ -609,10 +609,27 @@ def show_config(ctx: typer.Context) -> None: def list_blueprints() -> None: """List all available blueprints.""" from dimos.robot.all_blueprints import all_blueprints + from dimos.robot.external_blueprints import ( + ExternalBlueprintError, + list_external_blueprint_names, + ) blueprints = [name for name in all_blueprints.keys() if not name.startswith("demo-")] + typer.echo("Built-in blueprints:") for blueprint_name in sorted(blueprints): - typer.echo(blueprint_name) + typer.echo(f" {blueprint_name}") + + try: + external_blueprints = list_external_blueprint_names() + except ExternalBlueprintError as exc: + typer.echo(typer.style(str(exc), fg=typer.colors.RED), err=True) + raise typer.Exit(1) from exc + + if external_blueprints: + typer.echo("") + typer.echo("External blueprints:") + for blueprint_name in external_blueprints: + typer.echo(f" {blueprint_name}") @main.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/dimos/robot/cli/test_dimos.py b/dimos/robot/cli/test_dimos.py index b925d4ebd9..c6abf16a9a 100644 --- a/dimos/robot/cli/test_dimos.py +++ b/dimos/robot/cli/test_dimos.py @@ -15,10 +15,12 @@ from typing import Literal import pytest +from typer.testing import CliRunner from dimos.core.coordination.blueprints import autoconnect from dimos.core.module import Module, ModuleConfig -from dimos.robot.cli.dimos import _normalize_simulation_argv, arg_help +from dimos.robot import external_blueprints as external +from dimos.robot.cli.dimos import _normalize_simulation_argv, arg_help, main @pytest.mark.parametrize( @@ -144,3 +146,72 @@ class TestModule(Module): " * testmodule.spam: str (default: eggs)", "", ] + + +def test_list_blueprints_groups_builtin_and_external(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + external, + "list_external_blueprint_names", + lambda: ["my-test-stack.demo", "my-test-stack.keyboard-teleop"], + ) + + result = CliRunner().invoke(main, ["list"]) + + assert result.exit_code == 0 + assert "Built-in blueprints:" in result.output + assert " unitree-go2" in result.output + assert "demo-agent" not in result.output + assert "External blueprints:" in result.output + assert " my-test-stack.demo" in result.output + assert " my-test-stack.keyboard-teleop" in result.output + + +def test_list_blueprints_without_external_names(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(external, "list_external_blueprint_names", lambda: []) + + result = CliRunner().invoke(main, ["list"]) + + assert result.exit_code == 0 + assert "Built-in blueprints:" in result.output + assert "External blueprints:" not in result.output + + +def test_list_blueprints_reports_external_discovery_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def raise_error() -> list[str]: + raise external.ExternalBlueprintError("external metadata is invalid") + + monkeypatch.setattr(external, "list_external_blueprint_names", raise_error) + + result = CliRunner().invoke(main, ["list"]) + + assert result.exit_code == 1 + assert "external metadata is invalid" in result.output + + +def test_run_reports_external_resolution_errors(monkeypatch: pytest.MonkeyPatch) -> None: + def raise_error(name: str): + raise external.ExternalBlueprintError( + "Failed to load external blueprint " + f"{name!r} from entry point 'my_test_stack.missing:demo_blueprint': " + "ModuleNotFoundError: No module named 'my_test_stack.missing'" + ) + + monkeypatch.setattr( + "dimos.robot.get_all_blueprints.resolve_external_blueprint_by_name", + raise_error, + ) + + result = CliRunner().invoke(main, ["run", "my-test-stack.demo"]) + + assert result.exit_code == 1 + assert "Failed to load external blueprint 'my-test-stack.demo'" in result.output + assert "my_test_stack.missing:demo_blueprint" in result.output + + +def test_run_reports_unknown_bare_blueprint() -> None: + result = CliRunner().invoke(main, ["run", "missing-bare-blueprint"]) + + assert result.exit_code == 1 + assert "Unknown blueprint or module: missing-bare-blueprint" in result.output diff --git a/dimos/robot/external_blueprints.py b/dimos/robot/external_blueprints.py new file mode 100644 index 0000000000..ddb94825d9 --- /dev/null +++ b/dimos/robot/external_blueprints.py @@ -0,0 +1,231 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +import importlib.metadata as importlib_metadata +import re + +from packaging.utils import canonicalize_name + +from dimos.core.coordination.blueprints import Blueprint +from dimos.core.module import is_module_type + +ENTRY_POINT_GROUP = "dimos.blueprints" +LOCAL_BLUEPRINT_NAME_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") + + +class ExternalBlueprintError(ValueError): + """Base class for external blueprint discovery and resolution errors.""" + + +def _invalid_external_blueprint_name_error( + local_name: str, distribution_name: str +) -> ExternalBlueprintError: + return ExternalBlueprintError( + "Invalid external blueprint entry point name " + f"{local_name!r} in distribution {distribution_name!r}. " + "External local blueprint names must be lowercase kebab-case " + "and match ^[a-z0-9]+(-[a-z0-9]+)*$." + ) + + +def _invalid_external_blueprint_request_name_error(local_name: str) -> ExternalBlueprintError: + return ExternalBlueprintError( + f"Invalid external blueprint local name {local_name!r}. " + "External local blueprint names must be lowercase kebab-case " + "and match ^[a-z0-9]+(-[a-z0-9]+)*$." + ) + + +def _external_blueprint_namespace_not_found_error( + namespace: str, available_namespaces: Iterable[str] +) -> ExternalBlueprintError: + msg = f"External blueprint namespace {namespace!r} was not discovered." + available = sorted(set(available_namespaces)) + if available: + msg += f" Available external namespaces: {', '.join(available)}." + return ExternalBlueprintError(msg) + + +def _external_blueprint_local_name_not_found_error( + namespace: str, local_name: str, available_local_names: Iterable[str] +) -> ExternalBlueprintError: + msg = f"External blueprint namespace {namespace!r} has no local blueprint {local_name!r}." + available = sorted(set(available_local_names)) + if available: + msg += f" Available local blueprints: {', '.join(available)}." + return ExternalBlueprintError(msg) + + +def _external_blueprint_load_error( + name: str, target: str, cause: Exception +) -> ExternalBlueprintError: + return ExternalBlueprintError( + f"Failed to load external blueprint {name!r} from entry point {target!r}: " + f"{type(cause).__name__}: {cause}" + ) + + +def _invalid_external_blueprint_target_error(name: str, target: object) -> ExternalBlueprintError: + return ExternalBlueprintError( + f"External blueprint {name!r} loaded unsupported target {target!r}. " + "Entry point targets must be a Blueprint object or a DimOS Module class." + ) + + +@dataclass(frozen=True) +class ExternalBlueprintEntry: + namespace: str + local_name: str + distribution_name: str + entry_point: importlib_metadata.EntryPoint + + @property + def qualified_name(self) -> str: + return f"{self.namespace}.{self.local_name}" + + @property + def target(self) -> str: + return self.entry_point.value + + +@dataclass(frozen=True) +class InvalidExternalBlueprintEntry: + namespace: str + local_name: str + distribution_name: str + + +@dataclass(frozen=True) +class _ExternalBlueprintCollection: + entries_by_namespace: dict[str, list[ExternalBlueprintEntry]] + invalid_entries_by_namespace: dict[str, list[InvalidExternalBlueprintEntry]] + + +def canonicalize_distribution_namespace(distribution_name: str) -> str: + """Normalize a Python distribution name for use as an external blueprint namespace.""" + + return str(canonicalize_name(distribution_name)) + + +def is_valid_external_local_blueprint_name(name: str) -> bool: + """Return whether a local external blueprint name uses DimOS-style kebab-case.""" + + return LOCAL_BLUEPRINT_NAME_PATTERN.fullmatch(name) is not None + + +def is_namespaced_blueprint_name(name: str) -> bool: + """Return whether a runnable blueprint name has an external namespace separator.""" + + return "." in name + + +def list_external_blueprint_names() -> list[str]: + """List namespaced external blueprint names from installed package metadata.""" + + return sorted(entry.qualified_name for entry in list_external_blueprints()) + + +def list_external_blueprints() -> list[ExternalBlueprintEntry]: + """List external blueprint entry point metadata without loading targets.""" + + namespace_entries = _collect_external_blueprints().entries_by_namespace + return sorted( + (entry for entries in namespace_entries.values() for entry in entries), + key=lambda entry: entry.qualified_name, + ) + + +def resolve_external_blueprint_by_name(name: str) -> Blueprint: + """Resolve a fully-qualified external blueprint name to a Blueprint.""" + + namespace, sep, local_name = name.partition(".") + if not sep: + raise _external_blueprint_namespace_not_found_error(name, []) + if not is_valid_external_local_blueprint_name(local_name): + raise _invalid_external_blueprint_request_name_error(local_name) + + collection = _collect_external_blueprints() + namespace_entries = collection.entries_by_namespace + if namespace not in namespace_entries: + invalid_entries = collection.invalid_entries_by_namespace.get(namespace) + if invalid_entries: + invalid_entry = invalid_entries[0] + raise _invalid_external_blueprint_name_error( + invalid_entry.local_name, invalid_entry.distribution_name + ) + raise _external_blueprint_namespace_not_found_error(namespace, namespace_entries.keys()) + + entries = namespace_entries[namespace] + matches = [entry for entry in entries if entry.local_name == local_name] + if not matches: + raise _external_blueprint_local_name_not_found_error( + namespace, local_name, (entry.local_name for entry in entries) + ) + + entry = matches[0] + try: + target = entry.entry_point.load() + except Exception as exc: + raise _external_blueprint_load_error(entry.qualified_name, entry.target, exc) from exc + + return _target_to_blueprint(entry.qualified_name, target) + + +def _target_to_blueprint(name: str, target: object) -> Blueprint: + if isinstance(target, Blueprint): + return target + if is_module_type(target): + return target.blueprint() + raise _invalid_external_blueprint_target_error(name, target) + + +def _collect_external_blueprints() -> _ExternalBlueprintCollection: + entries_by_namespace: dict[str, list[ExternalBlueprintEntry]] = {} + invalid_entries_by_namespace: dict[str, list[InvalidExternalBlueprintEntry]] = {} + + for entry_point in importlib_metadata.entry_points(group=ENTRY_POINT_GROUP): + distribution = entry_point.dist + if distribution is None: + continue + distribution_name = distribution.metadata.get("Name") + if not distribution_name: + continue + namespace = canonicalize_distribution_namespace(distribution_name) + local_name = entry_point.name + if not is_valid_external_local_blueprint_name(local_name): + invalid_entries_by_namespace.setdefault(namespace, []).append( + InvalidExternalBlueprintEntry( + namespace=namespace, + local_name=local_name, + distribution_name=distribution_name, + ) + ) + continue + entries_by_namespace.setdefault(namespace, []).append( + ExternalBlueprintEntry( + namespace=namespace, + local_name=local_name, + distribution_name=distribution_name, + entry_point=entry_point, + ) + ) + + return _ExternalBlueprintCollection( + entries_by_namespace=entries_by_namespace, + invalid_entries_by_namespace=invalid_entries_by_namespace, + ) diff --git a/dimos/robot/get_all_blueprints.py b/dimos/robot/get_all_blueprints.py index 98d5e2d98f..3e816c1413 100644 --- a/dimos/robot/get_all_blueprints.py +++ b/dimos/robot/get_all_blueprints.py @@ -14,13 +14,17 @@ import difflib import re -import sys from typing import NoReturn import typer from dimos.core.coordination.blueprints import Blueprint from dimos.robot.all_blueprints import all_blueprints, all_modules +from dimos.robot.external_blueprints import ( + ExternalBlueprintError, + is_namespaced_blueprint_name, + resolve_external_blueprint_by_name, +) all_names = sorted(set(all_blueprints.keys()) | set(all_modules.keys())) @@ -61,6 +65,8 @@ def get_by_name(name: str) -> Blueprint: return get_blueprint_by_name(name) elif name in all_modules: return get_module_by_name(name) + elif is_namespaced_blueprint_name(name): + return resolve_external_blueprint_by_name(name) else: _raise_unknown(name, all_names) @@ -72,7 +78,12 @@ def _fail_or_exit(name: str, candidates: list[str]) -> NoReturn: typer.echo("Did you mean one of these?", err=True) for s in suggestions: typer.echo(f" {s}", err=True) - sys.exit(1) + raise typer.Exit(1) + + +def _exit_with_error(message: str) -> NoReturn: + typer.echo(typer.style(message, fg=typer.colors.RED), err=True) + raise typer.Exit(1) def get_by_name_or_exit(name: str) -> Blueprint: @@ -80,6 +91,11 @@ def get_by_name_or_exit(name: str) -> Blueprint: return get_blueprint_by_name(name) elif name in all_modules: return get_module_by_name(name) + elif is_namespaced_blueprint_name(name): + try: + return resolve_external_blueprint_by_name(name) + except ExternalBlueprintError as exc: + _exit_with_error(str(exc)) else: _fail_or_exit(name, all_names) diff --git a/dimos/robot/test_external_blueprints.py b/dimos/robot/test_external_blueprints.py new file mode 100644 index 0000000000..ac559a7f15 --- /dev/null +++ b/dimos/robot/test_external_blueprints.py @@ -0,0 +1,384 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import Any + +import pytest + +from dimos.core.coordination.blueprints import Blueprint, autoconnect +from dimos.core.module import Module +from dimos.robot import external_blueprints as external +from dimos.robot.get_all_blueprints import get_by_name + + +class ExternalTestModule(Module): + pass + + +@dataclass(frozen=True) +class FakeEntryPoint: + name: str + value: str + target: Any = None + error: Exception | None = None + group: str = external.ENTRY_POINT_GROUP + dist: Any = None + + def load(self) -> Any: + if self.error is not None: + raise self.error + return self.target + + +@dataclass(frozen=True) +class FakeDistribution: + name: str + entry_points: tuple[FakeEntryPoint, ...] + metadata_name: str | None = None + + @property + def metadata(self) -> dict[str, str]: + if self.metadata_name is None: + return {"Name": self.name} + if self.metadata_name == "": + return {} + return {"Name": self.metadata_name} + + +def patch_distributions(monkeypatch: pytest.MonkeyPatch, *distributions: FakeDistribution) -> None: + entry_points = [ + replace(entry_point, dist=distribution) + for distribution in distributions + for entry_point in distribution.entry_points + ] + + monkeypatch.setattr( + external.importlib_metadata, + "entry_points", + lambda *, group=None: [ + entry_point + for entry_point in entry_points + if group is None or entry_point.group == group + ], + ) + + +@pytest.mark.parametrize( + ("distribution_name", "expected"), + [ + ("My_Robot.Stack", "my-robot-stack"), + ("my---robot___stack", "my-robot-stack"), + ("my.robot_stack", "my-robot-stack"), + ("my-robot-stack", "my-robot-stack"), + ], +) +def test_canonicalize_distribution_namespace(distribution_name: str, expected: str) -> None: + assert external.canonicalize_distribution_namespace(distribution_name) == expected + + +@pytest.mark.parametrize("name", ["go2", "keyboard-teleop", "g1-sim2"]) +def test_valid_external_local_blueprint_names(name: str) -> None: + assert external.is_valid_external_local_blueprint_name(name) + + +@pytest.mark.parametrize("name", ["", "Go2", "go2_sim", "go2.sim", "go2/real", "go2--sim"]) +def test_invalid_external_local_blueprint_names(name: str) -> None: + assert not external.is_valid_external_local_blueprint_name(name) + + +def test_list_external_blueprint_names_without_loading_targets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + entry_point = FakeEntryPoint( + name="demo", + value="external_stack.demo:demo_blueprint", + error=AssertionError("list must not load targets"), + ) + patch_distributions(monkeypatch, FakeDistribution("My_Test.Stack", (entry_point,))) + + assert external.list_external_blueprint_names() == ["my-test-stack.demo"] + + +def test_resolve_external_blueprint_object(monkeypatch: pytest.MonkeyPatch) -> None: + blueprint = ExternalTestModule.blueprint() + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:demo_blueprint", blueprint),), + ), + ) + + assert external.resolve_external_blueprint_by_name("my-test-stack.demo") is blueprint + + +def test_resolve_external_module_class(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + ( + FakeEntryPoint( + "module-demo", "external_stack.demo:ExternalTestModule", ExternalTestModule + ), + ), + ), + ) + + blueprint = external.resolve_external_blueprint_by_name("my-test-stack.module-demo") + + assert isinstance(blueprint, Blueprint) + assert blueprint.blueprints[0].module is ExternalTestModule + + +@pytest.mark.parametrize("target", [lambda: ExternalTestModule.blueprint(), 42, object()]) +def test_rejects_unsupported_external_targets(monkeypatch: pytest.MonkeyPatch, target: Any) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", (FakeEntryPoint("demo", "external_stack.demo:target", target),) + ), + ) + + with pytest.raises(external.ExternalBlueprintError): + external.resolve_external_blueprint_by_name("my-test-stack.demo") + + +def test_unknown_external_namespace(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions(monkeypatch, FakeDistribution("My-Test-Stack", ())) + + with pytest.raises(external.ExternalBlueprintError): + external.resolve_external_blueprint_by_name("missing-stack.demo") + + +def test_unknown_external_namespace_lists_available_namespaces( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + with pytest.raises(external.ExternalBlueprintError) as exc_info: + external.resolve_external_blueprint_by_name("missing-stack.demo") + + assert "Available external namespaces: my-test-stack" in str(exc_info.value) + + +def test_resolve_external_name_requires_namespace_separator( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions(monkeypatch) + + with pytest.raises(external.ExternalBlueprintError): + external.resolve_external_blueprint_by_name("my-test-stack") + + +def test_namespace_exists_but_local_name_missing(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + ( + FakeEntryPoint( + "demo", "external_stack.demo:demo_blueprint", ExternalTestModule.blueprint() + ), + ), + ), + ) + + with pytest.raises(external.ExternalBlueprintError): + external.resolve_external_blueprint_by_name("my-test-stack.arm") + + +def test_local_name_missing_message_omits_empty_available_names() -> None: + error = external._external_blueprint_local_name_not_found_error("my-test-stack", "arm", []) + + assert "Available local blueprints" not in str(error) + + +def test_entry_point_load_failure(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + ( + FakeEntryPoint( + "demo", "external_stack.demo:demo_blueprint", error=ImportError("boom") + ), + ), + ), + ) + + with pytest.raises(external.ExternalBlueprintError, match="ImportError: boom"): + external.resolve_external_blueprint_by_name("my-test-stack.demo") + + +def test_invalid_external_metadata_name(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution("My-Test-Stack", (FakeEntryPoint("Go2", "external_stack.demo:go2"),)), + ) + + assert external.list_external_blueprint_names() == [] + with pytest.raises(external.ExternalBlueprintError): + external.resolve_external_blueprint_by_name("my-test-stack.demo") + + +def test_invalid_requested_external_local_name(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + with pytest.raises(external.ExternalBlueprintError) as exc_info: + external.resolve_external_blueprint_by_name("my-test-stack.Go2") + + message = str(exc_info.value) + assert "Invalid external blueprint local name 'Go2'" in message + assert "entry point name" not in message + assert "distribution" not in message + + +def test_invalid_external_metadata_does_not_block_unrelated_valid_package( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution("Broken-Stack", (FakeEntryPoint("BadName", "broken_stack.demo:demo"),)), + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + assert external.list_external_blueprint_names() == ["my-test-stack.demo"] + blueprint = external.resolve_external_blueprint_by_name("my-test-stack.demo") + + assert blueprint.blueprints[0].module is ExternalTestModule + + +def test_entry_points_without_distribution_are_ignored(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + external.importlib_metadata, + "entry_points", + lambda *, group=None: [ + FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule) + ], + ) + + assert external.list_external_blueprint_names() == [] + + +def test_entry_points_without_distribution_name_are_ignored( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + metadata_name="", + ), + ) + + assert external.list_external_blueprint_names() == [] + + +def test_all_invalid_colliding_distribution_does_not_block_valid_package( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution("my_robot.stack", (FakeEntryPoint("Go2", "invalid_stack.demo:go2"),)), + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + assert external.list_external_blueprint_names() == ["my-test-stack.demo"] + blueprint = external.resolve_external_blueprint_by_name("my-test-stack.demo") + + assert blueprint.blueprints[0].module is ExternalTestModule + + +def test_colliding_external_namespace_uses_matching_valid_entry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", (FakeEntryPoint("demo", "a:b", ExternalTestModule.blueprint()),) + ), + FakeDistribution( + "my_test.stack", (FakeEntryPoint("other", "c:d", ExternalTestModule.blueprint()),) + ), + ) + + blueprint = external.resolve_external_blueprint_by_name("my-test-stack.demo") + + assert isinstance(blueprint, Blueprint) + + +def test_bare_names_never_search_external_entry_points(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + external.importlib_metadata, + "entry_points", + lambda: (_ for _ in ()).throw(AssertionError("bare lookup searched external metadata")), + ) + + with pytest.raises(ValueError, match="Unknown blueprint or module"): + get_by_name("missing-bare-blueprint") + + +def test_get_by_name_resolves_external_names(monkeypatch: pytest.MonkeyPatch) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + blueprint = get_by_name("my-test-stack.demo") + + assert blueprint.blueprints[0].module is ExternalTestModule + + +def test_mixed_builtin_and_external_names_resolve_before_composition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + patch_distributions( + monkeypatch, + FakeDistribution( + "My-Test-Stack", + (FakeEntryPoint("demo", "external_stack.demo:ExternalTestModule", ExternalTestModule),), + ), + ) + + mixed_blueprint = autoconnect( + get_by_name("demo-mcp-stress-test"), get_by_name("my-test-stack.demo") + ) + + assert any(atom.module is ExternalTestModule for atom in mixed_blueprint.blueprints) diff --git a/dimos/robot/test_get_all_blueprints.py b/dimos/robot/test_get_all_blueprints.py index 4982c2ac64..2b06bdc9a1 100644 --- a/dimos/robot/test_get_all_blueprints.py +++ b/dimos/robot/test_get_all_blueprints.py @@ -16,7 +16,7 @@ import pytest -from dimos.robot.get_all_blueprints import get_by_name +from dimos.robot.get_all_blueprints import get_by_name, get_by_name_or_exit def test_resolve_string_blueprint(): @@ -29,6 +29,16 @@ def test_resolve_string_module(): assert bp is not None +def test_resolve_string_blueprint_or_exit(): + bp = get_by_name_or_exit("demo-mcp-stress-test") + assert bp is not None + + +def test_resolve_string_module_or_exit(): + bp = get_by_name_or_exit("camera-module") + assert bp is not None + + def test_resolve_unknown_name(): with pytest.raises(ValueError, match="Unknown blueprint or module"): get_by_name("nonexistent-blueprint-xyz") diff --git a/docs/coding-agents/code-quality-rules.md b/docs/coding-agents/code-quality-rules.md index 5b252fc968..ad87831e43 100644 --- a/docs/coding-agents/code-quality-rules.md +++ b/docs/coding-agents/code-quality-rules.md @@ -18,7 +18,7 @@ Rules dimos code is expected to follow. They address recurring issues found in c * No lambdas -- they can't be pickled to worker processes. Use named functions. * Do no work at import time: no subprocesses, viewers, model parsing, or network. In particular don't call `get_data(...)` (it blocks import until the download finishes) -- use `LfsPath` (resolved at access time) or build the config in `start`/`build`. Any process you start must be managed (shut down when not needed). * Blueprint files define blueprints, not modules/classes. -* Helper blueprints not meant to run alone must start with `_` (the `all_blueprints.py` generator skips them); demo/non-shared ones get a `demo_` prefix (hidden from `dimos list`). +* Helper blueprints not meant to run alone must start with `_` (the in-repo `all_blueprints.py` generator skips them); demo/non-shared built-in ones get a `demo_` prefix (hidden from `dimos list`). Externally packaged blueprints are not added to `all_blueprints.py`; expose them with `dimos.blueprints` Python package entry points. ## Concurrency and thread safety diff --git a/docs/development/conventions.md b/docs/development/conventions.md index 53259c74da..5484bb4125 100644 --- a/docs/development/conventions.md +++ b/docs/development/conventions.md @@ -9,4 +9,5 @@ This mostly to track when conventions change (with regard to codebase updates) b - To customize the way rerun renders something, right now we use a `rerun_config` dict. This will (hopefully) change very soon to be a per-module config instead of a per-blueprint config - Similar to the `rerun_config` the `rrb` (rerun blueprint) is defined at a blueprint level right now, but ideally would be a per-module contribution with only a per-blueprint override of the layout. - No `__init__.py` files -- Helper blueprints (like `_with_vis`) that should not be used on their own need to start with an underscore to avoid being picked up by the all_blueprints.py code generation step +- Helper blueprints (like `_with_vis`) that should not be used on their own need to start with an underscore to avoid being picked up by the in-repo `all_blueprints.py` code generation step +- Built-in runnable blueprints are registered through the generated in-repo `all_blueprints.py`; externally packaged blueprints are discovered through installed Python package entry points in the `dimos.blueprints` group instead. diff --git a/docs/usage/blueprints.md b/docs/usage/blueprints.md index bceb356cd7..919f446c02 100644 --- a/docs/usage/blueprints.md +++ b/docs/usage/blueprints.md @@ -84,6 +84,50 @@ expanded_blueprint = autoconnect( Blueprints are frozen data classes, and `autoconnect()` always constructs an expanded blueprint so you never have to worry about changes in one affecting the other. +## Publishing external blueprints + +DimOS can discover runnable blueprints from installed Python packages. External +packages declare entry points in the `dimos.blueprints` group: + +```toml +[project] +name = "my-robot-stack" + +[project.entry-points."dimos.blueprints"] +go2 = "my_robot_stack.go2:go2_blueprint" +keyboard-teleop = "my_robot_stack.teleop:KeyboardTeleop" +``` + +After the package is installed in the same Python environment as DimOS, users can run +those blueprints by fully qualified name: + +```bash +dimos run my-robot-stack.go2 +dimos run unitree-go2 my-robot-stack.keyboard-teleop +``` + +External names are always `.`: + +- The namespace comes from the installed distribution name. DimOS lowercases it and + collapses runs of `-`, `_`, and `.` into `-`, so `My_Robot.Stack` becomes + `my-robot-stack`. +- The local blueprint name is the entry point name. It must be lowercase kebab-case + matching `^[a-z0-9]+(-[a-z0-9]+)*$`, such as `go2` or `keyboard-teleop`. + +Entry point targets may be either: + +- a `Blueprint` object, such as a module-level `go2_blueprint`; or +- a DimOS `Module` class, such as `KeyboardTeleop`, which DimOS converts with + `.blueprint()`. + +`dimos list` includes external names from package metadata without importing the target +modules. `dimos run my-robot-stack.go2` imports only the requested entry point target. + +Remote coordinator resolution happens in the coordinator environment. If a client asks +a coordinator to load `my-robot-stack.go2`, the `my-robot-stack` package must be +installed where the coordinator performs name resolution; installing it only in the +client environment is not enough. + ### Duplicate module handling If the same module appears multiple times in `autoconnect`, the **later blueprint wins** and overrides earlier ones: diff --git a/docs/usage/cli.md b/docs/usage/cli.md index 7abd1fa7bb..894c6a0f5e 100644 --- a/docs/usage/cli.md +++ b/docs/usage/cli.md @@ -59,7 +59,9 @@ Environment variables and `.env` values must be prefixed with `DIMOS_`. ### `dimos run` -Start a robot blueprint. +Start one or more robot blueprints. Built-in DimOS blueprints use bare names such as +`unitree-go2`; external blueprints installed from Python packages use namespaced names +such as `my-robot-stack.go2`. ```bash dimos run [ ...] [--daemon] [--disable ...] @@ -89,10 +91,22 @@ dimos run unitree-go2-agentic --robot-ip 192.168.123.161 # Compose modules dynamically dimos run unitree-go2 keyboard-teleop +# Run an externally packaged blueprint +dimos run my-robot-stack.go2 + +# Compose built-in and external blueprints +dimos run unitree-go2 my-robot-stack.keyboard-teleop + # Disable specific modules dimos run unitree-go2-agentic --disable OsmSkill WebInput ``` +External blueprint names are always fully qualified as +`.`. The namespace is +derived from the installed Python distribution name by lowercasing it and collapsing +runs of `-`, `_`, and `.` into `-`. The local blueprint name is the entry point name +and must be lowercase kebab-case, for example `keyboard-teleop`. + When `--daemon` is used, the process: 1. Builds and starts all modules (foreground — you see errors) 2. Runs a health check (polls worker PIDs) @@ -101,13 +115,17 @@ When `--daemon` is used, the process: #### Adding a New Blueprint -Define a module-level `Blueprint` variable and register it in `all_blueprints.py`: +For an in-repository DimOS blueprint, define a module-level `Blueprint` variable and +regenerate the built-in registry: ```bash pytest dimos/robot/test_all_blueprints_generation.py ``` -This auto-generates the registry. See [blueprints](/docs/usage/blueprints.md) for composition details. +This auto-generates `dimos/robot/all_blueprints.py` for built-in blueprints. External +packages do not edit that file; they expose blueprints through Python package entry +points. See [blueprints](/docs/usage/blueprints.md) for composition and external +publishing details. ### `dimos status` @@ -179,12 +197,26 @@ dimos log --json | jq 'select(.logger | contains("RerunBridge"))' ### `dimos list` -List all available blueprints. +List all available blueprints. Built-in and external blueprints are grouped separately; +external names are read from installed package metadata without importing their target +modules. ```bash dimos list ``` +Example output: + +```text +Built-in blueprints: + unitree-go2 + unitree-go2-agentic + +External blueprints: + my-robot-stack.go2 + my-robot-stack.keyboard-teleop +``` + ### `dimos show-config` Print resolved GlobalConfig values and their sources. diff --git a/pyproject.toml b/pyproject.toml index 55e7ffcb73..18fae8ed25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ dependencies = [ "sortedcontainers==2.4.0", "pydantic", "python-dotenv", + "packaging>=24.0", "typing_extensions>=4.0; python_version < '3.11'", "annotation-protocol>=1.4.0", "lazy_loader", diff --git a/uv.lock b/uv.lock index 0736ec2961..91c674145d 100644 --- a/uv.lock +++ b/uv.lock @@ -1867,6 +1867,7 @@ dependencies = [ { name = "open3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" }, { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" }, { name = "opencv-python" }, + { name = "packaging" }, { name = "pin" }, { name = "plotext" }, { name = "plum-dispatch" }, @@ -2402,6 +2403,7 @@ requires-dist = [ { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", marker = "extra == 'apriltag'", specifier = "==4.10.0.84" }, { name = "opencv-python" }, + { name = "packaging", specifier = ">=24.0" }, { name = "pillow", marker = "extra == 'perception'" }, { name = "pin", specifier = ">=3.3.0" }, { name = "pin-pink", marker = "extra == 'manipulation'", specifier = ">=4.2.0" },