Stop the test suite writing into its own repository (#431)

.github/workflows/ci.yml

@@ -165,9 +165,27 @@ jobs:
           set -o pipefail
           uv run --no-sync python -m pytest \
             -m "not unit and not contract and not e2e and not live and not integration and not docker and not slow" \
-            -q --timeout=60 --timeout-method=signal --tb=no -rN \
+            -q --timeout=60 --timeout-method=signal --tb=no -rfE \
             | tee legacy-results.txt
 
+      - name: Fail if the suite dirtied the repository
+        # NOT continue-on-error, deliberately: the backlog is allowed to be
+        # red, but it is not allowed to modify tracked files. Tests used to
+        # write generated pipelines -- embedding whichever ephemeral port their
+        # mock server bound -- straight into tests/scenarios/test_pipelines/,
+        # so every run left a diff and whichever port was committed last became
+        # everyone else's fixture (#431). This is the guard that keeps it fixed.
+        if: always()
+        run: |
+          if ! git diff --quiet; then
+            echo "::error::The test suite modified tracked files:"
+            git diff --stat
+            echo
+            echo "Tests must write generated files to a temp directory"
+            echo "(pytest's tmp_path), never into the repository."
+            exit 1
+          fi
+
       - name: Publish backlog counts
         # Runs even when the step above fails, which is the normal case.
         #

tests/scenarios/test_network_failures.py

@@ -84,15 +84,32 @@ def stop(self):
             self.thread.join(timeout=1)
 
 
-class TestNetworkFailures:
+class _UsesTemporaryPipelineDir:
+    """Generated pipelines go to a temp directory, never into the repository.
+
+    These tests build pipeline YAML that embeds the ephemeral port their mock
+    server happened to bind. They used to write it into
+    tests/scenarios/test_pipelines/, which is tracked, so every run left the
+    working tree dirty with a different random port -- and whichever port was
+    committed last became the fixture everyone else got (#431).
+
+    The directory was also an absolute path to one developer's home directory,
+    so on any other machine `mkdir(exist_ok=True)` raised FileNotFoundError on
+    the missing parent and the whole class errored out.
+    """
+
+    @pytest.fixture(autouse=True)
+    def _pipeline_dir(self, tmp_path):
+        self.test_pipelines_dir = tmp_path
+
+
+class TestNetworkFailures(_UsesTemporaryPipelineDir):
     """Test network failure scenarios and timeout handling."""
-    
+
     def setup_method(self):
         """Set up test fixtures."""
         self.mock_server = MockSlowServer()
-        self.test_pipelines_dir = Path("/Users/jmanning/orchestrator/tests/scenarios/test_pipelines")
-        self.test_pipelines_dir.mkdir(exist_ok=True)
-
+
     def teardown_method(self):
         """Clean up test fixtures."""
         self.mock_server.stop()
@@ -393,7 +410,7 @@ async def test_rate_limiting_response(self):
         print("✓ Rate limiting scenario tested")
 
 
-class TestAPITimeoutScenarios:
+class TestAPITimeoutScenarios(_UsesTemporaryPipelineDir):
     """Test various API timeout scenarios with real services."""
 
     @pytest.mark.asyncio
@@ -425,9 +442,7 @@ async def test_model_api_timeout(self):
       - slow_model_call
 """
 
-        test_dir = Path("/Users/jmanning/orchestrator/tests/scenarios/test_pipelines")
-        test_dir.mkdir(exist_ok=True)
-        pipeline_path = test_dir / "model_timeout.yaml"
+        pipeline_path = self.test_pipelines_dir / "model_timeout.yaml"
         pipeline_path.write_text(pipeline_content)
 
         executor = Orchestrator()
@@ -479,8 +494,7 @@ async def test_web_search_timeout(self):
       - web_search_with_timeout
 """
 
-        test_dir = Path("/Users/jmanning/orchestrator/tests/scenarios/test_pipelines")
-        pipeline_path = test_dir / "web_search_timeout.yaml"
+        pipeline_path = self.test_pipelines_dir / "web_search_timeout.yaml"
         pipeline_path.write_text(pipeline_content)
 
         executor = Orchestrator()

tests/scenarios/test_real_world_examples.py

@@ -31,7 +31,9 @@ def setup_method(self):
         self.orchestrator = Orchestrator(model_registry=self.model_registry)
 
         # Path to examples directory
-        self.examples_dir = Path("/Users/jmanning/orchestrator/examples")
+        # Repo-relative: an absolute path to one developer's home directory
+        # meant every example silently `pytest.skip`ped on any other machine.
+        self.examples_dir = Path(__file__).resolve().parents[2] / "examples"
 
     def load_example_pipeline(self, filename: str) -> str:
         """Load an example pipeline from the examples directory."""

tests/test_control_flow_conditional.py

@@ -8,7 +8,9 @@
 import shutil
 import sys
 
-sys.path.insert(0, '/Users/jmanning/orchestrator/src')
+# Repo-relative: an absolute path to one developer's home directory made
+# this import work on exactly one machine.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
 from orchestrator.orchestrator import Orchestrator
 

tests/test_control_flow_dynamic.py

@@ -10,7 +10,9 @@
 import sys
 import json
 
-sys.path.insert(0, '/Users/jmanning/orchestrator/src')
+# Repo-relative: an absolute path to one developer's home directory made
+# this import work on exactly one machine.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
 from orchestrator.orchestrator import Orchestrator