From 7795959b6d3fd4f8511f54426329d5fd0ac8b603 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 18 Aug 2026 19:57:16 -0700 Subject: [PATCH] fix(ingestion): install the repository into the Dagster+ image database_connectivity failed with ModuleNotFoundError: No module named 'db'. The heartbeat diagnostic settled why: inside the running container the working directory is /opt/dagster/app and db/ is present in it, but find_spec reports db and domain as not findable -- that directory is not on sys.path in the process that executes a step. My two previous attempts treated the symptom. Inserting the repository root from automated_ingestion/__init__.py assumed the package's own location could bootstrap the path, and setting PYTHONPATH assumed the variable reached that process. Neither worked, and both left the image depending on path luck. The image now installs the repository via the post-install hook the build template already provides, so db, domain, services, core, and schemas resolve from site-packages regardless of working directory, PYTHONPATH, or which process is importing. --no-deps because the pinned, hashed requirements are already installed and this must not resolve on top of them. Verified in a locally built image: with cwd set to / and no PYTHONPATH, db and domain are findable, db resolves from site-packages, and both import. The sys.path insert is removed rather than left as redundant insurance -- it encoded a theory that turned out to be wrong, and keeping it would suggest the mechanism still matters. Co-Authored-By: Claude Opus 5 --- automated_ingestion/__init__.py | 23 ++++++------------- .../tests/test_connectivity.py | 15 ++---------- dagster_cloud_post_install.sh | 15 ++++++++++++ 3 files changed, 24 insertions(+), 29 deletions(-) create mode 100755 dagster_cloud_post_install.sh diff --git a/automated_ingestion/__init__.py b/automated_ingestion/__init__.py index de197f548..401409576 100644 --- a/automated_ingestion/__init__.py +++ b/automated_ingestion/__init__.py @@ -29,22 +29,13 @@ See ``docs/automated-ingestion-pipeline-plan.md``. -Importing this package puts the repository root on ``sys.path``. That is -unusual and deliberate. The Dagster+ image copies the repository to -``/opt/dagster/app`` but never installs it -- the generated requirements omit -the project, and the build template only runs ``pip install .`` when a -``setup.py`` exists -- so ``db`` and ``domain`` are importable only while that -directory happens to be on the path. It is, when Dagster loads the code -location; it is not guaranteed in the separate process that executes a step, -which is where the loader's imports actually run. Locally the editable install -hides the difference entirely, so the failure appears only once deployed. +The image installs this repository as a package (see +``dagster_cloud_post_install.sh``), so ``db``, ``domain``, and the rest resolve +from site-packages rather than from whatever happens to be on ``sys.path``. That +matters because the process that loads the code location and the process that +executes a step do not agree about the path, and the loader's imports run in the +second one. Locally an editable install produces the same result, which is why +the difference is invisible until deployment. """ -import sys as _sys -from pathlib import Path as _Path - -_REPOSITORY_ROOT = _Path(__file__).resolve().parent.parent -if str(_REPOSITORY_ROOT) not in _sys.path: - _sys.path.insert(0, str(_REPOSITORY_ROOT)) - # ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_connectivity.py b/automated_ingestion/tests/test_connectivity.py index 35d4d83ae..622f9f1e7 100644 --- a/automated_ingestion/tests/test_connectivity.py +++ b/automated_ingestion/tests/test_connectivity.py @@ -57,23 +57,12 @@ def test_loading_definitions_does_not_import_db_engine(): assert result.stdout.strip() == "False", result.stdout -def test_importing_the_package_makes_the_repository_importable(): - # The deployed image never installs this project, so `db` and `domain` - # resolve only if the repository root is on sys.path. Locally an editable - # install provides that and hides the difference, which is why this failed - # only once deployed -- the code location loaded fine and the step that - # imported db died. - import sys - - import automated_ingestion - - assert str(automated_ingestion._REPOSITORY_ROOT) in sys.path - - def test_db_imports_from_an_unrelated_working_directory(): # Reproduces the deployed condition: a process whose cwd is not the # repository. The lazy imports in the resource and the connectivity asset # run at step execution, not at load, so this is the path that broke. + # In the image this passes because the repository is installed; locally + # because the editable install has the same effect. import subprocess import sys diff --git a/dagster_cloud_post_install.sh b/dagster_cloud_post_install.sh new file mode 100755 index 000000000..c093ae1c8 --- /dev/null +++ b/dagster_cloud_post_install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Runs inside the Dagster+ image build, after the repository has been copied to +# /opt/dagster/app and the pinned requirements installed. +# +# Installs this repository as a package so `db`, `domain`, `services`, `core`, +# and `schemas` resolve from site-packages. Without it they are importable only +# while /opt/dagster/app happens to be on sys.path -- true for the process that +# loads the code location, not for the process that executes a step, which is +# where the loader's imports run. That difference is invisible locally, where an +# editable install puts the repository on the path unconditionally. +# +# --no-deps because the pinned, hashed requirements are already installed and +# this must not resolve anything on top of them. +set -euo pipefail +pip install --no-deps .