diff --git a/automated_ingestion/__init__.py b/automated_ingestion/__init__.py index de197f548..401409576 100644 --- a/automated_ingestion/__init__.py +++ b/automated_ingestion/__init__.py @@ -29,22 +29,13 @@ See ``docs/automated-ingestion-pipeline-plan.md``. -Importing this package puts the repository root on ``sys.path``. That is -unusual and deliberate. The Dagster+ image copies the repository to -``/opt/dagster/app`` but never installs it -- the generated requirements omit -the project, and the build template only runs ``pip install .`` when a -``setup.py`` exists -- so ``db`` and ``domain`` are importable only while that -directory happens to be on the path. It is, when Dagster loads the code -location; it is not guaranteed in the separate process that executes a step, -which is where the loader's imports actually run. Locally the editable install -hides the difference entirely, so the failure appears only once deployed. +The image installs this repository as a package (see +``dagster_cloud_post_install.sh``), so ``db``, ``domain``, and the rest resolve +from site-packages rather than from whatever happens to be on ``sys.path``. That +matters because the process that loads the code location and the process that +executes a step do not agree about the path, and the loader's imports run in the +second one. Locally an editable install produces the same result, which is why +the difference is invisible until deployment. """ -import sys as _sys -from pathlib import Path as _Path - -_REPOSITORY_ROOT = _Path(__file__).resolve().parent.parent -if str(_REPOSITORY_ROOT) not in _sys.path: - _sys.path.insert(0, str(_REPOSITORY_ROOT)) - # ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_connectivity.py b/automated_ingestion/tests/test_connectivity.py index 35d4d83ae..622f9f1e7 100644 --- a/automated_ingestion/tests/test_connectivity.py +++ b/automated_ingestion/tests/test_connectivity.py @@ -57,23 +57,12 @@ def test_loading_definitions_does_not_import_db_engine(): assert result.stdout.strip() == "False", result.stdout -def test_importing_the_package_makes_the_repository_importable(): - # The deployed image never installs this project, so `db` and `domain` - # resolve only if the repository root is on sys.path. Locally an editable - # install provides that and hides the difference, which is why this failed - # only once deployed -- the code location loaded fine and the step that - # imported db died. - import sys - - import automated_ingestion - - assert str(automated_ingestion._REPOSITORY_ROOT) in sys.path - - def test_db_imports_from_an_unrelated_working_directory(): # Reproduces the deployed condition: a process whose cwd is not the # repository. The lazy imports in the resource and the connectivity asset # run at step execution, not at load, so this is the path that broke. + # In the image this passes because the repository is installed; locally + # because the editable install has the same effect. import subprocess import sys diff --git a/dagster_cloud_post_install.sh b/dagster_cloud_post_install.sh new file mode 100755 index 000000000..c093ae1c8 --- /dev/null +++ b/dagster_cloud_post_install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Runs inside the Dagster+ image build, after the repository has been copied to +# /opt/dagster/app and the pinned requirements installed. +# +# Installs this repository as a package so `db`, `domain`, `services`, `core`, +# and `schemas` resolve from site-packages. Without it they are importable only +# while /opt/dagster/app happens to be on sys.path -- true for the process that +# loads the code location, not for the process that executes a step, which is +# where the loader's imports run. That difference is invisible locally, where an +# editable install puts the repository on the path unconditionally. +# +# --no-deps because the pinned, hashed requirements are already installed and +# this must not resolve anything on top of them. +set -euo pipefail +pip install --no-deps .