feat: support global OS prefix

diracx-cli/src/diracx/cli/internal/legacy.py

@@ -311,6 +311,9 @@ def generate_helm_values(
         },
     }
 
+    if global_os_prefix := cfg["Systems"].get("NoSQLDatabases", {}).get("IndexPrefix"):
+        os_dbs["GlobalIndexPrefix"] = global_os_prefix
+
     for entry_point in select_from_extension(group=DiracEntryPoint.OS_DB):
         db_name = entry_point.name
         db_config = all_db_configs.get(db_name, {})

diracx-core/src/diracx/core/settings.py

@@ -5,6 +5,7 @@
 __all__ = [
     "AuthSettings",
     "DevelopmentSettings",
+    "FactorySettings",
     "LocalFileUrl",
     "SandboxStoreSettings",
     "ServiceSettingsBase",
@@ -390,6 +391,11 @@ class FactorySettings(ServiceSettingsBase):
     )
     """The url for the redis server to manage tasks"""
 
+    os_global_prefix: str = Field(
+        default="", validation_alias="DIRACX_FACTORY_OS_GLOBAL_PREFIX"
+    )
+    """Global prefix for OpenSearch database indices."""
+
     enabled_services: dict[str, bool] = Field(default_factory=dict)
     """The following environment variables dictates which routers are enabled."""
 

diracx-db/pyproject.toml

@@ -65,4 +65,5 @@ addopts = [
 asyncio_mode = "auto"
 markers = [
     "enabled_dependencies: List of dependencies which should be available to the FastAPI test client",
+    "os_global_prefix: test for global prefix",
 ]

diracx-db/src/diracx/db/os/utils.py

@@ -81,13 +81,17 @@ class BaseOSDB(metaclass=ABCMeta):
     @abstractmethod
     def index_name(self, vo: str, doc_id: int) -> str: ...
 
-    def __init__(self, connection_kwargs: dict[str, Any]) -> None:
+    def __init__(
+        self, connection_kwargs: dict[str, Any], *, global_prefix: str = ""
+    ) -> None:
         self._client: AsyncOpenSearch | None = None
         self._connection_kwargs = connection_kwargs
         # We use a ContextVar to make sure that self._conn
         # is specific to each context, and avoid parallel
         # route executions to overlap
         self._conn: ContextVar[bool] = ContextVar("_conn", default=False)
+        if global_prefix:
+            self.index_prefix = f"{global_prefix}_{self.index_prefix}"
 
     @classmethod
     def available_implementations(cls, db_name: str) -> list[type[BaseOSDB]]:

diracx-db/tests/opensearch/test_index_template.py

@@ -49,3 +49,10 @@
     assert list(index_mapping) == [index_name]
     assert list(index_mapping[index_name]) == ["mappings"]
     return index_mapping[index_name]["mappings"]
+
+
+@pytest.mark.os_global_prefix("myglobalprefix")
+async def test_global_prefix(sql_opensearch_db: DummyOSDB):
+    """Check the global prefix."""
+    idx_name = sql_opensearch_db.index_name("myvo", 123)
+    assert idx_name.startswith("myglobalprefix"), idx_name

diracx-routers/src/diracx/routers/factory.py

@@ -239,11 +239,14 @@ def create_app_inner(
             app.dependency_overrides[source_cls.create] = source.read
 
     # Add the OpenSearch DBs to the application
+    os_global_prefix = FactorySettings().os_global_prefix
     available_os_db_classes: set[type[BaseOSDB]] = set()
     for db_name, connection_kwargs in os_database_conn_kwargs.items():
         os_db_classes = BaseOSDB.available_implementations(db_name)
         # The first DB is the highest priority one
-        os_db = os_db_classes[0](connection_kwargs=connection_kwargs)
+        os_db = os_db_classes[0](
+            connection_kwargs=connection_kwargs, global_prefix=os_global_prefix
+        )
         app.lifetime_functions.append(os_db.client_context)
         # Add overrides for all the DB classes, including those from extensions
         # This means vanilla DiracX routers get an instance of the extension's DB

diracx-tasks/src/diracx/tasks/plumbing/factory.py

@@ -286,6 +286,8 @@ async def setup_dependency_overrides(
     from diracx.db.os.utils import BaseOSDB
     from diracx.db.sql.utils import BaseSQLDB
 
+    factory_settings = FactorySettings()
+
     overrides: dict[Callable, Callable] = {}
     async with AsyncExitStack() as stack:
         # --- SQL databases ---
@@ -300,15 +302,18 @@ async def setup_dependency_overrides(
                 )
 
         # --- OS databases ---
+        os_global_prefix = factory_settings.os_global_prefix
         for db_name, conn_kwargs in BaseOSDB.available_urls().items():
             os_db_classes = BaseOSDB.available_implementations(db_name)
-            os_db = os_db_classes[0](connection_kwargs=conn_kwargs)
+            os_db = os_db_classes[0](
+                connection_kwargs=conn_kwargs, global_prefix=os_global_prefix
+            )
             await stack.enter_async_context(os_db.client_context())
             for os_db_class in os_db_classes:
                 overrides[os_db_class.session] = partial(_db_context, os_db)
 
         # --- Config ---
-        config_url = FactorySettings().config_backend_url
+        config_url = factory_settings.config_backend_url
         if config_url:
             from diracx.core.config import ConfigSource
 

diracx-testing/src/diracx/testing/dummy_osdb.py

@@ -23,9 +23,14 @@ class DummyOSDB(BaseOSDB):
         "TextField": {"type": "text"},
     }
 
+    index_prefix = "hastobedefinedasabstractinthebase"
+
     def __init__(self, *args, **kwargs):
         # Randomize the index prefix to ensure tests are independent
         self.index_prefix = f"dummy_{secrets.token_hex(8)}"
+        # Since we mangle with the index prefix,
+        # and that the global_prefix is managed in the base class
+        # Keep the super call after the index_prefix overwrite
         super().__init__(*args, **kwargs)
 
     def index_name(self, vo: str, doc_id: int) -> str:

diracx-testing/src/diracx/testing/mock_osdb.py

@@ -36,7 +36,10 @@ class JobParametersDB(MockOSDBMixin, JobParametersDB):
         JobParametersDB = type("JobParametersDB", (MockOSDBMixin, JobParametersDB), {})
     """
 
-    def __init__(self, connection_kwargs: dict[str, Any]) -> None:
+    def __init__(
+        self, connection_kwargs: dict[str, Any], *, global_prefix: str = ""
+    ) -> None:
+        super().__init__(connection_kwargs, global_prefix=global_prefix)
         from sqlalchemy import JSON, Column, DateTime, Integer, MetaData, String, Table
 
         # Dynamically create a subclass of BaseSQLDB so we get clearer errors

diracx-testing/src/diracx/testing/osdb.py

@@ -92,14 +92,18 @@ async def dummy_opensearch_db(dummy_opensearch_db_without_template):
 
 
 @pytest.fixture
-async def sql_opensearch_db():
+async def sql_opensearch_db(request):
     """Fixture which returns a SQLOSDB object."""
+    os_global_prefix = None
+    if (marker := request.node.get_closest_marker("os_global_prefix")) is not None:
+        os_global_prefix = marker.args[0]
 
     class MockDummyOSDB(MockOSDBMixin, DummyOSDB):
         pass
 
     db = MockDummyOSDB(
-        connection_kwargs={"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"}
+        connection_kwargs={"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"},
+        global_prefix=os_global_prefix,
     )
     async with db.client_context():
         await db.create_index_template()

docs/admin/reference/env-variables.md

@@ -33,6 +33,11 @@ The hashed API key for the legacy exchange endpoint.
 *Optional*, default value: `redis://localhost`
 The url for the redis server to manage tasks
 
+### `DIRACX_FACTORY_OS_GLOBAL_PREFIX`
+
+*Optional*, default value: \`\`
+Global prefix for OpenSearch database indices.
+
 ### `ENABLED_SERVICES`
 
 *Optional*