Cam nosub

.github/workflows/main.yaml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.12", "3.13", "3.14"]
       fail-fast: false
 
     steps:
@@ -30,7 +30,7 @@ jobs:
         with:
           persist-credentials: false
           submodules: 'recursive' # grab submodules
-          
+
       # https://github.com/astral-sh/setup-uv
       - name: Install uv
         uses: astral-sh/setup-uv@v8.2.0

.github/workflows/release.yaml

@@ -0,0 +1,82 @@
+name: Model Release
+
+on:
+  push:
+    tags:
+      - "v*"
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag to release (e.g. v1.2.3)"
+        required: true
+
+env:
+  FORCE_COLOR: "1"
+  RELEASE_TAG: ${{ github.ref_name || github.event.inputs.tag }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # Required to create GitHub Releases and upload assets
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.1
+        with:
+          fetch-depth: 0 # Required for uv-dynamic-versioning to read git tags
+          persist-credentials: false
+          submodules: 'recursive' # grab submodules
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v7.0.0
+        with:
+          python-version: "3.10"
+
+      - name: Install just
+        run: uv tool install rust-just
+
+      # Install gen-sqla before uv sync so the hatch build hook can find it
+      # when building the editable install during `uv sync --no-install-project`.
+      - name: Install gen-sqla
+        run: uv tool install linkml
+
+      - name: Install dependencies
+        run: uv sync --dev --no-progress --no-install-project
+
+      - name: Install project
+        run: uv sync --dev --no-progress
+
+      - name: Generate all model artifacts
+        run: |
+          just site
+        # Runs gen-project, gen-doc, _gen_ftddd, _gen_sqla, _gen_harmony
+        # Produces: project/sqlalchemy/, project/harmony/, project/owl/, etc.
+        # hatch_build.py will skip re-generating the SQLAlchemy file since
+        # just site already created it (dest_file.exists() guard).
+
+      - name: Build wheel and sdist
+        run: uv build
+
+      - name: Zip full project artifacts
+        run: zip -r project-artifacts.zip project/
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v3
+        with:
+          tag_name: ${{ env.RELEASE_TAG }}
+          name: "Release ${{ env.RELEASE_TAG }}"
+          generate_release_notes: true
+          update_release: true
+          files: |
+            dist/*.whl
+            dist/*.tar.gz
+            project-artifacts.zip
+          fail_on_unmatched_files: true

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/include_access_model/schema/common-access-model"]
-	path = src/include_access_model/schema/common-access-model
-	url = https://github.com/include-dcc/common-access-model

.linkml-linter.yaml

@@ -0,0 +1,3 @@
+rules:
+  permissible_values_format:
+    format: '^[a-zA-Z0-9_\-:]+$'

COLLABORATORS.md

@@ -0,0 +1,72 @@
+## Welcome to the INCLUDE Access Model 🚀
+
+This repository serves as a downstream model designed to specialize the
+[Common Access Model](https://github.com/include-dcc/common-access-model). A
+pinned version of this upstream model is captured in
+src/kf_access_model/schema/upstream-models/common_access_model*.yaml.
+
+The purpose of this model is to build out a program-specific extension (or
+profile) by importing the core elements and layering the INCLUDE unique data
+requirements on top.
+
+## Key Integration Guidelines
+
+- The upstream-models/common_access_model.yaml should never be updated. This
+  should be treated like any other machine generated file and left alone. If
+  changes must be made to the upstream model, they should be made directly to
+  that model, a release published and the local version updated using the
+  tooling proved.
+- Leverage Imports: At this time, the current model imports the
+  common_access_model.yaml directly within the main model definition.
+- Extend via Inheritance: Use the is_a or mixins keys to create program-specific
+  subclasses that inherit core slots while allowing you to add local attributes.
+- Refine via Slot Usage: If you need to restrict or change the behavior of an
+  inherited core slot just for your program's classes, use the slot_usage
+  feature.
+
+## Updating to the newest common_access_model
+
+Upon new releases in the common_access_model, a flattened version of the model
+is generated and made available immediately. Tooling is provided to
+automatically pull the newest version down and link it correctly to work without
+making any changes to the local model's yaml.
+
+```bash
+just update-cam
+```
+
+This recipe will download the newest version to the directory,
+src/kf_access_model/schema/upstream-models and create a symbolic link from the
+common name, common_access_model.yaml. As a result, once that one entry has been
+made to the local model, subsequent updates should work without further updates
+to the model itself.
+
+```model-yaml
+imports:
+  - upstream-models/common_access_model
+```
+
+The current version of the common_access_model and the symbolic link should be
+properly managed by git so that all collaborators will be working on the same
+core model.
+
+> [WARNING] This will bring the upstream model to the most current version and,
+> as such, should only be performed according to planned upgrades.
+
+## Release Artifacts
+
+There are a number of artifacts which are used by various scripts including the
+dbt utilities which are built via github actions during release. To trigger the
+build, create releases linked to a semantic version preceded with a v (i.e.
+v1.0.1).
+
+These artifacts include:
+
+- SQL Alchemy model
+- dbt model yml file
+- SQL Schema
+- data dictionary conformant to the current FTD spec
+- enumerations csv file extracted from all of the permissible values
+
+The last two are used by this group's dbt utilities tooling. The SQL Alchemy
+model is used by a handful of other scripts.

config.yaml

@@ -6,6 +6,7 @@
 # By default all generators are run. Specify here the ones to skip.
 excludes:
   - markdown
+  - graphql
 
 # Alternatively only specify the generators to run
 # (comment out the above "excludes" section)

docs/dbt_artifacts.md

@@ -0,0 +1,90 @@
+# Custom Artifact Generation (dbt)
+
+We have a handful of custom scripts that build various artifacts required by
+different components of the various data pipeline components. These should all
+be handled by the CI/CD pipeline and added to the release's Assets list.
+
+- The SQL Alchemy model is available directly from the assets list as a whl file
+  and can be added to your python distribution using @git+{url} syntax.
+- The all artifacts generated by the _just site_ recipe can be found inside the
+  release assets' _project_artifacts.zip_ file.
+
+To trigger the production of these assets, a new release should be created with
+a semantic version based tag that starts with v: v1.0.0, etc.
+
+## The dbt Recipe
+
+A dumb wrapper recipe has been created with dependencies to trigger the
+production of each of those dbt related artifacts.
+
+This recipe has been added as a dependency of the standard recipe, _site_.
+
+```bash
+just dbt
+```
+
+Example command to add the model to your uv based python project:
+
+```bash
+uv add common_access_model@https://github.com/include-dcc/common-access-model/releases/download/0.0.1/common_access_model-0.0.0.post89.dev0+d378cee-py3-none-any.whl
+```
+
+Please note the _releases/download/0.0.1_ portion of the url. That is to a
+specific version of the model itself, which is by design. You will need to
+update your project each time you want to update to a new model.
+
+In a similar fashion, you can install it via pip or add it to your
+pyproject.toml file using a similar pattern.
+
+### SQL Alchemy Model
+
+Many of our tools rely on the SQL Alchemy models produced using LinkML's SQL
+Alchemy generator. We've added just recipe's to run build these:
+
+```bash
+just sqla
+```
+
+The resulting file is found inside _project/sqlalchemy/{model_name}.py_
+
+### Data Dictionary csv files
+
+The pipeline utitilies created by the FTD team rely on csv data dictionaries to
+the target .sql files.
+
+```bash
+just _gen_ftddd
+```
+
+Resulting csv files can be found inside: _project/data-dictionary/\*.csv_
+
+Each class from the model and any of the SQL specific join tables will be found
+in their own file, conforming to the
+[FTD DD Format](https://nih-ncpi.github.io/map-dragon/#/datadictionary).
+
+### Model Enums
+
+This is a CSV file that contains each enumeration and it's home within the
+datamodel. It is used by the dbt pipeline utitilies. This is a secondary
+artifact from the _gen_ftddd recipe.
+
+The resulting file is found inside _project/enumerations/model_enums.csv_
+
+### dbt Model
+
+A complete model that can be immediately dropped into your dbt pipeline can be
+generated that features various dbt tests based on the model's definition
+including:
+
+- valid_uri_format
+- foreign key target must exist
+- uri-or-curie
+- multi-column keys
+- not null
+- unique
+- allowable values
+
+The resulting file is found inside _project/dbt/src_dev_include_access.yml_
+
+The filename is purely a placeholder and should be renamed to whatever makes
+sense for your needs.

justfile

@@ -76,12 +76,7 @@ clean: _clean_project
 
 # (Re-)Generate project and documentation locally
 [group('model development')]
-site: gen-project gen-doc gensqla
-
-# SQL Alchemy model
-[group('model development')]
-gensqla: _gen_sqla
-
+site: gen-project gen-doc dbt
 
 # Deploy documentation site to Github Pages
 [group('deployment')]
@@ -95,7 +90,7 @@ test: _test-schema _test-python _test-examples
 # Run linting
 [group('model development')]
 lint:
-  uv run linkml-lint {{source_schema_dir}}
+  uv run linkml-lint --config .linkml-linter.yaml  {{source_schema_dir}}
 
 # Generate md documentation for the schema
 [group('model development')]
@@ -118,7 +113,7 @@ gen-python:
 
 # Generate project files including Python data model
 [group('model development')]
-gen-project: _merge_common
+gen-project:
   uv run gen-project {{config_yaml}} -d {{dest}} {{source_schema_path}}
   mv {{dest}}/*.py {{pymodel}}
   uv run gen-pydantic {{gen_pydantic_args}} {{source_schema_path}} > {{pymodel}}/{{schema_name}}_pydantic.py

mkdocs.yml

@@ -21,13 +21,41 @@ plugins:
   #     scripts:
   #       - gen_pages.py  # or any other name or path
 
+
+markdown_extensions:
+  - pymdownx.superfences:
+      # https://mkdocs-mermaid2.readthedocs.io/en/latest/superfences
+      # custom code highlighting rules
+      custom_fences:
+        - name: mermaid
+          # disable code highlighting to display diagrams
+          format: !!python/name:pymdownx.superfences.fence_code_format
+          class: mermaid
+  # https://facelessuser.github.io/pymdown-extensions/extensions/magiclink/
+  # auto-link HTML, FTP, and email links
+  - pymdownx.magiclink
+  # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#admonition
+  # support for call-outs
+  - admonition
+  # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#tables
+  # create tables
+  - tables
+  # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#attribute-lists
+  # enables use of HTML attributes and CSS classes to applicable elements
+  - attr_list
+  # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/#markdown-in-html
+  # write markdown inside of HTML
+  - md_in_html
+
+
 watch:
   - src/include_access_model/schema
 
 nav:
 #  - Home: index.md
   - Schema: elements/index.md
   - ER Diagram: elements/erdiagram.md
+  - dbt: dbt_artifacts
   - About: about.md
 
 exclude_docs: |

project.justfile

@@ -3,11 +3,26 @@
 # Overriding recipes from the root justfile by adding a recipe with the same
 # name in this file is not possible until a known issue in just is fixed,
 # https://github.com/casey/just/issues/2540
+
+# Support for various dbt related tasks
 [group('model development')]
-_merge_common:
-    git submodule update --init --recursive
+dbt: gen-sqla gen-ftddd gen-dbtmodel
 
+# SQL Alchemy model
 [group('model development')]
-_gen_sqla:
+gen-sqla:
     mkdir -p {{dest}}/sqlalchemy && \
     uv run gen-sqla {{source_schema_path}} --declarative > {{dest}}/sqlalchemy/{{schema_name}}.py
+
+
+[group('model development')]
+gen-ftddd:
+  uv run linkml_extract_dd {{source_schema_path}}
+
+[group('model development')]
+gen-dbtmodel:
+  uv run gen-dbtmodel
+
+[group('model development')]
+update-cam:
+  uv run update-cam -l src/include_access_model/schema/include_access_model.yaml -d src/include_access_model/schema/upstream-models