From df0dddde8e9dd0bae342f10719a722e95e8f2ff6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:42:37 +0800 Subject: [PATCH 001/180] ci: add release promotion automation --- .github/workflows/pack-bundle.yml | 4 +- .github/workflows/promote-develop-to-main.yml | 142 ++++++++++++++++++ .github/workflows/release-please.yml | 65 ++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/promote-develop-to-main.yml create mode 100644 .github/workflows/release-please.yml diff --git a/.github/workflows/pack-bundle.yml b/.github/workflows/pack-bundle.yml index 2d80de1a..96a2557f 100644 --- a/.github/workflows/pack-bundle.yml +++ b/.github/workflows/pack-bundle.yml @@ -157,8 +157,8 @@ jobs: BUNDLE=$(ls auplc-bundle-*.tar.gz) TAG="${{ github.event.workflow_run.head_branch }}" - # Upload to the existing release. Releases are created manually with - # proper release notes before tagging; CI only attaches the bundle. + # Upload to the release created by the release workflow. If the + # release is not available yet, keep the bundle artifact for retry. if gh release view "${TAG}" &>/dev/null; then gh release upload "${TAG}" "${BUNDLE}" --clobber echo "Bundle uploaded to release ${TAG}" diff --git a/.github/workflows/promote-develop-to-main.yml b/.github/workflows/promote-develop-to-main.yml new file mode 100644 index 00000000..13bbc6de --- /dev/null +++ b/.github/workflows/promote-develop-to-main.yml @@ -0,0 +1,142 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +name: Promote Develop to Main + +on: + workflow_dispatch: + inputs: + head_branch: + description: 'Integration branch to promote' + required: true + default: develop + type: string + base_branch: + description: 'Release branch that receives the promotion PR' + required: true + default: main + type: string + enable_auto_merge: + description: 'Enable auto-merge for the promotion PR after checks pass' + required: true + default: false + type: boolean + +permissions: + contents: read + pull-requests: write + issues: write + +concurrency: + group: promote-${{ inputs.head_branch }}-to-${{ inputs.base_branch }} + cancel-in-progress: false + +jobs: + open-promotion-pr: + name: Open promotion PR + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.RELEASE_AUTOMATION_TOKEN || github.token }} + BASE_BRANCH: ${{ inputs.base_branch }} + HEAD_BRANCH: ${{ inputs.head_branch }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create or reuse promotion PR + id: promote + run: | + set -euo pipefail + + git fetch --no-tags origin "${BASE_BRANCH}" "${HEAD_BRANCH}" + + ahead=$(git rev-list --count "origin/${BASE_BRANCH}..origin/${HEAD_BRANCH}") + behind=$(git rev-list --count "origin/${HEAD_BRANCH}..origin/${BASE_BRANCH}") + + echo "ahead=${ahead}" >> "${GITHUB_OUTPUT}" + echo "behind=${behind}" >> "${GITHUB_OUTPUT}" + + if [[ "${ahead}" == "0" ]]; then + echo "${HEAD_BRANCH} has no commits to promote into ${BASE_BRANCH}." + echo "pr_url=" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + existing_pr=$( + gh pr list \ + --base "${BASE_BRANCH}" \ + --head "${HEAD_BRANCH}" \ + --state open \ + --json url \ + --jq '.[0].url // ""' + ) + + if [[ -n "${existing_pr}" ]]; then + echo "Reusing existing promotion PR: ${existing_pr}" + echo "pr_url=${existing_pr}" >> "${GITHUB_OUTPUT}" + gh pr comment "${existing_pr}" --body \ + "Promotion check refreshed: ${HEAD_BRANCH} is ${ahead} commit(s) ahead of ${BASE_BRANCH} and ${behind} commit(s) behind." + exit 0 + fi + + body_file=$(mktemp) + cat > "${body_file}" <> "${GITHUB_OUTPUT}" + + - name: Enable auto-merge + if: inputs.enable_auto_merge && steps.promote.outputs.pr_url != '' + run: gh pr merge --auto --merge "${{ steps.promote.outputs.pr_url }}" + + - name: Summarize promotion + run: | + { + echo "## Promotion summary" + echo + echo "- Head branch: \`${HEAD_BRANCH}\`" + echo "- Base branch: \`${BASE_BRANCH}\`" + echo "- Commits ahead: \`${{ steps.promote.outputs.ahead }}\`" + echo "- Commits behind: \`${{ steps.promote.outputs.behind }}\`" + if [[ -n "${{ steps.promote.outputs.pr_url }}" ]]; then + echo "- Promotion PR: ${{ steps.promote.outputs.pr_url }}" + else + echo "- Promotion PR: not created because there are no commits to promote" + fi + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 00000000..02451ad6 --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,65 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +name: Release Please + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: write + issues: write + pull-requests: write + +concurrency: + group: release-please-${{ github.ref }} + cancel-in-progress: false + +jobs: + release-please: + name: Prepare or publish release + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + version: ${{ steps.release.outputs.version }} + steps: + - name: Run release-please + id: release + uses: googleapis/release-please-action@v4 + with: + token: ${{ secrets.RELEASE_AUTOMATION_TOKEN || github.token }} + target-branch: main + release-type: simple + + - name: Summarize release-please result + run: | + { + echo "## Release Please summary" + echo + echo "- Release created: \`${{ steps.release.outputs.release_created || 'false' }}\`" + echo "- Tag: \`${{ steps.release.outputs.tag_name || 'not created' }}\`" + echo "- Version: \`${{ steps.release.outputs.version || 'not created' }}\`" + echo + echo "Use a repository secret named \`RELEASE_AUTOMATION_TOKEN\` from a" + echo "GitHub App or fine-grained PAT if tag or release events must" + echo "trigger downstream workflows." + } >> "${GITHUB_STEP_SUMMARY}" From 4efa5c66134b003aaf077d621b2904a1b3d9347f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:30:39 +0800 Subject: [PATCH 002/180] ci: harden release automation workflows --- .github/workflows/promote-develop-to-main.yml | 10 +++++++--- .github/workflows/release-please.yml | 20 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/promote-develop-to-main.yml b/.github/workflows/promote-develop-to-main.yml index 13bbc6de..4e99c690 100644 --- a/.github/workflows/promote-develop-to-main.yml +++ b/.github/workflows/promote-develop-to-main.yml @@ -26,12 +26,16 @@ on: description: 'Integration branch to promote' required: true default: develop - type: string + type: choice + options: + - develop base_branch: description: 'Release branch that receives the promotion PR' required: true default: main - type: string + type: choice + options: + - main enable_auto_merge: description: 'Enable auto-merge for the promotion PR after checks pass' required: true @@ -52,7 +56,7 @@ jobs: name: Open promotion PR runs-on: ubuntu-latest env: - GH_TOKEN: ${{ secrets.RELEASE_AUTOMATION_TOKEN || github.token }} + GH_TOKEN: ${{ github.token }} BASE_BRANCH: ${{ inputs.base_branch }} HEAD_BRANCH: ${{ inputs.head_branch }} steps: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 02451ad6..f9e83772 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -42,11 +42,22 @@ jobs: tag_name: ${{ steps.release.outputs.tag_name }} version: ${{ steps.release.outputs.version }} steps: + - name: Verify release automation token + env: + RELEASE_AUTOMATION_TOKEN: ${{ secrets.RELEASE_AUTOMATION_TOKEN }} + run: | + if [[ -z "${RELEASE_AUTOMATION_TOKEN}" ]]; then + echo "RELEASE_AUTOMATION_TOKEN is required for release automation." >&2 + echo "Use a repository-scoped GitHub App token or fine-grained PAT." >&2 + echo "The token must create release tags that trigger downstream workflows." >&2 + exit 1 + fi + - name: Run release-please id: release - uses: googleapis/release-please-action@v4 + uses: googleapis/release-please-action@8b8fd2cc23b2e18957157a9d923d75aa0c6f6ad5 # v4 with: - token: ${{ secrets.RELEASE_AUTOMATION_TOKEN || github.token }} + token: ${{ secrets.RELEASE_AUTOMATION_TOKEN }} target-branch: main release-type: simple @@ -59,7 +70,6 @@ jobs: echo "- Tag: \`${{ steps.release.outputs.tag_name || 'not created' }}\`" echo "- Version: \`${{ steps.release.outputs.version || 'not created' }}\`" echo - echo "Use a repository secret named \`RELEASE_AUTOMATION_TOKEN\` from a" - echo "GitHub App or fine-grained PAT if tag or release events must" - echo "trigger downstream workflows." + echo "\`RELEASE_AUTOMATION_TOKEN\` is required so release-created tags" + echo "can trigger downstream Docker image and bundle workflows." } >> "${GITHUB_STEP_SUMMARY}" From f9693454405588301f7b8aa9288229128f255adc Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:46:26 +0800 Subject: [PATCH 003/180] feat(config): add 9600 GRE accelerator baseline --- runtime/values-multi-nodes.yaml.example | 7 +++++++ runtime/values.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 25c90da6..20e6d217 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -189,6 +189,13 @@ custom: amd.com/gpu.product-name: "AMD_Radeon_AI_PRO_R9700" env: {} quotaRate: 4 + 9600gre: + displayName: "AMD Radeon™ RX 9600 GRE (Desktop GPU)" + description: "RDNA 4.0 (gfx120x) | Compute Units 32 | 12GB GDDR6" + nodeSelector: + amd.com/gpu.product-name: "AMD_Radeon_RX_9600_GRE" + env: {} + quotaRate: 4 # -------------------------------------------------------------------------- # Course Resources diff --git a/runtime/values.yaml b/runtime/values.yaml index d1455e20..383eb8bd 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -268,6 +268,13 @@ custom: amd.com/gpu.product-name: "AMD_Radeon_AI_PRO_R9700" env: {} quotaRate: 4 + 9600gre: + displayName: "AMD Radeon™ RX 9600 GRE (Desktop GPU)" + description: "RDNA 4.0 (gfx120x) | Compute Units 32 | 12GB GDDR6" + nodeSelector: + amd.com/gpu.product-name: "AMD_Radeon_RX_9600_GRE" + env: {} + quotaRate: 4 # ============================================================================ # Course Resources Configuration From a27dc758cb3abcf74c01d1b5331f66f79f4c3aaa Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:46:33 +0800 Subject: [PATCH 004/180] feat(installer): treat 9600 GRE as a curated SKU --- auplc_installer/gpu.py | 4 ++-- tests/installer/test_gpu.py | 6 +++++- tests/installer/test_overlay.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/auplc_installer/gpu.py b/auplc_installer/gpu.py index b967a90d..3c10305f 100644 --- a/auplc_installer/gpu.py +++ b/auplc_installer/gpu.py @@ -48,8 +48,8 @@ # Accelerator keys defined in runtime/values.yaml custom.accelerators. When # the resolved accel_key is not in this list, ``overlay.py`` injects a full # minimal accelerator stanza so helm install succeeds without values.yaml -# edits (useful for ad-hoc SKUs like 9600gre). -GPU_CURATED_SKU_KEYS = ("phx", "strix", "strix-halo", "9070xt", "r9700") +# edits (useful for ad-hoc SKUs not yet promoted to the default values). +GPU_CURATED_SKU_KEYS = ("phx", "strix", "strix-halo", "9070xt", "r9700", "9600gre") def is_curated_sku(key: str) -> bool: diff --git a/tests/installer/test_gpu.py b/tests/installer/test_gpu.py index af20b4a4..32f4285d 100644 --- a/tests/installer/test_gpu.py +++ b/tests/installer/test_gpu.py @@ -136,8 +136,12 @@ def test_is_curated_sku(key: str) -> None: assert is_curated_sku(key) +def test_is_curated_sku_true_for_9600gre() -> None: + assert is_curated_sku("9600gre") + + def test_is_curated_sku_false_for_unknown() -> None: - assert not is_curated_sku("9600gre") + assert not is_curated_sku("totally-unknown") def test_resolve_gpu_config_known_short_name() -> None: diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index 7d1d1c9f..9fb69757 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -122,6 +122,18 @@ def test_curated_sku_with_product_name_emits_node_selector() -> None: assert accelerators["strix-halo"]["nodeSelector"]["amd.com/gpu.product-name"] == "AMD_Radeon_8060S_Graphics" +def test_9600gre_uses_curated_overlay_path() -> None: + cfg = GpuConfig() + append_product(cfg, "AMD_Radeon_RX_9600_GRE") + text, parsed = _render(cfg, courses=CourseSelection.default()) + accel = parsed["custom"]["accelerators"]["9600gre"] + assert accel["nodeSelector"]["amd.com/gpu.product-name"] == "AMD_Radeon_RX_9600_GRE" + assert "displayName" not in accel + assert "description" not in accel + assert "quotaRate" not in accel + assert "SKU '9600gre' is not curated in values.yaml" not in text + + def test_basic_emits_filtered_teams_mapping() -> None: _, parsed = _render( _strix_halo_cfg(), From 1f6e23f56b13ebb9ed6811883c4028ede7b7663d Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:58:41 +0800 Subject: [PATCH 005/180] feat(config): add GPU resource image overrides --- runtime/values-multi-nodes.yaml.example | 117 +++++++++++++++++-- runtime/values.yaml | 113 +++++++++++++++++- tests/installer/test_values_gpu_overrides.py | 66 +++++++++++ 3 files changed, 282 insertions(+), 14 deletions(-) create mode 100644 tests/installer/test_values_gpu_overrides.py diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 20e6d217..5084148e 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -284,7 +284,25 @@ custom: subDescription: "GPU Accelerated Environment" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" allowGitClone: true resourceType: "notebook" code-gpu: @@ -293,7 +311,25 @@ custom: subDescription: "GPU-accelerated development workspace" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" allowGitClone: true launchMode: "code-server" resourceType: "browser-ide" @@ -303,41 +339,104 @@ custom: subDescription: "Suitable for CV experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre resourceType: "notebook" - # acceleratorOverrides: (optional) per-accelerator image and env overrides - # Use this when one course is available on multiple GPU targets with - # different image tags. - # 9070xt: - # image: "ghcr.io/your-org/auplc-cv:latest-gfx120x" - # r9700: - # image: "ghcr.io/your-org/auplc-cv:latest-gfx120x" - # strix-halo: - # image: "ghcr.io/your-org/auplc-cv:latest-gfx1151" + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" Course-DL: group: "TEACHING LABS" description: "Deep Learning Course" subDescription: "Suitable for DL experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre resourceType: "notebook" + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" Course-LLM: group: "TEACHING LABS" description: "Large Language Models Course" subDescription: "Suitable for LLM experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre resourceType: "notebook" + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" Course-PhySim: group: "TEACHING LABS" description: "Genesis Physical Simulation Course" subDescription: "Suitable for physical simulation experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre resourceType: "notebook" + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" # -------------------------------------------------------------------------- # Team Permission Configuration diff --git a/runtime/values.yaml b/runtime/values.yaml index 383eb8bd..ed7f1bc5 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -371,7 +371,25 @@ custom: subDescription: "GPU Accelerated Environment" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-base:latest-gfx120x" allowGitClone: true defaultPath: "/home/jovyan" resourceType: "notebook" @@ -381,7 +399,25 @@ custom: subDescription: "GPU-accelerated development workspace" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx120x" allowGitClone: true launchMode: "code-server" defaultPath: "/home/jovyan" @@ -392,12 +428,25 @@ custom: subDescription: "Suitable for CV experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo - # acceleratorOverrides: (optional) per-accelerator image and env overrides - # 9070xt: - # image: "ghcr.io/your-org/auplc-cv:" - # r9700: - # image: "ghcr.io/your-org/auplc-cv:" + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-cv:latest-gfx120x" defaultPath: "/opt/workspace/CV" resourceType: "notebook" Course-DL: @@ -406,7 +455,25 @@ custom: subDescription: "Suitable for DL experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-dl:latest-gfx120x" defaultPath: "/opt/workspace/DL" resourceType: "notebook" Course-LLM: @@ -415,7 +482,25 @@ custom: subDescription: "Suitable for LLM experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-llm:latest-gfx120x" defaultPath: "/opt/workspace/LLM" resourceType: "notebook" Course-PhySim: @@ -424,7 +509,25 @@ custom: subDescription: "Suitable for physical simulation experiments with GPU" accelerator: "GPU" acceleratorKeys: + - phx + - strix - strix-halo + - 9070xt + - r9700 + - 9600gre + acceleratorOverrides: + phx: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx110x" + strix: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx1150" + strix-halo: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx1151" + 9070xt: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" + r9700: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" + 9600gre: + image: "ghcr.io/amdresearch/auplc-physim:latest-gfx120x" defaultPath: "/opt/workspace/PhySim" resourceType: "notebook" diff --git a/tests/installer/test_values_gpu_overrides.py b/tests/installer/test_values_gpu_overrides.py new file mode 100644 index 00000000..1f3934cf --- /dev/null +++ b/tests/installer/test_values_gpu_overrides.py @@ -0,0 +1,66 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] + +VALUES_FILES = ( + ROOT / "runtime" / "values.yaml", + ROOT / "runtime" / "values-multi-nodes.yaml.example", +) + +GPU_ACCELERATOR_TAGS = { + "phx": "gfx110x", + "strix": "gfx1150", + "strix-halo": "gfx1151", + "9070xt": "gfx120x", + "r9700": "gfx120x", + "9600gre": "gfx120x", +} + +GPU_RESOURCE_IMAGES = { + "gpu": "auplc-base", + "code-gpu": "auplc-code-gpu", + "Course-CV": "auplc-cv", + "Course-DL": "auplc-dl", + "Course-LLM": "auplc-llm", + "Course-PhySim": "auplc-physim", +} + + +def _load_values(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def test_default_values_expose_supported_gpu_accelerators() -> None: + expected_keys = list(GPU_ACCELERATOR_TAGS) + + for values_file in VALUES_FILES: + values = _load_values(values_file) + accelerators = values["custom"]["accelerators"] + metadata = values["custom"]["resources"]["metadata"] + + for accelerator_key in expected_keys: + assert accelerator_key in accelerators, values_file + + for resource_key in GPU_RESOURCE_IMAGES: + assert metadata[resource_key]["acceleratorKeys"] == expected_keys, values_file + + +def test_default_values_route_gpu_resources_to_supported_image_tags() -> None: + for values_file in VALUES_FILES: + values = _load_values(values_file) + metadata = values["custom"]["resources"]["metadata"] + + for resource_key, image_name in GPU_RESOURCE_IMAGES.items(): + overrides = metadata[resource_key]["acceleratorOverrides"] + assert set(overrides) == set(GPU_ACCELERATOR_TAGS), values_file + + for accelerator_key, gpu_target in GPU_ACCELERATOR_TAGS.items(): + assert overrides[accelerator_key]["image"] == ( + f"ghcr.io/amdresearch/{image_name}:latest-{gpu_target}" + ), values_file From e5555072ee562c1fda3efe5ccf09652ab1c65678 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:49:24 +0800 Subject: [PATCH 006/180] fix(config): keep GPU accelerator visibility conservative --- runtime/values-multi-nodes.yaml.example | 34 +++----------------- runtime/values.yaml | 34 +++----------------- tests/installer/test_values_gpu_overrides.py | 9 ++++-- 3 files changed, 15 insertions(+), 62 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 5084148e..473c16ad 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -283,13 +283,12 @@ custom: description: "Basic GPU Environment" subDescription: "GPU Accelerated Environment" accelerator: "GPU" + # Add only accelerators that exist in this cluster and are validated for + # this resource. Image overrides for the curated accelerators are + # preconfigured below, so enabling one normally only requires adding its + # key here. acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-base:latest-gfx110x" @@ -311,12 +310,7 @@ custom: subDescription: "GPU-accelerated development workspace" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx110x" @@ -339,12 +333,7 @@ custom: subDescription: "Suitable for CV experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre resourceType: "notebook" acceleratorOverrides: phx: @@ -365,12 +354,7 @@ custom: subDescription: "Suitable for DL experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre resourceType: "notebook" acceleratorOverrides: phx: @@ -391,12 +375,7 @@ custom: subDescription: "Suitable for LLM experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre resourceType: "notebook" acceleratorOverrides: phx: @@ -417,12 +396,7 @@ custom: subDescription: "Suitable for physical simulation experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre resourceType: "notebook" acceleratorOverrides: phx: diff --git a/runtime/values.yaml b/runtime/values.yaml index ed7f1bc5..f50dc60d 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -370,13 +370,12 @@ custom: description: "Basic GPU Environment" subDescription: "GPU Accelerated Environment" accelerator: "GPU" + # Add only accelerators that this deployment should expose to users. + # Image overrides for the curated accelerators are preconfigured below, + # so enabling another GPU family normally only requires adding its key + # here from custom.accelerators. acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-base:latest-gfx110x" @@ -399,12 +398,7 @@ custom: subDescription: "GPU-accelerated development workspace" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-code-gpu:latest-gfx110x" @@ -428,12 +422,7 @@ custom: subDescription: "Suitable for CV experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-cv:latest-gfx110x" @@ -455,12 +444,7 @@ custom: subDescription: "Suitable for DL experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-dl:latest-gfx110x" @@ -482,12 +466,7 @@ custom: subDescription: "Suitable for LLM experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-llm:latest-gfx110x" @@ -509,12 +488,7 @@ custom: subDescription: "Suitable for physical simulation experiments with GPU" accelerator: "GPU" acceleratorKeys: - - phx - - strix - strix-halo - - 9070xt - - r9700 - - 9600gre acceleratorOverrides: phx: image: "ghcr.io/amdresearch/auplc-physim:latest-gfx110x" diff --git a/tests/installer/test_values_gpu_overrides.py b/tests/installer/test_values_gpu_overrides.py index 1f3934cf..7f6ef6ad 100644 --- a/tests/installer/test_values_gpu_overrides.py +++ b/tests/installer/test_values_gpu_overrides.py @@ -42,13 +42,18 @@ def test_default_values_expose_supported_gpu_accelerators() -> None: for values_file in VALUES_FILES: values = _load_values(values_file) accelerators = values["custom"]["accelerators"] - metadata = values["custom"]["resources"]["metadata"] for accelerator_key in expected_keys: assert accelerator_key in accelerators, values_file + +def test_default_values_keep_visible_gpu_accelerators_conservative() -> None: + for values_file in VALUES_FILES: + values = _load_values(values_file) + metadata = values["custom"]["resources"]["metadata"] + for resource_key in GPU_RESOURCE_IMAGES: - assert metadata[resource_key]["acceleratorKeys"] == expected_keys, values_file + assert metadata[resource_key]["acceleratorKeys"] == ["strix-halo"], values_file def test_default_values_route_gpu_resources_to_supported_image_tags() -> None: From 9d89191b2c23f3beda22166e2e5b7b8be655abe7 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:09:00 +0800 Subject: [PATCH 007/180] fix(installer): override preconfigured accelerator images --- auplc_installer/overlay.py | 15 +++++---------- tests/installer/test_overlay.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index b2d984d9..5815f391 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -122,16 +122,11 @@ def emit_overlay( buf.write(" acceleratorKeys:\n") for sku in cfg.skus: buf.write(f" - {sku.accel_key}\n") - if not homogeneous_target: - base_name = _RESOURCE_IMAGE_BASE[resource] - wrote_overrides = False - for sku in cfg.skus: - if sku.gpu_target != cfg.gpu_target: - if not wrote_overrides: - buf.write(" acceleratorOverrides:\n") - wrote_overrides = True - buf.write(f" {sku.accel_key}:\n") - buf.write(f' image: "{image_registry}/{base_name}:{image_tag}-{sku.gpu_target}"\n') + base_name = _RESOURCE_IMAGE_BASE[resource] + buf.write(" acceleratorOverrides:\n") + for sku in cfg.skus: + buf.write(f" {sku.accel_key}:\n") + buf.write(f' image: "{image_registry}/{base_name}:{image_tag}-{sku.gpu_target}"\n') # --- teams.mapping filter (only when course selection is in effect) --- if filter_courses: diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index 9fb69757..b028cdd9 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -115,6 +115,17 @@ def test_resource_images_use_primary_tag() -> None: assert images["Course-PhySim"] == "ghcr.io/amdresearch/auplc-physim:v1.0-gfx1151" +def test_homogeneous_target_emits_matching_accelerator_overrides() -> None: + _, parsed = _render(_strix_halo_cfg(), courses=CourseSelection.default()) + gpu_metadata = parsed["custom"]["resources"]["metadata"]["gpu"] + overrides = gpu_metadata["acceleratorOverrides"] + assert overrides == { + "strix-halo": { + "image": "ghcr.io/amdresearch/auplc-base:v1.0-gfx1151", + }, + } + + def test_curated_sku_with_product_name_emits_node_selector() -> None: _, parsed = _render(_strix_halo_cfg(), courses=CourseSelection.default()) accelerators = parsed["custom"]["accelerators"] @@ -206,6 +217,7 @@ def test_mixed_targets_emit_accelerator_overrides() -> None: gpu_metadata = parsed["custom"]["resources"]["metadata"]["gpu"] assert "acceleratorOverrides" in gpu_metadata overrides = gpu_metadata["acceleratorOverrides"] + assert overrides["strix-halo"]["image"] == "ghcr.io/amdresearch/auplc-base:v1.0-gfx1151" assert "r9700" in overrides assert overrides["r9700"]["image"] == "ghcr.io/amdresearch/auplc-base:v1.0-gfx120x" From 972b4db25b715bc5e78a5f899f0bd5e4bf14ff8a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:14:49 +0800 Subject: [PATCH 008/180] docs(config): note accelerator override tag compatibility --- runtime/values-multi-nodes.yaml.example | 3 +++ runtime/values.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 473c16ad..3db3cffa 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -287,6 +287,9 @@ custom: # this resource. Image overrides for the curated accelerators are # preconfigured below, so enabling one normally only requires adding its # key here. + # If you override GPU resource images with custom tags or registries, + # also override the matching acceleratorOverrides entries so accelerator + # selection does not fall back to the default latest-* images. acceleratorKeys: - strix-halo acceleratorOverrides: diff --git a/runtime/values.yaml b/runtime/values.yaml index f50dc60d..da79243a 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -374,6 +374,9 @@ custom: # Image overrides for the curated accelerators are preconfigured below, # so enabling another GPU family normally only requires adding its key # here from custom.accelerators. + # If you override GPU resource images with custom tags or registries, + # also override the matching acceleratorOverrides entries so accelerator + # selection does not fall back to the default latest-* images. acceleratorKeys: - strix-halo acceleratorOverrides: From 91cf3e8753ce6586352a2d22d83a242468f70ae3 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <11815099+mariodruiz@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:36:36 +0100 Subject: [PATCH 009/180] fix(scripts): use quota REST API in manage_users instead of kubectl exec --- scripts/manage_users.py | 189 +++++++++++----------------------------- 1 file changed, 51 insertions(+), 138 deletions(-) diff --git a/scripts/manage_users.py b/scripts/manage_users.py index a6d63c9c..8808e039 100644 --- a/scripts/manage_users.py +++ b/scripts/manage_users.py @@ -69,7 +69,6 @@ import os import secrets import string -import subprocess import sys import pandas as pd @@ -168,6 +167,49 @@ def batch_set_passwords(self, users: list[dict], force_change: bool = True) -> t except requests.exceptions.RequestException as e: return False, {"error": str(e)} + def get_quota(self, username: str) -> int | None: + """Get a user's quota balance via the admin API.""" + username = self.normalize_username(username) + try: + response = requests.get(f"{self.hub_url}/hub/admin/api/quota/{username}", headers=self.headers) + if response.status_code == 200: + return response.json().get("balance") + return None + except requests.exceptions.RequestException: + return None + + def set_quota(self, username: str, amount: int) -> tuple[bool, str]: + """Set a user's quota balance via the admin API.""" + return self._modify_quota(username, {"action": "set", "amount": amount}) + + def add_quota(self, username: str, amount: int) -> tuple[bool, str]: + """Add to a user's quota balance via the admin API.""" + return self._modify_quota(username, {"action": "add", "amount": amount}) + + def _modify_quota(self, username: str, payload: dict) -> tuple[bool, str]: + """Post a quota modification and return (success, message).""" + username = self.normalize_username(username) + try: + response = requests.post( + f"{self.hub_url}/hub/admin/api/quota/{username}", headers=self.headers, json=payload + ) + data = response.json() + if response.status_code == 200: + return True, str(data.get("balance", "")) + return False, data.get("error", f"HTTP {response.status_code}") + except requests.exceptions.RequestException as e: + return False, str(e) + + def list_quotas(self) -> list[dict] | None: + """List all user quota balances via the admin API.""" + try: + response = requests.get(f"{self.hub_url}/hub/admin/api/quota", headers=self.headers) + if response.status_code == 200: + return response.json().get("users", []) + return None + except requests.exceptions.RequestException: + return None + def _check_connection(self) -> bool: """Check if connection to JupyterHub is working""" try: @@ -664,125 +706,8 @@ def cmd_set_passwords(args, manager: JupyterHubUserManager): # ============ Quota Management Commands ============ -def set_quota_in_pod(username: str, amount: int, namespace: str = "jupyterhub") -> bool: - """Set quota for a user via kubectl exec.""" - username = username.strip().lower() - - python_code = f''' -import sys -sys.path.insert(0, "/etc/jupyterhub") -from quota_manager import get_quota_manager - -qm = get_quota_manager() -qm.set_balance("{username}", {amount}, "cli_admin") -print("OK") -''' - - try: - result = subprocess.run( - ["kubectl", "--namespace", namespace, "exec", "deployment/hub", "--", "python3", "-c", python_code], - capture_output=True, - text=True, - timeout=30, - ) - return result.returncode == 0 and "OK" in result.stdout - except Exception as e: - print(f" Error: {e}") - return False - - -def add_quota_in_pod(username: str, amount: int, namespace: str = "jupyterhub") -> bool: - """Add quota to a user via kubectl exec.""" - username = username.strip().lower() - - python_code = f''' -import sys -sys.path.insert(0, "/etc/jupyterhub") -from quota_manager import get_quota_manager - -qm = get_quota_manager() -qm.add_quota("{username}", {amount}, "cli_admin") -print("OK") -''' - - try: - result = subprocess.run( - ["kubectl", "--namespace", namespace, "exec", "deployment/hub", "--", "python3", "-c", python_code], - capture_output=True, - text=True, - timeout=30, - ) - return result.returncode == 0 and "OK" in result.stdout - except Exception as e: - print(f" Error: {e}") - return False - - -def get_quota_from_pod(username: str, namespace: str = "jupyterhub") -> int | None: - """Get quota balance for a user via kubectl exec.""" - username = username.strip().lower() - - python_code = f''' -import sys -sys.path.insert(0, "/etc/jupyterhub") -from quota_manager import get_quota_manager - -qm = get_quota_manager() -balance = qm.get_balance("{username}") -print(f"BALANCE:{{balance}}") -''' - - try: - result = subprocess.run( - ["kubectl", "--namespace", namespace, "exec", "deployment/hub", "--", "python3", "-c", python_code], - capture_output=True, - text=True, - timeout=30, - ) - if result.returncode == 0: - for line in result.stdout.split("\n"): - if line.startswith("BALANCE:"): - return int(line.split(":")[1]) - return None - except Exception: - return None - - -def list_quota_from_pod(namespace: str = "jupyterhub") -> list[dict] | None: - """Get all user quota balances via kubectl exec.""" - python_code = """ -import sys -import json -sys.path.insert(0, "/etc/jupyterhub") -from quota_manager import get_quota_manager - -qm = get_quota_manager() -balances = qm.get_all_balances() -print("JSON:" + json.dumps(balances)) -""" - - try: - result = subprocess.run( - ["kubectl", "--namespace", namespace, "exec", "deployment/hub", "--", "python3", "-c", python_code], - capture_output=True, - text=True, - timeout=30, - ) - if result.returncode == 0: - import json - - for line in result.stdout.split("\n"): - if line.startswith("JSON:"): - return json.loads(line[5:]) - return None - except Exception: - return None - - def cmd_set_quota(args, manager: JupyterHubUserManager): """Set quota for users""" - namespace = args.namespace - if args.file: users = load_users_from_file(args.file) print(f"📄 Loaded {len(users)} users from {args.file}") @@ -802,14 +727,14 @@ def cmd_set_quota(args, manager: JupyterHubUserManager): print(f" ⚠️ Skipping {username}: no quota amount specified") continue - success = set_quota_in_pod(username, int(amount), namespace) + success, message = manager.set_quota(username, int(amount)) if success: print(f" ✅ Set {amount} quota for: {username}") results["success"] += 1 output_data.append({"username": username, "quota": amount}) else: - print(f" ❌ Failed: {username}") + print(f" ❌ Failed: {username}: {message}") results["failed"] += 1 print("\n" + "=" * 50) @@ -821,7 +746,6 @@ def cmd_set_quota(args, manager: JupyterHubUserManager): def cmd_add_quota(args, manager: JupyterHubUserManager): """Add quota to users""" - namespace = args.namespace amount = args.amount if args.file: @@ -839,12 +763,12 @@ def cmd_add_quota(args, manager: JupyterHubUserManager): if not username: continue - success = add_quota_in_pod(username, amount, namespace) + success, message = manager.add_quota(username, amount) if success: print(f" ✅ Added {amount} quota to: {username}") results["success"] += 1 else: - print(f" ❌ Failed: {username}") + print(f" ❌ Failed: {username}: {message}") results["failed"] += 1 print("\n" + "=" * 50) @@ -856,9 +780,7 @@ def cmd_add_quota(args, manager: JupyterHubUserManager): def cmd_list_quota(args, manager: JupyterHubUserManager): """List all user quota balances""" - namespace = args.namespace - - balances = list_quota_from_pod(namespace) + balances = manager.list_quotas() if balances is None: print("❌ Failed to retrieve quota balances") @@ -978,28 +900,19 @@ def main(): setpw_parser.add_argument("--output", "-o", help="Output file to save usernames and passwords") # Set-quota command - setquota_parser = subparsers.add_parser("set-quota", help="Set quota for users (requires kubectl)") + setquota_parser = subparsers.add_parser("set-quota", help="Set quota for users") setquota_parser.add_argument("users", nargs="*", help="Username(s) to set quota for") setquota_parser.add_argument("--file", "-f", help="CSV or Excel file with username,quota columns") setquota_parser.add_argument("--amount", "-a", type=int, help="Quota amount (when using usernames)") - setquota_parser.add_argument( - "--namespace", "-n", default="jupyterhub", help="Kubernetes namespace (default: jupyterhub)" - ) # Add-quota command - addquota_parser = subparsers.add_parser("add-quota", help="Add quota to users (requires kubectl)") + addquota_parser = subparsers.add_parser("add-quota", help="Add quota to users") addquota_parser.add_argument("users", nargs="*", help="Username(s) to add quota to") addquota_parser.add_argument("--file", "-f", help="CSV or Excel file with usernames") addquota_parser.add_argument("--amount", "-a", type=int, required=True, help="Quota amount to add") - addquota_parser.add_argument( - "--namespace", "-n", default="jupyterhub", help="Kubernetes namespace (default: jupyterhub)" - ) # List-quota command - listquota_parser = subparsers.add_parser("list-quota", help="List all user quota balances (requires kubectl)") - listquota_parser.add_argument( - "--namespace", "-n", default="jupyterhub", help="Kubernetes namespace (default: jupyterhub)" - ) + subparsers.add_parser("list-quota", help="List all user quota balances") args = parser.parse_args() From cf2fa91f18e627bc5dee9bb064d9565ff5478d04 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <11815099+mariodruiz@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:13:49 +0100 Subject: [PATCH 010/180] Make code more robust --- scripts/manage_users.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/scripts/manage_users.py b/scripts/manage_users.py index 8808e039..2a989c42 100644 --- a/scripts/manage_users.py +++ b/scripts/manage_users.py @@ -167,17 +167,6 @@ def batch_set_passwords(self, users: list[dict], force_change: bool = True) -> t except requests.exceptions.RequestException as e: return False, {"error": str(e)} - def get_quota(self, username: str) -> int | None: - """Get a user's quota balance via the admin API.""" - username = self.normalize_username(username) - try: - response = requests.get(f"{self.hub_url}/hub/admin/api/quota/{username}", headers=self.headers) - if response.status_code == 200: - return response.json().get("balance") - return None - except requests.exceptions.RequestException: - return None - def set_quota(self, username: str, amount: int) -> tuple[bool, str]: """Set a user's quota balance via the admin API.""" return self._modify_quota(username, {"action": "set", "amount": amount}) @@ -221,7 +210,7 @@ def _check_connection(self) -> bool: print(f"❌ Connection failed with status {response.status_code}") print(f"Response: {response.text}") return False - except Exception as e: + except requests.exceptions.RequestException as e: print(f"❌ Connection error: {e}") return False @@ -382,7 +371,7 @@ def get_user(self, username: str) -> dict | None: if response.status_code == 200: return response.json() return None - except Exception: + except requests.exceptions.RequestException: return None def set_admin(self, username: str, admin: bool = True) -> bool: @@ -714,6 +703,14 @@ def cmd_set_quota(args, manager: JupyterHubUserManager): else: users = [{"username": u} for u in args.users] + if not users: + print("❌ No users specified") + return + + if not args.file and args.amount is None: + print("❌ --amount is required when specifying usernames (or use --file with a quota column)") + return + results = {"success": 0, "failed": 0} output_data = [] @@ -727,7 +724,14 @@ def cmd_set_quota(args, manager: JupyterHubUserManager): print(f" ⚠️ Skipping {username}: no quota amount specified") continue - success, message = manager.set_quota(username, int(amount)) + try: + amount = int(amount) + except (TypeError, ValueError): + print(f" ⚠️ Skipping {username}: invalid quota amount '{amount}'") + results["failed"] += 1 + continue + + success, message = manager.set_quota(username, amount) if success: print(f" ✅ Set {amount} quota for: {username}") @@ -754,6 +758,10 @@ def cmd_add_quota(args, manager: JupyterHubUserManager): else: usernames = args.users + if not usernames: + print("❌ No users specified") + return + print(f"\n🔄 Adding {amount} quota to {len(usernames)} users...") results = {"success": 0, "failed": 0} From a48b95f86c458be02abf5a59456f1afbbba00438 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:14:10 +0800 Subject: [PATCH 011/180] feat(admin): add suffix width control for user generation --- .../admin/src/components/CreateUserModal.tsx | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx index a1b87c9f..ff4f4f42 100644 --- a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx +++ b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx @@ -51,13 +51,17 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, const [prefix, setPrefix] = useState(''); const [count, setCount] = useState(10); const [startNum, setStartNum] = useState(1); + const [suffixWidth, setSuffixWidth] = useState(2); const [quotaValue, setQuotaValue] = useState(String(defaultQuota || 0)); const handleGenerateNames = useCallback(() => { if (!prefix.trim()) return; - const names = Array.from({ length: count }, (_, i) => `${prefix.trim()}${startNum + i}`); + const names = Array.from({ length: count }, (_, i) => { + const suffix = String(startNum + i); + return `${prefix.trim()}${suffixWidth > 0 ? suffix.padStart(suffixWidth, '0') : suffix}`; + }); setUsernames(names.join('\n')); - }, [prefix, count, startNum]); + }, [prefix, count, startNum, suffixWidth]); const generateRandomPassword = () => { const chars = 'ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz23456789'; @@ -214,6 +218,7 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, setPrefix(''); setCount(10); setStartNum(1); + setSuffixWidth(2); setQuotaValue(String(defaultQuota || 0)); onHide(); }; @@ -276,11 +281,29 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, min={0} max={9999} value={startNum} - onChange={(e) => setStartNum(parseInt(e.target.value) || 1)} + onChange={(e) => { + const value = parseInt(e.target.value); + setStartNum(Number.isNaN(value) ? 1 : value); + }} style={{ width: 70 }} /> + + + digits (0 = none) + { + const value = parseInt(e.target.value); + setSuffixWidth(Number.isNaN(value) ? 0 : value); + }} + style={{ width: 60 }} + /> + + count From 7cbb27af4f0d420a30cab6ba168be49007b7a57c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:17:22 +0800 Subject: [PATCH 012/180] fix(admin): bound user generator numeric inputs --- .../admin/src/components/CreateUserModal.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx index ff4f4f42..898bf2da 100644 --- a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx +++ b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx @@ -38,6 +38,12 @@ interface CreatedUser { error?: string; } +const parseBoundedInteger = (value: string, fallback: number, min: number, max: number) => { + const parsed = Number.parseInt(value, 10); + if (Number.isNaN(parsed)) return fallback; + return Math.min(Math.max(parsed, min), max); +}; + export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, defaultQuota = 0 }: Props) { const [usernames, setUsernames] = useState(''); const [password, setPassword] = useState(''); @@ -281,10 +287,7 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, min={0} max={9999} value={startNum} - onChange={(e) => { - const value = parseInt(e.target.value); - setStartNum(Number.isNaN(value) ? 1 : value); - }} + onChange={(e) => setStartNum(parseBoundedInteger(e.target.value, 1, 0, 9999))} style={{ width: 70 }} /> @@ -295,11 +298,9 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, { - const value = parseInt(e.target.value); - setSuffixWidth(Number.isNaN(value) ? 0 : value); - }} + onChange={(e) => setSuffixWidth(parseBoundedInteger(e.target.value, 0, 0, 6))} style={{ width: 60 }} /> @@ -312,7 +313,7 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, min={1} max={1000} value={count} - onChange={(e) => setCount(parseInt(e.target.value) || 1)} + onChange={(e) => setCount(parseBoundedInteger(e.target.value, 1, 1, 1000))} style={{ width: 70 }} /> From c5dc614f552c8db8186fff737a304e67f19f008b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:16:48 +0800 Subject: [PATCH 013/180] feat(frontend): add shared password policy helpers --- .../packages/shared/src/utils/index.ts | 1 + .../shared/src/utils/password.test.ts | 56 +++++++++++++ .../packages/shared/src/utils/password.ts | 82 +++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 runtime/hub/frontend/packages/shared/src/utils/password.test.ts create mode 100644 runtime/hub/frontend/packages/shared/src/utils/password.ts diff --git a/runtime/hub/frontend/packages/shared/src/utils/index.ts b/runtime/hub/frontend/packages/shared/src/utils/index.ts index 212851a7..0912789d 100644 --- a/runtime/hub/frontend/packages/shared/src/utils/index.ts +++ b/runtime/hub/frontend/packages/shared/src/utils/index.ts @@ -19,3 +19,4 @@ export * from "./xsrf.js"; export * from "./user.js"; +export * from "./password.js"; diff --git a/runtime/hub/frontend/packages/shared/src/utils/password.test.ts b/runtime/hub/frontend/packages/shared/src/utils/password.test.ts new file mode 100644 index 00000000..1883ed32 --- /dev/null +++ b/runtime/hub/frontend/packages/shared/src/utils/password.test.ts @@ -0,0 +1,56 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { generateStrongPassword, getPasswordError, isStrongPassword } from "./password.js"; + +describe("password helpers", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("accepts passwords that satisfy the native password policy", () => { + expect(isStrongPassword("Valid-Password1")).toBe(true); + expect(getPasswordError("Valid-Password1")).toBeNull(); + }); + + it.each([ + ["Short1!", "At least 8 characters"], + ["lowercase1!", "One uppercase letter"], + ["UPPERCASE1!", "One lowercase letter"], + ["NoDigits!", "One digit"], + ["NoSpecial1", "One special character"], + ])("rejects %s with %s", (password, label) => { + expect(isStrongPassword(password)).toBe(false); + expect(getPasswordError(password)).toBe(`Password requirement not met: ${label}`); + }); + + it("generates passwords that satisfy every rule", () => { + for (let i = 0; i < 50; i += 1) { + expect(isStrongPassword(generateStrongPassword())).toBe(true); + } + }); + + it("does not fall back to non-secure random generation", () => { + vi.stubGlobal("crypto", undefined); + + expect(() => generateStrongPassword()).toThrow("Secure random password generation is not available"); + }); +}); diff --git a/runtime/hub/frontend/packages/shared/src/utils/password.ts b/runtime/hub/frontend/packages/shared/src/utils/password.ts new file mode 100644 index 00000000..a0b70c34 --- /dev/null +++ b/runtime/hub/frontend/packages/shared/src/utils/password.ts @@ -0,0 +1,82 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +export interface PasswordRule { + label: string; + test: (password: string) => boolean; +} + +const UPPERCASE = "ABCDEFGHJKLMNPQRSTUVWXYZ"; +const LOWERCASE = "abcdefghijkmnpqrstuvwxyz"; +const DIGITS = "23456789"; +const SPECIAL = "!@#$%^&*_+-="; + +export const PASSWORD_RULES: PasswordRule[] = [ + { test: (password: string) => password.length >= 8, label: "At least 8 characters" }, + { test: (password: string) => /[A-Z]/.test(password), label: "One uppercase letter" }, + { test: (password: string) => /[a-z]/.test(password), label: "One lowercase letter" }, + { test: (password: string) => /\d/.test(password), label: "One digit" }, + { + test: (password: string) => /[!@#$%^&*()_+\-=[\]{};':"\\|,.<>/?`~]/.test(password), + label: "One special character", + }, +]; + +function randomIndex(length: number): number { + if (globalThis.crypto?.getRandomValues) { + const value = new Uint32Array(1); + globalThis.crypto.getRandomValues(value); + return value[0] % length; + } + throw new Error("Secure random password generation is not available in this browser"); +} + +function pick(chars: string): string { + return chars[randomIndex(chars.length)]; +} + +function shuffle(chars: string[]): string[] { + const result = [...chars]; + for (let i = result.length - 1; i > 0; i -= 1) { + const j = randomIndex(i + 1); + [result[i], result[j]] = [result[j], result[i]]; + } + return result; +} + +export function getPasswordError(password: string): string | null { + const failedRule = PASSWORD_RULES.find((rule) => !rule.test(password)); + return failedRule ? `Password requirement not met: ${failedRule.label}` : null; +} + +export function isStrongPassword(password: string): boolean { + return password.length > 0 && getPasswordError(password) === null; +} + +export function generateStrongPassword(length = 16): string { + const passwordLength = Math.max(length, 8); + const all = UPPERCASE + LOWERCASE + DIGITS + SPECIAL; + const chars = [pick(UPPERCASE), pick(LOWERCASE), pick(DIGITS), pick(SPECIAL)]; + + while (chars.length < passwordLength) { + chars.push(pick(all)); + } + + return shuffle(chars).join(""); +} From 27ed4fda3f66f9b19f6275d6edd1d34a2e1bd34b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:17:01 +0800 Subject: [PATCH 014/180] feat(hub): add best-effort user provisioning api --- runtime/hub/core/authenticators/firstuse.py | 20 ++ runtime/hub/core/handlers.py | 253 +++++++++++++++++++- 2 files changed, 267 insertions(+), 6 deletions(-) diff --git a/runtime/hub/core/authenticators/firstuse.py b/runtime/hub/core/authenticators/firstuse.py index 3de29fda..0ac3765c 100644 --- a/runtime/hub/core/authenticators/firstuse.py +++ b/runtime/hub/core/authenticators/firstuse.py @@ -26,6 +26,7 @@ from __future__ import annotations +import secrets from concurrent.futures import ThreadPoolExecutor import bcrypt @@ -80,6 +81,10 @@ def _get_user_password(self, username: str) -> UserPassword | None: session.close() MIN_PASSWORD_LENGTH = 8 + UPPERCASE_CHARS = "ABCDEFGHJKLMNPQRSTUVWXYZ" + LOWERCASE_CHARS = "abcdefghijkmnpqrstuvwxyz" + DIGIT_CHARS = "23456789" + SPECIAL_CHARS = "!@#$%^&*_+-=" @staticmethod def _check_password_strength(password: str) -> str | None: @@ -99,6 +104,21 @@ def _check_password_strength(password: str) -> str | None: return "Password must contain at least one special character" return None + @classmethod + def generate_password(cls, length: int = 16) -> str: + """Generate a password that satisfies the native password policy.""" + password_length = max(length, cls.MIN_PASSWORD_LENGTH) + all_chars = cls.UPPERCASE_CHARS + cls.LOWERCASE_CHARS + cls.DIGIT_CHARS + cls.SPECIAL_CHARS + chars = [ + secrets.choice(cls.UPPERCASE_CHARS), + secrets.choice(cls.LOWERCASE_CHARS), + secrets.choice(cls.DIGIT_CHARS), + secrets.choice(cls.SPECIAL_CHARS), + ] + chars.extend(secrets.choice(all_chars) for _ in range(password_length - len(chars))) + secrets.SystemRandom().shuffle(chars) + return "".join(chars) + def _validate_password(self, password): """Validate password meets strength requirements.""" return self._check_password_strength(password) is None diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index 964a42fd..dbd839e4 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -37,6 +37,7 @@ from jupyterhub.apihandlers import APIHandler from jupyterhub.handlers import BaseHandler +from jupyterhub.scopes import needs_scope from multiauthenticator import MultiAuthenticator from pydantic import ValidationError from tornado import web @@ -77,6 +78,9 @@ } +MAX_NATIVE_PASSWORD_BYTES = 72 + + def _serialize_dismissed_at(value: datetime | None) -> str | None: """Serialize onboarding dismissal timestamps for API responses.""" if value is None: @@ -113,6 +117,17 @@ def _dismiss_onboarding(username: str) -> str: return _serialize_dismissed_at(dismissed_at) or "" +def _find_firstuse_authenticator(authenticator: Any) -> CustomFirstUseAuthenticator | None: + """Find the native password authenticator inside the active auth stack.""" + if isinstance(authenticator, CustomFirstUseAuthenticator): + return authenticator + if isinstance(authenticator, MultiAuthenticator): + for candidate in authenticator._authenticators: + if isinstance(candidate, CustomFirstUseAuthenticator): + return candidate + return None + + def configure_handlers( accelerator_options: dict[str, Any] | None = None, quota_rates: dict[str, int] | None = None, @@ -491,12 +506,7 @@ async def get(self): self.set_header("Content-Type", "application/json") return self.finish(json.dumps({"error": "Admin access required"})) - import secrets - import string - - chars = string.ascii_letters + string.digits - chars = chars.replace("l", "").replace("I", "").replace("O", "").replace("0", "") - password = "".join(secrets.choice(chars) for _ in range(16)) + password = CustomFirstUseAuthenticator.generate_password() self.set_header("Content-Type", "application/json") self.finish(json.dumps({"password": password})) @@ -573,6 +583,235 @@ async def post(self): self.finish(json.dumps({"error": "Internal server error"})) +class AdminAPIProvisionUsersHandler(APIHandler): + """Best-effort native user provisioning for the admin UI.""" + + @web.authenticated + @needs_scope("admin:users") + async def post(self): + """Create users, set initial passwords, and optionally set quota. + + This endpoint deliberately provides per-user best-effort consistency, not + crash-safe transactional atomicity. JupyterHub users, native password + rows, and quota rows are owned by different modules and are not updated + under one database transaction in this deployment. A rare orphan or + partially provisioned user is acceptable operationally and can be fixed + by an administrator; the goal here is to keep orchestration and failure + semantics out of the frontend while preventing predictable password + policy failures before creating users. + """ + assert self.current_user is not None + if not self.current_user.admin: + self.set_status(403) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "Admin access required"})) + + try: + data = json.loads(self.request.body.decode("utf-8")) + if not isinstance(data, dict): + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "Request body must be a JSON object"})) + + users = data.get("users", []) + admin = data.get("admin", False) + force_change = data.get("force_change", True) + quota = data.get("quota") + + if not users or not isinstance(users, list): + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "users array is required"})) + if len(users) > 1000: + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "Maximum 1000 users per batch"})) + if quota is not None and not isinstance(quota, dict): + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "quota must be an object"})) + if not isinstance(admin, bool) or not isinstance(force_change, bool): + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "admin and force_change must be booleans"})) + + firstuse_auth = _find_firstuse_authenticator(self.authenticator) + if not firstuse_auth: + self.set_status(500) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "Password management not available"})) + + results = {"success": 0, "failed": 0, "skipped": 0, "results": []} + quota_manager = get_quota_manager() if quota else None + quota_amount = 0 + quota_unlimited = False + if quota: + raw_unlimited = quota.get("unlimited", False) + raw_amount = quota.get("amount", 0) + if not isinstance(raw_unlimited, bool): + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "quota.unlimited must be a boolean"})) + if isinstance(raw_amount, bool) or not isinstance(raw_amount, int) or raw_amount < 0: + self.set_status(400) + self.set_header("Content-Type", "application/json") + return self.finish(json.dumps({"error": "quota.amount must be a non-negative integer"})) + quota_unlimited = raw_unlimited + quota_amount = raw_amount + + from jupyterhub.roles import assign_default_roles + from jupyterhub.utils import maybe_future + + for entry in users: + if not isinstance(entry, dict) or "username" not in entry or "password" not in entry: + results["failed"] += 1 + results["results"].append( + { + "username": "", + "requested_username": "", + "status": "failed", + "created": False, + "password_set": False, + "quota_set": False, + "error": "Each entry must have username and password", + } + ) + continue + + raw_username = entry.get("username") + password = entry.get("password") + requested_username = raw_username.strip() if isinstance(raw_username, str) else "" + username = firstuse_auth.normalize_username(requested_username) + + result = { + "username": username, + "requested_username": requested_username, + "status": "failed", + "created": False, + "password_set": False, + "quota_set": False, + } + + if not username: + result["error"] = "Username is required" + results["failed"] += 1 + results["results"].append(result) + continue + if self.find_user(username) is not None: + result["status"] = "existed" + results["skipped"] += 1 + results["results"].append(result) + continue + if not isinstance(password, str): + result["error"] = "Password must be a string" + results["failed"] += 1 + results["results"].append(result) + continue + if len(password.encode("utf-8")) > MAX_NATIVE_PASSWORD_BYTES: + result["error"] = f"Password must be at most {MAX_NATIVE_PASSWORD_BYTES} bytes" + results["failed"] += 1 + results["results"].append(result) + continue + if username.startswith(GITHUB_USERNAME_PREFIX): + result["error"] = "Cannot provision native password for GitHub users" + results["failed"] += 1 + results["results"].append(result) + continue + if not self.authenticator.validate_username(username): + result["error"] = f"Invalid username: {username}" + results["failed"] += 1 + results["results"].append(result) + continue + + strength_error = firstuse_auth._check_password_strength(password) + if strength_error: + result["error"] = strength_error + results["failed"] += 1 + results["results"].append(result) + continue + + try: + loop = asyncio.get_event_loop() + password_result = await loop.run_in_executor( + None, + lambda username=username, password=password: firstuse_auth.set_password( + username, password, force_change=force_change + ), + ) + if not password_result.startswith("Password set for"): + result["error"] = password_result + results["failed"] += 1 + results["results"].append(result) + continue + result["password_set"] = True + except Exception as e: + self.log.error( + "Failed to set password during provisioning for %s: %s", + username, + e.__class__.__name__, + ) + result["error"] = "Failed to set password" + results["failed"] += 1 + results["results"].append(result) + continue + + user = None + try: + user = self.user_from_username(username) + if admin: + user.admin = True + assign_default_roles(self.db, entity=user) + self.db.commit() + await maybe_future(self.authenticator.add_user(user)) + result["created"] = True + except Exception as e: + self.log.error("Failed to create user during provisioning: %s", username, exc_info=True) + if user is not None: + try: + self.users.delete(user) + except Exception: + self.log.warning("Failed to remove partially registered user: %s", username, exc_info=True) + result["error"] = f"Password stored, but failed to create user: {e}" + results["failed"] += 1 + results["results"].append(result) + continue + + if quota_manager and (quota_unlimited or quota_amount > 0): + try: + if quota_unlimited: + quota_manager.set_unlimited(username, True, self.current_user.name) + else: + quota_manager.set_balance(username, quota_amount, self.current_user.name) + result["quota_set"] = True + except Exception: + self.log.error("Failed to set quota during provisioning: %s", username, exc_info=True) + result["error"] = "User and password created, but quota setup failed" + results["failed"] += 1 + results["results"].append(result) + continue + + result["status"] = "success" + results["success"] += 1 + results["results"].append(result) + + self.set_header("Content-Type", "application/json") + self.finish(json.dumps(results)) + + except json.JSONDecodeError: + self.set_status(400) + self.set_header("Content-Type", "application/json") + self.finish(json.dumps({"error": "Invalid JSON"})) + except (TypeError, ValueError): + self.set_status(400) + self.set_header("Content-Type", "application/json") + self.finish(json.dumps({"error": "Invalid quota value"})) + except Exception: + self.log.error("Failed to provision users", exc_info=True) + self.set_status(500) + self.set_header("Content-Type", "application/json") + self.finish(json.dumps({"error": "Internal server error"})) + + # ============================================================================= # Quota Management Handlers # ============================================================================= @@ -1643,6 +1882,7 @@ def get_handlers() -> list[tuple[str, type]]: (r"/admin/api/set-password", AdminAPISetPasswordHandler), (r"/admin/api/batch-set-password", AdminAPIBatchSetPasswordHandler), (r"/admin/api/generate-password", AdminAPIGeneratePasswordHandler), + (r"/admin/api/provision-users", AdminAPIProvisionUsersHandler), # Group management API (r"/admin/api/groups/?", GroupsAPIHandler), (r"/admin/api/groups/sync/?", GroupSyncAPIHandler), @@ -1694,6 +1934,7 @@ def get_handlers() -> list[tuple[str, type]]: "AdminUIHandler", "AdminAPISetPasswordHandler", "AdminAPIGeneratePasswordHandler", + "AdminAPIProvisionUsersHandler", # Quota handlers "QuotaAPIHandler", "QuotaBatchAPIHandler", From 297b79f11d68e1a4462a6998ce701eaead37141f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:17:15 +0800 Subject: [PATCH 015/180] feat(frontend): expose user provisioning client --- .../frontend/packages/shared/src/api/users.ts | 11 +++++++ .../packages/shared/src/types/user.ts | 32 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/runtime/hub/frontend/packages/shared/src/api/users.ts b/runtime/hub/frontend/packages/shared/src/api/users.ts index 0f38e9a1..ccd5e74a 100644 --- a/runtime/hub/frontend/packages/shared/src/api/users.ts +++ b/runtime/hub/frontend/packages/shared/src/api/users.ts @@ -21,6 +21,8 @@ import type { User, UsersResponse, Group, + ProvisionUsersRequest, + ProvisionUsersResponse, SetPasswordRequest, } from "../types/user.js"; import type { HubInfo } from "../types/hub.js"; @@ -102,6 +104,15 @@ export async function createUsers( }); } +export async function provisionUsers( + data: ProvisionUsersRequest +): Promise { + return adminApiRequest("/provision-users", { + method: "POST", + body: JSON.stringify(data), + }); +} + export async function deleteUser(username: string): Promise { return apiRequest(`/users/${encodeURIComponent(username)}`, { method: "DELETE", diff --git a/runtime/hub/frontend/packages/shared/src/types/user.ts b/runtime/hub/frontend/packages/shared/src/types/user.ts index 0a69f61d..e7dbd056 100644 --- a/runtime/hub/frontend/packages/shared/src/types/user.ts +++ b/runtime/hub/frontend/packages/shared/src/types/user.ts @@ -66,6 +66,38 @@ export interface SetPasswordRequest { force_change?: boolean; } +export interface ProvisionUserEntry { + username: string; + password: string; +} + +export interface ProvisionUsersRequest { + users: ProvisionUserEntry[]; + admin?: boolean; + force_change?: boolean; + quota?: { + amount?: number; + unlimited?: boolean; + }; +} + +export interface ProvisionUserResult { + username: string; + requested_username: string; + status: "success" | "failed" | "existed"; + created: boolean; + password_set: boolean; + quota_set: boolean; + error?: string; +} + +export interface ProvisionUsersResponse { + success: number; + failed: number; + skipped: number; + results: ProvisionUserResult[]; +} + export interface Group { name: string; users: string[]; From 6956c197893f8a3a0dbe1d040688733c4e74a516 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:17:26 +0800 Subject: [PATCH 016/180] fix(admin): use provisioning api for user creation --- .../admin/src/components/CreateUserModal.tsx | 179 ++++++++---------- 1 file changed, 78 insertions(+), 101 deletions(-) diff --git a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx index a1b87c9f..3b4b5523 100644 --- a/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx +++ b/runtime/hub/frontend/apps/admin/src/components/CreateUserModal.tsx @@ -20,6 +20,7 @@ import { useState, useCallback, useMemo } from 'react'; import { Modal, Button, Form, Alert, Spinner, InputGroup, Row, Col, Badge } from 'react-bootstrap'; import * as api from '@auplc/shared'; +import { generateStrongPassword, getPasswordError, isStrongPassword, PASSWORD_RULES } from '@auplc/shared'; interface Props { show: boolean; @@ -59,19 +60,9 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, setUsernames(names.join('\n')); }, [prefix, count, startNum]); - const generateRandomPassword = () => { - const chars = 'ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnpqrstuvwxyz23456789'; - let result = ''; - for (let i = 0; i < 16; i++) { - result += chars.charAt(Math.floor(Math.random() * chars.length)); - } - return result; - }; - const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); setError(null); - setLoading(true); try { const names = usernames @@ -81,110 +72,75 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, if (names.length === 0) { setError('Please enter at least one username'); - setLoading(false); return; } + const passwordError = generateRandom ? null : getPasswordError(password); + if (passwordError) { + setError(passwordError); + return; + } + + setLoading(true); + // Generate passwords for all users upfront const passwordMap = new Map( names.map(username => [ username, - generateRandom ? generateRandomPassword() : password, - ]) - ); - - // Initialize result tracking - const results: Map = new Map( - names.map(username => [ - username, - { username, password: passwordMap.get(username)!, status: 'created' as const, passwordSet: false, quotaSet: false }, + generateRandom ? generateStrongPassword() : password, ]) ); const warnings: string[] = []; - // Step 1: Batch create users - let createdNames: string[] = []; - try { - const created = await api.createUsers(names, isAdmin); - // API returns only newly created users; existing ones are silently skipped - createdNames = created.map(u => u.name); - const existedNames = names.filter(n => !createdNames.includes(n)); - for (const name of existedNames) { - const r = results.get(name)!; - r.status = 'existed'; - } - if (existedNames.length > 0) { - warnings.push(`${existedNames.length} user(s) already existed: ${existedNames.join(', ')}`); - } - } catch (err) { - const msg = err instanceof Error ? err.message : 'Unknown error'; - // If 409 (all users exist), mark them all as existed and continue with password/quota - if (msg.includes('already exist')) { - for (const name of names) { - results.get(name)!.status = 'existed'; - } - createdNames = []; - warnings.push(`All ${names.length} user(s) already existed`); - } else { - // Fatal error - can't determine which users were created - setError(`Failed to create users: ${msg}`); - setLoading(false); + let quota: { amount?: number; unlimited?: boolean } | undefined; + if (quotaEnabled) { + const input = quotaValue.trim(); + const isUnlimited = input === '-1' || input === '∞' || input.toLowerCase() === 'unlimited'; + if (!isUnlimited && input !== '' && !/^\d+$/.test(input)) { + setError('Initial quota must be a non-negative integer, -1, or unlimited'); return; } + const amount = isUnlimited ? 0 : (Number(input) || 0); + if (isUnlimited || amount > 0) { + quota = isUnlimited ? { amount: 0, unlimited: true } : { amount }; + } } - // Step 2: Set passwords (only for newly created users) - if (createdNames.length > 0) { - const passwordEntries = createdNames.map(username => ({ - username, - password: passwordMap.get(username)!, - })); - - try { - const pwResult = await api.batchSetPasswords(passwordEntries, forceChange); - for (const r of pwResult.results) { - const entry = results.get(r.username); - if (entry) { - if (r.status === 'success') { - entry.passwordSet = true; - } else { - entry.error = r.error || 'Password set failed'; - } - } - } - if (pwResult.failed > 0) { - warnings.push(`${pwResult.failed} password(s) failed to set`); - } - } catch (err) { - const msg = err instanceof Error ? err.message : 'Unknown error'; - warnings.push(`Password setting failed: ${msg}`); - } + const userEntries = names.map(username => ({ username, password: passwordMap.get(username)! })); + + const response = await api.provisionUsers({ + users: userEntries, + admin: isAdmin, + force_change: forceChange, + quota, + }); + + const results = new Map(); + for (const [index, entry] of userEntries.entries()) { + const r = response.results[index]; + const displayUsername = r?.username || entry.username; + results.set(`${index}:${entry.username}`, { + username: displayUsername, + password: entry.password, + status: r?.status === 'existed' + ? 'existed' + : r?.created || r?.status === 'success' + ? 'created' + : 'failed', + passwordSet: r?.password_set ?? false, + quotaSet: r?.quota_set ?? false, + error: r?.error, + }); } - // Step 3: Set quota if enabled (only for newly created users) - if (quotaEnabled && createdNames.length > 0) { - const input = quotaValue.trim(); - const isUnlimited = input === '-1' || input === '∞' || input.toLowerCase() === 'unlimited'; - const amount = isUnlimited ? 0 : (parseInt(input) || 0); - if (isUnlimited || amount > 0) { - try { - await api.batchSetQuota( - createdNames.map(username => ({ - username, - amount, - ...(isUnlimited ? { unlimited: true } : {}), - })) - ); - for (const name of createdNames) { - const entry = results.get(name); - if (entry) entry.quotaSet = true; - } - } catch (err) { - const msg = err instanceof Error ? err.message : 'Unknown error'; - warnings.push(`Quota setting failed: ${msg}`); - } - } + const existedNames = response.results.filter(r => r.status === 'existed').map(r => r.username); + if (existedNames.length > 0) { + warnings.push(`${existedNames.length} user(s) already existed: ${existedNames.join(', ')}`); + } + const failedResults = response.results.filter(r => r.status === 'failed'); + if (failedResults.length > 0) { + warnings.push(...failedResults.map(r => `${r.username}: ${r.error || 'Provisioning failed'}`)); } // Set warnings as non-fatal error for display @@ -244,6 +200,9 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, URL.revokeObjectURL(url); }; + const manualPasswordError = generateRandom ? null : getPasswordError(password); + const canSubmit = !loading && (generateRandom || isStrongPassword(password)); + return ( @@ -343,6 +302,13 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, /> + +
Native passwords must meet all rules before users are created.
+
+ {PASSWORD_RULES.map((rule) => rule.label).join(' · ')} +
+ + {!generateRandom && ( Password (same for all users) @@ -354,17 +320,28 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, placeholder="Enter password" required={!generateRandom} minLength={8} + isInvalid={Boolean(manualPasswordError)} + isValid={isStrongPassword(password)} /> + {manualPasswordError && ( + + {manualPasswordError} + + )} - - Minimum 8 characters - +
+ {PASSWORD_RULES.map((rule, i) => ( +
+ {rule.test(password) ? '✓' : '●'} {rule.label} +
+ ))} +
)} @@ -485,7 +462,7 @@ export function CreateUserModal({ show, onHide, onSuccess, quotaEnabled = false, + {manualPasswordError && ( + + {manualPasswordError} + + )} - - Minimum 8 characters - +
+ {PASSWORD_RULES.map((rule, i) => ( +
+ {rule.test(password) ? '✓' : '●'} {rule.label} +
+ ))} +
)} @@ -269,7 +289,7 @@ export function BatchPasswordModal({ show, usernames, onHide }: Props) { From 649f24acdfce05a2c7715a57029954157b3b11e9 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:37:07 +0800 Subject: [PATCH 018/180] feat(admin): support batched group membership API --- .../frontend/packages/shared/src/api/client.ts | 4 ++++ .../frontend/packages/shared/src/api/users.ts | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/runtime/hub/frontend/packages/shared/src/api/client.ts b/runtime/hub/frontend/packages/shared/src/api/client.ts index 10ac7c2f..321bee9f 100644 --- a/runtime/hub/frontend/packages/shared/src/api/client.ts +++ b/runtime/hub/frontend/packages/shared/src/api/client.ts @@ -70,5 +70,9 @@ export async function adminApiRequest( throw new Error(body.error || body.message || `API Error: ${response.status}`); } + if (response.status === 202 || response.status === 204) { + return undefined as T; + } + return response.json(); } diff --git a/runtime/hub/frontend/packages/shared/src/api/users.ts b/runtime/hub/frontend/packages/shared/src/api/users.ts index 0f38e9a1..ecd6880e 100644 --- a/runtime/hub/frontend/packages/shared/src/api/users.ts +++ b/runtime/hub/frontend/packages/shared/src/api/users.ts @@ -217,19 +217,33 @@ export async function updateGroup( export async function addUserToGroup( groupName: string, username: string +): Promise { + return addUsersToGroup(groupName, [username]); +} + +export async function addUsersToGroup( + groupName: string, + usernames: string[] ): Promise { return adminApiRequest(`/groups/${encodeURIComponent(groupName)}/users`, { method: "POST", - body: JSON.stringify({ users: [username] }), + body: JSON.stringify({ users: usernames }), }); } export async function removeUserFromGroup( groupName: string, username: string +): Promise { + return removeUsersFromGroup(groupName, [username]); +} + +export async function removeUsersFromGroup( + groupName: string, + usernames: string[] ): Promise { return adminApiRequest(`/groups/${encodeURIComponent(groupName)}/users`, { method: "DELETE", - body: JSON.stringify({ users: [username] }), + body: JSON.stringify({ users: usernames }), }); } From 2a7075e791eedd74f4824851fcfa6a29a91dc3c8 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:37:23 +0800 Subject: [PATCH 019/180] feat(admin): add bulk user group actions --- .../apps/admin/src/pages/UserList.tsx | 236 ++++++++++++++++-- 1 file changed, 215 insertions(+), 21 deletions(-) diff --git a/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx b/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx index 051d3c4a..eb53f1c7 100644 --- a/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx +++ b/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx @@ -19,7 +19,7 @@ import React, { useState, useEffect, useCallback, useMemo, memo } from 'react'; import { Table, Button, Form, InputGroup, Badge, Spinner, Alert, ButtonGroup, Modal, Dropdown } from 'react-bootstrap'; -import type { User, UserQuota, Server } from '@auplc/shared'; +import type { User, UserQuota, Server, Group } from '@auplc/shared'; import * as api from '@auplc/shared'; import { isGitHubUser, isNativeUser as isNativeUsername } from '@auplc/shared'; import { CreateUserModal } from '../components/CreateUserModal'; @@ -358,6 +358,7 @@ const ServerDetails = memo(function ServerDetails({ serverName, server, userName export function UserList() { const [users, setUsers] = useState([]); + const [groups, setGroups] = useState([]); const [totalUsers, setTotalUsers] = useState(0); const [loading, setLoading] = useState(true); const [initialLoading, setInitialLoading] = useState(true); @@ -387,6 +388,9 @@ export function UserList() { const [userToDelete, setUserToDelete] = useState(null); const [showBatchDeleteModal, setShowBatchDeleteModal] = useState(false); const [showBatchPasswordModal, setShowBatchPasswordModal] = useState(false); + const [showBatchGroupModal, setShowBatchGroupModal] = useState(false); + const [batchGroupMode, setBatchGroupMode] = useState<'add' | 'remove'>('add'); + const [batchGroupName, setBatchGroupName] = useState(''); const [showQuotaRefreshModal, setShowQuotaRefreshModal] = useState(false); const [usageUsername, setUsageUsername] = useState(null); @@ -410,6 +414,31 @@ export function UserList() { }); }, [selectedUsers, users]); + const mutableGroups = useMemo( + () => groups.filter(group => group.source !== 'system'), + [groups] + ); + + const selectedUsernames = useMemo( + () => Array.from(selectedUsers), + [selectedUsers] + ); + + const allCurrentPageSelected = useMemo( + () => users.length > 0 && users.every(user => selectedUsers.has(user.name)), + [selectedUsers, users] + ); + + const selectedOnCurrentPage = useMemo( + () => users.filter(user => selectedUsers.has(user.name)).length, + [selectedUsers, users] + ); + + const selectedBatchGroup = useMemo( + () => groups.find(group => group.name === batchGroupName), + [batchGroupName, groups] + ); + // Debounce search input useEffect(() => { const timer = setTimeout(() => { @@ -476,6 +505,15 @@ export function UserList() { } }, []); + const loadGroups = useCallback(async () => { + try { + const response = await api.getGroups(); + setGroups(response.groups); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load groups'); + } + }, []); + // Load users when pagination, search, sort, or filter changes useEffect(() => { loadUsers(); @@ -486,6 +524,10 @@ export function UserList() { loadQuota(); }, [loadQuota]); + useEffect(() => { + loadGroups(); + }, [loadGroups]); + const handleQuotaEdit = (username: string, currentBalance: number, isUnlimited: boolean) => { setEditingQuota(username); setQuotaInput(isUnlimited ? '∞' : currentBalance.toString()); @@ -566,6 +608,44 @@ export function UserList() { } }; + const openBatchGroupModal = (mode: 'add' | 'remove') => { + if (selectedUsers.size === 0) { + setError('Please select users first'); + return; + } + setBatchGroupMode(mode); + setBatchGroupName(''); + setError(null); + setShowBatchGroupModal(true); + }; + + const handleBatchGroupSave = async () => { + if (selectedUsernames.length === 0) { + setError('Please select users first'); + return; + } + if (!batchGroupName) { + setError('Please select a group'); + return; + } + + try { + setActionLoading('batch-group'); + if (batchGroupMode === 'add') { + await api.addUsersToGroup(batchGroupName, selectedUsernames); + } else { + await api.removeUsersFromGroup(batchGroupName, selectedUsernames); + } + await Promise.all([loadUsers(true), loadGroups()]); + setShowBatchGroupModal(false); + setSelectedUsers(new Set()); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to update group membership'); + } finally { + setActionLoading(null); + } + }; + // Handle sort column click - only allow sortable columns const handleSort = (column: typeof sortColumn) => { // Only allow sorting by columns the API supports @@ -613,7 +693,7 @@ export function UserList() { setQuotaInput(value); }, []); - const handleStartServer = async (user: User) => { + const handleStartServer = useCallback(async (user: User) => { try { setActionLoading(`start-${user.name}`); await api.startServer(user.name); @@ -623,9 +703,9 @@ export function UserList() { } finally { setActionLoading(null); } - }; + }, [loadUsers]); - const handleStopServer = async (user: User) => { + const handleStopServer = useCallback(async (user: User) => { try { setActionLoading(`stop-${user.name}`); await api.stopServer(user.name); @@ -635,7 +715,7 @@ export function UserList() { } finally { setActionLoading(null); } - }; + }, [loadUsers]); const handleStartAll = async () => { const usersToStart = selectedUsers.size > 0 @@ -679,13 +759,15 @@ export function UserList() { const toggleSelectAll = useCallback(() => { setSelectedUsers(prev => { - if (prev.size === users.length) { - return new Set(); + const newSelected = new Set(prev); + if (allCurrentPageSelected) { + users.forEach(user => newSelected.delete(user.name)); } else { - return new Set(users.map(u => u.name)); + users.forEach(user => newSelected.add(user.name)); } + return newSelected; }); - }, [users]); + }, [allCurrentPageSelected, users]); const openPasswordModal = useCallback((user: User) => { setSelectedUser(user); @@ -740,15 +822,6 @@ export function UserList() { } }; - // Memoize start/stop server handlers with useCallback - const handleStartServerCallback = useCallback((user: User) => { - handleStartServer(user); - }, []); - - const handleStopServerCallback = useCallback((user: User) => { - handleStopServer(user); - }, []); - // Only show full-screen spinner on initial load if (initialLoading) { return ( @@ -809,6 +882,29 @@ export function UserList() { > Reset PW ({nativeSelected.length}) + + + Group Actions ({selectedUsers.size}) + + + openBatchGroupModal('add')}> + Add to group + + openBatchGroupModal('remove')}> + Remove from group + + + + + )} + {/* User Table */} @@ -902,8 +1013,9 @@ export function UserList() { @@ -221,46 +105,35 @@ const GroupRow = memo(function GroupRow({ group, onEdit, onMembersChange, loadUs Manual )} -
- {group.users.length} {group.users.length === 1 ? 'member' : 'members'} - {(group.resources?.length ?? 0) > 0 && ` · ${group.resources!.length} resources`} -
+ {(group.resources?.length ?? 0) > 0 && ( +
+ {group.resources!.length} resources +
+ )} - ); @@ -325,33 +198,6 @@ export function GroupList() { setShowEditModal(true); }, []); - // Load user options for AsyncSelect - const loadUserOptions = useCallback(async (inputValue: string, excludeUsers: string[]): Promise => { - if (!inputValue || inputValue.length < 1) { - return []; - } - try { - const response = await api.getUsers({ offset: 0, limit: 20, nameFilter: inputValue }); - const users = response.items || []; - return users - .filter(user => !excludeUsers.includes(user.name)) - .map(user => ({ - value: user.name, - label: user.admin ? `${user.name} (Admin)` : user.name, - })); - } catch (err) { - console.error('Failed to load users:', err); - return []; - } - }, []); - - // Handle members change from GroupRow - const handleMembersChange = useCallback((groupName: string, newMembers: string[]) => { - setGroups(prev => prev.map(g => - g.name === groupName ? { ...g, users: newMembers } : g - )); - }, []); - const handleCreateGroup = async () => { if (!newGroupName.trim()) { setCreateError('Group name cannot be empty'); @@ -535,8 +381,6 @@ export function GroupList() { key={group.name} group={group} onEdit={handleEditGroup} - onMembersChange={handleMembersChange} - loadUserOptions={loadUserOptions} /> ))} From a21760a882bf0890625fb4cd6b578396dc5a95d2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:43:38 +0800 Subject: [PATCH 021/180] fix(admin): serve nested group detail routes --- runtime/hub/core/handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index 964a42fd..de5f4fe0 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -407,7 +407,7 @@ class AdminUIHandler(BaseHandler): """Serve the custom admin UI (React app).""" @web.authenticated - async def get(self): + async def get(self, *args): """Serve admin UI page.""" assert self.current_user is not None if not self.current_user.admin: @@ -1640,6 +1640,7 @@ def get_handlers() -> list[tuple[str, type]]: # Admin UI (r"/admin/users", AdminUIHandler), (r"/admin/groups", AdminUIHandler), + (r"/admin/groups/(.*)", AdminUIHandler), (r"/admin/api/set-password", AdminAPISetPasswordHandler), (r"/admin/api/batch-set-password", AdminAPIBatchSetPasswordHandler), (r"/admin/api/generate-password", AdminAPIGeneratePasswordHandler), From 53e203363276fcd417436fb5da7f27f7b8aab49e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:06:03 +0800 Subject: [PATCH 022/180] feat(admin): streamline bulk group selection --- .../apps/admin/src/pages/UserList.tsx | 161 +++++++++++------- 1 file changed, 104 insertions(+), 57 deletions(-) diff --git a/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx b/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx index eb53f1c7..0a312040 100644 --- a/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx +++ b/runtime/hub/frontend/apps/admin/src/pages/UserList.tsx @@ -434,9 +434,35 @@ export function UserList() { [selectedUsers, users] ); + const batchGroupNameTrimmed = batchGroupName.trim(); + + const existingBatchGroup = useMemo( + () => groups.find(group => group.name === batchGroupNameTrimmed), + [batchGroupNameTrimmed, groups] + ); + const selectedBatchGroup = useMemo( - () => groups.find(group => group.name === batchGroupName), - [batchGroupName, groups] + () => mutableGroups.find(group => group.name === batchGroupNameTrimmed), + [batchGroupNameTrimmed, mutableGroups] + ); + + const canCreateBatchGroup = useMemo( + () => ( + batchGroupMode === 'add' + && batchGroupNameTrimmed.length > 0 + && !existingBatchGroup + && /^[a-zA-Z0-9_-]+$/.test(batchGroupNameTrimmed) + ), + [batchGroupMode, batchGroupNameTrimmed, existingBatchGroup] + ); + + const batchGroupInputInvalid = useMemo( + () => ( + batchGroupNameTrimmed.length > 0 + && !selectedBatchGroup + && !canCreateBatchGroup + ), + [batchGroupNameTrimmed, canCreateBatchGroup, selectedBatchGroup] ); // Debounce search input @@ -624,17 +650,20 @@ export function UserList() { setError('Please select users first'); return; } - if (!batchGroupName) { - setError('Please select a group'); + if (!selectedBatchGroup && !canCreateBatchGroup) { + setError(batchGroupMode === 'add' + ? 'Select a mutable group or enter a new group name' + : 'Please select a mutable group from the list'); return; } try { setActionLoading('batch-group'); + const targetGroup = selectedBatchGroup ?? await api.createGroup(batchGroupNameTrimmed); if (batchGroupMode === 'add') { - await api.addUsersToGroup(batchGroupName, selectedUsernames); + await api.addUsersToGroup(targetGroup.name, selectedUsernames); } else { - await api.removeUsersFromGroup(batchGroupName, selectedUsernames); + await api.removeUsersFromGroup(targetGroup.name, selectedUsernames); } await Promise.all([loadUsers(true), loadGroups()]); setShowBatchGroupModal(false); @@ -856,63 +885,59 @@ export function UserList() { {actionLoading === 'stop-all' ? : 'Stop All'} {quotaEnabled && ( - <> - - - + )} - - Group Actions ({selectedUsers.size}) + Selected ({selectedUsers.size}) - openBatchGroupModal('add')}> + {quotaEnabled && ( + setShowBatchQuotaModal(true)}> + Set Quota + + )} + setShowBatchPasswordModal(true)} + disabled={nativeSelected.length === 0} + title={nativeSelected.length === 0 ? 'No native users selected' : `Reset passwords for ${nativeSelected.length} users`} + > + Reset PW ({nativeSelected.length}) + + + openBatchGroupModal('add')} + > Add to group - openBatchGroupModal('remove')}> + openBatchGroupModal('remove')} + disabled={mutableGroups.length === 0} + title={mutableGroups.length === 0 ? 'No mutable groups available' : undefined} + > Remove from group + + setShowBatchDeleteModal(true)} + disabled={deletableSelected.length === 0} + title={deletableSelected.length === 0 ? 'No deletable users selected' : `Delete ${deletableSelected.length} users`} + > + Delete ({deletableSelected.length}) + - From 719d3f64bb86fd3e7cc340d187d41c663b4b2dee Mon Sep 17 00:00:00 2001 From: KerwinTsaiii Date: Fri, 17 Jul 2026 17:39:42 +0800 Subject: [PATCH 023/180] feat: integrate auplc-skills into the repository as a bundled plugin Bring the auplc-skills catalog into aup-learning-cloud so the whole repo is the plugin (marketplace source "./"), installable directly from GitHub via claude-code/cursor. Skills, templates, docs, plugin manifests, and the validation scripts now live at the repo root; the skills README is kept as README-SKILL.md and linked from the main README. - Add .claude-plugin/ and .cursor-plugin/ manifests (plugin "auplc") - Add skills/, templates/, plugin-metadata.json and skill authoring docs - Merge skill validation workflow into .github/workflows/validate-skills.yml and scripts under .github/scripts/ - Add scripts/check_skills_version.py to keep version fields in sync with pyproject.toml (strategy A + C: shared project version, pinned via git tags) - Point skill install instructions at AMDResearch/aup-learning-cloud Co-authored-by: Cursor --- .claude-plugin/marketplace.json | 18 + .claude-plugin/plugin.json | 21 + .cursor-plugin/marketplace.json | 18 + .cursor-plugin/plugin.json | 19 + .github/scripts/check.sh | 32 ++ .../scripts/generate_cursor_marketplace.py | 207 +++++++++ .github/scripts/publish.sh | 47 ++ .github/scripts/validate_skills.py | 395 +++++++++++++++++ .github/workflows/validate-skills.yml | 106 +++++ README-SKILL.md | 220 ++++++++++ README.md | 1 + docs/adding-a-skill.md | 73 +++ docs/skill-cards.md | 41 ++ docs/skill-categories.md | 80 ++++ plugin-metadata.json | 22 + scripts/check_skills_version.py | 93 ++++ .../build-aup-learning-cloud-images/SKILL.md | 106 +++++ .../reference.md | 95 ++++ .../skill-card.md | 9 + .../SKILL.md | 104 +++++ .../reference.md | 156 +++++++ .../skill-card.md | 9 + .../SKILL.md | 85 ++++ .../reference.md | 149 +++++++ .../skill-card.md | 9 + .../SKILL.md | 104 +++++ .../reference.md | 112 +++++ .../skill-card.md | 9 + skills/deploy-aup-learning-cloud/SKILL.md | 203 +++++++++ skills/deploy-aup-learning-cloud/reference.md | 415 ++++++++++++++++++ .../scripts/README.md | 46 ++ .../scripts/detect_cluster.sh | 151 +++++++ .../scripts/detect_hardware.sh | 155 +++++++ .../scripts/gen_configs.py | 299 +++++++++++++ .../scripts/validate.py | 248 +++++++++++ .../deploy-aup-learning-cloud/skill-card.md | 9 + .../SKILL.md | 98 +++++ .../reference.md | 106 +++++ .../skill-card.md | 9 + skills/expose-aup-learning-cloud/SKILL.md | 109 +++++ skills/expose-aup-learning-cloud/reference.md | 174 ++++++++ .../expose-aup-learning-cloud/skill-card.md | 9 + .../SKILL.md | 106 +++++ .../reference.md | 128 ++++++ .../skill-card.md | 9 + .../manage-aup-learning-cloud-users/SKILL.md | 147 +++++++ .../reference.md | 235 ++++++++++ .../scripts/hub-api-env.sh | 45 ++ .../skill-card.md | 9 + skills/monitor-aup-learning-cloud/SKILL.md | 129 ++++++ .../monitor-aup-learning-cloud/reference.md | 103 +++++ .../scripts/verify_monitoring.sh | 66 +++ .../monitor-aup-learning-cloud/skill-card.md | 9 + .../SKILL.md | 148 +++++++ .../reference.md | 289 ++++++++++++ .../skill-card.md | 9 + .../troubleshoot-aup-learning-cloud/SKILL.md | 93 ++++ .../reference.md | 75 ++++ .../skill-card.md | 9 + skills/upgrade-aup-learning-cloud/SKILL.md | 81 ++++ .../upgrade-aup-learning-cloud/reference.md | 96 ++++ .../upgrade-aup-learning-cloud/skill-card.md | 9 + templates/skill-template/SKILL.md | 41 ++ templates/skill-template/reference.md | 19 + templates/skill-template/skill-card.md | 9 + 65 files changed, 6235 insertions(+) create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json create mode 100644 .cursor-plugin/marketplace.json create mode 100644 .cursor-plugin/plugin.json create mode 100755 .github/scripts/check.sh create mode 100755 .github/scripts/generate_cursor_marketplace.py create mode 100755 .github/scripts/publish.sh create mode 100755 .github/scripts/validate_skills.py create mode 100644 .github/workflows/validate-skills.yml create mode 100644 README-SKILL.md create mode 100644 docs/adding-a-skill.md create mode 100644 docs/skill-cards.md create mode 100644 docs/skill-categories.md create mode 100644 plugin-metadata.json create mode 100644 scripts/check_skills_version.py create mode 100644 skills/build-aup-learning-cloud-images/SKILL.md create mode 100644 skills/build-aup-learning-cloud-images/reference.md create mode 100644 skills/build-aup-learning-cloud-images/skill-card.md create mode 100644 skills/configure-aup-learning-cloud-auth/SKILL.md create mode 100644 skills/configure-aup-learning-cloud-auth/reference.md create mode 100644 skills/configure-aup-learning-cloud-auth/skill-card.md create mode 100644 skills/configure-aup-learning-cloud-courses/SKILL.md create mode 100644 skills/configure-aup-learning-cloud-courses/reference.md create mode 100644 skills/configure-aup-learning-cloud-courses/skill-card.md create mode 100644 skills/configure-aup-learning-cloud-repos/SKILL.md create mode 100644 skills/configure-aup-learning-cloud-repos/reference.md create mode 100644 skills/configure-aup-learning-cloud-repos/skill-card.md create mode 100644 skills/deploy-aup-learning-cloud/SKILL.md create mode 100644 skills/deploy-aup-learning-cloud/reference.md create mode 100644 skills/deploy-aup-learning-cloud/scripts/README.md create mode 100755 skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh create mode 100755 skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh create mode 100755 skills/deploy-aup-learning-cloud/scripts/gen_configs.py create mode 100755 skills/deploy-aup-learning-cloud/scripts/validate.py create mode 100644 skills/deploy-aup-learning-cloud/skill-card.md create mode 100644 skills/develop-aup-learning-cloud-courses/SKILL.md create mode 100644 skills/develop-aup-learning-cloud-courses/reference.md create mode 100644 skills/develop-aup-learning-cloud-courses/skill-card.md create mode 100644 skills/expose-aup-learning-cloud/SKILL.md create mode 100644 skills/expose-aup-learning-cloud/reference.md create mode 100644 skills/expose-aup-learning-cloud/skill-card.md create mode 100644 skills/install-aup-learning-cloud-single-node/SKILL.md create mode 100644 skills/install-aup-learning-cloud-single-node/reference.md create mode 100644 skills/install-aup-learning-cloud-single-node/skill-card.md create mode 100644 skills/manage-aup-learning-cloud-users/SKILL.md create mode 100644 skills/manage-aup-learning-cloud-users/reference.md create mode 100644 skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh create mode 100644 skills/manage-aup-learning-cloud-users/skill-card.md create mode 100644 skills/monitor-aup-learning-cloud/SKILL.md create mode 100644 skills/monitor-aup-learning-cloud/reference.md create mode 100755 skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh create mode 100644 skills/monitor-aup-learning-cloud/skill-card.md create mode 100644 skills/plan-aup-learning-cloud-deployment/SKILL.md create mode 100644 skills/plan-aup-learning-cloud-deployment/reference.md create mode 100644 skills/plan-aup-learning-cloud-deployment/skill-card.md create mode 100644 skills/troubleshoot-aup-learning-cloud/SKILL.md create mode 100644 skills/troubleshoot-aup-learning-cloud/reference.md create mode 100644 skills/troubleshoot-aup-learning-cloud/skill-card.md create mode 100644 skills/upgrade-aup-learning-cloud/SKILL.md create mode 100644 skills/upgrade-aup-learning-cloud/reference.md create mode 100644 skills/upgrade-aup-learning-cloud/skill-card.md create mode 100644 templates/skill-template/SKILL.md create mode 100644 templates/skill-template/reference.md create mode 100644 templates/skill-template/skill-card.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 00000000..87b46405 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,18 @@ +{ + "name": "auplc-skills", + "owner": { + "name": "AMD Research" + }, + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "metadata": { + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "auplc", + "source": "./", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 00000000..1ecc51b4 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,21 @@ +{ + "name": "auplc", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", + "version": "0.1.0", + "author": { + "name": "AMD Research" + }, + "homepage": "https://github.com/AMDResearch/aup-learning-cloud", + "repository": "https://github.com/AMDResearch/auplc-skills", + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json new file mode 100644 index 00000000..87b46405 --- /dev/null +++ b/.cursor-plugin/marketplace.json @@ -0,0 +1,18 @@ +{ + "name": "auplc-skills", + "owner": { + "name": "AMD Research" + }, + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "metadata": { + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "auplc", + "source": "./", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs." + } + ] +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 00000000..3325c0ca --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "auplc", + "version": "0.1.0", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", + "author": { + "name": "AMD Research" + }, + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh new file mode 100755 index 00000000..588fe900 --- /dev/null +++ b/.github/scripts/check.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Validate every SKILL.md and that generated plugin manifests are up to date. +# +# Usage: +# ./.github/scripts/check.sh Validate every skill and check manifests. +# ./.github/scripts/check.sh -h|--help Print this help. +# +# Requires `uv` (https://github.com/astral-sh/uv). + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +usage() { + sed -n 's/^# \{0,1\}//p' "${BASH_SOURCE[0]}" | sed -n '/^Usage:/,/^Requires/p' +} + +case "${1:-}" in + "") + uv run .github/scripts/validate_skills.py + uv run .github/scripts/generate_cursor_marketplace.py --check + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/generate_cursor_marketplace.py b/.github/scripts/generate_cursor_marketplace.py new file mode 100755 index 00000000..b08e0bc0 --- /dev/null +++ b/.github/scripts/generate_cursor_marketplace.py @@ -0,0 +1,207 @@ +#!/usr/bin/env -S uv run --quiet +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Generate the Cursor plugin manifests from the canonical sources. + +`auplc-skills` ships as a single bundled plugin: the whole repository is one +plugin whose `skills/` folder every supported agent discovers automatically +(this mirrors how `cloudflare/skills` is published). To avoid drift, the Cursor +manifests are generated from the Claude manifests rather than hand-maintained. + +Sources of truth: +- `plugin-metadata.json` (repo root): shared identity and discovery metadata + (name, description, version, author, homepage, repository, + keywords). This is the vendor-neutral metadata file, reused by every + marketplace/manifest target. It is NOT a plugin manifest. +- `.claude-plugin/marketplace.json`: the marketplace catalog with the single + bundled plugin entry and its human-readable description (hand-maintained, + since the catalog blurb intentionally differs from the SKILL.md routing + descriptions). +- `.claude-plugin/plugin.json`: the bundled plugin manifest (hand-maintained). + +Outputs: +- `.cursor-plugin/marketplace.json`: a mirror of the Claude marketplace so + Cursor exposes exactly the same plugin as Claude. +- `.cursor-plugin/plugin.json`: the Cursor plugin manifest derived from the + Claude plugin manifest + `plugin-metadata.json`. + +Usage: + uv run .github/scripts/generate_cursor_marketplace.py # write + uv run .github/scripts/generate_cursor_marketplace.py --check # validate only + +`--check` fails if any generated file is stale or if the Claude manifests' +top-level identity has drifted from `plugin-metadata.json`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +PLUGIN_METADATA = ROOT / "plugin-metadata.json" +CLAUDE_MARKETPLACE = ROOT / ".claude-plugin" / "marketplace.json" +CLAUDE_PLUGIN = ROOT / ".claude-plugin" / "plugin.json" +CURSOR_MARKETPLACE = ROOT / ".cursor-plugin" / "marketplace.json" +CURSOR_PLUGIN = ROOT / ".cursor-plugin" / "plugin.json" + + +def load_json(path: Path) -> dict: + if not path.exists(): + raise FileNotFoundError(f"Missing required file: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def check_identity_consistency( + metadata: dict, claude: dict, claude_plugin: dict +) -> list[str]: + """Return error strings if the Claude manifests' identity has drifted from + the canonical `plugin-metadata.json`.""" + errors: list[str] = [] + + name = metadata.get("name") + description = metadata.get("description") + version = metadata.get("version") + + if claude.get("name") != name: + errors.append( + f".claude-plugin/marketplace.json `name` ({claude.get('name')!r}) " + f"must match plugin-metadata.json `name` ({name!r})." + ) + if claude.get("description") != description: + errors.append( + ".claude-plugin/marketplace.json `description` must match " + "plugin-metadata.json `description`." + ) + claude_version = (claude.get("metadata") or {}).get("version") + if claude_version != version: + errors.append( + f".claude-plugin/marketplace.json metadata.version " + f"({claude_version!r}) must match plugin-metadata.json `version` " + f"({version!r})." + ) + + # The single bundled plugin entry's name must match the plugin manifest. + plugins = claude.get("plugins") + if not isinstance(plugins, list) or len(plugins) != 1: + errors.append( + ".claude-plugin/marketplace.json must list exactly one bundled " + "plugin (source `./`)." + ) + else: + entry_name = plugins[0].get("name") + if entry_name != claude_plugin.get("name"): + errors.append( + f".claude-plugin/marketplace.json plugin `name` ({entry_name!r}) " + f"must match .claude-plugin/plugin.json `name` " + f"({claude_plugin.get('name')!r})." + ) + + if claude_plugin.get("version") != version: + errors.append( + f".claude-plugin/plugin.json `version` " + f"({claude_plugin.get('version')!r}) must match plugin-metadata.json " + f"`version` ({version!r})." + ) + return errors + + +def build_cursor_marketplace(metadata: dict, claude: dict) -> dict: + author = metadata.get("author") or {} + owner_name = author.get("name") if isinstance(author, dict) else None + + return { + "name": metadata["name"], + "owner": {"name": owner_name} if owner_name else {}, + "description": metadata["description"], + "metadata": { + "description": metadata["description"], + "version": metadata["version"], + }, + "plugins": claude.get("plugins", []), + } + + +def build_cursor_plugin(metadata: dict, claude_plugin: dict) -> dict: + return { + "name": claude_plugin["name"], + "version": metadata["version"], + "description": claude_plugin.get("description", metadata["description"]), + "author": metadata.get("author") or {}, + "keywords": metadata.get("keywords", []), + } + + +def render_json(data: dict) -> str: + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def write_or_check(path: Path, content: str, check: bool) -> bool: + """Return True when the file is already up to date.""" + current = path.read_text(encoding="utf-8") if path.exists() else None + if current == content: + return True + if check: + return False + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Generate the .cursor-plugin/ manifests from the canonical " + "Claude manifests and plugin-metadata.json." + ) + parser.add_argument( + "--check", + action="store_true", + help="Validate the generated manifests are up to date without writing.", + ) + args = parser.parse_args(argv) + + metadata = load_json(PLUGIN_METADATA) + claude = load_json(CLAUDE_MARKETPLACE) + claude_plugin = load_json(CLAUDE_PLUGIN) + + identity_errors = check_identity_consistency(metadata, claude, claude_plugin) + if identity_errors: + print("Plugin manifest identity is inconsistent:", file=sys.stderr) + for err in identity_errors: + print(f" - {err}", file=sys.stderr) + return 1 + + targets = { + CURSOR_MARKETPLACE: render_json(build_cursor_marketplace(metadata, claude)), + CURSOR_PLUGIN: render_json(build_cursor_plugin(metadata, claude_plugin)), + } + + stale = [ + path + for path, content in targets.items() + if not write_or_check(path, content, check=args.check) + ] + + if args.check: + if stale: + for path in stale: + print(f"{path.relative_to(ROOT)} is out of date.", file=sys.stderr) + print( + "Run: uv run .github/scripts/generate_cursor_marketplace.py", + file=sys.stderr, + ) + return 1 + print("Cursor plugin manifests are up to date.") + return 0 + + for path in targets: + print(f"Wrote {path.relative_to(ROOT)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/publish.sh b/.github/scripts/publish.sh new file mode 100755 index 00000000..0732dce5 --- /dev/null +++ b/.github/scripts/publish.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Regenerate every committed artifact derived from skills/ and the +# canonical marketplace + metadata sources. +# +# Usage: +# ./.github/scripts/publish.sh Regenerate all derived artifacts. +# ./.github/scripts/publish.sh --check Verify derived artifacts are up to date. +# ./.github/scripts/publish.sh -h|--help Print this help. +# +# Currently regenerates: +# - .cursor-plugin/marketplace.json (mirror of .claude-plugin/marketplace.json) +# - .cursor-plugin/plugin.json (derived from .claude-plugin/plugin.json +# + plugin-metadata.json) +# +# The `.claude-plugin/` manifests are hand-maintained because the human-facing +# plugin description intentionally differs from the SKILL.md routing +# descriptions; ./.github/scripts/check.sh enforces that they stay consistent +# with plugin-metadata.json. +# +# Requires `uv` (https://github.com/astral-sh/uv). + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +usage() { + sed -n 's/^# \{0,1\}//p' "${BASH_SOURCE[0]}" | sed -n '/^Usage:/,/^Requires/p' +} + +case "${1:-}" in + "") + uv run .github/scripts/generate_cursor_marketplace.py + echo "Publish artifacts generated successfully." + ;; + --check) + uv run .github/scripts/generate_cursor_marketplace.py --check + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py new file mode 100755 index 00000000..85384665 --- /dev/null +++ b/.github/scripts/validate_skills.py @@ -0,0 +1,395 @@ +#!/usr/bin/env -S uv run --quiet +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml>=6.0"] +# /// +"""Validate auplc-skills against the standardized Agent Skills format. + +Enforces the rules documented in CONTRIBUTING.md: + + - SKILL.md exists at the skill root + - YAML frontmatter is parseable + - `name` is lowercase-with-hyphens, <=64 chars, no `anthropic`/`claude` + substrings, and matches the directory name + - `description` is a non-empty string <=1024 chars + - SKILL.md body is <=500 lines + - skill-card.md exists at the skill root and has non-empty + `## Description` and `## Owner` sections + +Also validates the bundled-plugin manifests: `.claude-plugin/marketplace.json` +must list exactly one plugin whose `source` is `./` (the whole repo is one +plugin, mirroring how `cloudflare/skills` is published), and +`.claude-plugin/plugin.json` must exist with a matching `name`. + +Run from the repo root: + + ./.github/scripts/check.sh # used locally; thin wrapper + uv run .github/scripts/validate_skills.py # validate every skill + manifest + uv run .github/scripts/validate_skills.py --skills-dir skills + uv run .github/scripts/validate_skills.py --list # print skill names as JSON + uv run .github/scripts/validate_skills.py --skill deploy-aup-learning-cloud + uv run .github/scripts/validate_skills.py --marketplace-only # manifest only + +The `--list` / `--skill` options let CI validate each skill in its own job +(see .github/workflows/validate.yml) so a single bad skill doesn't mask the +status of the others. + +Exits non-zero if any validated skill (or the marketplace check) fails. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_SKILLS_DIR = REPO_ROOT / "skills" +CLAUDE_MARKETPLACE = REPO_ROOT / ".claude-plugin" / "marketplace.json" +CLAUDE_PLUGIN = REPO_ROOT / ".claude-plugin" / "plugin.json" + +# Limits from CONTRIBUTING.md and the standardized Agent Skills format. +MAX_NAME_LEN = 64 +MAX_DESCRIPTION_LEN = 1024 +MAX_BODY_LINES = 500 + +NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +FRONTMATTER_RE = re.compile( + r"\A---\r?\n(?P.*?)\r?\n---\r?\n?(?P.*)\Z", + re.DOTALL, +) +RESERVED_NAME_SUBSTRINGS = ("anthropic", "claude") + +# Per-skill governance card (see docs/skill-cards.md). Each section must be a +# top-level `##` heading followed by some non-empty body text. +CARD_FILENAME = "skill-card.md" +REQUIRED_CARD_SECTIONS = ("Description", "Owner") + + +@dataclass +class SkillReport: + skill: str + errors: list[str] = field(default_factory=list) + + +def validate_skill(skill_dir: Path) -> SkillReport: + """Run every validation rule against `skill_dir` and return a report.""" + report = SkillReport(skill=skill_dir.name) + skill_md = skill_dir / "SKILL.md" + + if not skill_md.exists(): + report.errors.append("Missing SKILL.md.") + return report + + text = skill_md.read_text(encoding="utf-8") + match = FRONTMATTER_RE.match(text) + if match is None: + report.errors.append( + "SKILL.md must start with a `---` YAML frontmatter block " + "followed by `---` on its own line." + ) + return report + + try: + frontmatter = yaml.safe_load(match.group("frontmatter")) + except yaml.YAMLError as exc: + report.errors.append(f"YAML frontmatter is invalid: {exc}") + return report + + if not isinstance(frontmatter, dict): + report.errors.append( + "YAML frontmatter must be a mapping with at least `name` " + "and `description`." + ) + return report + + _validate_name(frontmatter.get("name"), skill_dir.name, report) + _validate_description(frontmatter.get("description"), report) + _validate_body(match.group("body"), report) + _validate_card(skill_dir, report) + return report + + +def _validate_name(name: object, dir_name: str, report: SkillReport) -> None: + if not isinstance(name, str) or not name: + report.errors.append("Frontmatter `name` is missing or not a non-empty string.") + return + + if len(name) > MAX_NAME_LEN: + report.errors.append( + f"`name` length {len(name)} exceeds {MAX_NAME_LEN} characters." + ) + if not NAME_RE.match(name): + report.errors.append( + f"`name` `{name}` must be lowercase-with-hyphens " + "(letters, digits, single hyphens between segments)." + ) + for sub in RESERVED_NAME_SUBSTRINGS: + if sub in name.lower(): + report.errors.append(f"`name` may not contain `{sub}`.") + if name != dir_name: + report.errors.append( + f"`name` `{name}` must match the skill directory name `{dir_name}`." + ) + + +def _validate_description(description: object, report: SkillReport) -> None: + if not isinstance(description, str) or not description: + report.errors.append( + "Frontmatter `description` is missing or not a non-empty string." + ) + return + if len(description) > MAX_DESCRIPTION_LEN: + report.errors.append( + f"`description` length {len(description)} exceeds " + f"{MAX_DESCRIPTION_LEN} characters." + ) + + +def _validate_body(body: str, report: SkillReport) -> None: + # Skip surrounding blank lines so the blank line after `---` doesn't + # inflate the count. + lines = body.splitlines() + while lines and not lines[0].strip(): + lines.pop(0) + while lines and not lines[-1].strip(): + lines.pop() + if len(lines) > MAX_BODY_LINES: + report.errors.append( + f"SKILL.md body is {len(lines)} lines; max is {MAX_BODY_LINES}. " + "Move reference material into sibling files (reference.md, " + "examples.md, ...) and link to them from SKILL.md." + ) + + +def _validate_card(skill_dir: Path, report: SkillReport) -> None: + """Require a skill-card.md with non-empty Description, Owner.""" + card = skill_dir / CARD_FILENAME + if not card.exists(): + report.errors.append( + f"Missing {CARD_FILENAME} (governance card). See docs/skill-cards.md; " + "it needs `## Description` and `## Owner` sections." + ) + return + + sections = _parse_card_sections(card.read_text(encoding="utf-8")) + for name in REQUIRED_CARD_SECTIONS: + body = sections.get(name.lower()) + if body is None: + report.errors.append(f"{CARD_FILENAME} is missing a `## {name}` section.") + elif not body.strip(): + report.errors.append(f"{CARD_FILENAME} `## {name}` section is empty.") + + +def _parse_card_sections(text: str) -> dict[str, str]: + """Map each `##` heading (lowercased) to the text until the next heading.""" + sections: dict[str, str] = {} + current: str | None = None + buffer: list[str] = [] + + def flush() -> None: + if current is not None: + sections[current] = "\n".join(buffer).strip() + + for line in text.splitlines(): + heading = re.match(r"^##\s+(?P.+?)\s*$", line) + if heading: + flush() + current = heading.group("title").lower() + buffer = [] + elif current is not None: + buffer.append(line) + flush() + return sections + + +def discover_skills(root: Path) -> list[Path]: + """List skill directories under `root`, ignoring dotfiles.""" + if not root.exists(): + return [] + return sorted( + p for p in root.iterdir() if p.is_dir() and not p.name.startswith(".") + ) + + +def validate_claude_marketplace() -> list[str]: + """Validate the single bundled-plugin manifests. + + `auplc-skills` is published as one plugin whose `source` is `./` (the whole + repo), so the marketplace must list exactly one plugin and a matching + `.claude-plugin/plugin.json` must exist. The marketplace's human-readable + `description` is intentionally allowed to differ from the SKILL.md + descriptions (per CONTRIBUTING.md), so its text is not cross-checked. + """ + errors: list[str] = [] + + if not CLAUDE_MARKETPLACE.exists(): + return [ + f"Missing {CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}; expected a " + "single bundled-plugin entry (source `./`)." + ] + + try: + data = json.loads(CLAUDE_MARKETPLACE.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: invalid JSON: {exc}"] + + plugins = data.get("plugins") if isinstance(data, dict) else None + if not isinstance(plugins, list): + return [ + f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: top-level `plugins` " + "array is missing." + ] + if len(plugins) != 1: + return [ + f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: expected exactly one " + f"bundled plugin (source `./`), found {len(plugins)}." + ] + + entry = plugins[0] + if not isinstance(entry, dict): + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: plugins[0] must be an object."] + + name = entry.get("name") + source = entry.get("source") + description = entry.get("description") + + if not isinstance(name, str) or not name: + errors.append("plugins[0] is missing a non-empty `name`.") + if source != "./": + errors.append(f"plugins[0]: `source` must be `./`, got `{source}`.") + if not isinstance(description, str) or not description.strip(): + errors.append("plugins[0] is missing a non-empty `description`.") + + if not CLAUDE_PLUGIN.exists(): + errors.append( + f"Missing {CLAUDE_PLUGIN.relative_to(REPO_ROOT)}; the bundled plugin " + "needs a `.claude-plugin/plugin.json` manifest." + ) + else: + try: + plugin = json.loads(CLAUDE_PLUGIN.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + errors.append(f"{CLAUDE_PLUGIN.relative_to(REPO_ROOT)}: invalid JSON: {exc}") + else: + plugin_name = plugin.get("name") if isinstance(plugin, dict) else None + if plugin_name != name: + errors.append( + f"{CLAUDE_PLUGIN.relative_to(REPO_ROOT)} `name` " + f"({plugin_name!r}) must match the marketplace plugin " + f"`name` ({name!r})." + ) + + return errors + + +def _print_report(report: SkillReport) -> int: + """Print a single skill report and return its error count.""" + status = "OK " if not report.errors else "FAIL" + print(f"[{status}] {report.skill}") + for err in report.errors: + print(f" {err}") + return len(report.errors) + + +def list_skills(skills_dir: Path) -> int: + """Print discovered skill names as a compact JSON array (for CI matrices).""" + skills = discover_skills(skills_dir) + if not skills: + print(f"No skills found under {skills_dir}", file=sys.stderr) + return 1 + print(json.dumps([p.name for p in skills], separators=(",", ":"))) + return 0 + + +def run_single(skills_dir: Path, name: str) -> int: + """Validate a single skill directory by name (no marketplace cross-check).""" + skill_dir = skills_dir / name + if not skill_dir.is_dir(): + print(f"No such skill directory: {skill_dir}", file=sys.stderr) + return 1 + + errors = _print_report(validate_skill(skill_dir)) + print(f"\nSummary: {errors} error(s) in skill `{name}`") + return 0 if errors == 0 else 1 + + +def run_marketplace(skills_dir: Path) -> int: + """Validate only the bundled-plugin manifests.""" + marketplace_errors = validate_claude_marketplace() + status = "OK " if not marketplace_errors else "FAIL" + print(f"[{status}] .claude-plugin/marketplace.json") + for err in marketplace_errors: + print(f" {err}") + print(f"\nSummary: {len(marketplace_errors)} error(s) in marketplace manifest") + return 0 if not marketplace_errors else 1 + + +def run(skills_dir: Path) -> int: + skills = discover_skills(skills_dir) + if not skills: + print(f"No skills found under {skills_dir}", file=sys.stderr) + return 1 + + print(f"Validating {len(skills)} skill(s) in {skills_dir}\n") + total_errors = 0 + for skill_dir in skills: + total_errors += _print_report(validate_skill(skill_dir)) + + marketplace_errors = validate_claude_marketplace() + marketplace_status = "OK " if not marketplace_errors else "FAIL" + print(f"\n[{marketplace_status}] .claude-plugin/marketplace.json") + for err in marketplace_errors: + print(f" {err}") + total_errors += len(marketplace_errors) + + print(f"\nSummary: {total_errors} error(s) across {len(skills)} skill(s)") + return 0 if total_errors == 0 else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--skills-dir", + type=Path, + default=DEFAULT_SKILLS_DIR, + help=f"Directory containing skill folders (default: {DEFAULT_SKILLS_DIR}).", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list", + action="store_true", + help="Print discovered skill names as a JSON array and exit.", + ) + group.add_argument( + "--skill", + metavar="NAME", + help="Validate only the named skill directory (skips the marketplace " + "cross-check, which is repo-wide).", + ) + group.add_argument( + "--marketplace-only", + action="store_true", + help="Only validate that marketplace.json is in sync with skills/.", + ) + args = parser.parse_args(argv) + skills_dir = args.skills_dir.resolve() + + if args.list: + return list_skills(skills_dir) + if args.skill: + return run_single(skills_dir, args.skill) + if args.marketplace_only: + return run_marketplace(skills_dir) + return run(skills_dir) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml new file mode 100644 index 00000000..c1b2f4cd --- /dev/null +++ b/.github/workflows/validate-skills.yml @@ -0,0 +1,106 @@ +name: validate-skills + +on: + push: + branches: [main] + paths: + - "skills/**" + - "templates/**" + - ".claude-plugin/**" + - ".cursor-plugin/**" + - ".github/scripts/**" + - ".github/workflows/validate-skills.yml" + pull_request: + paths: + - "skills/**" + - "templates/**" + - ".claude-plugin/**" + - ".cursor-plugin/**" + - ".github/scripts/**" + - ".github/workflows/validate-skills.yml" + workflow_dispatch: + +# Least privilege: these jobs only read the repo to validate skills/manifests. +permissions: + contents: read + +jobs: + # Enumerate the skills so the validation job can fan out over them with a + # matrix. Running each skill in its own job (with fail-fast disabled) means + # one broken skill shows up as a single red check instead of failing the + # whole suite and hiding the status of every other skill. + discover-skills: + name: Discover skills + runs-on: ubuntu-latest + outputs: + skills: ${{ steps.discover.outputs.skills }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: List skills + id: discover + run: echo "skills=$(uv run .github/scripts/validate_skills.py --list)" >> "$GITHUB_OUTPUT" + + validate-skill: + name: Validate skill + needs: discover-skills + runs-on: ubuntu-latest + strategy: + # Don't cancel the other skills when one fails; we want to see every + # skill's status in a single run. + fail-fast: false + matrix: + skill: ${{ fromJson(needs.discover-skills.outputs.skills) }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Validate skill + run: uv run .github/scripts/validate_skills.py --skill "${{ matrix.skill }}" + + # Repo-wide checks that aren't tied to a single skill: the generated plugin + # manifests. + validate-manifests: + name: Validate plugin manifests + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Validate marketplace manifest + run: uv run .github/scripts/validate_skills.py --marketplace-only + + - name: Validate generated Cursor manifest + run: uv run .github/scripts/generate_cursor_marketplace.py --check + + # Single gate that aggregates the per-skill matrix and the repo-wide manifest + # checks. Branch protection can require just this one check: it only passes + # when every skill validated and the manifest job succeeded. Because matrix + # jobs always succeed individually under `fail-fast: false`, we inspect the + # job results explicitly rather than relying on `needs` short-circuiting. + validate: + name: Validate skills and plugin manifests + needs: [validate-skill, validate-manifests] + if: always() + runs-on: ubuntu-latest + steps: + - name: Verify all validation jobs passed + run: | + echo "validate-skill result: ${{ needs.validate-skill.result }}" + echo "validate-manifests result: ${{ needs.validate-manifests.result }}" + if [ "${{ needs.validate-skill.result }}" != "success" ] || \ + [ "${{ needs.validate-manifests.result }}" != "success" ]; then + echo "One or more validation jobs failed." >&2 + exit 1 + fi + echo "All skill and manifest validations passed." diff --git a/README-SKILL.md b/README-SKILL.md new file mode 100644 index 00000000..e9eb4a03 --- /dev/null +++ b/README-SKILL.md @@ -0,0 +1,220 @@ +# AUP Learning Cloud Skills (`auplc-skills`) + +Agent Skills that help any coding agent deploy and maintain +[AUP Learning Cloud](https://github.com/AMDResearch/aup-learning-cloud) — the +multi-node JupyterHub-on-k3s teaching platform for AMD GPUs. + +Skills follow the standardized [Agent Skills](https://github.com/anthropics/skills) +format and interoperate with the major coding agents: Cursor, Claude Code, +OpenAI Codex, and Gemini CLI. + +> **Tech preview.** The catalog spans install → deploy → configure → build → +> upgrade → troubleshoot. Expect frequent changes while the foundations settle; +> the skills are a first draft to review with the operators who own each area. + +## The catalog + +The skills are organized into three groups so an agent can start in the right +place for a task. It is still **one bundled plugin** — installing it brings +every skill at once; the groups are a routing aid (each skill's `description` +also carries its `Group:` tag). See +[docs/skill-categories.md](docs/skill-categories.md) for the full taxonomy and +routing guidance. + +### Plan and deploy AUP Learning Cloud + +Bring the platform into existence: size it, install or deploy it, and build its +images. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`plan-aup-learning-cloud-deployment`](skills/plan-aup-learning-cloud-deployment/SKILL.md) | Size a new deployment for a prospective adopter: interview course/headcount needs and the network, research current AMD silicon, then recommend how many AIPCs/workstations/servers and routers/switches to buy, the topology, an IP plan, and a buyer-facing bill of materials. | in-repo | +| [`install-aup-learning-cloud-single-node`](skills/install-aup-learning-cloud-single-node/SKILL.md) | Install on a single AMD GPU/APU box with the `./auplc-installer` flow: prerequisites, GPU/courses/image flags, gated install, verify at `localhost:30890`. | in-repo | +| [`deploy-aup-learning-cloud`](skills/deploy-aup-learning-cloud/SKILL.md) | Deploy end to end on a multi-AIPC PXE/k3s cluster: interview the operator, generate the Ansible inventory + PXE vars + Helm values (helper scripts), then drive the install with confirmation gates at risky steps. | in-repo | +| [`build-aup-learning-cloud-images`](skills/build-aup-learning-cloud-images/SKILL.md) | Build and publish the Hub and notebook/course Docker images with `img build`, incl. GPU-target tagging and registry push. | in-repo | + +### Maintain AUP Learning Cloud + +Operate and keep a running deployment healthy: upgrade, debug, observe, secure +logins, manage users, and control network/storage exposure. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`upgrade-aup-learning-cloud`](skills/upgrade-aup-learning-cloud/SKILL.md) | Upgrade the JupyterHub chart/values/images and the k3s cluster on a running deployment, in a safe order with rollback. | in-repo | +| [`troubleshoot-aup-learning-cloud`](skills/troubleshoot-aup-learning-cloud/SKILL.md) | Diagnose netboot, node-join, GPU scheduling, storage, and auth failures from runtime evidence, then hand off the fix. | in-repo | +| [`monitor-aup-learning-cloud`](skills/monitor-aup-learning-cloud/SKILL.md) | Wire the Hub into Prometheus + Grafana: ServiceMonitor, authenticated metrics, dashboards, alert rules, and the metrics NetworkPolicy. | in-repo | +| [`configure-aup-learning-cloud-auth`](skills/configure-aup-learning-cloud-auth/SKILL.md) | Configure the auth mode (auto-login/dummy/github/multi), the GitHub App / OAuth + team sync, native accounts, and admin bootstrap. | in-repo | +| [`manage-aup-learning-cloud-users`](skills/manage-aup-learning-cloud-users/SKILL.md) | Day-2 user/group/quota operations via the admin console and `manage_users.py`: bulk onboarding, passwords, admins, and quota grants/refresh. | in-repo | +| [`expose-aup-learning-cloud`](skills/expose-aup-learning-cloud/SKILL.md) | Take a deployment past the local defaults: NodePort/LoadBalancer/ingress + TLS, CORS origins, externally-terminated TLS, and shared NFS storage. | in-repo | + +### Course and other editor + +Edit what lives inside the platform: the course catalog, new course content, and +per-user repository cloning. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`configure-aup-learning-cloud-courses`](skills/configure-aup-learning-cloud-courses/SKILL.md) | Edit the course catalog, spawn-UI metadata, GPU accelerator selectors, team mappings, and quota in `values.yaml`, then re-apply. | in-repo | +| [`develop-aup-learning-cloud-courses`](skills/develop-aup-learning-cloud-courses/SKILL.md) | Author a new course end to end: notebooks under `projects/`, a course image, and catalog registration, then build + wire it in. | in-repo | +| [`configure-aup-learning-cloud-repos`](skills/configure-aup-learning-cloud-repos/SKILL.md) | Configure per-user Git repo cloning: the spawn-form repo field/picker, private-repo tokens, provider allowlist, and clone persistence. | in-repo | + +## What is a skill? + +A skill is a self-contained folder that bundles everything an agent needs to +perform a focused task: instructions, helper scripts, and references. At its +core is a `SKILL.md` file with YAML frontmatter — a `name` and a short +`description` that tells the agent *when* the skill should activate — followed +by the guidance the agent reads while the skill is in use. + +``` +skills/ + deploy-aup-learning-cloud/ + SKILL.md # routing frontmatter + workflow + skill-card.md # governance card (Description, Owner) + reference.md # full step-by-step commands + troubleshooting + scripts/ # executable helpers +``` + +When an agent decides a skill is relevant (or you invoke it explicitly), it +loads `SKILL.md` and follows the instructions inside. Descriptions stay in +context cheaply; the full body loads only when the task actually matches. + +## Installation + +The whole catalog ships as a single bundled plugin (`auplc`), so any of the +methods below installs every skill at once. Pick the one that matches your +agent. + +### Claude Code + +Install with the [plugin marketplace](https://code.claude.com/docs/en/plugin-marketplaces): + +``` +/plugin marketplace add AMDResearch/aup-learning-cloud +/plugin install auplc@auplc-skills +``` + +### Cursor + +Install from the Cursor Marketplace, or add manually via **Settings → Rules → +Add Rule → Remote Rule (Github)** with `AMDResearch/aup-learning-cloud`. Cursor scans +the repo and copies the skills into `.cursor/skills/`. + +### npx skills + +Install with the [`npx skills`](https://skills.sh) CLI (works with any agent +that follows the Agent Skills standard): + +``` +npx skills add https://github.com/AMDResearch/aup-learning-cloud +``` + +### Clone / Copy + +Clone this repo and copy (or symlink) the skill folders you want from `skills/` +into your agent's skills directory. Each agent discovers `SKILL.md` +automatically. + +```bash +git clone https://github.com/AMDResearch/aup-learning-cloud.git +cp -r aup-learning-cloud/skills/deploy-aup-learning-cloud <agent-skills-dir>/ +``` + +| Agent | Skills directory (personal / project) | +| --- | --- | +| Cursor | `~/.cursor/skills/` / `.cursor/skills/` | +| Claude Code | `~/.claude/skills/` / `.claude/skills/` | +| Codex | `$HOME/.agents/skills` / `$REPO_ROOT/.agents/skills` | + +## Recommended models + +These skills drive long, gated workflows — Ansible runs, `kubectl`/`helm` +rollouts, netboot setup — where the agent has to hold a plan across many phases +and stop at each confirmation gate. They work best on a frontier reasoning model +with the reasoning effort turned up. + +| Agent | Model | Reasoning effort | +| --- | --- | --- | +| Claude Code | Opus 4.8 | high | +| Codex | GPT-5.6-Sol | high | +| OpenCode | DeepSeek V4 Flash | high | + +Any agent that follows the Agent Skills standard can load the catalog. If yours +isn't listed, pick its strongest reasoning model and raise the effort/thinking +setting to high. + +## Using a skill + +Once installed, reference it in plain language while talking to your agent. In +most cases the agent picks the right skill on its own from the description. + +### Example prompts — the three ways to stand up a deployment + +There are three deployment paths. Pick the prompt that matches your hardware; +the agent routes to the right skill and interviews you for the rest. + +- **Single node** (one AMD GPU/APU box → `install-aup-learning-cloud-single-node`): + + > *"Install AUP Learning Cloud on this single AMD GPU workstation with the + > `./auplc-installer` flow and verify it at `localhost:30890`."* + +- **Multi-node, PXE diskless netboot** (one service machine netboots diskless + agents → `deploy-aup-learning-cloud`, `topology: pxe-diskless`): + + > *"Deploy AUP Learning Cloud across my 3 AIPCs over PXE — the machine I'm on + > right now is the head/service node, and the other two are diskless agents + > that should netboot and auto-join k3s."* + +- **Multi-node, SSH pre-installed** (every node already runs Ubuntu, reachable + over SSH → `deploy-aup-learning-cloud`, `topology: ssh-preinstalled`): + + > *"Deploy AUP Learning Cloud on my 4 nodes that already run Ubuntu 24.04 and + > are reachable over SSH — the machine I'm on right now is the head/server + > node, install k3s and ROCm on all of them with Ansible, no PXE."* + +The two multi-node prompts both drive `deploy-aup-learning-cloud`; its Phase 1a +gate asks you to confirm the topology (`pxe-diskless` vs `ssh-preinstalled`) +before touching any machine. + +> **Tip — watch every command live in your own tmux.** These deploy/install +> skills run a lot of shell commands (Ansible, `kubectl`, `helm`, netboot +> setup). To see exactly what an agent runs, have it drive a tmux session you +> keep open instead of its hidden shell. First, open the session: +> +> ```bash +> tmux new -s auplc +> ``` +> +> Then tell the agent to send commands to it, e.g.: +> +> > *"Run every shell command by sending it to my tmux session `auplc` with +> > `tmux send-keys -t auplc '<command>' Enter`, then read the pane with +> > `tmux capture-pane -t auplc -p` to check the result — don't use your own +> > shell."* +> +> You watch the commands and their output scroll in the `auplc` pane in real +> time, and can hit `Ctrl-C` there to stop anything that looks wrong. This works +> in any agent that has terminal access (Claude Code, Cursor, Codex). + +## Repository layout + +``` +skills/ # All skills the agent can load +templates/skill-template # Starting point for a new skill +docs/ # Authoring + governance docs +.claude-plugin/ # Claude marketplace + bundled-plugin manifest (hand-maintained) +.cursor-plugin/ # Cursor marketplace + plugin manifest (generated) +plugin-metadata.json # Vendor-neutral identity/discovery metadata +.github/scripts/ # Validation + publish scripts +.github/workflows/ # CI that validates skills and manifests +``` + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for authoring conventions and +[docs/adding-a-skill.md](docs/adding-a-skill.md) for the step-by-step procedure +to add a new skill. Run the same checks CI runs before opening a PR: + +```bash +./.github/scripts/check.sh +``` diff --git a/README.md b/README.md index 5b3ce3a6..781aab70 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ Current environments are set up as `RESOURCE_IMAGES` in `runtime/chart/files/hub - [User Quota System](docs/jupyterhub/quota-system.md) - Resource usage tracking and quota management - [GitHub OAuth Setup](docs/jupyterhub/How_to_Setup_GitHub_OAuth.md) - OAuth configuration - [Maintenance Manual](docs/user-manual/aup-remote-lab-user-manual-admin-new.md) - Operations guide +- [AUP Learning Cloud Skills](README-SKILL.md) - Agent Skills for deploying and maintaining AUP Learning Cloud ## Contributing diff --git a/docs/adding-a-skill.md b/docs/adding-a-skill.md new file mode 100644 index 00000000..231f6aa5 --- /dev/null +++ b/docs/adding-a-skill.md @@ -0,0 +1,73 @@ +# Adding a skill + +This catalog is designed to grow. Each capability for working with AUP Learning +Cloud (deploying, course configuration, image building, upgrades, troubleshooting) +is its own self-contained skill folder under `skills/`. Adding one is a fixed, +five-step procedure. + +## 1. Copy the template + +```bash +cp -r templates/skill-template skills/<your-skill-name> +``` + +Use a `lowercase-with-hyphens` name tied to the outcome (e.g. +`configure-aup-learning-cloud-courses`, `build-aup-learning-cloud-images`, +`upgrade-aup-learning-cloud`). Avoid generic names like `helper` or `utils`. + +## 2. Write `SKILL.md` + +- Set `name:` in the frontmatter to **exactly** the directory name. +- Write a `description:` in the third person that states **what** the skill + produces and **when** an agent should reach for it, including the trigger + words a user is likely to say. Keep it under 1024 characters. +- **Assign a category.** Pick exactly one group from + [skill-categories.md](skill-categories.md) and **prepend its `Group:` tag** to + the start of the `description` (e.g. `Group: Maintain AUP Learning Cloud.`). + This is what lets an agent route to the right group first. +- Keep the body under 500 lines. Push long reference material into sibling + files (`reference.md`, `examples.md`, ...) linked one level deep. + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for the full authoring conventions. + +## 3. Write `skill-card.md` + +Fill in the `## Description` and `## Owner` sections. See +[skill-cards.md](skill-cards.md). + +## 4. List the skill in the catalog + +The repo ships as a single bundled plugin (`source: "./"`), so the plugin +manifests do **not** need a per-skill entry — dropping the folder under +`skills/` is enough for every install method to pick it up. Add a row to the +catalog table in the [README](../README.md) **under the skill's group section**, +list it under that group in [skill-categories.md](skill-categories.md) so people +can discover it, then keep the Cursor manifests in sync: + +```bash +./.github/scripts/publish.sh # regenerates .cursor-plugin/ from the canonical sources +``` + +## 5. Validate + +```bash +./.github/scripts/check.sh # same command CI runs +``` + +CI runs the same validation on every pull request via +`.github/workflows/validate.yml`, fanning out one job per skill so a single +broken skill is easy to spot. + +## Ideas for future skills + +The catalog now covers install, deploy, configure (courses), build, upgrade, +and troubleshoot, plus auth, user/quota management, monitoring, network/storage +exposure, per-user repo cloning, and course authoring (see the +[README](../README.md)). Natural next additions, each following the same +procedure: + +| Skill | Outcome | +| --- | --- | +| `backup-aup-learning-cloud` | Back up and restore the Hub DB PVC and user home data (snapshot, off-cluster copy, restore drill). | +| `offline-aup-learning-cloud` | Drive the air-gapped `pack`/`pack --local` bundle workflow end to end, including registry/PyPI/npm mirrors. | +| `tune-aup-learning-cloud-resources` | Right-size per-course CPU/memory/GPU requirements, prePuller, and node scheduling for a given fleet. | diff --git a/docs/skill-cards.md b/docs/skill-cards.md new file mode 100644 index 00000000..3690a292 --- /dev/null +++ b/docs/skill-cards.md @@ -0,0 +1,41 @@ +# Skill cards + +Every skill in this catalog ships a `skill-card.md` next to its `SKILL.md`. The card is a short, human-facing governance record: it tells a reviewer *what* the skill is and *who* owns it, without making them read the source first. + +A `SKILL.md` is written for the agent (routing and instructions). A skill card is written for the people deciding whether to trust, install, or maintain the skill. + +## Required sections + +The card is intentionally minimal. Two sections are required, each a top-level `##` heading with non-empty body text: + +| Section | Question it answers | +| --- | --- | +| Description | What does this skill do, in one sentence? | +| Owner | Who is accountable for maintaining it? | + +The validator (`.github/scripts/validate_skills.py`) fails any skill whose card is missing or whose required sections are absent or empty. + +## Template + +Copy this into `skills/<your-skill>/skill-card.md`: + +```markdown +# Skill Card + +## Description + +<one sentence: what the skill does, for whom> + +## Owner + +<team or org accountable for maintenance, e.g. AMD Research> +``` + +## Writing a good Description + +Keep it to one sentence that states the outcome, matching the marketplace blurb. Avoid restating internal mechanics (that belongs in `SKILL.md`). + +``` +Good: Deploy AUP Learning Cloud onto a multi-AIPC PXE/k3s cluster end to end. +Bad: Runs a series of Ansible playbooks and Helm commands in order. +``` diff --git a/docs/skill-categories.md b/docs/skill-categories.md new file mode 100644 index 00000000..b9799819 --- /dev/null +++ b/docs/skill-categories.md @@ -0,0 +1,80 @@ +# Skill categories + +The catalog is one bundled plugin, but its skills are organized into three +groups so an agent (or a person) can start in the right place for a task. The +grouping is a routing aid, not a packaging boundary: installing the plugin +brings every skill, and each skill's `SKILL.md` `description` begins with a +`Group:` tag so the category travels with the routing signal the agent loads. + +When you get a task, identify the group first, then pick the skill within it. + +## Plan and deploy AUP Learning Cloud + +Bring the platform into existence: size it, install or deploy it, and build the +images it runs. Reach for this group when nothing is running yet (or you are +adding/replacing infrastructure) and the goal is to stand the platform up. + +- `plan-aup-learning-cloud-deployment` — pre-purchase sizing, topology, network + plan, and bill of materials. +- `install-aup-learning-cloud-single-node` — single-box `./auplc-installer` + install. +- `deploy-aup-learning-cloud` — multi-node PXE / SSH + Ansible + Helm cluster + deploy. +- `build-aup-learning-cloud-images` — build/publish the Hub and notebook/course + images. + +Tag: `Group: Plan & deploy AUP Learning Cloud.` + +## Maintain AUP Learning Cloud + +Operate and keep a running deployment healthy. Reach for this group when the +platform already exists and the goal is day-2 operations: upgrades, debugging, +observability, login security, user/quota administration, and how the Hub is +exposed and stored. + +- `upgrade-aup-learning-cloud` — chart/values/image and k3s upgrades with + rollback. +- `troubleshoot-aup-learning-cloud` — evidence-first diagnosis of a broken + deployment. +- `monitor-aup-learning-cloud` — Prometheus/Grafana, ServiceMonitor, alerts. +- `configure-aup-learning-cloud-auth` — auth mode, GitHub App/OAuth, team sync, + native accounts, admin bootstrap. +- `manage-aup-learning-cloud-users` — users/groups/quota operations and class + onboarding. +- `expose-aup-learning-cloud` — NodePort/LoadBalancer/ingress + TLS, CORS, and + NFS storage. + +Tag: `Group: Maintain AUP Learning Cloud.` + +## Course and other editor + +Edit what lives inside the platform. Reach for this group when the cluster is +fine and the goal is content: the spawnable course catalog, authoring new course +material, or the per-user repositories learners pull into their workspaces. + +- `configure-aup-learning-cloud-courses` — edit the course catalog, spawn-UI + metadata, accelerators, team mapping, and quota knobs in `values.yaml`. +- `develop-aup-learning-cloud-courses` — author a new course end to end + (notebooks → image → catalog registration). +- `configure-aup-learning-cloud-repos` — per-user Git repo cloning (spawn-form + field/picker, private-repo tokens, persistence). + +Tag: `Group: Course & other editor.` + +## Cross-group handoffs + +Tasks often cross a boundary; hand off rather than stretch a skill: + +- Sizing/planning (plan) hands off to install or deploy once a plan is agreed. +- Authoring a course (develop, editor group) hands off to + `build-aup-learning-cloud-images` (deploy group) to build the image, then to + `configure-aup-learning-cloud-courses` (editor group) to wire it in. +- `troubleshoot` (maintain) diagnoses, then hands the fix to the matching + deploy/install/configure/upgrade skill. + +## Adding a new skill + +Assign exactly one group, prepend the group's `Group:` tag to the new skill's +`description`, add its row under that group's table in the +[README](../README.md), and list it here. See +[adding-a-skill.md](adding-a-skill.md) for the full procedure. diff --git a/plugin-metadata.json b/plugin-metadata.json new file mode 100644 index 00000000..f562b4e5 --- /dev/null +++ b/plugin-metadata.json @@ -0,0 +1,22 @@ +{ + "name": "auplc-skills", + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "version": "0.1.0", + "author": { + "name": "AMD Research" + }, + "homepage": "https://github.com/AMDResearch/aup-learning-cloud", + "repository": "https://github.com/AMDResearch/auplc-skills", + "license": "MIT", + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/scripts/check_skills_version.py b/scripts/check_skills_version.py new file mode 100644 index 00000000..1e682333 --- /dev/null +++ b/scripts/check_skills_version.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +""" +Check that the bundled auplc-skills version fields stay in sync. + +The project version in pyproject.toml is the single source of truth (strategy +A + C: skills share the main project version and are pinned at install time via +git tags/refs). This script is READ-ONLY: it only compares the version strings +declared across the plugin manifests against pyproject.toml and reports any +mismatch. It never edits skills or any other file. + +Checked version fields: + - pyproject.toml -> [project].version (source of truth) + - .claude-plugin/marketplace.json -> metadata.version + - .cursor-plugin/marketplace.json -> metadata.version + - .claude-plugin/plugin.json -> version + - .cursor-plugin/plugin.json -> version + - plugin-metadata.json -> version + +Usage: + python scripts/check_skills_version.py + +Exits non-zero if any version field does not match pyproject.toml. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def read_pyproject_version(path: Path) -> str: + text = path.read_text(encoding="utf-8") + # Match the version key inside the [project] table without adding a TOML dep. + match = re.search(r'(?m)^\s*version\s*=\s*"([^"]+)"', text) + if not match: + raise ValueError(f"could not find a version in {path}") + return match.group(1) + + +def read_json_field(path: Path, *keys: str) -> str: + data = json.loads(path.read_text(encoding="utf-8")) + node = data + for key in keys: + node = node[key] + return node + + +def main() -> int: + source_version = read_pyproject_version(REPO_ROOT / "pyproject.toml") + + # (relative path, (nested json keys ...)) + targets = [ + (".claude-plugin/marketplace.json", ("metadata", "version")), + (".cursor-plugin/marketplace.json", ("metadata", "version")), + (".claude-plugin/plugin.json", ("version",)), + (".cursor-plugin/plugin.json", ("version",)), + ("plugin-metadata.json", ("version",)), + ] + + mismatches: list[str] = [] + print(f"source of truth: pyproject.toml version = {source_version}") + for rel_path, keys in targets: + path = REPO_ROOT / rel_path + if not path.exists(): + mismatches.append(f"missing file: {rel_path}") + continue + try: + value = read_json_field(path, *keys) + except (KeyError, TypeError): + mismatches.append(f"missing field {'.'.join(keys)} in {rel_path}") + continue + status = "ok" if value == source_version else "MISMATCH" + print(f" [{status}] {rel_path} ({'.'.join(keys)}) = {value}") + if value != source_version: + mismatches.append(f"{rel_path}: {'.'.join(keys)} = {value}, expected {source_version}") + + if mismatches: + print("\nversion check failed:", file=sys.stderr) + for item in mismatches: + print(f" - {item}", file=sys.stderr) + return 1 + + print("\nall skill version fields match pyproject.toml") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/build-aup-learning-cloud-images/SKILL.md b/skills/build-aup-learning-cloud-images/SKILL.md new file mode 100644 index 00000000..132506bb --- /dev/null +++ b/skills/build-aup-learning-cloud-images/SKILL.md @@ -0,0 +1,106 @@ +--- +name: build-aup-learning-cloud-images +description: >- + Group: Plan & deploy AUP Learning Cloud. Builds and publishes the AUP Learning + Cloud Docker images — the Hub image and + the CPU/GPU notebook and course images — with ./auplc-installer img build. + Use when the user wants to build, rebuild, tag, or push AUPLC images, mentions + img build / img pull, the dockerfiles/ directory, auplc-hub / auplc-base / + auplc-default / auplc-cv / auplc-dl / auplc-llm / auplc-physim / code-cpu / + code-gpu, a gfx-specific image tag, the GHCR registry, code-server VS Code + extensions, or preparing images for an offline/registry deployment. Covers + GPU-target tagging and pushing to a registry. Do not use to install or deploy + a cluster (install-/deploy-aup-learning-cloud) or to edit the course catalog + in values.yaml (configure-aup-learning-cloud-courses). +--- + +# Build AUP Learning Cloud images + +Produce the container images the platform runs: the Hub image plus the notebook +and course images, GPU-tagged per accelerator family, and (optionally) pushed +to a registry for a multi-node or offline deployment. + +`./auplc-installer img build` is the source of truth and wraps +`dockerfiles/`. Your job is to pick the right targets + GPU tag, run the build, +and (if asked) push. Target list, tag scheme, and the push flow are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; Docker with enough disk (GPU images are + large) and, for course images, network access to base layers. +- For pushing: `docker login` to the target registry (default + `ghcr.io/amdresearch`). +- Know the **GPU target** for GPU images (`phx`, `strix`, `strix-halo`, + `9070xt`, `r9700`, …) — GPU images are tagged `:<tag>-<gpu_target>`. + +## Targets at a glance + +| Target | Image | GPU-tagged? | +| --- | --- | --- | +| `hub` | `auplc-hub` | no (infra image) | +| `base-cpu` | `auplc-default` | no | +| `base-rocm` | `auplc-base` | yes | +| `code-cpu` / `code-gpu` | `auplc-code-cpu` / `auplc-code-gpu` | gpu only | +| `cv` / `dl` / `llm` / `physim` | `auplc-cv` / `-dl` / `-llm` / `-physim` | yes | +| `all` | hub + selected courses | mixed | + +## Workflow + +1. **Decide scope.** Which targets, and the GPU target for ROCm images. For a + demo rebuild of one course, build just that target; avoid `all` unless + needed. +2. **Build.** + + ```bash + ./auplc-installer img build hub + ./auplc-installer img build base-rocm --gpu=strix + ./auplc-installer img build cv dl --gpu=strix-halo + ./auplc-installer img build --image-tag=develop base-rocm --gpu=strix-halo + ``` + +3. **(Optional) Push** to the registry referenced by `custom.resources.images`: + + ```bash + docker push ghcr.io/amdresearch/auplc-hub:latest + docker push ghcr.io/amdresearch/auplc-base:latest-gfx1151 # GPU-tagged example + ``` + +4. **Wire the tag in.** If you changed the tag, update + `custom.resources.images` (and `prePuller.extraImages` if used) — that's the + configure-aup-learning-cloud-courses skill — then `rt upgrade` / `helm + upgrade`. + +## Editing the Hub image — preserve attribution + +If a change touches Hub source, **all four attribution layers from the project +`AGENTS.md` must stay intact** (do not remove/rename any): + +1. `X-Powered-By: AUP Learning Cloud` header in + `runtime/hub/core/jupyterhub_config.py`. +2. `PlatformInfoHandler` (`/api/platform`, unauthenticated) in + `runtime/hub/core/handlers.py`. +3. The `<footer id="auplc-powered-by-footer">` in + `runtime/hub/frontend/templates/page.html` (kept outside all Jinja blocks). +4. `PLATFORM_NAME` / `PLATFORM_VENDOR` / `PLATFORM_WEBSITE` in + `runtime/hub/frontend/packages/shared/src/branding.ts` (import, never + hardcode the platform string). + +Also keep the `Copyright (C) … Advanced Micro Devices, Inc.` header on every +source file (MIT requirement). + +## Safety + +- **Disk + time.** GPU/course image builds are large and slow — confirm before + `all` or `--image-source=build` on a small box. +- **Pushing is publishing.** Confirm the registry, repo, and tag before any + `docker push`; never push secrets baked into a layer. +- **code-server safety.** The code images run `code-server --auth none` on port + 8888; this is safe only behind the Hub proxy. Never expose that port via + NodePort/LoadBalancer/ingress. Confirm VS Code/OpenVSX extension licenses + before adding to `dockerfiles/Code/extensions.txt`. + +## Reference + +Full target list, the gfx tag scheme, `img pull` for offline, registry/mirror +flags, and troubleshooting: [reference.md](reference.md). diff --git a/skills/build-aup-learning-cloud-images/reference.md b/skills/build-aup-learning-cloud-images/reference.md new file mode 100644 index 00000000..e7ce3c90 --- /dev/null +++ b/skills/build-aup-learning-cloud-images/reference.md @@ -0,0 +1,95 @@ +# Build AUP Learning Cloud images — Reference + +Target list, tag scheme, push/pull flows, and troubleshooting for +`./auplc-installer img build`. Workflow and the attribution rules are in +[SKILL.md](SKILL.md). + +## Source + +- Repo README "Available Notebook and Coding Environments" + `./auplc-installer help`. +- `auplc_installer/catalog.py` (course → image basename + make target). +- `dockerfiles/` (the actual build context, incl. `dockerfiles/Code/extensions.txt`). + +## Target → image map + +| `img build` target | Image basename | GPU-tagged | Make target | +| --- | --- | --- | --- | +| `hub` | `auplc-hub` | no | (hub) | +| `base-cpu` | `auplc-default` | no | `base-cpu` | +| `base-rocm` | `auplc-base` | yes | `base-rocm` | +| `code-cpu` | `auplc-code-cpu` | no | `code-cpu` | +| `code-gpu` | `auplc-code-gpu` | yes | `code-gpu` | +| `cv` | `auplc-cv` | yes | `cv` | +| `dl` | `auplc-dl` | yes | `dl` | +| `llm` | `auplc-llm` | yes | `llm` | +| `physim` | `auplc-physim` | yes | `physim` | +| `all` | hub + selected courses | mixed | — | +| `code` | both code-server images | — | — | + +## Tag scheme + +- Plain (non-GPU) images: `:<IMAGE_TAG>` (default `IMAGE_TAG=latest`). +- GPU images: `:<IMAGE_TAG>-<gpu_target>` — the GPU suffix is appended + automatically from `--gpu` (e.g. `auplc-base:latest-gfx1151` for strix-halo). +- Registry prefix: `--image-registry` / `IMAGE_REGISTRY` + (default `ghcr.io/amdresearch`). + +## Build examples + +```bash +./auplc-installer img build hub +./auplc-installer img build base-rocm --gpu=strix +./auplc-installer img build cv dl llm physim --gpu=strix-halo +./auplc-installer img build --image-tag=develop base-rocm --gpu=strix-halo +./auplc-installer img build all --gpu=strix-halo # hub + all courses +``` + +Relevant global flags (see install skill for the full table): `--gpu`, +`--image-tag`, `--image-registry`, `--mirror=`, `--mirror-pip=`, `--mirror-npm=`, +`-v/--verbose`. + +## Push to a registry + +```bash +docker login ghcr.io +docker push ghcr.io/amdresearch/auplc-hub:latest +docker push ghcr.io/amdresearch/auplc-default:latest +docker push ghcr.io/amdresearch/auplc-base:latest-gfx1151 +docker push ghcr.io/amdresearch/auplc-cv:latest-gfx1151 +``` + +Then point `custom.resources.images` (and `prePuller.extraImages` if used) at +the pushed tags — see configure-aup-learning-cloud-courses. + +## Offline: pull external images + +```bash +./auplc-installer img pull # fetch external (non-custom) images for offline use +``` + +For a full air-gapped bundle (custom + external + installer), use +`./auplc-installer pack` (see install-aup-learning-cloud-single-node). + +## code-server images + +The `code-cpu` / `code-gpu` images launch `code-server --auth none` on port +**8888**, safe only behind the JupyterHub proxy auth boundary — never expose +that port directly. Built-in extensions come from +`dockerfiles/Code/extensions.txt` plus local `.vsix` packages (e.g. the AUPLC +Back-to-Hub extension). Confirm extension licenses / marketplace terms before +adding any. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Build fails pulling base layers | Network / mirror | `--mirror=`, `--mirror-pip=`, retry; check Docker daemon proxy | +| `no space left on device` | GPU/course images are large | Free disk, build fewer targets, prune `docker image prune` | +| Wrong gfx kernels at runtime | Built for the wrong `--gpu` target | Rebuild with the correct `--gpu`; for Phoenix note `HSA_OVERRIDE_GFX_VERSION` | +| Pushed image not used by Hub | `custom.resources.images` tag not updated | Update the overlay + `rt upgrade`/`helm upgrade` | +| Attribution check fails in review | A Hub-source edit dropped a layer | Restore all four `AGENTS.md` layers + file copyright headers | + +## Out of scope + +Installing/deploying a cluster, editing the values course catalog, and authoring +new course curricula (notebooks). This skill builds and publishes the images. diff --git a/skills/build-aup-learning-cloud-images/skill-card.md b/skills/build-aup-learning-cloud-images/skill-card.md new file mode 100644 index 00000000..520c81ae --- /dev/null +++ b/skills/build-aup-learning-cloud-images/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Build and publish the AUP Learning Cloud Hub and notebook/course Docker images with ./auplc-installer img build, for maintainers. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-auth/SKILL.md b/skills/configure-aup-learning-cloud-auth/SKILL.md new file mode 100644 index 00000000..12003da8 --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/SKILL.md @@ -0,0 +1,104 @@ +--- +name: configure-aup-learning-cloud-auth +description: >- + Group: Maintain AUP Learning Cloud. Configures authentication for AUP Learning + Cloud: auth modes (auto-login/dummy/github/multi), GitHub App / OAuth, GitHub + team-to-group sync, native local accounts, password policy and forced + first-login change, and admin bootstrap. Use when the user wants to set or + switch custom.authMode, enable GitHub login, create or migrate a GitHub + App, set oauth_callback_url / client_id / client_secret / app_id / + private_key_file, sync GitHub teams into JupyterHub groups, enable native + accounts, bootstrap the initial admin (custom.adminUser), or debug "Resource + not accessible by integration", a login 404, or OAuth callback errors. + Triggers include custom.authMode, GitHubOAuthenticator, custom.githubOrgName, + allowed_organizations, jupyterhub-admin-credentials. Do not use to map which + resources a group sees (configure-aup-learning-cloud-courses), to + bulk-manage users (manage-aup-learning-cloud-users), or to configure + private-repo cloning (configure-aup-learning-cloud-repos). +--- + +# Configure AUP Learning Cloud authentication + +Choose and wire the Hub's login path: pick the `custom.authMode`, set up the +GitHub App (OAuth + server-to-server team sync) and/or native local accounts, +and bootstrap the initial admin — then re-apply with the installer or Helm. + +Edit a **values overlay** (`runtime/values.yaml`, `values-multi-nodes.yaml`, or +`values.local.yaml`), never hardcode secrets into tracked files. The full +GitHub App walkthrough, value blocks, and troubleshooting are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; a running (or about-to-deploy) Hub. +- `helm` + `kubectl` against the cluster, or `./auplc-installer` on a + single-node box. +- For `github` / `multi`: a GitHub **organization** you own (the App is created + under the org, not a personal account) and admin access to its settings. + +## Pick the auth mode + +| Mode | When to use | Notes | +| --- | --- | --- | +| `auto-login` | Local demo / single dev box | No credentials; quota auto-disabled unless forced. The checked-in default. | +| `dummy` | Throwaway testing only | Accepts any user/password; not for real use; its login can 404 in normal setups. | +| `github` | Org-backed SSO | GitHub App only; team membership syncs into Hub groups. | +| `multi` | GitHub + local accounts | Combined login page; native accounts for users without GitHub. | + +`custom.authMode` is the single switch. Confirm the target mode with the user +before changing a live Hub (a `helm upgrade` restarts the Hub pod, a brief +login blip). + +## Workflow + +1. **Read current state.** Check `custom.authMode`, `custom.adminUser.enabled`, + `custom.githubOrgName`, and `hub.config.GitHubOAuthenticator` in the active + overlay. +2. **Set the mode** in the overlay. For `auto-login`/`dummy` you are done with + credentials; skip to step 6. +3. **GitHub App (github/multi).** Create the App under the org with the exact + callback URL for the mode and `Members: Read-only` + `Contents: Read-only` + permissions, then fill `hub.config.GitHubOAuthenticator` (`app_id`, + `client_id`, `client_secret`, `private_key_file`, `allowed_organizations`, + `scope: []`) and `custom.githubOrgName`. Step-by-step in + [reference.md](reference.md). + - **Callback URL must match the mode exactly:** `multi` uses + `…/hub/github/oauth_callback`; single `github` uses `…/hub/oauth_callback`. +4. **Team sync.** Team-to-group sync uses the App installation token; the org + teams are intersected with `custom.teams.mapping`. Mapping *which resource* a + group sees stays in the configure-courses skill — this skill only makes the + groups exist. +5. **Native accounts (multi).** The first-use authenticator has + `create_users = False`, so accounts must be created by an admin before login + (see manage-users skill). Password policy: ≥8 chars with upper, lower, digit, + and special; users can be forced to change on first login. +6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` to have + the chart mint the `jupyterhub-admin-credentials` secret and the `admin` + user. +7. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed. +8. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: + `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f + runtime/values.yaml -f <overlay>`. +9. **Verify.** Load the Hub: the expected login page appears, a GitHub user + lands in the right groups, and (if bootstrapped) the admin can log in. Read + the secret with the commands in [reference.md](reference.md). + +## Safety + +- **Secrets never go in tracked files.** `client_secret`, the App private key, + and `jupyterhub-admin-credentials` must come from a mounted K8s secret or an + untracked overlay. Never commit them. +- **Avoid `dummy` outside isolated testing** — it accepts any credentials. +- **Switching modes is disruptive.** `auto-login` → `github`/`multi` forces + every user through login and changes who can spawn; confirm timing for a live + class. +- A `helm upgrade` / `rt upgrade` restarts the Hub pod (brief auth blip). +- If Hub source is touched, preserve the four attribution layers and per-file + copyright headers (see the project `AGENTS.md`). + +## Reference + +GitHub App creation walkthrough, every `GitHubOAuthenticator` field, the +OAuth-App→GitHub-App migration, native-account/password details, admin secret +retrieval, and the troubleshooting table: [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md new file mode 100644 index 00000000..8c659dab --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -0,0 +1,156 @@ +# Configure AUP Learning Cloud authentication — Reference + +Full GitHub App setup, every `GitHubOAuthenticator` field, the OAuth-App → +GitHub-App migration, native accounts, admin bootstrap, and troubleshooting. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Authentication Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/authentication-guide.html> +- GitHub App Setup: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/github-app-setup.html> +- Configuration Reference: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> + +The live `runtime/values.yaml` and `runtime/chart/values.schema.yaml` are the +source of truth; verify keys against them. + +## 1. Auth modes (`custom.authMode`) + +```yaml +custom: + authMode: "auto-login" # auto-login | dummy | github | multi +``` + +- `auto-login` — shared, no credentials. Quota auto-disables unless explicitly + enabled. Checked-in single-node default. +- `dummy` — accepts any username/password. Testing only. +- `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. +- `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends + in `/hub/github/oauth_callback`. + +## 2. Admin bootstrap (`custom.adminUser`) + +```yaml +custom: + adminUser: + enabled: true +``` + +The chart creates the `jupyterhub-admin-credentials` secret and bootstraps the +`admin` user. Retrieve: + +```bash +kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.admin-password}' | base64 -d && echo +kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' | base64 -d && echo +``` + +## 3. GitHub App — create it (github/multi) + +1. **Create the App under the organization** (not a personal account): + `https://github.com/organizations/<ORG>/settings/apps/new`. +2. **Basic info:** name (e.g. `auplc-hub`), Homepage = Hub URL, **Callback URL** + matching the mode: + - `multi`: `https://<domain>/hub/github/oauth_callback` + - single `github`: `https://<domain>/hub/oauth_callback` +3. Check **Expire user authorization tokens** and **Request user authorization + (OAuth) during installation**. Uncheck **Webhook → Active**. +4. **Permissions:** + - Repository → `Contents`: Read-only (private-repo cloning), `Metadata`: + Read-only (default). + - Organization → `Members`: **Read-only** (required for team sync/group + mapping — without it the Hub logs `Resource not accessible by + integration`). +5. **Installation scope:** Any account. Create the App. +6. Record **App ID**, **Client ID** (`Iv23li…`, different from App ID), + generate a **Client secret**, and generate a **private key** (`.pem`). Mount + the `.pem` into the Hub pod and record the path. +7. **Install the App on the org** configured as `custom.githubOrgName`; pick the + repos users may access if private cloning is used. + +## 4. GitHub App — configure the Hub + +```yaml +custom: + githubOrgName: "<YOUR-ORG-NAME>" + + gitClone: + githubAppName: "your-app-slug" # only if private-repo cloning is wanted (see repos skill) + +hub: + config: + GitHubOAuthenticator: + oauth_callback_url: "https://<domain>/hub/github/oauth_callback" + app_id: "<GitHub App App ID>" + installation_id: "" # blank = auto-discover from the org installation + private_key_file: "/path/to/mounted/github-app-private-key.pem" + # private_key: "" # alternative; prefer a mounted secret + team_sync_ttl_seconds: 3600 + client_id: "<GitHub App Client ID>" + client_secret: "<GitHub App Client Secret>" + allowed_organizations: + - <YOUR-ORG-NAME> + scope: [] # GitHub App uses App permissions, not OAuth scopes +``` + +`scope: []` is correct for a GitHub App. `installation_id` can stay blank when +the App is installed on the org (auto-discovered via `GET /orgs/{org}/installation`). + +## 5. Team-to-group sync + +The Hub lists actual org teams, intersects them with `custom.teams.mapping`, +and batches member lookups through GitHub GraphQL using the App installation +token. Team keys correspond to GitHub team slugs (e.g. `AUP` is queried as +`aup`, but the JupyterHub group stays `AUP`). Missing teams are logged and +skipped rather than failing the whole sync. Assigning *resources* to those +groups is the configure-courses skill. + +GitHub users without a matched team fall into a `github-users` fallback group; +native users can be assigned `native-users`. + +## 6. Native accounts (multi) + +- The first-use authenticator sets `create_users = False` — accounts must exist + before login (create them via the manage-users skill or `/hub/admin`). +- **Password policy:** ≥8 chars, ≥1 uppercase, ≥1 lowercase, ≥1 digit, ≥1 + special. Applies to admin-set and user-changed passwords. +- **Forced first-login change** uses `/auth/check-force-password-change` and + `/auth/change-password`. + +## 7. Migrating OAuth App → GitHub App + +Keep `oauth_callback_url` and `allowed_organizations`. Change `client_id` / +`client_secret` to the App's, add `app_id`, `installation_id` (blank ok), +`private_key_file`, `team_sync_ttl_seconds`, set `scope: []`, and set +`gitClone.githubAppName`. Existing sessions keep working; new logins use the +App. Delete the old OAuth App after everyone has re-logged. + +## 8. Apply and verify + +```bash +# render check +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +# single-node +sudo ./auplc-installer rt upgrade +# multi-node / manual +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +kubectl logs -n jupyterhub deployment/hub | grep -i -E 'admin|github|oauth' +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Login 404 / no login page | `authMode: dummy`, or wrong mode for the deploy | Set `github`/`multi`/`auto-login`; re-apply | +| OAuth callback error | `oauth_callback_url` mismatch (mode or http/https) | Match the App's Callback URL exactly to the mode | +| `Resource not accessible by integration` | App missing `Members: Read-only` | Add the org permission; an org owner must approve the updated install | +| GitHub users see no/wrong resources | `githubOrgName`, `allowed_organizations`, `teams.mapping`, or team membership | Verify all four; confirm the user's GitHub teams | +| Configured team skipped in sync | Team doesn't exist on GitHub | The Hub only syncs teams that exist; create it or fix the key | +| Installation token unavailable | `app_id`/`private_key_file` wrong or App not installed on org | Verify both and the org installation | +| No admin user created | `custom.adminUser.enabled` not true | Set it, re-apply, `kubectl logs … | grep -i admin` | +| Native user can't log in | Not `multi`, user not pre-created, or no local password | Confirm mode + that an admin created the account | +| Password change keeps failing | New password fails the strength policy | Re-check length + upper/lower/digit/special | diff --git a/skills/configure-aup-learning-cloud-auth/skill-card.md b/skills/configure-aup-learning-cloud-auth/skill-card.md new file mode 100644 index 00000000..5dde230e --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud authentication — auth mode, GitHub App / OAuth, team-to-group sync, native accounts, and admin bootstrap — for operators standing up or securing a Hub. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-courses/SKILL.md b/skills/configure-aup-learning-cloud-courses/SKILL.md new file mode 100644 index 00000000..ff8a079f --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/SKILL.md @@ -0,0 +1,85 @@ +--- +name: configure-aup-learning-cloud-courses +description: >- + Group: Course & other editor. Edits the AUP Learning Cloud course catalog and + access control in the + JupyterHub values.yaml: course images, resource requirements, spawn-UI + metadata, group ordering, GPU accelerator selectors, team-to-course mappings, + and the quota knobs. Use when the user wants to add/remove a course or + notebook environment, show/hide an option in the spawn picker, map a GitHub + team or group to courses, set per-course CPU/memory/amd.com/gpu requirements, + add or retune an accelerator (custom.accelerators), or configure quota + (cpuRate, quotaRate, minimumToStart, refresh rules). Triggers include + values.yaml, custom.resources.images, custom.teams.mapping, + custom.accelerators, custom.quota, acceleratorKeys, launchMode. Do not use to + build the images themselves (build-aup-learning-cloud-images) or to install a + cluster (install-/deploy-aup-learning-cloud). +--- + +# Configure AUP Learning Cloud courses + +Change what users can spawn and who can see it, by editing the `custom:` block +of the JupyterHub values and re-applying with Helm. One coherent surface: +course images, their resource requirements, the spawn-UI metadata, accelerator +selectors, team mappings, and quota. + +Edit a **values overlay** (e.g. `runtime/values-basic-example.yaml` or +`values.local.yaml`), never the chart defaults blindly. The key map and the +full field guide are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; a running Hub (single- or multi-node). +- `helm` + `kubectl` against the cluster, or `./auplc-installer` on a + single-node box. +- Know which keys already exist: `custom.resources.images` is the catalog; + course keys are `cpu`, `gpu`, `code-cpu`, `code-gpu`, and `Course-CV`, + `Course-DL`, `Course-LLM`, `Course-PhySim`. + +## The four places a course lives + +A course key must be consistent across **all** of these or the spawn UI breaks: + +1. `custom.resources.images.<key>` — the container image. +2. `custom.resources.requirements.<key>` — `cpu`, `memory`, and `amd.com/gpu`. +3. `custom.resources.metadata.<key>` — spawn-UI `group`, `description`, + `accelerator`, `acceleratorKeys`, `allowGitClone`, `launchMode`, + `resourceType`. +4. `custom.teams.mapping.<team>` — the teams allowed to launch it. + +## Workflow + +1. **Read the current state.** Open `runtime/values.yaml` for the canonical + shape, and the active overlay for what is deployed. Confirm the exact key + you are changing. +2. **Make the edit in the overlay.** Add/modify the key in all four places + above (or, for accelerators/quota, the relevant block). Keep `acceleratorKeys` + pointing at real `custom.accelerators` keys (`phx`, `strix`, `strix-halo`, + `9070xt`, `r9700`). +3. **Keep accelerator selectors honest.** Each `custom.accelerators.<key>.nodeSelector` + must equal a real node label — confirm with + `kubectl describe node <node> | grep amd.com/gpu.product-name`. +4. **Validate the render before applying.** `helm template jupyterhub + ./runtime/chart -f runtime/values.yaml -f <overlay>` must succeed; the repo + also ships `runtime/chart/values.schema.json`. +5. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: + `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f + runtime/values.yaml -f <overlay>`. +6. **Verify.** Reload the spawn page; the course appears in its `group` for the + mapped teams only, and a launched pod gets the expected resources/node. + +## Safety + +- **Edit overlays, not secrets.** Never put OAuth secrets or tokens in tracked + files. Never commit a `values.local.yaml` that carries site config. +- **Removing a course** hides it and can strand running servers on that image — + confirm with the user and check for active spawns first. +- **Quota changes apply cluster-wide.** Lowering `minimumToStart` / `cpuRate` + or editing `refreshRules` affects every user; confirm before applying. +- A `helm upgrade` restarts the Hub pod (brief auth blip). Confirm timing for a + live class. + +## Reference + +Course-key map, every `metadata`/`requirements` field, the accelerator block, +team-mapping semantics, and the quota knobs: [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-courses/reference.md b/skills/configure-aup-learning-cloud-courses/reference.md new file mode 100644 index 00000000..eb773a9e --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/reference.md @@ -0,0 +1,149 @@ +# Configure AUP Learning Cloud courses — Reference + +The course-key map, every field under `custom.resources`, the accelerator and +team blocks, and the quota knobs, as they appear in `runtime/values.yaml`. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (`runtime/values.yaml`): <https://amdresearch.github.io/aup-learning-cloud/> +- Overview (resource selection, teams, quota): <https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html> + +The live `runtime/values.yaml` is the source of truth; verify keys against it. + +## Course catalog (default keys) + +| Key | Default image | HW | Notes | +| --- | --- | --- | --- | +| `cpu` | `ghcr.io/amdresearch/auplc-default:latest` | CPU | Basic Python notebook | +| `gpu` | `ghcr.io/amdresearch/auplc-base:latest` | GPU | Basic GPU notebook | +| `code-cpu` | `ghcr.io/amdresearch/auplc-code-cpu:latest` | CPU | code-server (`launchMode: code-server`) | +| `code-gpu` | `ghcr.io/amdresearch/auplc-code-gpu:latest` | GPU | code-server | +| `Course-CV` | `ghcr.io/amdresearch/auplc-cv:latest` | GPU | Computer Vision | +| `Course-DL` | `ghcr.io/amdresearch/auplc-dl:latest` | GPU | Deep Learning | +| `Course-LLM` | `ghcr.io/amdresearch/auplc-llm:latest` | GPU | LLM from scratch | +| `Course-PhySim` | `ghcr.io/amdresearch/auplc-physim:latest` | GPU | Genesis physics sim | + +These keys must match across `custom.resources.{images,requirements,metadata}` +and be referenced by `custom.teams.mapping`. The installer mirrors this in +`auplc_installer/catalog.py`; keep both consistent if you add a course used by +`./auplc-installer --courses`. + +## custom.resources.requirements.<key> + +```yaml +gpu: + cpu: "0" # "0" = no explicit request/limit (best-effort) + memory: "0Gi" + amd.com/gpu: "1" # present only for GPU courses +``` + +## custom.resources.metadata.<key> + +```yaml +Course-CV: + group: "TEACHING LABS" # spawn-UI grouping (see groupOrder) + description: "Computer Vision Course" + subDescription: "Suitable for CV experiments with GPU" + accelerator: "GPU" # "" for CPU courses + acceleratorKeys: # which custom.accelerators entries apply + - strix-halo + allowGitClone: true + launchMode: "code-server" # only for browser-IDE resources; omit for notebooks + resourceType: "notebook" # or "browser-ide" + # acceleratorOverrides: # optional per-accelerator image/env override + # 9070xt: + # image: "ghcr.io/your-org/auplc-cv:<tag-for-9070xt>" +``` + +`custom.resources.groupOrder` is a list controlling spawn/Home group order +(e.g. `TEACHING LABS`, `DEVELOPMENT ENVIRONMENT`, `CUSTOM REPOS`). Unlisted +groups follow alphabetically. + +## custom.accelerators.<key> + +```yaml +strix-halo: + displayName: "AMD Radeon™ 8060S (Strix Halo iGPU)" + description: "RDNA 3.5 (gfx1151) | Compute Units 40 | 64GB LPDDR5X" + nodeSelector: + amd.com/gpu.product-name: "AMD_Radeon_8060S_Graphics" # MUST match a real node label + env: {} # e.g. HSA_OVERRIDE_GFX_VERSION for Phoenix (phx) + quotaRate: 3 # quota consumed per hour when this accelerator is used +``` + +Default accelerator keys → product label: + +| Key | `amd.com/gpu.product-name` | +| --- | --- | +| `phx` | `AMD_Radeon_780M_Graphics` (sets `HSA_OVERRIDE_GFX_VERSION: 11.0.0`) | +| `strix` | `AMD_Radeon_890M_Graphics` | +| `strix-halo` | `AMD_Radeon_8060S_Graphics` | +| `9070xt` | `AMD_Radeon_RX_9070_XT` | +| `r9700` | `AMD_Radeon_AI_PRO_R9700` | + +If your fleet normalizes a product name differently, change the `nodeSelector` +to the exact string from `kubectl describe node`. + +## custom.teams.mapping.<team> + +A team name maps to the list of course keys its members can launch. Built-in +teams seen in defaults include `cpu`, `gpu`, `official`, `AUP`, `native-users`, +`github-users`. In GitHub auth, GitHub team membership syncs into these groups. + +```yaml +teams: + mapping: + gpu: + - code-gpu + - Course-CV + - Course-DL + - Course-LLM + - Course-PhySim +``` + +When the installer is run with `--courses=<subset>`, each team's list is +rewritten as the intersection with the selection, so unselected courses +disappear from the UI. + +## custom.quota + +```yaml +quota: + enabled: null # null = auto (disabled for auto-login/dummy unless set true) + cpuRate: 1 # quota/hour for CPU-only sessions + minimumToStart: 10 # min balance required to spawn anything + defaultQuota: 0 # initial allocation for new users (0 = none) + refreshRules: {} # each rule becomes a K8s CronJob that tops up balances +``` + +Per-accelerator consumption is `custom.accelerators.<key>.quotaRate`. + +## Apply and verify + +```bash +# render check +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +# single-node +./auplc-installer rt upgrade +# multi-node / manual +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +``` + +Reload the spawn page: the course shows in its `group` for mapped teams only; +a launched pod gets the declared `requirements` and lands on a node matching +the accelerator `nodeSelector`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Course missing from spawn UI | Key absent from `metadata`/`images`, or team mapping | Confirm the key in all four places + `teams.mapping` | +| GPU course Pending | `acceleratorKeys` → `nodeSelector` label mismatch | `kubectl describe node | grep amd.com/gpu.product-name` | +| code-server resource opens as a notebook | `launchMode`/`resourceType` not set | `launchMode: code-server`, `resourceType: browser-ide` | +| Quota blocks all spawns | `minimumToStart` too high or `defaultQuota: 0` | Review `custom.quota`, grant balance via Admin console | +| `helm upgrade` schema error | Value violates `values.schema.json` | Read the error; fix the offending key's type | diff --git a/skills/configure-aup-learning-cloud-courses/skill-card.md b/skills/configure-aup-learning-cloud-courses/skill-card.md new file mode 100644 index 00000000..cd5985ee --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Edit the AUP Learning Cloud course catalog, team mappings, accelerator selectors, and quota in the JupyterHub values.yaml, for platform admins. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-repos/SKILL.md b/skills/configure-aup-learning-cloud-repos/SKILL.md new file mode 100644 index 00000000..4eaceb0b --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/SKILL.md @@ -0,0 +1,104 @@ +--- +name: configure-aup-learning-cloud-repos +description: >- + Group: Course & other editor. Configures per-user Git repository cloning: the + custom.gitClone block (githubAppName repo picker, defaultAccessToken for + private repos, allowedProviders, maxCloneTimeout, defaultPersistence, + allowPersistenceChoice) and the per-resource metadata.allowGitClone gate that + clones a repo into a user's workspace at spawn time. Use when the user wants + to let learners clone a Git repo on startup, enable the spawn-form repo + URL/branch field or GitHub repo picker, give access to a private repo (bot PAT + or GitHub App token), choose whether cloned repos persist, allow + GitLab/Bitbucket, or debug "Repository URL ignored" or a failed clone init + container. Triggers include custom.gitClone, allowGitClone, githubAppName, + defaultAccessToken, allowedProviders, init-clone-repo. Do not use to set up + GitHub login itself (configure-aup-learning-cloud-auth), to publish a course + to the catalog (configure-/develop-aup-learning-cloud-courses), or to build + images (build-aup-learning-cloud-images). +--- + +# Configure AUP Learning Cloud repository cloning + +Enable the runtime, per-user feature where a learner pastes a Git URL on the +spawn form (or picks a private repo) and the Hub clones it into their home PVC +via an init container. This is **not** how you publish a course to the catalog +(that is develop-/configure-courses); it brings *each user's own* repo into +*their own* workspace. + +Edit a **values overlay** and re-apply. The token model, the persistence rules, +and the GitHub App requirement are subtle and partly silent — read the gates +below. Full details and troubleshooting are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` and a running (or about-to-deploy) Hub; + `helm` + `kubectl` or `./auplc-installer`. +- For private repos via GitHub App: the App configured in the auth skill + (`hub.config.GitHubOAuthenticator` + `custom.githubOrgName`). +- For private repos via a shared token: a read-only bot/service-account PAT. + +## The two gates (both required, one is silent) + +A repo URL is only cloned when **both** are true: + +1. `custom.gitClone` is configured (at minimum the feature is on; private repos + need a token source). +2. The **selected resource** has `custom.resources.metadata.<key>.allowGitClone: + true`. + +If `allowGitClone` is false for the chosen resource, the Hub **silently drops** +the repo URL (it logs a warning but shows no user error). Always set both. + +## Token priority (private repos) + +`OAuth token (GitHub App) > defaultAccessToken > none (public only)` + +- `githubAppName` — enables the repo picker + automatic per-repo OAuth token, + but **only for GitHub-App users**. No effect for auto-login/native users. +- `defaultAccessToken` — a bot PAT applied transparently to **all** users + (including auto-login); right for single-node/classroom shared private repos. + Helm base64s it into the `jupyterhub-git-default-token` secret. + +## Workflow + +1. **Read current state.** Inspect `custom.gitClone` and which + `metadata.<key>.allowGitClone` are already true. +2. **Turn on cloning** in the overlay; set `allowedProviders` (defaults + `github.com`, `gitlab.com`, `bitbucket.org`) and `maxCloneTimeout` as needed. +3. **Open the gate per resource.** Set `allowGitClone: true` on each course/env + that should accept a user repo (configure-courses owns the rest of that + metadata block). +4. **Private repos (optional).** Pick a token source: + - GitHub App: ensure the auth skill's App is set, then + `custom.gitClone.githubAppName: "<app-slug>"`. + - Shared PAT: `custom.gitClone.defaultAccessToken: "<read-only PAT>"` (keep + it out of tracked files — see Safety). +5. **Persistence policy.** Decide `defaultPersistence` (default `true`; cloned + repos survive server stop, no auto-pull after first clone) and whether to let + users choose with `allowPersistenceChoice`. +6. **Pre-flight + apply.** `helm template …` must succeed; then + `./auplc-installer rt upgrade` (single) or `helm upgrade --install …` + (multi). +7. **Verify.** On the spawn page for an allowed resource, the repo URL/branch + field (and picker, if `githubAppName`) appears; launch with a repo and + confirm `init-clone-repo` succeeds and the repo lands under + `/home/jovyan/<repo>`. + +## Safety + +- **`defaultAccessToken` is a secret.** It is base64'd into a K8s secret — never + commit it in a tracked values file. Scope the PAT **read-only** to the + specific repos to limit blast radius. +- **Persistence has destructive edges.** Ephemeral mode deletes the clone via a + `preStop` hook; the script refuses to touch a directory it didn't create and + refuses to replace a persistent clone for an ephemeral request. Don't flip + `defaultPersistence` casually on a class with in-progress work. +- **Provider allowlist is a security control.** Only add providers you trust; + cloning runs inside the user's pod. +- A `helm upgrade` restarts the Hub pod (brief login blip). + +## Reference + +Every `custom.gitClone` field, the init-container/token mechanics, the +persistence state machine, the `allowGitClone` gate, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-repos/reference.md b/skills/configure-aup-learning-cloud-repos/reference.md new file mode 100644 index 00000000..6198bbc0 --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/reference.md @@ -0,0 +1,112 @@ +# Configure AUP Learning Cloud repository cloning — Reference + +Every `custom.gitClone` field, the init-container/token mechanics, the +persistence state machine, and troubleshooting. Workflow and gates are in +[SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (section 4, custom.gitClone): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> +- Authentication Guide (GitHub App for repos): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/authentication-guide.html> + +The live `runtime/values.yaml` (`custom.gitClone`) and +`runtime/hub/core/scripts/git-clone.sh` are the source of truth. + +## custom.gitClone fields + +```yaml +custom: + gitClone: + # -- Private repo access -- + githubAppName: "" # GitHub App slug; enables repo picker + OAuth token. + # Only effective for GitHub-App users. + defaultAccessToken: "" # Bot/service-account PAT for ALL users (incl. auto-login). + # Helm creates secret jupyterhub-git-default-token from it. + # -- Clone behavior -- + allowedProviders: # subdomains of these are also accepted + - github.com + - gitlab.com + - bitbucket.org + maxCloneTimeout: 300 # seconds per clone/fetch + initContainerImage: "alpine/git:2.47.2" # must contain git + sh + # -- Persistence -- + defaultPersistence: true # keep clones after the server stops + allowPersistenceChoice: false # expose a per-user persist toggle on the spawn form +``` + +## The per-resource gate + +```yaml +custom: + resources: + metadata: + gpu: + allowGitClone: true # REQUIRED for this resource to accept a repo URL +``` + +If the selected resource's `allowGitClone` is false, the spawner discards the +submitted `repo_url` and logs `Repository URL ignored … does not allow git +cloning` — no user-visible error. This metadata block otherwise belongs to the +configure-courses skill; this skill only flips the clone gate. + +## Token model + +Priority: **OAuth (GitHub App) > defaultAccessToken > none (public only)**. + +- The spawner injects the chosen token as `GIT_ACCESS_TOKEN` into the + `init-clone-repo` container via a `secretKeyRef`. +- `git-clone.sh` rewrites the HTTPS remote to + `https://x-access-token:<token>@<host>/…`, so any provider/token type works. +- `githubAppName` users authorize specific private repos through the GitHub App + UI on the spawn page; the token comes from their OAuth session. +- `defaultAccessToken` is applied transparently to everyone — ideal for a shared + classroom private repo with no GitHub login. + +## Persistence state machine + +`git-clone.sh` writes repo-external metadata under `~/.auplc/git-clones` and: + +- **persistent** (default): reuses a compatible existing clone; **does not + auto-pull/reset/sync** after the first successful clone. +- **ephemeral**: a `preStop` hook `rm -rf`s the clone when the session ends. +- Refuses to modify a directory lacking compatible AUPLC metadata (won't clobber + a user's own folder). +- Refuses to replace a persistent managed clone for an ephemeral request. + +`allowPersistenceChoice: true` exposes the choice to users; otherwise +`defaultPersistence` is enforced. + +## Branch selection + +Users can pass a branch, or paste a `/tree/<branch>` URL — the spawner extracts +the branch from `https://host/owner/repo/tree/<branch>`. `git-clone.sh` does a +`--depth 1` clone of that branch (or the default branch). + +## Apply and verify + +```bash +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null +sudo ./auplc-installer rt upgrade # single-node +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> # multi-node + +# after a user spawns with a repo: +kubectl get pods -n jupyterhub -o wide +kubectl logs -n jupyterhub <user-pod> -c init-clone-repo +``` + +The repo URL/branch field (and picker if `githubAppName`) shows on the spawn +page for allowed resources; a successful spawn has the repo under +`/home/jovyan/<repo>`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Repo field absent on spawn | `allowGitClone` not true for that resource | Set `metadata.<key>.allowGitClone: true`, re-apply | +| "Repository URL ignored" in Hub logs | Same gate — resource disallows cloning | Same as above | +| Private clone fails (auth) | No usable token for that user | GitHub-App user must authorize the repo; or set `defaultAccessToken` | +| Clone fails ("could not be cloned") | Bad URL/branch, provider not allowed, timeout | Check URL, `allowedProviders`, raise `maxCloneTimeout`; read `init-clone-repo` logs | +| Server fails to start, `repo_clone_failed` | Init container clone error | `kubectl logs … -c init-clone-repo`; verify repo access/network | +| "Refusing to modify existing directory" | Target dir exists without AUPLC metadata | User has a same-named folder; choose another path or remove it | +| Changes to persistence not taking | Switched mode under a managed clone | Persistent↔ephemeral has refusal rules; clear the clone or keep the mode | diff --git a/skills/configure-aup-learning-cloud-repos/skill-card.md b/skills/configure-aup-learning-cloud-repos/skill-card.md new file mode 100644 index 00000000..74a69f5b --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud's per-user Git repository cloning — the spawn-form repo field, private-repo tokens, provider allowlist, and persistence — for operators enabling bring-your-own-repo workspaces. + +## Owner + +AMD Research diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..92522f1c --- /dev/null +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -0,0 +1,203 @@ +--- +name: deploy-aup-learning-cloud +description: >- + Group: Plan & deploy AUP Learning Cloud. Deploys AUP Learning Cloud (a + multi-node JupyterHub-on-k3s platform for AMD + GPUs) onto physical hardware end to end. Use when the user wants to install, + deploy, set up, or stand up AUP Learning Cloud, AUPLC, or "the learning + cloud" on a cluster; mentions a multi-AIPC or 3-node mini-cluster, PXE / + netboot / diskless agents, the Ansible inventory.yml, pb-pxe-controller, + pb-k3s-site, the ROCm GPU device plugin/labeller, an NFS provisioner, or a + JupyterHub values.yaml / Helm chart for this project. Covers both the + PXE-diskless topology and the SSH-preinstalled multi-node topology. Do not + use for the single-node "./auplc-installer install" flow, for building + notebook images, or for non-AUPLC JupyterHub or k3s installs. +--- + +# Deploy AUP Learning Cloud + +Stand up AUP Learning Cloud on a multi-node k3s cluster: build the cluster with +Ansible, expose AMD GPUs, provide shared storage, and deploy the JupyterHub +chart with Helm so users can log in and spawn GPU notebooks. + +This skill is written for any coding agent. Run the commands and edit the files +as described; the full, copy-runnable command sequence and the troubleshooting +table live in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` on the operator/service machine. +- The service machine runs Ubuntu 24.04 with a reserved/static IP and internet + access. +- `ansible` on the operator machine; `kubectl` and `helm` for the cluster + (reference.md has the Helm install command). +- For GPU scheduling: AMD GPU nodes with a working in-kernel NIC driver. +- The user supplies the physical hardware. **No site values (IPs, subnet, SSH + keys, tokens) ship in the repo** — this skill generates them. + +## Phase 1 — Interview + +Work through this in order. **The deployment-method choice (1a) is a hard gate: +ask it first and get an explicit answer before collecting anything else or +touching the machines.** + +### Phase 1a — Choose the deployment method (ask first, always) + +Ask the user to pick one. **Never assume or auto-select** — even when the +machines "look like" one case, present both options and let the user decide (you +may recommend, but you still need an explicit choice before continuing): + +| Choose | When | +| --- | --- | +| **PXE Diskless Netboot** (`topology: pxe-diskless`) — one service machine netboots diskless agents | Agents have no OS installed; you want zero per-machine install; small teaching lab. This is the [3-node mini-cluster guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html). | +| **Multi Node SSH Installation** (`topology: ssh-preinstalled`) — every node already runs Ubuntu | Each node has an OS and is reachable over SSH; closer to a long-running lab. This is the [multi-node guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). | + +The value in parentheses is the `topology` field for `gen_configs.py` (Phase 3) +and selects the matching section in [reference.md](reference.md). + +### Phase 1b — Collect the rest (some items branch on the choice above) + +Collect, and confirm back to the user, before touching anything: + +1. **Courses** wanted — drives the `values.yaml` course keys + team mappings + (full catalog setup lives in `configure-aup-learning-cloud-courses`). +2. **Node count** and which node is the controller/server, plus its static IP. + - *SSH path only:* also the hostname + IP of every agent node, and confirm + passwordless root SSH already reaches each one. +3. **GPU — do not ask the user to name the model.** Let the tooling find it: the + detectors report the GPUs (`detect_hardware.sh` in Phase 2) and the real ROCm + `amd.com/gpu.product-name` label (`detect_cluster.sh` in Phase 5). Then + **confirm the detected GPU → accelerator-key mapping with the user** before it + goes into the values file. +4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers + (also auto-detected in Phase 2 and cross-checked), plus at least one SSH + public key for the rootfs and the apache web port. + +Login mode (`custom.authMode`) is unchanged — it stays at its `auto-login` +default; switch it later with `configure-aup-learning-cloud-auth` if needed. The +detailed steps for both paths are in [reference.md](reference.md). + +## Phase 2 — Discover + +On the service machine, run the bundled detector and cross-check its JSON +against the Phase 1 answers: + +```bash +scripts/detect_hardware.sh # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] +``` + +It reports the default-route NIC, the service-machine IP + subnet CIDR, the +gateway, DNS servers, and each AMD GPU (`lspci`, vendor `1002`) with the bound +`kernel_driver`. If a GPU's `kernel_driver` is empty, note its module for +`pxe_initramfs_modules` (PXE path only). Empty fields come back in `warnings` +so you know exactly what to ask the operator for. The detected GPUs are the +source of truth for the accelerator mapping — Phase 1 does not ask the user to +name them, so surface the detected list and confirm it with the user. + +## Phase 3 — Generate config + +Drive `scripts/gen_configs.py` rather than hand-writing YAML — it keeps the +three artifacts consistent, mints the k3s token locally with a CSPRNG (never +printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. + +```bash +scripts/gen_configs.py --print-schema > spec.json # fill from Phase 1 + 2 +scripts/gen_configs.py --spec spec.json --out-dir ./generated +``` + +It writes, into `--out-dir`: + +1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for + PXE; listed for SSH) plus the `pxe_controller` group for PXE. +2. `pb-pxe-controller.vars.yml` — PXE path only: the `vars:` to merge into + `deploy/ansible/playbooks/pb-pxe-controller.yml` (`pxe_network_interface`, + `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, + `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, + `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). +3. `values-basic-example.yaml` — `custom.accelerators.*.nodeSelector` (matched + to real GPU labels in Phase 5), `custom.resources.images`, the storage class + (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). + +Review the artifacts, then copy them into the `aup-learning-cloud` checkout. +**Never commit `inventory.yml` — it holds the token.** Field-by-field guidance +is in [reference.md](reference.md). + +## Phase 4 — Execute (with confirmation gates) + +Run the install in order. **Pause for explicit user confirmation before each +risky/irreversible step** (see Safety). The PXE path is, in brief: + +1. Install host packages on the service machine. +2. `pb-pxe-controller.yml` to build the PXE/NFS rootfs, then verify the + controller (dnsmasq, NFS, apache2, TFTP boot files). +3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. +4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. +5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. + +The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against +the inventory instead. Full commands for both paths are in [reference.md](reference.md). + +## Phase 5 — GPU, storage, and chart + +1. Install the AMD GPU device plugin + ROCm labeller, then read the **real** + cluster state: + + ```bash + scripts/detect_cluster.sh > cluster.json # nodes[], gpu_product_names[], storage_classes[] + ``` + + Confirm the detected GPU → accelerator-key mapping with the user, then patch + `custom.accelerators.*.nodeSelector` so each `amd.com/gpu.product-name` + matches a value in `gpu_product_names`. Gate the install on a clean + pre-flight (exits non-zero on any mismatch): + + ```bash + scripts/validate.py --repo ~/aup-learning-cloud \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --cluster cluster.json --helm-dry-run + ``` + +2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` + (storage class `nfs-client`). +3. Deploy the chart: + +```bash +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +## Phase 6 — Validate end to end + +```bash +kubectl get nodes -o wide # server + agents Ready +kubectl get pods -A # nothing CrashLoopBackOff/Pending/ImagePullBackOff +kubectl get storageclass # nfs-client present +``` + +Then open the Hub (NodePort example: `http://<SERVICE_IP>:30890`), log in, +spawn a CPU notebook, confirm file persistence across a restart, then spawn a +GPU notebook and confirm its pod lands on a GPU node +(`kubectl get pods -n jupyterhub -o wide`). + +## Safety + +These steps are destructive or hard to reverse — **stop and get explicit user +confirmation before each one**, and never run them silently: + +- Building/rebuilding the PXE rootfs (`pxe_rootfs_force_rebuild: true`). +- Editing `/etc/exports` and restarting `nfs-kernel-server`. +- `kubectl delete node <name>` (debugging only). +- `helm uninstall` or a cluster reset (`pb-k3s-reset.yml`). +- Changing firmware boot order / disabling Secure Boot on agents. + +Never commit or push. Never write the k3s token, OAuth secrets, or SSH private +keys into tracked files. Preserve the four AUP Learning Cloud attribution +layers (see the project `AGENTS.md`) if any chart/Hub source is touched. + +## Reference + +Full step-by-step commands for both topologies, the GPU-label-to-accelerator +mapping, the `values.yaml` field guide, and the troubleshooting table: +[reference.md](reference.md). diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md new file mode 100644 index 00000000..c032738f --- /dev/null +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -0,0 +1,415 @@ +# Deploy AUP Learning Cloud — Reference + +Full, copy-runnable commands for both deployment topologies, the GPU label +mapping, the `values.yaml` field guide, and the troubleshooting table. The +workflow and confirmation gates are in [SKILL.md](SKILL.md). + +## Contents + +- [Source guides](#source-guides) +- [PXE-diskless topology (3-node mini-cluster)](#pxe-diskless-topology-3-node-mini-cluster) +- [SSH-preinstalled topology (standard multi-node)](#ssh-preinstalled-topology-standard-multi-node) +- [GPU label to accelerator key](#gpu-label-to-accelerator-key) +- [values.yaml field guide](#valuesyaml-field-guide) +- [Troubleshooting](#troubleshooting) + +## Source guides + +- 3-node mini-cluster (PXE diskless): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html> +- Standard multi-node (SSH): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> + +Treat the live docs as the source of truth for version pins; this file +condenses the opinionated path. + +The two topology sections below are the two branches of the Phase 1a gate in +[SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → +[PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node +SSH Installation** (`topology: ssh-preinstalled`) → +[SSH-preinstalled topology](#ssh-preinstalled-topology-standard-multi-node). + +## PXE-diskless topology (3-node mini-cluster) + +One service machine (AIPC 1) runs the PXE controller, the single-node k3s +server, NFS, and the apache k3s-credential endpoint. The other machines are +diskless agents that netboot and auto-join. Only AIPC 1 is Ansible-managed. + +### Step 1 — Prepare the service machine + +```bash +sudo apt update +sudo apt install -y git ansible curl ca-certificates jq \ + dnsmasq pxelinux syslinux-common apache2 \ + nfs-kernel-server debootstrap \ + grub-efi-amd64-signed shim-signed + +ip -br addr # record the NIC and IP +ip route # record the gateway +``` + +Give the local `root` a passwordless SSH login (or add `ansible_connection: +local` to the host vars to skip SSH entirely): + +```bash +sudo install -d -m 0700 /root/.ssh +sudo tee -a /root/.ssh/authorized_keys < ~/.ssh/id_ed25519.pub >/dev/null +sudo chmod 0600 /root/.ssh/authorized_keys +ssh root@<SERVICE_IP> true && echo root-ssh-ok +``` + +### Step 2 — Configure the inventory + +Edit `deploy/ansible/inventory.yml`. AIPC 1 is the only host; the `agent` group +stays empty (netboot agents are not Ansible-managed). Generate the token with +`openssl rand -base64 64` and keep it out of chat/VCS. + +```yaml +k3s_cluster: + children: + server: + hosts: + aipc1: + ansible_host: <SERVICE_IP> + agent: + hosts: {} # diskless netboot agents auto-join; do NOT list them here + vars: + ansible_user: root + k3s_version: v1.32.3+k3s1 + token: "<paste-a-strong-random-token>" # openssl rand -base64 64 + api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" + +pxe_controller: + hosts: + aipc1: + ansible_host: <SERVICE_IP> + vars: + ansible_port: 22 + ansible_user: root +``` + +### Step 3 — Configure the PXE controller playbook + +Edit the `vars:` block in `deploy/ansible/playbooks/pb-pxe-controller.yml`. The +network, controller, server-IP, and SSH-key values are empty by default and the +role asserts on them. `scripts/gen_configs.py` emits this exact block as +`pb-pxe-controller.vars.yml` — generate it and merge, or hand-edit: + +```yaml +pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) +pxe_network_interface: "enp1s0" # service-machine NIC (Step 1) +pxe_subnet: "192.168.1.0/24" # node subnet, CIDR +pxe_gateway: "192.168.1.1" # default gateway (informational) +pxe_dns_servers: "8.8.8.8,8.8.4.4" +pxe_controller_ip: "192.168.1.10" # this service machine's IP +pxe_k3s_server_ips: + - "192.168.1.10" +pxe_k3s_version: "v1.32.3+k3s1" # MUST match inventory k3s_version +pxe_web_port: 8080 # apache port for the k3s token/kubeconfig (not 80) +pxe_rootfs_password: "" # optional; empty disables password login (use ansible-vault if set) +pxe_rootfs_authorized_keys: + - "ssh-ed25519 AAAA... you@host" # at least one key required +``` + +Set `pxe_rootfs_force_rebuild: false` after the first stable build so you do +not rebuild the rootfs under running agents. The playbook also exposes +`pxe_apt_mirror`, `pxe_rootfs_packages`, and `pxe_initramfs_modules` (add your +NIC module here if it lacks an in-kernel driver) — leave these at their defaults +unless discovery flagged a need. + +### Step 4 — Run the PXE controller playbook + +```bash +cd ~/aup-learning-cloud/deploy/ansible +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml +``` + +### Step 5 — Verify the controller + +```bash +systemctl is-active dnsmasq nfs-kernel-server apache2 +showmount -e localhost +ls -l /srv/tftp/pxelinux.0 /srv/tftp/grubnetx64.efi /srv/tftp/vmlinuz /srv/tftp/initrd.img +curl -I http://127.0.0.1:8080/k3s/ # 403 expected (dir exists, empty) +``` + +The `/k3s/` endpoint is served on port 8080 (k3s owns 80/443 for ingress). + +### Step 6 — Install the single-node k3s server + +Run **without** `sudo` (key-based root SSH already connects as root): + +```bash +cd ~/aup-learning-cloud/deploy/ansible +ansible-playbook -i inventory.yml playbooks/pb-base.yml +ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +export KUBECONFIG=~/.kube/config # add to ~/.bashrc to persist +kubectl get nodes -o wide +``` + +### Step 7 — Publish k3s credentials for agents + +```bash +sudo install -d -m 0755 /var/www/html/k3s +sudo install -m 0644 /var/lib/rancher/k3s/server/token /var/www/html/k3s/token +sudo sed "s#https://127.0.0.1:6443#https://<SERVICE_IP>:6443#g" \ + /etc/rancher/k3s/k3s.yaml | sudo tee /var/www/html/k3s/kubeconfig >/dev/null +sudo chmod 0644 /var/www/html/k3s/token /var/www/html/k3s/kubeconfig +sudo systemctl reload apache2 + +curl -fsS http://127.0.0.1:8080/k3s/token >/dev/null && echo token-ok +curl -fsS http://127.0.0.1:8080/k3s/kubeconfig >/dev/null && echo kubeconfig-ok +``` + +### Step 8 — Netboot the agents + +On each agent: disable Secure Boot, enable network boot, and put PXE before the +local disk in the firmware boot order. Boot, then watch them register: + +```bash +watch kubectl get nodes -o wide +``` + +Agents appear as `agent-<mac>` nodes and become `Ready`. + +### Step 9 — Validate agent persistence + +Reboot one agent; confirm it rejoins with the same identity. On the agent: + +```bash +mount | grep /var/lib/rancher/k3s +test -f /var/lib/rancher/k3s/node-password && echo node-password-ok +systemctl status mount-local-disk k3s-agent --no-pager +``` + +`kubectl delete node <name>` clears a stale node object — **debugging only**, +confirm with the user first. + +Continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). + +## SSH-preinstalled topology (standard multi-node) + +Every node already runs Ubuntu 24.04 and is reachable over passwordless SSH. + +### Prepare SSH and inventory + +Helper scripts in `deploy/scripts/` enable root SSH and distribute kubeconfig: + +```bash +./deploy/scripts/edit_sshd.sh +./deploy/scripts/setup_ssh_root_access.sh +./deploy/scripts/deploy-kubeconfig.sh +``` + +Edit `deploy/ansible/inventory.yml` — list every node under `server`/`agent`: + +```yaml +k3s_cluster: + children: + server: + hosts: + <SERVER-HOSTNAME>: + agent: + hosts: + <AGENT-HOSTNAME-1>: + <AGENT-HOSTNAME-2>: + vars: + ansible_port: 22 + ansible_user: root + k3s_version: v1.32.3+k3s1 + token: "<strong-random-token>" # openssl rand -base64 64 + api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" +``` + +### Build the cluster + +```bash +cd deploy/ansible +sudo ansible-playbook playbooks/pb-base.yml # base OS / packages +sudo ansible-playbook playbooks/pb-k3s-site.yml # deploy k3s +sudo ansible-playbook playbooks/pb-rocm.yml # ROCm on GPU nodes +``` + +Related: `pb-k3s-upgrade.yml` (upgrade), `pb-k3s-reset.yml` (reset — RISKY). +Then install `kubectl`/`helm` on the operator machine (see Helm command below) +and continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). + +### Install Helm + +```bash +wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz +cd /tmp && tar -zxvf helm.tar.gz +sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm +``` + +## Step 10 — AMD GPU device plugin and labeller + +```bash +kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml +kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml + +kubectl get pods -A | grep -i amd +kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu +``` + +Use the labels that actually appear. Common keys: +`amd.com/gpu.product-name`, `amd.com/gpu.family`, `amd.com/gpu.device-id`. + +## Step 11 — Shared NFS storage for notebook PVCs + +This is separate from the PXE rootfs export. Append the export directly to +`/etc/exports` (on Ubuntu 24.04 `/etc/exports.d/*.conf` is ignored): + +```bash +sudo mkdir -p <NFS_EXPORT> +sudo chown -R nobody:nogroup <NFS_EXPORT> +sudo chmod 0777 <NFS_EXPORT> +echo "<NFS_EXPORT> <CLUSTER_SUBNET>(rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports +sudo exportfs -ra +sudo systemctl restart nfs-kernel-server +showmount -e localhost +``` + +Install the provisioner (storage class `nfs-client`): + +```bash +cd ~/aup-learning-cloud +cp deploy/k8s/nfs-provisioner/values.yaml deploy/k8s/nfs-provisioner/values.local.yaml +# edit values.local.yaml: nfs.server, nfs.path, storageClass.name = nfs-client +helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ +helm repo update +helm upgrade --install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --namespace nfs-provisioner --create-namespace \ + -f deploy/k8s/nfs-provisioner/values.local.yaml +kubectl get storageclass +``` + +## Step 12 — Configure JupyterHub values + +```bash +cd ~/aup-learning-cloud/runtime +cp values-multi-nodes.yaml.example values-basic-example.yaml +``` + +Minimum edits (see the [field guide](#valuesyaml-field-guide)): + +```yaml +custom: + authMode: "auto-login" # single-machine default; avoid "dummy" (login 404s) + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: "<GPU_PRODUCT_LABEL>" # from Step 10 + quotaRate: 3 + resources: + images: + cpu: "<CPU_NOTEBOOK_IMAGE>" + gpu: "<GPU_NOTEBOOK_IMAGE>" +hub: + db: + pvc: + storageClassName: nfs-client +singleuser: + storage: + dynamic: + storageClass: nfs-client +proxy: + service: + type: NodePort + nodePorts: + http: 30890 +``` + +## Step 13 — Deploy AUP Learning Cloud + +```bash +cd ~/aup-learning-cloud +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml + +kubectl get pods -n jupyterhub -o wide +kubectl get svc -n jupyterhub +``` + +For later config changes, re-run the same `helm upgrade --install`. + +## Step 14 — End-to-end validation + +```bash +kubectl get nodes -o wide +kubectl get pods -A +kubectl get storageclass +kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu +``` + +Then browse to `http://<SERVICE_IP>:30890` (or your ingress host), log in, +spawn a CPU notebook, create a file, restart and confirm it persists, then +spawn a GPU notebook and confirm its pod lands on a GPU node. + +## GPU label to accelerator key + +The chart's accelerator catalog (`runtime/values.yaml`) is keyed by accelerator +names; map the ROCm labeller's `amd.com/gpu.product-name` to the right key. The +GPU is auto-detected (Phase 2 and Phase 5), not named by the user in the +interview — use this table to confirm the detected product-name → key mapping +with the user. Verify against the live values file — product names can normalize +differently per fleet. + +| `amd.com/gpu.product-name` (example) | Accelerator key | +| --- | --- | +| `AMD_Radeon_780M_Graphics` | `phx` | +| `AMD_Radeon_890M_Graphics` | `strix` | +| `AMD_Radeon_8060S_Graphics` | `strix-halo` | +| `AMD_Radeon_RX_9070_XT` | `9070xt` | +| `AMD_Radeon_AI_PRO_R9700` | `r9700` | + +If your labeller reports a different product name, update the matching +`custom.accelerators.*.nodeSelector` entry to that exact string. + +## values.yaml field guide + +Sections to review in `values-basic-example.yaml` (from +`values-multi-nodes.yaml.example`): + +| Field | Purpose | +| --- | --- | +| `custom.authMode` | `auto-login` for the single-machine example; OAuth modes for real auth | +| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | GitHub OAuth (when not auto-login) | +| `custom.adminUser` | Hub admin | +| `custom.accelerators.*.nodeSelector` | Must match real `amd.com/gpu.*` labels | +| `custom.resources.images` | CPU/GPU/course notebook images | +| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Per-team resources and quotas | +| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | `nfs-client` for multi-node | +| `proxy.service`, `ingress` | NodePort (e.g. 30890) or ingress host | + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Playbook fails immediately on an assert | A required PXE var is empty | Re-check `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, and at least one SSH key | +| Agent never shows the PXE menu | Firmware boot order, network boot disabled, or Proxy-DHCP not reaching the client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | +| Agent gets an IP but cannot load boot files | TFTP blocked, missing files, or Secure Boot still on | `/srv/tftp`, firewall, Secure Boot disabled, `dnsmasq` logs | +| Agent has no network during netboot | NIC has no in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | +| Agent kernel boots but cannot mount rootfs | NFS export, subnet ACL, or wrong `pxe_controller_ip` | `showmount -e <SERVICE_IP>`, `/etc/exports`, rootfs kernel args | +| Agent waits for the k3s token | Token not published or apache ACL blocks the subnet | `curl http://<SERVICE_IP>:8080/k3s/token`, apache config | +| Agent joins once but fails after reboot | Missing local k3s persistence or lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | +| Agent fails to join with a version error | Agent rootfs k3s newer than the server | Align `pxe_k3s_version` with `k3s_version`, rebuild rootfs | +| Agent node does not join (SSH path) | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent`, `/etc/hosts` | +| GPU notebook stays Pending | Chart `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub`, node labels | +| PVC stays Pending | StorageClass name mismatch or NFS provisioner cannot mount | `kubectl get storageclass`, provisioner logs, NFS export | +| `kubectl` permission denied on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | + +For a complete reset (RISKY — confirm with the user): + +```bash +cd deploy/ansible +sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster +sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node> # single node +``` + +## Out of scope + +Zot registry mirror, Cloudflare Tunnel ingress, monitoring/Grafana, HA k3s, +external databases, and NPU setup. Add them only after the minimal deployment +boots agents, schedules GPU notebooks, and persists notebook storage. diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md new file mode 100644 index 00000000..5d571b4d --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -0,0 +1,46 @@ +# Helper scripts + +Deterministic helpers the deploy skill runs instead of generating commands ad +hoc. They are dependency-light (`bash` + `python3`, plus the obvious system +tools) and agent-agnostic, and follow the script conventions in +[../../../CONTRIBUTING.md](../../../CONTRIBUTING.md). Each emits JSON or a clear +report and uses exit codes the agent can branch on. + +| Script | Run when | What it does | +| --- | --- | --- | +| `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | +| `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | +| `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | +| `validate.py` | Before each `ansible-playbook` / `helm` run | Checks required PXE vars are non-empty, `k3s_version == pxe_k3s_version`, that each `nodeSelector` GPU label matches a real node (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | + +## Quick reference + +```bash +# Phase 2 — discover the host +./detect_hardware.sh # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] + +# Phase 3 — generate config from a spec +./gen_configs.py --print-schema > spec.json # then edit spec.json +./gen_configs.py --spec spec.json --out-dir ./generated + +# Phase 5 — after k3s + device plugin are up +./detect_cluster.sh > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] + +# Before running playbooks / helm +./validate.py --repo ~/aup-learning-cloud \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --cluster cluster.json --helm-dry-run +``` + +## Conventions + +- **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON + object; partial detection is reported via empty fields + a `warnings` array + rather than failing, so the agent can decide what to ask the operator. +- **Exit codes mean something.** `0` success (warnings allowed), `1` a real + validation failure, `2` a usage / missing-tooling error. +- **Secrets never touch stdout or VCS.** `gen_configs.py` mints the k3s token + with a CSPRNG, writes it only into `inventory.yml`, and `chmod 600`s it. +- **No third-party Python.** `gen_configs.py` / `validate.py` use the stdlib + only (no PyYAML), so they run on a bare operator machine. YAML is emitted + from templates and parsed with targeted scanning. diff --git a/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh b/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh new file mode 100755 index 00000000..eda1cc65 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# detect_cluster.sh -- after k3s is up, emit a JSON snapshot of the cluster the +# deploy skill needs to align custom.accelerators.*.nodeSelector with the REAL +# amd.com/gpu.* labels, confirm storage, and check the ROCm device plugin + +# labeller are running. Read-only: it only runs `kubectl get`. +# +# Usage: +# ./detect_cluster.sh # uses current KUBECONFIG +# KUBECONFIG=~/.kube/config ./detect_cluster.sh +# ./detect_cluster.sh --kubeconfig /path/to/k3s.yaml +# ./detect_cluster.sh -h | --help +# +# Output (stdout) is a single JSON object: +# { +# "nodes": [ +# {"name":"aipc1","ready":true,"roles":["control-plane"], +# "internal_ip":"192.168.0.140","gpu_product_names":["AMD_Radeon_8060S_Graphics"], +# "gpu_allocatable":"1","gpu_labels":{...}} +# ], +# "gpu_product_names": ["AMD_Radeon_8060S_Graphics"], +# "storage_classes": [{"name":"local-path","default":true}], +# "amdgpu_device_plugin": true, +# "amdgpu_labeller": true, +# "warnings": ["..."] +# } +# +# Exit codes: 0 on success (including "cluster reachable but nothing labelled +# yet"); 2 if kubectl/python3 missing or the API server is unreachable. +# +# Dependencies: bash, kubectl, python3 (stdlib only -- parses `kubectl -o json`). + +set -uo pipefail + +KCFG="" +while [[ $# -gt 0 ]]; do + case "$1" in + --kubeconfig) KCFG="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "detect_cluster: unknown arg $1" >&2; exit 2 ;; + esac +done + +command -v kubectl >/dev/null 2>&1 || { echo "detect_cluster: kubectl is required" >&2; exit 2; } +command -v python3 >/dev/null 2>&1 || { echo "detect_cluster: python3 is required" >&2; exit 2; } +[[ -n "$KCFG" ]] && export KUBECONFIG="$KCFG" + +kc() { kubectl "$@" 2>/dev/null; } + +# Fail fast (exit 2) if we cannot reach the API server at all -- this is the +# single most common "ran too early / wrong kubeconfig" case. +if ! kc version --request-timeout=10s >/dev/null; then + echo "detect_cluster: cannot reach the Kubernetes API server. Check KUBECONFIG / that k3s is up." >&2 + exit 2 +fi + +NODES_JSON="$(kc get nodes -o json || echo '{}')" +SC_JSON="$(kc get storageclass -o json || echo '{}')" +# The device plugin + labeller are DaemonSets; their names/namespaces can vary, +# so we scan all daemonsets and match on the amdgpu substring. +DS_JSON="$(kc get ds -A -o json || echo '{}')" + +export DC_NODES="$NODES_JSON" DC_SC="$SC_JSON" DC_DS="$DS_JSON" + +python3 <<'PY' +import json, os + +def load(name): + try: + return json.loads(os.environ.get(name, "") or "{}") + except json.JSONDecodeError: + return {} + +nodes_raw = load("DC_NODES").get("items", []) +sc_raw = load("DC_SC").get("items", []) +ds_raw = load("DC_DS").get("items", []) + +warnings = [] +nodes = [] +all_products = set() +for n in nodes_raw: + meta = n.get("metadata", {}) + name = meta.get("name", "") + labels = meta.get("labels", {}) or {} + status = n.get("status", {}) + ready = False + for c in status.get("conditions", []) or []: + if c.get("type") == "Ready": + ready = (c.get("status") == "True") + roles = sorted( + k.split("/", 1)[1] or "node" + for k in labels + if k.startswith("node-role.kubernetes.io/") + ) + internal_ip = "" + for a in status.get("addresses", []) or []: + if a.get("type") == "InternalIP": + internal_ip = a.get("address", "") + gpu_labels = {k: v for k, v in labels.items() if k.startswith("amd.com/gpu")} + products = [v for k, v in gpu_labels.items() if k == "amd.com/gpu.product-name"] + all_products.update(products) + alloc = (status.get("allocatable", {}) or {}).get("amd.com/gpu", "0") + nodes.append({ + "name": name, + "ready": ready, + "roles": roles, + "internal_ip": internal_ip, + "gpu_product_names": products, + "gpu_allocatable": alloc, + "gpu_labels": gpu_labels, + }) + +storage_classes = [] +for sc in sc_raw: + meta = sc.get("metadata", {}) + ann = meta.get("annotations", {}) or {} + is_default = ann.get("storageclass.kubernetes.io/is-default-class") == "true" + storage_classes.append({"name": meta.get("name", ""), "default": is_default}) + +def has_ds(substr): + for ds in ds_raw: + if substr in ds.get("metadata", {}).get("name", "").lower(): + return True + return False + +device_plugin = has_ds("device-plugin") or has_ds("amdgpu-dp") or ( + any("amdgpu" in ds.get("metadata", {}).get("name", "").lower() + and "label" not in ds.get("metadata", {}).get("name", "").lower() + for ds in ds_raw) +) +labeller = has_ds("labeller") or has_ds("labeler") or has_ds("amdgpu-labeller") + +if not nodes: + warnings.append("no nodes returned; cluster may still be initialising") +if not all_products: + warnings.append("no amd.com/gpu.product-name labels yet; install the ROCm device plugin + labeller, then re-run") +if not device_plugin: + warnings.append("AMD GPU device plugin DaemonSet not detected") +if not labeller: + warnings.append("ROCm node labeller DaemonSet not detected") + +print(json.dumps({ + "nodes": nodes, + "gpu_product_names": sorted(all_products), + "storage_classes": storage_classes, + "amdgpu_device_plugin": bool(device_plugin), + "amdgpu_labeller": bool(labeller), + "warnings": warnings, +}, indent=2)) +PY diff --git a/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh b/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh new file mode 100755 index 00000000..67d5a87c --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# detect_hardware.sh -- inspect the service machine and emit a JSON snapshot of +# the network and AMD GPU facts the deploy skill needs to fill in PXE / inventory +# variables. Read-only: it never changes the host. +# +# Usage: +# ./detect_hardware.sh # auto-detect the default-route NIC +# ./detect_hardware.sh --nic enp1s0 # force a specific NIC +# ./detect_hardware.sh -h | --help +# +# Output (stdout) is a single JSON object: +# { +# "nic": "enp1s0", +# "ip": "192.168.0.140", +# "subnet_cidr": "192.168.0.0/24", +# "gateway": "192.168.0.1", +# "dns_servers": "8.8.8.8,8.8.4.4", +# "gpus": [ {"pci":"c5:00.0","vendor":"1002","description":"...","kernel_driver":"amdgpu"} ], +# "warnings": [ "..." ] +# } +# +# Exit codes: 0 always (partial detection is reported via empty fields + +# warnings so the agent can decide what to ask the operator). Hard tooling +# failures (no python3) exit 2. +# +# Dependencies: bash, iproute2 (ip), pciutils (lspci), python3 (stdlib only). +# python3 is used purely to serialise JSON safely (lspci descriptions contain +# brackets, quotes, commas). No third-party packages. + +set -uo pipefail + +NIC="" +while [[ $# -gt 0 ]]; do + case "$1" in + --nic) NIC="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "detect_hardware: unknown arg $1" >&2; exit 2 ;; + esac +done + +command -v python3 >/dev/null 2>&1 || { echo "detect_hardware: python3 is required" >&2; exit 2; } + +warnings=() + +# --- NIC: default to the interface owning the default route --------------- +if [[ -z "$NIC" ]]; then + NIC="$(ip -o route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')" +fi +[[ -z "$NIC" ]] && warnings+=("no default-route NIC found; pass --nic explicitly") + +# --- IPv4 address + CIDR on that NIC -------------------------------------- +IP=""; CIDR="" +if [[ -n "$NIC" ]]; then + # e.g. "192.168.0.140/24" + addr="$(ip -o -f inet addr show "$NIC" 2>/dev/null | awk '{print $4; exit}')" + if [[ -n "$addr" ]]; then + IP="${addr%/*}" + prefix="${addr#*/}" + # Network address for the CIDR (zero the host bits) via python ipaddress. + CIDR="$(python3 - "$addr" <<'PY' 2>/dev/null +import ipaddress, sys +net = ipaddress.ip_interface(sys.argv[1]).network +print(net.with_prefixlen) +PY +)" + fi +fi +[[ -z "$IP" ]] && warnings+=("no IPv4 address on NIC '$NIC'") + +# --- Default gateway ------------------------------------------------------ +GATEWAY="$(ip -o route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="via"){print $(i+1); exit}}')" +[[ -z "$GATEWAY" ]] && warnings+=("no default gateway found") + +# --- DNS servers ---------------------------------------------------------- +# Prefer systemd-resolved when present; fall back to /etc/resolv.conf. +DNS="" +if command -v resolvectl >/dev/null 2>&1; then + DNS="$(resolvectl dns 2>/dev/null | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}' | sort -u | paste -sd, -)" +fi +if [[ -z "$DNS" && -r /etc/resolv.conf ]]; then + DNS="$(awk '/^nameserver/{print $2}' /etc/resolv.conf | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | paste -sd, -)" +fi +[[ -z "$DNS" ]] && warnings+=("no DNS servers detected; defaulting suggestion is 8.8.8.8,8.8.4.4") + +# --- AMD GPUs via lspci --------------------------------------------------- +# AMD/ATI PCI vendor id is 1002. We hand the full `lspci -D -nnk` dump to +# python3 (below) and parse device blocks there: mawk (Ubuntu's default awk) +# does not support {n} interval regexes, so block parsing in python is far more +# portable. We record the bound kernel driver (amdgpu = the in-kernel driver is +# loaded, which the PXE rootfs needs for GPU scheduling). +LSPCI_RAW="" +if command -v lspci >/dev/null 2>&1; then + LSPCI_RAW="$(lspci -D -nnk 2>/dev/null)" +else + warnings+=("lspci not found (install pciutils); GPU detection skipped") +fi + +# Hand everything to python3 for safe JSON assembly. +export DH_NIC="$NIC" DH_IP="$IP" DH_CIDR="$CIDR" DH_GW="$GATEWAY" DH_DNS="$DNS" +export DH_LSPCI="$LSPCI_RAW" +DH_WARNINGS="$(printf '%s\n' "${warnings[@]:-}")" +export DH_WARNINGS + +python3 <<'PY' +import json, os, re + +# PCI classes we treat as a GPU/accelerator: VGA (0300), 3D (0302), +# Display (0380), Processing accelerator (1200). +GPU_CLASSES = ("0300", "0302", "0380", "1200") + +def amd_gpus(raw): + out = [] + cur = None + for line in (raw or "").splitlines(): + # Device header lines start at column 0 with a PCI address. + if re.match(r"^[0-9a-fA-F]{4}:", line): + if cur: + out.append(cur) + cur = None + m = re.match( + r"^(\S+)\s+.*?\[(?P<cls>[0-9a-f]{4})\]:\s+(?P<desc>.*?)\s*" + r"\[(?P<vendor>[0-9a-f]{4}):(?P<dev>[0-9a-f]{4})\]", + line) + if not m: + continue + if m.group("vendor") != "1002" or m.group("cls") not in GPU_CLASSES: + continue + cur = { + "pci": m.group(1), + "vendor": "1002", + "device_id": m.group("dev"), + "description": m.group("desc").strip(), + "kernel_driver": "", + } + elif cur is not None: + dm = re.search(r"Kernel driver in use:\s*(\S+)", line) + if dm: + cur["kernel_driver"] = dm.group(1) + if cur: + out.append(cur) + return out + +warnings = [w for w in (os.environ.get("DH_WARNINGS", "").splitlines()) if w.strip()] +print(json.dumps({ + "nic": os.environ.get("DH_NIC", ""), + "ip": os.environ.get("DH_IP", ""), + "subnet_cidr": os.environ.get("DH_CIDR", ""), + "gateway": os.environ.get("DH_GW", ""), + "dns_servers": os.environ.get("DH_DNS", ""), + "gpus": amd_gpus(os.environ.get("DH_LSPCI", "")), + "warnings": warnings, +}, indent=2)) +PY diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py new file mode 100755 index 00000000..e480c936 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. + +Given a JSON cluster-spec (see ``--print-schema``), write the three files the +deploy skill needs, keeping them mutually consistent: + + 1. ``inventory.yml`` -- Ansible inventory (server + token + + k3s_version; agents listed for the + SSH topology, empty for PXE). + 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: the ``vars:`` values + to merge into + deploy/ansible/playbooks/pb-pxe-controller.yml. + 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, + storage class, proxy NodePort, authMode. + +Design choices (deliberate): + + * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this + runs on a bare operator machine. YAML is emitted from templates, not a + serialiser -- the output is small, fixed-shape, and carries the copyright header. + * The k3s token is generated locally with ``secrets`` (CSPRNG) and written + ONLY into inventory.yml. It is never printed to stdout/stderr. Pass + ``--token-file`` to reuse an existing token instead of minting one. + * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never + be newer than the server (k3s refuses that). + * Existing files are not overwritten unless ``--force`` is given. + +Usage: + gen_configs.py --print-schema + gen_configs.py --spec spec.json --out-dir ./generated + cat spec.json | gen_configs.py --spec - --out-dir ./generated --force + +Exit codes: 0 on success; 1 on a spec/validation error; 2 on a usage error. +""" +from __future__ import annotations + +import argparse +import base64 +import json +import secrets +import sys +from pathlib import Path + +HEADER_HASH = ( + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" + "# Generated by auplc-skills gen_configs.py -- review before use.\n" +) + +# Default GPU product-name labels, keyed by the accelerator key used in +# runtime/values.yaml (custom.accelerators.<key>). Verified against the chart's +# values.yaml; override per fleet via spec["accelerators"][key]["product_name"]. +DEFAULT_ACCEL_LABELS = { + "phx": "AMD_Radeon_780M_Graphics", + "strix": "AMD_Radeon_890M_Graphics", + "strix-halo": "AMD_Radeon_8060S_Graphics", + "9070xt": "AMD_Radeon_RX_9070_XT", + "r9700": "AMD_Radeon_AI_PRO_R9700", +} + +SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", + "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} + + +def die(msg: str, code: int = 1) -> "None": + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def gen_token() -> str: + # Mirror `openssl rand -base64 64`: 64 random bytes, base64-encoded. + return base64.b64encode(secrets.token_bytes(64)).decode("ascii") + + +def require(spec: dict, path: str): + cur = spec + for part in path.split("."): + if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): + die(f"spec is missing required field '{path}'") + cur = cur[part] + return cur + + +def yaml_quote(s: str) -> str: + return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def render_inventory(spec: dict, token: str) -> str: + topo = spec["topology"] + server = spec["server"] + k3s_version = spec["k3s_version"] + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " agent:", + ] + if topo == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for a in spec["agents"]: + lines.append(f" {a['name']}:") + lines.append(f" ansible_host: {a['ip']}") + else: + # PXE diskless agents auto-join by netboot; do NOT list them here. + lines.append(" hosts: {}") + lines += [ + " vars:", + " ansible_port: 22", + " ansible_user: root", + f" k3s_version: {k3s_version}", + f" token: {yaml_quote(token)}", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host']" + " | default(groups['server'][0]) }}\"", + ] + if topo == "pxe-diskless": + lines += [ + "", + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + ] + return "\n".join(lines) + "\n" + + +def render_pxe_vars(spec: dict) -> str: + net = require(spec, "network") + pxe = spec.get("pxe", {}) + keys = pxe.get("authorized_keys", []) + if not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + server_ip = spec["server"]["ip"] + k3s_version = spec["k3s_version"] + lines = [ + HEADER_HASH, + "# Merge these into the vars: block of", + "# deploy/ansible/playbooks/pb-pxe-controller.yml", + "# pxe_k3s_version is pinned to k3s_version so agents are never newer", + "# than the server.", + "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", + f"pxe_network_interface: {yaml_quote(net['interface'])}", + f"pxe_subnet: {yaml_quote(net['subnet'])}", + f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", + f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", + f"pxe_controller_ip: {yaml_quote(server_ip)}", + "pxe_k3s_server_ips:", + f" - {yaml_quote(server_ip)}", + f"pxe_k3s_version: {yaml_quote(k3s_version)}", + f"pxe_web_port: {int(pxe.get('web_port', 8080))}", + f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", + "pxe_rootfs_authorized_keys:", + ] + for k in keys: + lines.append(f" - {yaml_quote(k)}") + return "\n".join(lines) + "\n" + + +def render_values(spec: dict) -> str: + accel = spec.get("accelerators") or {} + storage_class = (spec.get("storage") or {}).get("class", "nfs-client") + node_port = (spec.get("proxy") or {}).get("node_port", 30890) + auth_mode = spec.get("auth_mode", "auto-login") + images = spec.get("images") or {} + + lines = [ + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", + "# Helm overlay generated by auplc-skills gen_configs.py.", + "# Layer this on top of runtime/values.yaml:", + "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", + "# --create-namespace -f runtime/values.yaml -f <this file>", + "custom:", + f" authMode: {yaml_quote(auth_mode)}", + ] + if accel: + lines.append(" accelerators:") + for key, cfg in accel.items(): + product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) + if not product: + die(f"accelerator '{key}' has no product_name and no known default; " + "add accelerators.<key>.product_name from `kubectl describe node`") + lines += [ + f" {key}:", + " nodeSelector:", + f" amd.com/gpu.product-name: {yaml_quote(product)}", + ] + if images: + lines.append(" resources:") + lines.append(" images:") + for k, v in images.items(): + lines.append(f" {k}: {yaml_quote(v)}") + lines += [ + "hub:", + " db:", + " pvc:", + f" storageClassName: {yaml_quote(storage_class)}", + "singleuser:", + " storage:", + " dynamic:", + f" storageClass: {yaml_quote(storage_class)}", + "proxy:", + " service:", + " type: NodePort", + " nodePorts:", + f" http: {int(node_port)}", + ] + return "\n".join(lines) + "\n" + + +def write_file(path: Path, content: str, force: bool, secret: bool = False) -> None: + if path.exists() and not force: + die(f"refusing to overwrite existing {path} (use --force)", 1) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + # Tighten perms on files that carry the cluster token. + if secret: + path.chmod(0o600) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") + ap.add_argument("--out-dir", default="generated", + help="directory to write artifacts into (default: ./generated)") + ap.add_argument("--token-file", + help="read the k3s token from this file instead of generating one") + ap.add_argument("--force", action="store_true", help="overwrite existing files") + ap.add_argument("--print-schema", action="store_true", + help="print an example cluster-spec and exit") + args = ap.parse_args(argv) + + if args.print_schema: + print(json.dumps(SCHEMA, indent=2)) + return 0 + if not args.spec: + die("--spec is required (or use --print-schema)", 2) + + raw = sys.stdin.read() if args.spec == "-" else Path(args.spec).read_text(encoding="utf-8") + try: + spec = json.loads(raw) + except json.JSONDecodeError as exc: + die(f"spec is not valid JSON: {exc}") + + topo = spec.get("topology") + if topo not in ("pxe-diskless", "ssh-preinstalled"): + die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") + require(spec, "k3s_version") + require(spec, "server.name") + require(spec, "server.ip") + + if args.token_file: + token = Path(args.token_file).read_text(encoding="utf-8").strip() + if not token: + die("--token-file is empty") + else: + token = gen_token() + + out = Path(args.out_dir) + write_file(out / "inventory.yml", render_inventory(spec, token), args.force, secret=True) + if topo == "pxe-diskless": + write_file(out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), args.force) + write_file(out / "values-basic-example.yaml", render_values(spec), args.force) + + print("\nNext: review the files, then copy them into your aup-learning-cloud " + "checkout. Never commit inventory.yml -- it holds the k3s token.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py new file mode 100755 index 00000000..9e073d27 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Pre-flight validation for an AUP Learning Cloud deploy. + +Catches the mistakes that otherwise surface only after a long playbook or a +failed spawn: + + * required PXE vars empty (interface / subnet / controller_ip / dns / + k3s_server_ips / at least one authorized key); + * the k3s server version and the PXE agent rootfs version disagree + (agents must not be newer than the server); + * a custom.accelerators.*.nodeSelector that names a GPU product label no + node actually reports (the #1 cause of GPU notebooks stuck Pending) -- + checked against detect_cluster.sh output when supplied; + * (optional) the chart does not render: a `helm template` dry-run. + +This intentionally uses regex/line scanning rather than a YAML parser so it +runs on a bare operator machine with stdlib only. It is a linter, not a schema +validator: it reports what it can prove wrong, and says so when it cannot +inspect something. + +Usage: + validate.py --repo ~/aup-learning-cloud + validate.py --repo ~/aup-learning-cloud \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --cluster cluster.json --helm-dry-run + +Exit codes: 0 if every check passed (warnings allowed); 1 if any check failed; +2 on a usage error. +""" +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" +INVENTORY = "deploy/ansible/inventory.yml" +CHART = "runtime/chart" + +errors: list[str] = [] +warnings: list[str] = [] +passed: list[str] = [] + + +def ok(msg: str) -> None: + passed.append(msg) + + +def warn(msg: str) -> None: + warnings.append(msg) + + +def fail(msg: str) -> None: + errors.append(msg) + + +def scalar(text: str, key: str) -> str | None: + """First `key: value` scalar in `text` (ignores list/empty values).""" + m = re.search(rf"^\s*{re.escape(key)}\s*:\s*(.+?)\s*$", text, re.MULTILINE) + if not m: + return None + val = m.group(1).strip().strip('"').strip("'") + return val or None + + +def list_nonempty(text: str, key: str) -> bool: + """True if `key:` is a YAML list with at least one item, or an inline + non-empty flow list (``[...]`` with content).""" + # Inline flow list: key: ["a", "b"] or key: [] + m = re.search(rf"^\s*{re.escape(key)}\s*:\s*\[(.*?)\]\s*$", text, re.MULTILINE) + if m: + return bool(m.group(1).strip()) + # Block list: key:\n - item + m = re.search(rf"^(\s*){re.escape(key)}\s*:\s*$", text, re.MULTILINE) + if not m: + return False + indent = len(m.group(1)) + tail = text[m.end():].splitlines() + for line in tail: + if not line.strip(): + continue + cur_indent = len(line) - len(line.lstrip()) + if cur_indent <= indent: + break + if line.lstrip().startswith("- "): + return True + return False + + +def check_pxe_vars(repo: Path) -> None: + pb = repo / PXE_PLAYBOOK + if not pb.exists(): + warn(f"{PXE_PLAYBOOK} not found; skipping PXE checks (SSH topology?)") + return + text = pb.read_text(encoding="utf-8") + required_scalars = { + "pxe_network_interface": "service-machine NIC", + "pxe_subnet": "node subnet CIDR", + "pxe_controller_ip": "service host IP", + "pxe_dns_servers": "rootfs DNS servers", + } + for key, what in required_scalars.items(): + if scalar(text, key): + ok(f"PXE var {key} is set") + else: + fail(f"PXE var {key} ({what}) is empty -- the playbook asserts on this") + if list_nonempty(text, "pxe_k3s_server_ips"): + ok("PXE var pxe_k3s_server_ips has at least one IP") + else: + fail("PXE var pxe_k3s_server_ips is empty") + if list_nonempty(text, "pxe_rootfs_authorized_keys"): + ok("PXE var pxe_rootfs_authorized_keys has at least one key") + else: + fail("PXE var pxe_rootfs_authorized_keys is empty (rootfs would be unreachable)") + + +def check_version_sync(repo: Path) -> None: + inv = repo / INVENTORY + pb = repo / PXE_PLAYBOOK + if not inv.exists(): + warn(f"{INVENTORY} not found; skipping k3s version sync check") + return + server_ver = scalar(inv.read_text(encoding="utf-8"), "k3s_version") + if not server_ver: + warn("k3s_version not found in inventory.yml") + return + if not pb.exists(): + ok(f"k3s server version is {server_ver} (no PXE playbook to cross-check)") + return + agent_ver = scalar(pb.read_text(encoding="utf-8"), "pxe_k3s_version") + if not agent_ver: + warn("pxe_k3s_version not found in the PXE playbook") + return + if agent_ver == server_ver: + ok(f"k3s_version == pxe_k3s_version ({server_ver})") + else: + fail(f"version mismatch: inventory k3s_version={server_ver} but " + f"pxe_k3s_version={agent_ver}. Agents must not be newer than the server.") + + +def collect_values_text(repo: Path, values: list[str]) -> str: + paths = values or ["runtime/values.yaml"] + chunks = [] + for rel in paths: + p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if p.exists(): + chunks.append(p.read_text(encoding="utf-8")) + else: + warn(f"values file not found: {rel}") + return "\n".join(chunks) + + +def check_accelerator_labels(values_text: str, cluster: dict | None) -> None: + declared = sorted(set(re.findall( + r"amd\.com/gpu\.product-name\s*:\s*[\"']?([A-Za-z0-9_]+)[\"']?", values_text))) + if not declared: + warn("no amd.com/gpu.product-name nodeSelector found in the values overlay") + return + if cluster is None: + warn("no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}") + return + real = set(cluster.get("gpu_product_names", [])) + if not real: + warn("cluster snapshot reports no GPU product labels yet (device plugin / " + "labeller not ready?)") + return + for d in declared: + if d in real: + ok(f"nodeSelector '{d}' matches a real node label") + else: + fail(f"nodeSelector '{d}' matches no node label. Real labels: " + f"{', '.join(sorted(real))}") + + +def check_helm(repo: Path, values: list[str]) -> None: + if not shutil.which("helm"): + warn("helm not on PATH; skipped chart dry-run") + return + chart = repo / CHART + if not chart.exists(): + warn(f"chart not found at {CHART}; skipped dry-run") + return + cmd = ["helm", "template", "jupyterhub", str(chart)] + for rel in (values or ["runtime/values.yaml"]): + p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if p.exists(): + cmd += ["-f", str(p)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + ok("helm template rendered the chart successfully") + else: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] + fail("helm template failed:\n " + "\n ".join(tail)) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo", required=True, help="path to the aup-learning-cloud checkout") + ap.add_argument("--values", action="append", default=[], + help="values file (repeatable); defaults to runtime/values.yaml") + ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") + ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") + ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") + args = ap.parse_args(argv) + + repo = Path(args.repo).expanduser() + if not repo.exists(): + print(f"validate: repo not found: {repo}", file=sys.stderr) + return 2 + + cluster = None + if args.cluster: + try: + cluster = json.loads(Path(args.cluster).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) + return 2 + + check_pxe_vars(repo) + check_version_sync(repo) + values_text = collect_values_text(repo, args.values) + check_accelerator_labels(values_text, cluster) + if args.helm_dry_run: + check_helm(repo, args.values) + + if args.json: + print(json.dumps({"passed": passed, "warnings": warnings, "errors": errors, + "status": "ok" if not errors else "error"}, indent=2)) + else: + for m in passed: + print(f"[ OK ] {m}") + for m in warnings: + print(f"[WARN] {m}") + for m in errors: + print(f"[FAIL] {m}") + print(f"\n{len(passed)} ok, {len(warnings)} warning(s), {len(errors)} error(s)") + return 0 if not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/deploy-aup-learning-cloud/skill-card.md b/skills/deploy-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..640e1441 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Deploy AUP Learning Cloud end to end on a multi-node k3s cluster — either PXE-diskless netboot or SSH-preinstalled nodes — for operators standing up a K3s cluster with AUP Learning Cloud Service. + +## Owner + +AMD Research diff --git a/skills/develop-aup-learning-cloud-courses/SKILL.md b/skills/develop-aup-learning-cloud-courses/SKILL.md new file mode 100644 index 00000000..52101dad --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/SKILL.md @@ -0,0 +1,98 @@ +--- +name: develop-aup-learning-cloud-courses +description: >- + Group: Course & other editor. Authors a new learning toolkit for AUP Learning + Cloud end to end: write the + course notebooks under projects/<NAME>/, package them into a course Docker + image (dockerfiles/Courses/<NAME>/ + a Makefile target on the ROCm GPU base), + register the course in auplc_installer/catalog.py (COURSE_CATALOG + team + mapping), then hand off to build + values wiring. Use when an educator wants + to add a new course or lab set, create a toolkit like CV/DL/LLM/PhySim, turn a + notebook folder into a spawnable course, add a Dockerfile/build.sh for a + course, or add a course key to the catalog. Triggers include projects/CV, + projects/DL, projects/LLM, projects/PhySim, dockerfiles/Courses, + COURSE_CATALOG, "add a course", "new toolkit". Do not use to only build an + existing image (build-aup-learning-cloud-images), to only edit the values + catalog for an existing image (configure-aup-learning-cloud-courses), or to + clone a user's repo at runtime (configure-aup-learning-cloud-repos). +--- + +# Develop AUP Learning Cloud courses + +Create a brand-new course (a set of hands-on notebooks) and make it a spawnable +environment: author the curriculum, bake it into a course image, register the +course key, then build and wire it into the spawn UI. This is the +author/educator workflow that *produces* what configure-courses later tunes. + +The notebooks and the image build context are the source of truth; the catalog +keeps keys consistent. Per-file conventions, the new-course checklist, and the +directory map are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; Docker with enough disk (course images are + large); the ROCm GPU base image available (`auplc-base`, built by + build-images or pulled). +- Familiarity with the existing toolkits under `projects/{CV,DL,LLM,PhySim}` as + patterns. +- For the build + deploy hand-off: the build-images and configure-courses skills. + +## Where a course lives (four coordinated places) + +A new course `Course-<NAME>` must be consistent across: + +1. **Curriculum** — `projects/<NAME>/` (the `.ipynb` labs, README, assets). +2. **Image build context** — `dockerfiles/Courses/<NAME>/` (Dockerfile + + `build.sh`) layered on the GPU base, plus a `Makefile` target. +3. **Catalog** — a `Course(...)` entry in `auplc_installer/catalog.py` + `COURSE_CATALOG` (key, image basename, `gpu_required`, make target, display + name) and the mirrored bash `COURSE_CATALOG`, plus `BASE_TEAM_MAPPING`. +4. **Values** — `custom.resources.{images,requirements,metadata}.<key>` and + `custom.teams.mapping` (this is the configure-courses skill). + +## Workflow + +1. **Author the curriculum.** Add the notebooks under `projects/<NAME>/` + following the existing numbering/README pattern (e.g. `LLM01-…`). Keep the + per-file `Copyright (C) … Advanced Micro Devices, Inc.` header (MIT). +2. **Create the image build context.** Add `dockerfiles/Courses/<NAME>/` with a + `Dockerfile` + `build.sh` modeled on an existing course, `FROM` the GPU base + (`BASE_IMAGE=ghcr.io/amdresearch/auplc-base:latest`), and `COPY` the + `projects/<NAME>/` content into the image. Pin pip deps for reproducibility. +3. **Add the Makefile target.** Add a `<name>` target in `dockerfiles/Makefile` + that builds, GPU-tags (`:latest-$(GPU_TARGET)`), and `save-image`s — mirror + the `cv`/`dl` targets. Add it to the `courses` aggregate. +4. **Register the course key.** Add a `Course("Course-<NAME>", "auplc-<name>", + True, "<name>", "<Display Name>")` to `COURSE_CATALOG` in `catalog.py`, keep + the bash table byte-for-byte identical, and add the key to the relevant + `BASE_TEAM_MAPPING` groups. +5. **Build the image** (hand off to build-images): + + ```bash + ./auplc-installer img build <name> --gpu=<target> + ``` + +6. **Wire it into values** (hand off to configure-courses): add the key under + `custom.resources.images/requirements/metadata` and `custom.teams.mapping`, + then `rt upgrade` / `helm upgrade`. +7. **Verify.** The course appears in its spawn-UI `group` for mapped teams, and + a launched pod runs the new image with the notebooks present under the home + tree. + +## Safety + +- **Large/slow builds.** Course images are big; confirm disk and time before a + full build, and prefer building just the new `<name>` target. +- **Keep the catalog in sync.** `catalog.py` and the mirrored bash table must + match exactly, or `--courses` selection/overlay generation breaks. +- **Licensing.** Only bundle datasets, models, and third-party code whose + licenses permit redistribution; keep AMD copyright headers on new source. +- **Attribution.** If any change touches Hub source (not typical for a course), + preserve the four attribution layers from the project `AGENTS.md`. +- Never commit secrets or large binary blobs that belong in object storage. + +## Reference + +The new-course checklist, the `projects/`/`dockerfiles/Courses/` layout, the +`catalog.py` entry shape, GPU-tag rules, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/develop-aup-learning-cloud-courses/reference.md b/skills/develop-aup-learning-cloud-courses/reference.md new file mode 100644 index 00000000..c907f2d3 --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/reference.md @@ -0,0 +1,106 @@ +# Develop AUP Learning Cloud courses — Reference + +The new-course checklist, the directory layout, the `catalog.py` entry shape, +and troubleshooting. Workflow and gates are in [SKILL.md](SKILL.md). + +## Source + +- Repo README "Learning Solution" + `projects/{CV,DL,LLM,PhySim}/README.md`. +- `dockerfiles/Makefile` (course targets) and `dockerfiles/Courses/<NAME>/`. +- `auplc_installer/catalog.py` (the course catalog source of truth) and its + mirrored bash `COURSE_CATALOG` / `BASE_TEAM_MAPPING`. +- Build details: build-aup-learning-cloud-images. Values wiring: + configure-aup-learning-cloud-courses. + +## Directory layout + +``` +projects/<NAME>/ # curriculum: NN_*.ipynb labs, README.md, assets/ +dockerfiles/Courses/<NAME>/ # Dockerfile + build.sh (FROM the GPU base) +dockerfiles/Makefile # add a <name> target; add it to `courses` +auplc_installer/catalog.py # add a Course(...) to COURSE_CATALOG + team map +runtime/values.yaml # custom.resources.{images,requirements,metadata} +``` + +Existing toolkits to copy from: `projects/CV` (10 labs), `projects/DL` (12), +`projects/LLM` (9), `projects/PhySim` (Genesis robotics). + +## Makefile target (mirror cv/dl) + +```make +courses: cv dl llm physim <name> + +<name>: + cd Courses/<NAME> && BASE_IMAGE=$(GPU_BASE_IMAGE) bash ./build.sh + docker tag ghcr.io/amdresearch/auplc-<name>:latest ghcr.io/amdresearch/auplc-<name>:latest-$(GPU_TARGET) + $(MAKE) save-image IMAGE=ghcr.io/amdresearch/auplc-<name>:latest +``` + +GPU course images are tagged `:<IMAGE_TAG>-<gpu_target>` (e.g. `latest-gfx1151`). +`GPU_BASE_IMAGE` defaults to `ghcr.io/amdresearch/auplc-base:latest`. + +## catalog.py entry + +```python +COURSE_CATALOG: tuple[Course, ...] = ( + # ...existing entries... + Course("Course-<NAME>", "auplc-<name>", True, "<name>", "<Display Name> Course"), +) +``` + +`Course(key, image_basename, gpu_required, make_target, display_name)`: + +- `key` — matches `custom.resources.{images,requirements,metadata}` and + `custom.teams.mapping` (convention: `Course-<NAME>`). +- `image_basename` — `auplc-<name>` (no registry/tag). +- `gpu_required` — `True` → GPU-tagged build; `False` → plain `:<tag>`. +- `make_target` — the `dockerfiles/Makefile` target. + +Add the same row to the mirrored **bash** `COURSE_CATALOG` (byte-for-byte) and +add the key to the appropriate `BASE_TEAM_MAPPING` groups (e.g. `gpu`, +`official`, `AUP`, `native-users`, `github-users`). `COURSE_PRESET_BASIC` is +only `cpu, gpu, code-cpu, code-gpu`; new courses join `all`, not `basic`. + +## Build and wire (hand-offs) + +```bash +# build the new course image (build-images skill) +./auplc-installer img build <name> --gpu=<target> +# optional push for multi-node / offline +docker push ghcr.io/amdresearch/auplc-<name>:latest-<gpu_target> +``` + +Then, with configure-courses, add to the values overlay: + +```yaml +custom: + resources: + images: + Course-<NAME>: "ghcr.io/amdresearch/auplc-<name>:latest" + requirements: + Course-<NAME>: { cpu: "0", memory: "0Gi", amd.com/gpu: "1" } + metadata: + Course-<NAME>: + group: "TEACHING LABS" + description: "<Display Name> Course" + accelerator: "GPU" + acceleratorKeys: [strix-halo] + allowGitClone: true + resourceType: "notebook" + teams: + mapping: + gpu: [..., Course-<NAME>] +``` + +Apply with `./auplc-installer rt upgrade` (single) or `helm upgrade` (multi). + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `unknown course key` from installer | `catalog.py`/bash table out of sync, or key typo | Make both tables identical; use the exact `Course-<NAME>` key | +| Course image build fails | Missing base image or bad Dockerfile context | Build `base-rocm` first; verify `dockerfiles/Courses/<NAME>` paths | +| Notebooks missing in the pod | `COPY` path wrong in the course Dockerfile | Confirm `projects/<NAME>/` is copied into the image home tree | +| Course not in spawn UI | Values catalog/team mapping incomplete | Add the key in all of images/requirements/metadata + `teams.mapping` | +| GPU course Pending | `acceleratorKeys` → node label mismatch | `kubectl describe node | grep amd.com/gpu.product-name` | +| Wrong gfx kernels at runtime | Built for the wrong `--gpu` | Rebuild with the correct target | diff --git a/skills/develop-aup-learning-cloud-courses/skill-card.md b/skills/develop-aup-learning-cloud-courses/skill-card.md new file mode 100644 index 00000000..94ea9437 --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Author a new AUP Learning Cloud course end to end — notebooks, course image, and catalog registration — for educators and curriculum authors adding a learning toolkit. + +## Owner + +AMD Research diff --git a/skills/expose-aup-learning-cloud/SKILL.md b/skills/expose-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..9135bece --- /dev/null +++ b/skills/expose-aup-learning-cloud/SKILL.md @@ -0,0 +1,109 @@ +--- +name: expose-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Configures how AUP Learning Cloud is + exposed and stored for a real + deployment: the proxy service type (NodePort vs LoadBalancer/ingress), ingress + hostname and TLS, external-TLS handling (custom.security.publicScheme), CORS + origins (custom.hub/notebook.allowedOrigins), and + the shared NFS storage class for the Hub DB and user PVCs. Use when the user + wants to put the Hub behind a domain, enable HTTPS/TLS/certificates, set up + ingress, change the NodePort, move storage from local-path to NFS + (nfs-client / nfs-subdir-external-provisioner), fix mixed-content / _xsrf + cookie issues behind a reverse proxy, or allow embedding/CORS. Triggers + include ingress.enabled, proxy.service.type, nodePorts.http, publicScheme, + allowedOrigins, storageClassName, nfs-client, TLS, cert-manager. Do not use + for the first cluster build (deploy-/install-aup-learning-cloud), the GitHub + OAuth callback URL (configure-aup-learning-cloud-auth), or course/quota config + (configure-aup-learning-cloud-courses). +--- + +# Expose AUP Learning Cloud + +Take a deployment from the local NodePort/`local-path` defaults to a real +network and storage posture: choose how the proxy is reached (NodePort, +LoadBalancer, or ingress + TLS), tell the Hub about externally-terminated TLS, +set CORS origins, and move persistent data onto shared NFS. + +Edit a **values overlay** and re-apply with Helm / the installer. NFS, ingress, +and TLS are opt-in — the checked-in defaults are a plain HTTP NodePort. The full +value blocks, the NFS provisioner setup, and troubleshooting are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A running AUP Learning Cloud and `helm` + `kubectl` (or `./auplc-installer`). +- For ingress/TLS: an ingress controller in the cluster, a DNS record for the + hostname, and a certificate source (cert-manager issuer or a TLS secret). +- For NFS storage: an NFS server/export reachable from every node. + +## The defaults you are changing + +The checked-in `runtime/values.yaml` is local-oriented: `proxy.service.type: +NodePort` on `30890`, `ingress.enabled: false`, `hub.db.pvc.storageClassName: +local-path`, `singleuser.storage.dynamic.storageClass: local-path`. Treat NFS, +ingress, and TLS as deliberate additions. + +## Pick the exposure path + +| Path | When | Key values | +| --- | --- | --- | +| **NodePort** (default) | Lab on a known node IP | `proxy.service.type: NodePort`, `nodePorts.http` | +| **LoadBalancer** | Cloud / MetalLB | `proxy.service.type: LoadBalancer` | +| **Ingress + TLS** | Real domain, HTTPS | `ingress.enabled: true`, host, TLS secret/issuer | + +## Workflow + +1. **Read current state.** Note `proxy.service`, `ingress`, the two + `storageClassName`s, and whether TLS is terminated by the chart or upstream. +2. **Set exposure** in the overlay (one path above). For ingress, set the host + and the TLS config; point DNS at the controller. +3. **Handle TLS termination.** If TLS terminates **outside** the chart (LB or + external proxy), set `custom.security.publicScheme: "https"` so the Hub marks + `_xsrf` cookies secure and builds correct https URLs. +4. **CORS / embedding (only if needed).** Add origins to + `custom.hub.allowedOrigins` (Hub CORS) and/or `custom.notebook.allowedOrigins` + (single-user server args). Leave empty unless something embeds the Hub. +5. **Storage (multi-node / production).** Move the Hub DB and user PVCs to + `nfs-client`: install `nfs-subdir-external-provisioner` against your NFS + export, then set both `storageClassName`s. Provisioner setup is in + [reference.md](reference.md). +6. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed. +7. **Apply** with `helm upgrade --install … -n jupyterhub` (or `rt upgrade` + single-node) and **verify**: + + ```bash + kubectl get svc,ingress -n jupyterhub + kubectl get storageclass + kubectl get pvc -A + ``` + + Then load the public URL over HTTPS, log in, and confirm a spawned pod's PVC + binds on the new storage class. + +## code-server exposure safety + +code-server resources run `code-server --auth none` on port `8888` and are safe +**only** behind the JupyterHub proxy's auth boundary. Never expose that pod port +directly via NodePort, LoadBalancer, or ingress. Only the JupyterHub proxy +service should be public. + +## Safety + +- **Changing storage class does not migrate existing data.** Switching + `storageClassName` affects new PVCs; the Hub DB PVC and user homes do not move + automatically. Plan a migration/backup before changing it on a live Hub — + confirm with the user. +- **Editing `/etc/exports` + restarting `nfs-kernel-server`** is disruptive; + gate it (see deploy/troubleshoot skills for the NFS host side). +- **Exposing to the internet raises the stakes** — pair with HTTPS, a real auth + mode (configure-auth), and never expose code-server's raw port. +- A `helm upgrade` restarts the Hub pod (brief login blip). +- Never commit TLS private keys or put them in tracked values; use a K8s secret. + +## Reference + +NodePort/LoadBalancer/ingress value blocks, TLS + cert-manager options, +`publicScheme`/`allowedOrigins`, the NFS provisioner install and default-class +patch, and troubleshooting: [reference.md](reference.md). diff --git a/skills/expose-aup-learning-cloud/reference.md b/skills/expose-aup-learning-cloud/reference.md new file mode 100644 index 00000000..5937beb9 --- /dev/null +++ b/skills/expose-aup-learning-cloud/reference.md @@ -0,0 +1,174 @@ +# Expose AUP Learning Cloud — Reference + +Exposure value blocks (NodePort / LoadBalancer / ingress + TLS), externally +terminated TLS, CORS origins, and the NFS storage setup. Workflow and gates are +in [SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (sections 9, 10, 13): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> +- Multi-Node Cluster Deployment (storage, ingress): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- Single-Node Deployment (defaults): <https://amdresearch.github.io/aup-learning-cloud/installation/single-node.html> + +The chart follows zero-to-jupyterhub conventions; the live +`runtime/chart/values.schema.yaml` is the source of truth. + +## Local defaults (what you change) + +```yaml +proxy: + service: + type: NodePort + nodePorts: + http: 30890 +ingress: + enabled: false +hub: + db: + pvc: + storageClassName: local-path +singleuser: + storage: + dynamic: + storageClass: local-path +``` + +## Exposure option A — NodePort + +```yaml +proxy: + service: + type: NodePort + nodePorts: + http: 30890 # reach the Hub at http://<node-ip>:30890 +``` + +## Exposure option B — LoadBalancer + +```yaml +proxy: + service: + type: LoadBalancer # cloud LB or MetalLB + nodePorts: + http: null +``` + +## Exposure option C — Ingress + TLS (production) + +```yaml +proxy: + service: + type: ClusterIP # ingress fronts the proxy + nodePorts: + http: null + +ingress: + enabled: true + ingressClassName: traefik # or nginx + hosts: + - your.domain.com + tls: + - hosts: + - your.domain.com + secretName: jupyter-tls-cert # a K8s TLS secret, or one cert-manager creates + # annotations: # e.g. cert-manager issuer + # cert-manager.io/cluster-issuer: letsencrypt-prod +``` + +Point a DNS record for `your.domain.com` at the ingress controller. Provide the +TLS secret directly, or let cert-manager mint it via the annotation + an Issuer +you manage. + +## Externally terminated TLS + +If TLS is terminated by something outside the chart (cloud LB, external ingress, +Cloudflare tunnel) rather than the chart's `proxy.https`, tell the Hub the +public scheme is https so `_xsrf` cookies are marked Secure and URLs are https: + +```yaml +custom: + security: + publicScheme: "https" +``` + +## CORS / allowed origins + +Defaults are permissive (`["*"]`); tighten them for a public deployment. + +```yaml +custom: + hub: + allowedOrigins: ["https://portal.example.com"] # Access-Control-Allow-Origin on Hub responses + notebook: + allowedOrigins: ["https://portal.example.com"] # --ServerApp.allow_origin_pat (kernel WebSocket) +``` + +## Shared NFS storage + +### 1. NFS server/export (on a storage/controller node) + +```bash +sudo apt install nfs-kernel-server +sudo mkdir -p /nfs && sudo chown -R nobody:nogroup /nfs && sudo chmod 777 /nfs +echo "/nfs <subnet>/24(rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports +sudo systemctl restart nfs-kernel-server +# worker nodes: +sudo apt install nfs-common +``` + +### 2. Provisioner (creates the `nfs-client` storage class) + +```bash +helm repo add nfs-subdir-external-provisioner \ + https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ +helm repo update +helm install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --namespace nfs-provisioner --create-namespace \ + -f deploy/k8s/nfs-provisioner/values.yaml +# optional: make it default +kubectl patch storageclass nfs-client \ + -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +``` + +### 3. Point the chart at it + +```yaml +hub: + db: + pvc: + storageClassName: nfs-client +singleuser: + storage: + dynamic: + storageClass: nfs-client +``` + +Changing the class affects **new** PVCs only; existing Hub DB / user homes are +not migrated automatically. + +## Apply and verify + +```bash +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl get svc,ingress -n jupyterhub +kubectl get storageclass +kubectl get pvc -A +``` + +Load the public URL over HTTPS, log in, and confirm a spawned pod's PVC binds on +the intended storage class. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Ingress 404 / no route | Controller/class/host mismatch | `kubectl get ingress -n jupyterhub`; confirm `ingressClassName` + DNS | +| TLS cert not issued | cert-manager annotation/Issuer wrong, or secret missing | Describe the ingress + the Certificate; check the issuer | +| Login loops / `_xsrf` errors behind a proxy | External TLS without `publicScheme: https` | Set `custom.security.publicScheme: "https"`, re-apply | +| Mixed-content / blocked embed | `allowedOrigins` too strict/loose | Adjust `custom.hub`/`notebook.allowedOrigins` | +| PVC Pending | Storage class missing / NFS export wrong | `kubectl get storageclass`; provisioner logs; `showmount -e <nfs>` | +| code-server reachable without login | Pod port exposed directly | Only expose the JupyterHub proxy; never NodePort/ingress port `8888` | diff --git a/skills/expose-aup-learning-cloud/skill-card.md b/skills/expose-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..1490c4ed --- /dev/null +++ b/skills/expose-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud network exposure and storage — NodePort/LoadBalancer/ingress, TLS, CORS, and shared NFS — for operators taking a deployment beyond the local demo defaults. + +## Owner + +AMD Research diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md new file mode 100644 index 00000000..70f730b3 --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -0,0 +1,106 @@ +--- +name: install-aup-learning-cloud-single-node +description: >- + Group: Plan & deploy AUP Learning Cloud. Installs AUP Learning Cloud on a + single machine with the ./auplc-installer + flow (single-node k3s + JupyterHub for an AMD GPU/APU workstation). Use when + the user wants to install, set up, try, or demo AUP Learning Cloud / AUPLC on + one box, mentions ./auplc-installer, the installer TUI, "install" / "quick + start" / "single-node", --gpu / --courses / --image-source flags, a Ryzen AI + APU or Radeon dGPU dev box, localhost:30890, or uninstalling it. Also covers + the OEM kernel + Docker prerequisites and offline (pack) bundles. Do not use + for multi-node or PXE/netboot clusters (use deploy-aup-learning-cloud), for + building images (build-aup-learning-cloud-images), or for editing courses + (configure-aup-learning-cloud-courses). +--- + +# Install AUP Learning Cloud (single node) + +Stand up AUP Learning Cloud on one machine using the project's own installer: +detect the GPU, install single-node k3s, pull images, deploy the ROCm device +plugin, and `helm install` the Hub so the user can open `localhost:30890` and +spawn notebooks. This is the "quick start / dev / demo" path. + +The installer is the source of truth. Your job is to confirm prerequisites, +pick the right flags, run it (gating the risky steps), and verify. Full flag +table, offline flow, and troubleshooting are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` (run from its root). +- Hardware: a supported **Ryzen AI 300-series+ APU** or **Radeon 9000-series** + GPU; 32 GB+ RAM (64 GB recommended); 500 GB+ SSD. +- **Ubuntu 24.04**. Docker installed and usable without `sudo` + (`docker run hello-world` as the user). +- **Ryzen AI APU only:** the ROCm OEM kernel + (`sudo apt install linux-image-6.14.0-1018-oem`) and a reboot. Radeon dGPU + boxes typically use the stock kernel — confirm against ROCm docs. +- For the interactive TUI: `python3-questionary` + `python3-prompt-toolkit` + (apt), or `pip install questionary prompt_toolkit` in a venv. The + non-interactive `./auplc-installer install` does not need these. + +## Phase 1 — Interview (keep it short) + +1. **GPU**: let the installer auto-detect, or have the user name it so you can + pass `--gpu` (`phx`, `strix`, `strix-halo`, `9070xt`, `r9700`, `9600gre`, + `rdna4`). Confirm with `./auplc-installer detect-gpu`. +2. **Courses**: `all` (default), `basic` (cpu/gpu + code-server), `none` + (Hub only), or an explicit list (`cpu,gpu,Course-CV`). +3. **Image source**: `pull` (default, from `ghcr.io/amdresearch`) or `build` + (local from `dockerfiles/`). For a quick demo prefer `pull`. +4. **Online or offline**: a normal machine with internet, or an air-gapped one + that needs a `pack` bundle (see reference). + +## Phase 2 — Verify the environment + +```bash +docker run --rm hello-world # docker works rootless +uname -r # OEM kernel on Ryzen AI APU +./auplc-installer detect-gpu # installer agrees with the hardware +./auplc-installer install --dry-run # prints the Configuration summary, no changes +``` + +Read the `--dry-run` summary back to the user and **get confirmation before the +real install** — it installs k3s system-wide and needs sudo. + +## Phase 3 — Install (confirmation gate) + +Default, opinionated path: + +```bash +./auplc-installer install # auto GPU, all courses, pull images +# or pin choices: +./auplc-installer install --gpu=strix-halo --courses=basic --image-tag=develop +``` + +The installer runs 8 stages (detect GPU → values overlay → helm+k9s → k3s → +pull images → ROCm device plugin + labeller → refresh overlay from node labels +→ deploy Hub). It prompts for sudo once. Use `-y` only for scripted/CI runs. + +## Phase 4 — Verify + +```bash +kubectl get nodes # the node is Ready +kubectl get pods -n jupyterhub # hub + proxy Running, no CrashLoop/ImagePull +``` + +Open `http://localhost:30890` — the default values auto-log-in as `student` +(NodePort 30890, `local-path` storage, ingress disabled). Spawn a CPU notebook, +then a GPU notebook, and confirm the GPU pod schedules. + +## Safety + +Stop and get explicit confirmation before: + +- The real `install` (installs k3s + a containerd/Docker runtime, needs sudo). +- `./auplc-installer uninstall` (removes k3s **and** the runtime; data loss). +- Switching `--runtime` (docker ↔ containerd) on an existing install. +- Any `--image-source=build` run on a slow/low-disk box (large local builds). + +Never commit changes to the checkout. The installer writes a local values +overlay (e.g. `values.local.yaml`); do not commit it. + +## Reference + +Flag-by-flag table, the offline `pack`/air-gapped flow, `dev`/`rt` +subcommands, default-values facts, and troubleshooting: [reference.md](reference.md). diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md new file mode 100644 index 00000000..8faf677e --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -0,0 +1,128 @@ +# Install AUP Learning Cloud (single node) — Reference + +Full flag table, offline flow, subcommands, and troubleshooting for the +`./auplc-installer` single-node path. Workflow and gates are in +[SKILL.md](SKILL.md). + +## Source guides + +- Quick Start / Single-Node: <https://amdresearch.github.io/aup-learning-cloud/installation/> +- Repo README "Quick Start" section. + +Treat the installer's `--help` and the live docs as the source of truth for +flags and version pins; this file condenses the opinionated path. + +## Prerequisite commands + +```bash +# Ryzen AI APU only: ROCm OEM kernel (reboot afterwards) +sudo apt update && sudo apt install linux-image-6.14.0-1018-oem + +# Docker (rootless usage) +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker "$USER" && newgrp docker +sudo apt install build-essential + +# Interactive TUI deps (system Python) +sudo apt install python3-questionary python3-prompt-toolkit +``` + +## Commands + +| Command | What it does | +| --- | --- | +| `./auplc-installer` | Launch the interactive TUI (when a real terminal is attached). | +| `./auplc-installer install [--pull]` | Full install: k3s + images + runtime. Default pulls pre-built images. | +| `./auplc-installer install --dry-run` | Print the Configuration summary and exit. No sudo, no changes. | +| `./auplc-installer uninstall` | Remove everything (k3s + runtime). **Destructive.** | +| `./auplc-installer install-tools` | Install `helm` + `k9s` only. | +| `./auplc-installer detect-gpu` | Show the detected GPU configuration. | +| `./auplc-installer img build [target...]` | Build images (see build-aup-learning-cloud-images). | +| `./auplc-installer img pull` | Pull external images for offline use. | +| `./auplc-installer pack [--local]` | Create an offline deployment bundle. | +| `./auplc-installer rt install\|reinstall\|upgrade\|remove` | Runtime (Hub) only — for image/values changes without touching k3s. | +| `./auplc-installer dev [deploy\|upgrade\|reinstall]` | Dev cycle: rebuild hub image + restart, with a dev overlay (student=admin, pullPolicy=Never). | + +## Flags + +| Flag | Values / default | Notes | +| --- | --- | --- | +| `--gpu=TYPE` | `auto` (default), `phx`, `strix`, `strix-halo`, `9070xt`, `r9700`, `9600gre`, `rdna4`/`dgpu`, `gfxNNNN` | Auto-detect via rocminfo/KFD. Env `GPU_TYPE`. | +| `--courses=SPEC` | `all` (default), `basic`, `none`, or `cpu,gpu,Course-CV,...` | Restricts image build/pull **and** hides unselected courses in the spawn UI. Env `AUPLC_COURSES`. | +| `--image-source=SRC` | `pull` (default) or `build` | `pull` = registry; `build` = local from `dockerfiles/`. | +| `--image-registry=PREFIX` | default `ghcr.io/amdresearch` | Env `IMAGE_REGISTRY`. | +| `--image-tag=TAG` | default `latest` | GPU suffix appended automatically. Env `IMAGE_TAG`. Use `develop` for the preview UI. | +| `--runtime=MODE` | `docker` (default) or `containerd` | `docker` makes images visible to k3s immediately; `containerd` exports for offline. | +| `--courses`, `--mirror=`, `--mirror-pip=`, `--mirror-npm=` | — | Registry / PyPI / npm mirrors for restricted networks. | +| `-y`, `--yes` | — | Assume yes (scripted/CI). Env `AUPLC_YES=1`. | +| `--dry-run` (`--try-run`) | — | Preview only. | +| `-v`, `--verbose` | — | Stream every subprocess line. Env `AUPLC_VERBOSE=1`. | + +### Examples + +```bash +./auplc-installer install --dry-run +./auplc-installer install --image-source=pull --image-tag=develop +./auplc-installer install --gpu=strix-halo --courses=basic +./auplc-installer install --runtime=containerd --image-source=build +./auplc-installer install --mirror=mirror.example.com +``` + +## What a successful install looks like + +``` + ✓ [1/8] Detecting GPU + ✓ [2/8] Generating values overlay (initial) + ✓ [3/8] Installing helm + k9s + ✓ [4/8] Installing K3s (single-node) + ✓ [5/8] Pulling custom + external images + ✓ [6/8] Deploying ROCm GPU device plugin + node labeller + ✓ [7/8] Refreshing values overlay from node labels + ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) + + Open in your browser: http://localhost:30890 + (auto-logged-in as 'student' — no login needed) +``` + +## Default deployment facts + +The checked-in defaults describe a local deployment: NodePort **30890**, +`local-path` storage, ingress **disabled**, prePuller **disabled**, and +`custom.authMode: auto-login`. To change auth, courses, or accelerators, layer +a values overlay (see configure-aup-learning-cloud-courses) and +`./auplc-installer rt upgrade`. + +## Offline / air-gapped (pack) + +On a machine with Docker + internet: + +```bash +./auplc-installer pack --gpu=strix-halo # pull pre-built images into a bundle +./auplc-installer pack --gpu=strix-halo --local # or build locally first +``` + +Transfer the bundle, then on the air-gapped box: + +```bash +tar xzf auplc-bundle-gfx1151-*.tar.gz +cd auplc-bundle-gfx1151-* +sudo ./auplc-installer install +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `detect-gpu` shows the wrong/no GPU | ROCm not seeing the device, wrong kernel | OEM kernel installed + rebooted (`uname -r`), `rocminfo`, pass `--gpu=` explicitly | +| Install fails pulling images | Registry/network or wrong tag | `--image-tag`, `--mirror=`, or `--image-source=build` | +| Hub pod `ImagePullBackOff` | Tag mismatch between overlay and registry | `kubectl describe pod -n jupyterhub`, align `--image-tag` | +| GPU notebook stays Pending | Device plugin/labeller not ready or label mismatch | `kubectl get ds -A | grep amd`, `kubectl describe node | grep amd.com/gpu` | +| `localhost:30890` refused | Proxy not up or NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | +| `docker` permission denied | User not in docker group | re-run `usermod -aG docker $USER` then re-login / `newgrp docker` | +| Need to re-apply values only | Changed the overlay, not images | `./auplc-installer rt upgrade` (don't reinstall k3s) | + +## Out of scope + +Multi-node / PXE clusters (use deploy-aup-learning-cloud), GitHub OAuth and +production TLS/ingress hardening, image authoring, and course-catalog edits +(those are their own skills). This skill targets the one-box install. diff --git a/skills/install-aup-learning-cloud-single-node/skill-card.md b/skills/install-aup-learning-cloud-single-node/skill-card.md new file mode 100644 index 00000000..fcb03048 --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Install AUP Learning Cloud on a single AMD GPU/APU machine with the ./auplc-installer flow, for developers and demos. + +## Owner + +AMD Research diff --git a/skills/manage-aup-learning-cloud-users/SKILL.md b/skills/manage-aup-learning-cloud-users/SKILL.md new file mode 100644 index 00000000..7da07854 --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/SKILL.md @@ -0,0 +1,147 @@ +--- +name: manage-aup-learning-cloud-users +description: >- + Group: Maintain AUP Learning Cloud. Manages users, groups, passwords, admins, + and quota balances day to day with the built-in AUP Learning Cloud scripts + (scripts/generate_users_template.py and scripts/manage_users.py) plus the web + admin console (/hub/admin). Use when the user wants to onboard a class, + generate a roster CSV/Excel, bulk-create native users, export or back up users, + generate/reset passwords, force or skip first-login password changes, grant or + revoke admins, delete users, create or edit groups, run GitHub group sync, or + set/add/list quota balances and scheduled quota refresh rules. Triggers include + manage_users.py, generate_users_template.py, users.csv, passwords_output.csv, + /hub/admin, jupyterhub-admin-credentials, JUPYTERHUB_URL, JUPYTERHUB_TOKEN, + set-admin, set-passwords, set-quota, add-quota, list-quota, refreshRules, + "onboard a class", and "bulk users". Do not use to choose auth mode, configure + course visibility/quota rates, or install/deploy a cluster. +--- + +# Manage AUP Learning Cloud users + +Run the day-2 people operations: create and onboard users (including a whole +class), set/reset passwords, manage admins and groups, and grant or refresh +quota balances. Prefer the repository's deterministic scripts for bulk work and +use the web console for interactive inspection or one-off admin edits. + +The two built-in scripts are the primary automation surface: + +- `scripts/generate_users_template.py` creates CSV/Excel rosters with the + columns `manage_users.py` expects. +- `scripts/manage_users.py` performs API-backed user/admin/password work and + quota commands that also exec into the Hub pod. + +Exact command variants, file formats, env setup, and the quota field guide are +in **[reference.md](reference.md)**. + +## Prerequisites + +- A running Hub and an **admin** account (or `custom.adminUser.enabled: true` + and the bootstrapped `admin`). +- For CLI work: run from the `aup-learning-cloud` checkout and install + `pandas`, `openpyxl`, and `requests` in the Python environment that runs the + scripts. +- `manage_users.py` requires `JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN` for every + subcommand. The bundled `scripts/hub-api-env.sh` derives both from the + `jupyterhub-admin-credentials` secret and checks reachability. +- Quota subcommands also require `kubectl` access to the Hub namespace because + they call `kubectl exec deployment/hub` after the API-token preflight. +- Native-user creation/password reset requires `authMode: multi` (or another + mode with native accounts). Password actions never apply to GitHub identities. + +## Two surfaces + +| Task | Best surface | Command | +| --- | --- | --- | +| Generate roster | CLI | `generate_users_template.py --prefix student --count 50 -o users.csv` | +| Create users | CLI for bulk, web for one-off | `manage_users.py create users.csv` | +| Passwords | CLI for bulk native-user resets | `manage_users.py set-passwords users.csv --generate -o passwords_output.csv` | +| Admins | CLI or web | `manage_users.py set-admin [--file admins.csv] [--revoke]` | +| Groups | Web console | `/hub/admin` Groups view, including Sync Now | +| Quota | CLI for repeatable grants, web for inspection | `set-quota` / `add-quota` / `list-quota` | +| Export/backup | CLI | `manage_users.py export backup.xlsx` | + +Unlimited quota is entered as `-1`, `∞`, or `unlimited`. Admin users and the +current admin are protected from deletion. + +## Workflow — onboard a class (most common) + +1. **Confirm the live script surface.** The project can evolve; quickly check + help before composing a large batch command: + + ```bash + python scripts/generate_users_template.py --help + python scripts/manage_users.py --help + ``` +2. **Set env** so `manage_users.py` can reach the Hub API: + + ```bash + source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh + ``` + + (Or export `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` yourself — see reference.) + Use `HUB_URL="https://hub.example.com"` and `HUB_NAMESPACE=<namespace>` when + the Hub is not the default local NodePort in namespace `jupyterhub`. +3. **Generate a roster template**: + + ```bash + python scripts/generate_users_template.py --prefix student --count 50 --output users.csv + ``` +4. **Inspect the roster.** Confirm the `username` and optional `admin` columns, + and remember usernames are normalized to lowercase by `manage_users.py`. +5. **Create the users**: + + ```bash + python scripts/manage_users.py create users.csv + ``` +6. **Issue passwords** (generated, forced change on first login by default): + + ```bash + python scripts/manage_users.py set-passwords users.csv --generate -o passwords_output.csv + ``` +7. **Promote teaching staff** as needed: + + ```bash + python scripts/manage_users.py set-admin teacher01 teacher02 + ``` +8. **Grant starting quota** (if quota is enabled): + + ```bash + python scripts/manage_users.py set-quota student01 student02 --amount 1000 + ``` +9. **Deliver credentials securely** from `passwords_output.csv`, then verify in + `/hub/admin` (users appear, groups correct, balances set). + +## Quota operations + +This skill owns quota **operations** (granting/refreshing balances, scheduled +refresh). Quota **rates and enable/disable knobs** (`custom.quota.*`, +`accelerators.*.quotaRate`) live in the configure-courses skill. + +- One-off: `set-quota` (absolute) / `add-quota` (delta) / `list-quota`, or the + inline/batch editors and global "Refresh Quota" in `/hub/admin`. +- File-driven: `set-quota --file quotas.csv` expects `username,quota` columns; + `add-quota --file users.csv --amount 100` expects at least `username`. +- Scheduled: `custom.quota.refreshRules` become Kubernetes CronJobs. Verify with + `kubectl -n jupyterhub get cronjobs -l app.kubernetes.io/component=quota-refresh`. + The rule schema is in [reference.md](reference.md). + +## Safety + +- **Credentials are sensitive.** Generated passwords and `passwords_output.csv` + must be delivered securely and never committed. +- **Check rosters before writes.** Generated users are easy to create in bulk; + inspect the CSV/Excel and confirm count, prefix, admin flags, and target Hub + before running `create`, `set-passwords`, `set-admin`, or quota commands. +- **Bulk delete is destructive.** `manage_users.py delete … --yes` removes + accounts; confirm the list with the user first. Admins/current admin are + protected, but data on user PVCs can still be orphaned. +- **`set-admin` grants full platform control** — confirm the target list. +- **Quota refresh rules apply broadly.** A global Refresh Quota or a broad + `refreshRules` filter touches many users; confirm before applying. +- CLI quota commands run `kubectl exec` into `deployment/hub`; they need both a + valid API token for the script preflight and a healthy kube context/namespace. + +## Reference + +Env setup, every `manage_users.py` subcommand, the admin console views, +`refreshRules` schema, and troubleshooting: [reference.md](reference.md). diff --git a/skills/manage-aup-learning-cloud-users/reference.md b/skills/manage-aup-learning-cloud-users/reference.md new file mode 100644 index 00000000..929f7315 --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/reference.md @@ -0,0 +1,235 @@ +# Manage AUP Learning Cloud users — Reference + +Env setup, the built-in user-management script surface, roster file formats, +the admin console views, the `refreshRules` schema, and troubleshooting. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- User Management Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/user-management.html> +- User Quota System: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/quota-system.html> + +The live `scripts/generate_users_template.py` and `scripts/manage_users.py` in +`aup-learning-cloud` are the source of truth; verify subcommands/flags against +`--help` before large batches. + +```bash +python scripts/generate_users_template.py --help +python scripts/manage_users.py --help +python scripts/manage_users.py set-passwords --help +python scripts/manage_users.py set-quota --help +``` + +## API environment + +`manage_users.py` checks the Hub API before executing any subcommand. Set +`JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN`; the token comes from the +admin-credentials secret (requires `custom.adminUser.enabled`): + +```bash +export JUPYTERHUB_URL="http://localhost:30890" +export JUPYTERHUB_TOKEN=$(kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' | base64 -d) +``` + +The bundled `scripts/hub-api-env.sh` does this and probes `/hub/api/`. Source +it (don't execute) so the exports land in your shell: + +```bash +source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +# override the URL if not localhost:30890: +HUB_URL="https://hub.example.com" source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +``` + +CLI **quota** commands also use `kubectl exec` into `deployment/hub`, so they +need a working kube context + namespace in addition to the API token preflight. + +## Python dependencies + +```bash +pip install pandas openpyxl requests +``` + +## Generate roster templates + +Use `generate_users_template.py` to create the input files that +`manage_users.py` consumes. It supports numbered users or explicit names, CSV or +Excel output, optional admin flags, custom starting numbers, and digit padding. + +```bash +python scripts/generate_users_template.py --prefix student --count 50 --output users.csv +python scripts/generate_users_template.py --prefix AUP --count 30 --start 1 --output aup_users.xlsx +python scripts/generate_users_template.py --prefix student --count 100 --digits 3 --output users.csv +python scripts/generate_users_template.py --prefix admin --count 5 --admin --output admins.csv +python scripts/generate_users_template.py --names alice bob charlie --output custom_users.csv +``` + +Generated files contain at least: + +```csv +username,admin +student01,false +student02,false +``` + +You can add a `password` column before `set-passwords`, and `set-quota --file` +can read a `quota` column. + +## manage_users.py subcommands + +```bash +# Users +python scripts/manage_users.py create users.csv +python scripts/manage_users.py list +python scripts/manage_users.py export backup.xlsx +python scripts/manage_users.py delete remove_list.csv --yes + +# Admins +python scripts/manage_users.py set-admin teacher01 teacher02 +python scripts/manage_users.py set-admin --file admins.csv +python scripts/manage_users.py set-admin --revoke student01 + +# Passwords (native users only) +python scripts/manage_users.py set-passwords users.csv --generate -o passwords_output.csv +python scripts/manage_users.py set-passwords users.csv --generate --default-password "Welcome123" +python scripts/manage_users.py set-passwords users.csv --no-force-change + +# Quota +python scripts/manage_users.py set-quota user1 user2 --amount 1000 # absolute +python scripts/manage_users.py set-quota --file quotas.csv # username,quota columns +python scripts/manage_users.py add-quota user1 user2 --amount 100 # delta +python scripts/manage_users.py add-quota --file users.csv --amount 100 +python scripts/manage_users.py list-quota +``` + +Every command accepts `--url` and `--token`, but prefer exported env vars so +tokens do not appear in shell history: + +```bash +python scripts/manage_users.py --url "$JUPYTERHUB_URL" --token "$JUPYTERHUB_TOKEN" list +``` + +### Command behavior notes + +- Usernames are normalized to lowercase before API writes, matching JupyterHub's + default behavior. Avoid rosters that depend on case-sensitive usernames. +- `create` reads `username` and optional `admin`; it does not set passwords. + Run `set-passwords` after creating native users. +- `set-passwords` requires either a `password` column or `--generate`. Generated + passwords can be saved with `--output`; that file is sensitive. +- `set-passwords` forces first-login password change unless + `--no-force-change` is passed. +- `set-quota` with positional users requires `--amount`; with `--file`, the file + can provide per-user `quota` values. +- `delete --yes` skips the interactive confirmation and should only be used + after the exact roster has been reviewed. + +## Web admin console (`/hub/admin`) + +- **Users view:** search/page, filter to active servers, create native users + (single or many, random or shared password, force change, optional admin), + edit details, reset password (native), batch password reset, inline quota + edit, batch quota update, start/stop servers, batch delete, per-user usage. + Admins and the current admin are protected from deletion. +- **Groups view:** distinguishes GitHub-synced, system-managed, and manual + groups; create manual groups, edit membership of editable groups, review + group-to-resource mappings, and **Sync Now** (manual GitHub sync when + `custom.githubOrgName` is set). System-managed groups are read-only; + GitHub-synced groups are protected from deletion. +- **Dashboard view:** total users, active sessions, usage minutes, weekly active + users, usage trends, resource distribution, top users, live sessions, pending + spawns. + +Admin quota API endpoints used by the UI: `GET/POST /admin/api/quota/`, +`POST /admin/api/quota/batch`, `POST /admin/api/quota/refresh`, +`GET /api/quota/rates`, `GET /api/quota/me`. + +## Scheduled quota refresh (`refreshRules`) + +Configured under `custom.quota.refreshRules`; each rule becomes a CronJob. + +```yaml +custom: + quota: + refreshRules: + daily-topup: + enabled: true + schedule: "0 0 * * *" # cron + action: add # add | set + amount: 100 + maxBalance: 500 # also: minBalance + targets: + includeUnlimited: false + balanceBelow: 400 # also: balanceAbove, includeUsers, + # excludeUsers, usernamePattern +``` + +Verify: + +```bash +kubectl -n jupyterhub get cronjobs -l app.kubernetes.io/component=quota-refresh +kubectl -n jupyterhub get jobs -l app.kubernetes.io/component=quota-refresh +kubectl -n jupyterhub logs -l app.kubernetes.io/component=quota-refresh --tail=50 +``` + +Changing rate/enablement knobs (`custom.quota.enabled`, `cpuRate`, +`minimumToStart`, `defaultQuota`, `accelerators.*.quotaRate`) is the +configure-courses skill; re-apply with `rt upgrade` / `helm upgrade`. + +## Common runbooks + +### Onboard 50 native students + +```bash +pip install pandas openpyxl requests +source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +python scripts/generate_users_template.py --prefix student --count 50 --output users.csv +python scripts/manage_users.py create users.csv +python scripts/manage_users.py set-passwords users.csv --generate --output passwords_output.csv +python scripts/manage_users.py list +``` + +Review `passwords_output.csv`, distribute it through a secure channel, then +delete it when no longer needed. + +### Add teaching assistants as admins + +```bash +python scripts/generate_users_template.py --names ta01 ta02 --admin --output tas.csv +python scripts/manage_users.py create tas.csv +python scripts/manage_users.py set-passwords tas.csv --generate --output ta_passwords.csv +python scripts/manage_users.py set-admin --file tas.csv +``` + +### Grant class quota + +```bash +python scripts/manage_users.py set-quota --file quotas.csv +python scripts/manage_users.py add-quota --file users.csv --amount 100 +python scripts/manage_users.py list-quota +``` + +`quotas.csv` should contain `username,quota` when using `set-quota --file`. +`users.csv` only needs `username` for `add-quota --file`. + +## Apply config changes + +```bash +# single-node +sudo ./auplc-installer rt upgrade +# multi-node / manual +cd runtime && helm upgrade --install jupyterhub ./chart \ + -n jupyterhub --create-namespace -f values-multi-nodes.yaml +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Script cannot connect to the Hub | `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` wrong | Confirm both; `curl -H "Authorization: token $JUPYTERHUB_TOKEN" $JUPYTERHUB_URL/hub/api/` | +| Password reset fails | Target is a GitHub user, weak password, or session lacks perms | Native users only; meet the strength policy | +| Quota command passes API check but fails later | CLI uses `kubectl exec` into `deployment/hub` | Check kube context, namespace, and `kubectl -n jupyterhub get deploy/hub` | +| No api-token secret | `custom.adminUser.enabled: false` | Enable admin bootstrap, re-apply | +| Group membership can't be edited | System-managed or GitHub-synced group | Only manual/editable groups accept edits | +| Refresh rule didn't run | Rule disabled or absent from the applied values | `kubectl … get cronjobs -l …quota-refresh`; re-apply | +| Users log in with lowercase names | Script and JupyterHub normalize usernames | Keep rosters lowercase or communicate normalized usernames | diff --git a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh new file mode 100644 index 00000000..7894e08e --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh @@ -0,0 +1,45 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Derive the JupyterHub API environment for AUP Learning Cloud user-management +# scripts and probe reachability. SOURCE this file (do not execute) so the +# exports persist in your shell: +# +# source scripts/hub-api-env.sh +# HUB_URL="https://hub.example.com" source scripts/hub-api-env.sh +# +# Environment inputs (all optional): +# HUB_URL Hub base URL (default: http://localhost:30890) +# HUB_NAMESPACE Kubernetes namespace (default: jupyterhub) +# +# Exports on success: JUPYTERHUB_URL, JUPYTERHUB_TOKEN + +_auplc_ns="${HUB_NAMESPACE:-jupyterhub}" +_auplc_url="${HUB_URL:-http://localhost:30890}" + +_auplc_token="$(kubectl -n "$_auplc_ns" get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' 2>/dev/null | base64 -d 2>/dev/null)" + +if [ -z "$_auplc_token" ]; then + echo "hub-api-env: could not read api-token from secret 'jupyterhub-admin-credentials'" >&2 + echo " - is custom.adminUser.enabled: true and the Hub deployed?" >&2 + echo " - is your kube context/namespace ('$_auplc_ns') correct?" >&2 + return 1 2>/dev/null || exit 1 +fi + +export JUPYTERHUB_URL="$_auplc_url" +export JUPYTERHUB_TOKEN="$_auplc_token" + +# Probe the API (non-fatal: token may still be valid behind an auth proxy). +if command -v curl >/dev/null 2>&1; then + _auplc_code="$(curl -s -o /dev/null -w '%{http_code}' \ + -H "Authorization: token ${JUPYTERHUB_TOKEN}" \ + "${JUPYTERHUB_URL%/}/hub/api/" 2>/dev/null)" + case "$_auplc_code" in + 200) echo "hub-api-env: OK — $JUPYTERHUB_URL/hub/api/ reachable (200)" ;; + *) echo "hub-api-env: WARNING — $JUPYTERHUB_URL/hub/api/ returned '$_auplc_code'; check HUB_URL/network" >&2 ;; + esac +fi + +echo "hub-api-env: exported JUPYTERHUB_URL=$JUPYTERHUB_URL and JUPYTERHUB_TOKEN (hidden)" + +unset _auplc_ns _auplc_url _auplc_token _auplc_code diff --git a/skills/manage-aup-learning-cloud-users/skill-card.md b/skills/manage-aup-learning-cloud-users/skill-card.md new file mode 100644 index 00000000..39e91ca5 --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Manage AUP Learning Cloud users, groups, passwords, admins, and quota balances day to day with the built-in roster/template and user-management scripts, for operators and teaching staff running classes. + +## Owner + +AMD Research diff --git a/skills/monitor-aup-learning-cloud/SKILL.md b/skills/monitor-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..ae3bdc75 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/SKILL.md @@ -0,0 +1,129 @@ +--- +name: monitor-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Wires AUP Learning Cloud into a + Prometheus + Grafana monitoring stack: enables + the chart's monitoring resources (ServiceMonitor, authenticated metrics token, + Grafana dashboard ConfigMaps, PrometheusRule alerts, and the metrics + NetworkPolicy) and connects them to kube-prometheus-stack or an existing + Prometheus Operator. Use when the user wants to monitor the Hub, scrape + /hub/metrics, set up Prometheus/Grafana/alerts, install kube-prometheus-stack, + enable a ServiceMonitor, see the AUP Hub Grafana dashboards, or debug a hub + target that is DOWN / Unauthorized / not scraped. Triggers include + monitoring.enabled, serviceMonitor, releaseLabel, hubMetrics, + allowUnauthenticatedScrape, prometheusRule, grafana.dashboard, + kube-prometheus-stack, hub-metrics, hub_spawn_failed_total, + hub-metrics-token. Do not use to install/deploy the platform itself + (install-/deploy-aup-learning-cloud) or to edit courses/quota + (configure-aup-learning-cloud-courses). +--- + +# Monitor AUP Learning Cloud + +Turn on Hub observability: have the chart create the monitoring objects +(`ServiceMonitor`, authenticated token secret, Grafana dashboard ConfigMaps, +alert rules, metrics `NetworkPolicy`) and make a Prometheus Operator stack +scrape `/hub/metrics` so dashboards and alerts light up. + +Enable the `monitoring.*` block in a values overlay and re-apply with Helm. The +full value reference, the kube-prometheus-stack install, and troubleshooting are +in **[reference.md](reference.md)**. + +## Prerequisites + +- A running (or about-to-deploy) AUP Learning Cloud, plus `helm` + `kubectl`. +- Either install `kube-prometheus-stack` (reference) **or** an existing + Prometheus Operator + Grafana you can point at the `jupyterhub` namespace. +- Know the Prometheus Operator's selector label — the chart stamps `release: + <monitoring.releaseLabel>` on `ServiceMonitor`/`PrometheusRule`, and it must + match what the operator selects. + +## Decide the integration + +| Situation | Action | +| --- | --- | +| No monitoring stack yet | Install `kube-prometheus-stack` as release `monitoring` in namespace `monitoring`; keep `releaseLabel: monitoring` | +| Existing Prometheus Operator + Grafana | Set `monitoring.releaseLabel` to the operator's selector; confirm it watches `monitoring` ns and can scrape `jupyterhub` | + +## Workflow + +1. **Ensure a stack exists.** Confirm Prometheus Operator + Grafana are running + (install kube-prometheus-stack if not — see reference). +2. **Enable monitoring values** in the overlay. Recommended production shape: + + ```yaml + monitoring: + enabled: true + namespace: monitoring + releaseLabel: monitoring + hubMetrics: + enabled: true + allowUnauthenticatedScrape: false + serviceMonitor: + enabled: true + interval: 15s + authorization: + enabled: true + type: Bearer + hubServiceName: prometheus-metrics + secret: { create: true, name: "", key: token } + grafana: + dashboard: { enabled: true } + prometheusRule: + enabled: true + ``` + +3. **Keep `releaseLabel` honest.** It must equal the operator's rule/monitor + selector or nothing gets scraped. +4. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed (the chart validates that + `hubServiceName` exists under `hub.services` with a matching `read:metrics` + role). +5. **Apply.** + + ```bash + helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + ``` + +6. **Verify** the objects and the live target: + + ```bash + skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh + ``` + + It checks the `ServiceMonitor`, token secret, dashboard ConfigMap, and + metrics `NetworkPolicy`, then port-forwards Prometheus and confirms the + `hub` target is `UP`. Manual checks are in [reference.md](reference.md). + +## Authenticated scraping (default, recommended) + +`/hub/metrics` requires a JupyterHub token. The `ServiceMonitor` authorization +block makes the chart create a token secret (`<release>-metrics-token`) in the +monitoring namespace and scrape with a Bearer token. Annotation-based scraping +cannot attach the token — leave `serviceAnnotations` off in production. + +## Useful Hub metrics + +`hub_spawn_gpu_total`, `hub_spawn_failed_total`, `hub_active_sessions`, +`hub_session_runtime_minutes`, `hub_spawn_duration_seconds`, +`hub_quota_denied_total`, `hub_quota_deducted_total`, `hub_pod_failure_total`, +`hub_repo_clone_failed_total`. Alert rules cover `hub_spawn_failed_total` and +`hub_pod_failure_total`. + +## Safety + +- **Do not set `allowUnauthenticatedScrape: true` in production.** It exposes + `/hub/metrics` without a token; only safe in an isolated dev cluster where the + endpoint is never reachable via proxy/NodePort/LoadBalancer/Ingress. +- A `helm upgrade` restarts the Hub pod (brief login blip) — schedule around a + live class. +- Don't commit any real metrics token; the chart manages the secret. +- Read-only verification (`scripts/verify_monitoring.sh`) only port-forwards; + it makes no cluster changes. + +## Reference + +The full `monitoring.*` value reference, kube-prometheus-stack install, +existing-stack reuse, manual verification commands, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/monitor-aup-learning-cloud/reference.md b/skills/monitor-aup-learning-cloud/reference.md new file mode 100644 index 00000000..cacaf968 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/reference.md @@ -0,0 +1,103 @@ +# Monitor AUP Learning Cloud — Reference + +The kube-prometheus-stack install, the full `monitoring.*` value reference, +existing-stack reuse, manual verification, and troubleshooting. Workflow and +gates are in [SKILL.md](SKILL.md). + +## Source guide + +- Monitoring Deployment Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/monitoring.html> +- Configuration Reference (section 12): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> + +The chart ships the dashboards under `runtime/chart/dashboards/`; the live +`runtime/chart/values.schema.yaml` is the source of truth for the schema. + +## Install kube-prometheus-stack (reference stack) + +```bash +kubectl create namespace monitoring # AlreadyExists is safe to ignore + +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm upgrade --install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring + +kubectl -n monitoring get pods +``` + +The release name `monitoring` makes the operator select `release: monitoring`, +matching the default `monitoring.releaseLabel`. If you use a different release +name or selector, set `monitoring.releaseLabel` to match. + +## Reuse an existing Prometheus + Grafana + +Confirm with the monitoring owner that: + +- the Operator watches `ServiceMonitor` in the `monitoring` namespace, +- Prometheus may scrape services in `jupyterhub`, +- the operator's selector matches `release: <monitoring.releaseLabel>`, +- the Grafana sidecar reads dashboard ConfigMaps labelled `grafana_dashboard: "1"` + from `monitoring`. + +Example: if the stack selects `release: platform-monitoring`, set +`monitoring.releaseLabel: platform-monitoring`. + +## monitoring.* value reference + +| Value | Meaning | +| --- | --- | +| `monitoring.enabled` | Master switch for all monitoring objects | +| `monitoring.namespace` | Namespace the objects are created in (`monitoring`) | +| `monitoring.releaseLabel` | `release` label on ServiceMonitor/PrometheusRule; must match the operator selector | +| `monitoring.hubMetrics.enabled` | Hub metrics integration; also creates a metrics NetworkPolicy allowing the monitoring ns to reach the Hub on `8081` | +| `monitoring.hubMetrics.allowUnauthenticatedScrape` | Allow `/hub/metrics` without a token — dev only | +| `monitoring.hubMetrics.serviceAnnotations.enabled` | Adds `prometheus.io/*` annotations; cannot carry the token — prefer the ServiceMonitor path | +| `monitoring.serviceMonitor.enabled` | Creates `ServiceMonitor` `hub-metrics` selecting `component: hub`, port `8081`, path `<hub.baseUrl>/hub/metrics` | +| `monitoring.serviceMonitor.interval` | Scrape interval, e.g. `15s` | +| `monitoring.serviceMonitor.authorization.enabled` | Authenticated scraping (keep on) | +| `monitoring.serviceMonitor.authorization.type` | Default `Bearer` | +| `monitoring.serviceMonitor.authorization.hubServiceName` | Hub service account for the token; default `prometheus-metrics` must match `hub.services` + `hub.loadRoles` (`read:metrics`) | +| `monitoring.serviceMonitor.authorization.secret.create` | Create the token secret in the monitoring ns | +| `monitoring.serviceMonitor.authorization.secret.name` | Custom/existing secret name; blank = `<release>-metrics-token` | +| `monitoring.serviceMonitor.authorization.secret.key` | Secret key; default `token` | +| `monitoring.grafana.dashboard.enabled` | Creates dashboard ConfigMaps labelled `grafana_dashboard: "1"` | +| `monitoring.prometheusRule.enabled` | Creates alert rules for `hub_spawn_failed_total`, `hub_pod_failure_total` | + +## Apply + +```bash +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> +# include any local overlay too, e.g. -f runtime/values.local.yaml +``` + +## Manual verification + +```bash +kubectl -n monitoring get servicemonitor hub-metrics +kubectl -n monitoring get secret | grep metrics-token +kubectl -n monitoring get configmap grafana-dashboard-aup-hub +kubectl -n jupyterhub get networkpolicy hub-metrics +# alerts, if enabled: +kubectl -n monitoring get prometheusrule hub-alerts + +# Is the target UP? +kubectl -n monitoring port-forward svc/monitoring-kube-prometheus-prometheus 9090:9090 & +curl -fsSL 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22hub%22%7D' +# open http://127.0.0.1:9090/targets and look for hub-metrics = UP +``` + +A healthy query returns `"job":"hub"`, `"namespace":"jupyterhub"`, value `"1"`. +The dashboard ConfigMap should contain `aup-hub-operations.json` and +`aup-hub-notebook-resources.json`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| ServiceMonitor exists but no scraping | `release` label mismatch | `kubectl -n monitoring get servicemonitor hub-metrics --show-labels`; fix `releaseLabel`, re-apply | +| Target DOWN / Unauthorized | Annotation scraping or auth disabled | Use `serviceMonitor.authorization.enabled: true`, `serviceAnnotations` off | +| Token secret missing | Auth/secret create not enabled, or `hubServiceName` invalid | Enable `secret.create`; ensure `hubServiceName` exists under `hub.services` with `read:metrics` | +| Grafana dashboards absent | Sidecar not watching ns/label | ConfigMap label `grafana_dashboard: "1"`; sidecar must watch `monitoring` | +| Alerts absent | Rule ns/label not watched | `kubectl -n monitoring get prometheusrule hub-alerts --show-labels`; match the operator's rule selector | diff --git a/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh new file mode 100755 index 00000000..e27e428f --- /dev/null +++ b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Read-only verification that AUP Learning Cloud monitoring is wired up: +# checks the ServiceMonitor, metrics token secret, Grafana dashboard ConfigMap, +# and metrics NetworkPolicy, then port-forwards Prometheus and confirms the +# Hub target is UP. Makes no cluster changes. +# +# Usage: +# scripts/verify_monitoring.sh +# +# Environment (optional): +# MON_NS monitoring namespace (default: monitoring) +# HUB_NS jupyterhub namespace (default: jupyterhub) +# PROM_SVC Prometheus service (default: monitoring-kube-prometheus-prometheus) + +set -uo pipefail + +MON_NS="${MON_NS:-monitoring}" +HUB_NS="${HUB_NS:-jupyterhub}" +PROM_SVC="${PROM_SVC:-monitoring-kube-prometheus-prometheus}" + +rc=0 +pass() { printf ' [OK] %s\n' "$1"; } +warn() { printf ' [WARN] %s\n' "$1"; rc=1; } + +echo "Checking monitoring objects (mon ns=$MON_NS, hub ns=$HUB_NS)..." + +kubectl -n "$MON_NS" get servicemonitor hub-metrics >/dev/null 2>&1 \ + && pass "ServiceMonitor hub-metrics present" \ + || warn "ServiceMonitor hub-metrics missing (serviceMonitor.enabled?)" + +if kubectl -n "$MON_NS" get secret 2>/dev/null | grep -q 'metrics-token'; then + pass "metrics token secret present" +else + warn "metrics token secret missing (authorization.secret.create?)" +fi + +kubectl -n "$MON_NS" get configmap grafana-dashboard-aup-hub >/dev/null 2>&1 \ + && pass "Grafana dashboard ConfigMap present" \ + || warn "Grafana dashboard ConfigMap missing (grafana.dashboard.enabled?)" + +kubectl -n "$HUB_NS" get networkpolicy hub-metrics >/dev/null 2>&1 \ + && pass "metrics NetworkPolicy present" \ + || warn "metrics NetworkPolicy missing (hubMetrics.enabled?)" + +echo "Checking the live Prometheus target..." +if ! kubectl -n "$MON_NS" get svc "$PROM_SVC" >/dev/null 2>&1; then + warn "Prometheus service '$PROM_SVC' not found; set PROM_SVC to your service name" + echo "Done (with warnings)."; exit "$rc" +fi + +kubectl -n "$MON_NS" port-forward "svc/$PROM_SVC" 9090:9090 >/dev/null 2>&1 & +pf_pid=$! +trap 'kill "$pf_pid" 2>/dev/null' EXIT +sleep 3 + +result="$(curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22hub%22%7D' 2>/dev/null)" +case "$result" in + *'"job":"hub"'*'"1"'*) pass "Prometheus reports hub target UP" ;; + *'"job":"hub"'*) warn "hub target found but not UP (value != 1)" ;; + *) warn "hub target not found in Prometheus (label/selector mismatch?)" ;; +esac + +echo "Done." +exit "$rc" diff --git a/skills/monitor-aup-learning-cloud/skill-card.md b/skills/monitor-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..d8e45314 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Connect AUP Learning Cloud to Prometheus and Grafana — ServiceMonitor, authenticated metrics, dashboards, and alert rules — for operators who need Hub observability. + +## Owner + +AMD Research diff --git a/skills/plan-aup-learning-cloud-deployment/SKILL.md b/skills/plan-aup-learning-cloud-deployment/SKILL.md new file mode 100644 index 00000000..37e42259 --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/SKILL.md @@ -0,0 +1,148 @@ +--- +name: plan-aup-learning-cloud-deployment +description: >- + Group: Plan & deploy AUP Learning Cloud. Recommends the hardware sizing, + cluster topology, network plan, and a + buyer-facing bill of materials (BOM) for someone who saw an AUP Learning + Cloud / AUPLC demo and wants to stand up their own local deployment. Use + when the user asks how many AIPCs / machines / GPUs / routers they need, + wants sizing or a hardware recommendation, says "I saw the demo and want to + deploy this myself", "what should I buy", "bill of materials", "spec out a + lab", or describes a class headcount and a network (routers, subnets, static + IP vs DHCP) and wants a configuration. It interviews requirements + network, + researches current AMD silicon, and sizes the cluster. Do not use to actually + run the install (install-aup-learning-cloud-single-node), build a multi-node + cluster (deploy-aup-learning-cloud), or edit the course catalog + (configure-aup-learning-cloud-courses) — hand off to those once the plan is + agreed. +--- + +# Plan an AUP Learning Cloud deployment + +Turn a prospective adopter's needs into a concrete recommendation: how many +AMD machines (Ryzen AI AIPC, Radeon workstation, or server) and how much +networking gear to buy, which chips to pick, what cluster topology to use, and +an IP/network plan — ending in a sizing table and a buyer-facing bill of +materials (BOM). This is the pre-purchase advisory step that precedes the +install/deploy skills. + +The single measurable outcome: a defensible BOM + sizing/topology/network plan +the user can act on. Full sizing math, the hardware-research method, the +network decision table, worked examples, and the BOM template are in +**[reference.md](reference.md)**. + +## Prerequisites + +- Web access (to look up the latest AMD silicon and confirm ROCm support). +- No cluster or checkout is required — this skill produces a plan, not a + running system. +- Helpful context: the AUP Learning Cloud + [overview](https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html) + and [quick start](https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html). + +## Phase 1 — Interview the requirements + +Ask, and confirm back, before sizing anything: + +1. **Courses/toolkits** wanted: Computer Vision, Deep Learning, LLM-from-scratch, + Physics Sim, and/or generic CPU/GPU + code-server. This drives both the GPU + VRAM tier and which images to enable later. +2. **Total headcount** and the **session pattern**: a whole class on at the same + time (scheduled lab) vs self-paced/錯峰 usage. +3. **Peak concurrent users**, split into **GPU sessions vs CPU-only sessions**. + If the user only knows the total, estimate peak (see reference) and confirm. +4. **Persistence/storage** expectations (do notebooks need to survive reboots; + rough per-user disk). +5. **Budget band** and **online vs air-gapped**. + +## Phase 2 — Interview the network environment + +1. How many **routers**, and how many **subnets / CIDRs** with which IP ranges. +2. **Static IP vs DHCP**; can a stable/reserved IP be given to one machine. +3. A **managed switch** and how many **free ports** (PoE not needed). +4. **Internet access** from the would-be service machine; any VLANs/firewalls. +5. Whether the machines are **bare (can netboot)** or will each get an OS. + +## Phase 3 — Research current AMD hardware + +Do not rely on memory — **web-search the latest AMD silicon** and match it to +the requirements: + +1. Search current AMD options across form factors: **Ryzen AI APUs** (mini-PC / + laptop AIPC), **Radeon workstation dGPUs**, and **multi-GPU workstations or + servers**. Compare by **compute (CU/TFLOPs) and VRAM**, not marketing tier. +2. **Gate every candidate on ROCm support** — if a chip is not ROCm-supported it + cannot run the GPU notebooks. +3. **Map the chip to a chart accelerator key** (`phx`, `strix`, `strix-halo`, + `9070xt`, `r9700`, or the generic `rdna4`) and the expected + `amd.com/gpu.product-name` node label. A brand-new chip with no existing key + is a flag to raise with the user. +4. Prefer **multi-GPU chassis** (workstation/server) when peak concurrent GPU + users is high enough that many single-GPU AIPCs become impractical to cable, + power, and manage. Keep AIPCs for small labs and the demo-like experience. + +## Phase 4 — Size the cluster + +The full formulas and per-notebook config table are in +[reference.md](reference.md). The shape of it: + +1. **Concurrency, not headcount.** Convert total users to **peak concurrent** + (~40-60% of total for self-paced; ~100% for a whole-class scheduled lab). +2. **GPU drives machine count (whole-GPU, no sharing).** Each GPU notebook in + AUPLC claims a **whole, exclusive** `amd.com/gpu: "1"` (request == limit); + there is no time-slicing/MIG, and this is the same for every GPU course. So + `GPUs needed = peak concurrent GPU users`. An APU box = 1 GPU; a + workstation/server = N cards. +3. **RAM/CPU sets the per-machine spec.** CPU notebooks are best-effort and pack + densely (RAM-bound): `RAM ≈ (concurrent users on the node × max mem/user) + + overhead`. Pick per-user memory from the course type (reference table). +4. **VRAM picks the chip tier.** Exclude 4GB iGPUs (780M/890M) for LLM/large + models; steer to Strix Halo (64GB) or R9700 (32GB) for those. +5. **Add a control/service node.** PXE/NFS/k3s-server overhead; small labs may + co-locate it on a GPU node (state the single-point-of-failure trade-off). + +## Phase 5 — Plan topology and network + +1. **Choose the topology** (decision table in reference): + - **Single-node** (`./auplc-installer`) for one box / demo replica. + - **PXE-diskless cluster** for bare AIPCs on **one flat L2 subnet** that can + netboot (relies on the user's existing DHCP/router; the service machine + needs a static IP). + - **SSH-preinstalled cluster** when nodes already have an OS or the network is + routed/multi-subnet. +2. Derive the **switch-port count** (≈ nodes + uplink) and whether the existing + router(s) suffice or a managed switch is needed. +3. Produce an **IP plan**: the static service-machine IP, the node subnet/CIDR, + gateway, and DNS — consistent with the topology you chose. + +## Phase 6 — Deliver the recommendation + +Produce, for the user: + +- A **sizing table** (peak concurrency → GPU count → machine count + the chosen + chip/VRAM, with the assumptions spelled out). +- A **topology choice** and an **IP/network plan**. +- A **bill of materials**: machine model + quantity + GPU, plus switch/router and + cabling, framed so the user can purchase (this is what leads to AMD hardware + sales). Offer at least an AIPC-based option and a denser workstation/server + option when concurrency is non-trivial. +- A **handoff**: point to `install-aup-learning-cloud-single-node` (one box) or + `deploy-aup-learning-cloud` (cluster) to execute, and + `configure-aup-learning-cloud-courses` to enable the chosen courses. + +## Safety + +- **Advisory only.** This skill plans; it does not install, buy, or change any + system. Never run installer/deploy commands from here. +- **State every assumption** (concurrency ratio, per-user memory, GPUs per + chassis) so the user can correct them before spending money. +- **Always confirm ROCm support** for any recommended silicon; never recommend a + chip you could not verify is supported. +- **Flag single-point-of-failure** trade-offs of all-in-one small labs, and + storage durability (local-path/NFS-on-one-box is disposable without backups). + +## Reference + +Sizing formulas + per-notebook config table, the whole-GPU evidence, the +hardware-research method, the network/topology decision table, worked examples, +the BOM template, and the interview question bank: [reference.md](reference.md). diff --git a/skills/plan-aup-learning-cloud-deployment/reference.md b/skills/plan-aup-learning-cloud-deployment/reference.md new file mode 100644 index 00000000..fe2cb597 --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/reference.md @@ -0,0 +1,289 @@ +# Plan an AUP Learning Cloud deployment — Reference + +Sizing math, the whole-GPU evidence, the hardware-research method, the +network/topology decision table, worked examples, the BOM template, and the +interview question bank. The workflow and gates are in [SKILL.md](SKILL.md). + +## Contents + +- [Source guides](#source-guides) +- [Sizing model](#sizing-model) +- [Per-notebook resource config (typical)](#per-notebook-resource-config-typical) +- [Accelerator catalog and VRAM tiers](#accelerator-catalog-and-vram-tiers) +- [Researching current AMD hardware](#researching-current-amd-hardware) +- [Topology and network decision](#topology-and-network-decision) +- [Sizing procedure](#sizing-procedure) +- [Worked examples](#worked-examples) +- [BOM template](#bom-template) +- [Interview question bank](#interview-question-bank) +- [Handoff](#handoff) + +## Source guides + +- Overview: <https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html> +- Quick Start (single-node): <https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html> +- 3-node mini-cluster (PXE diskless): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html> +- Standard multi-node (SSH): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> + +Treat the live `aup-learning-cloud` repo (`runtime/values.yaml`, the spawner) +and AMD's current product pages as the sources of truth; this file condenses +the opinionated sizing path. + +## Sizing model + +The model is validated against industry JupyterHub capacity-planning practice. + +### Concurrency, not headcount + +Size on **peak concurrent users**, not total registrations — the always-on Hub +overhead is tiny and costs scale with simultaneously active users +([JupyterHub capacity planning](https://jupyterhub.readthedocs.io/en/stable/explanation/capacity-planning.html)). +Rule of thumb: peak concurrent ≈ **40-60% of total** for self-paced cohorts +([TLJH](https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html), +[UC Berkeley CDSS](https://cdss.berkeley.edu/choosing-right-jupyterhub-infrastructure)). +Use **~100%** when a whole class is scheduled on at the same time. + +### GPU dimension = machine count (whole-GPU, exclusive, no sharing) + +In AUP Learning Cloud every GPU notebook claims a **whole, exclusive GPU**. +The spawner sets both the guarantee (request) and the limit to the same +integer, in +[`runtime/hub/core/spawner/kubernetes.py`](https://github.com/AMDResearch/aup-learning-cloud/blob/main/runtime/hub/core/spawner/kubernetes.py) +(around lines 740-743): + +```python +if "amd.com/gpu" in requirements: + self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} + self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} +``` + +`amd.com/gpu` is a Kubernetes **integer extended resource** with request == +limit, so a pod takes whole cards only. There is **no fractional / time-slicing +/ MIG / MPS sharing** in this chart (those are NVIDIA-only: +[NVIDIA time-slicing](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/25.10/gpu-sharing.html), +[MIG/MPS](https://kubedojo.com/gpu-sharing-mig-time-slicing-k8s)); the AMD ROCm +k8s device plugin allocates whole devices. This is **universal across every GPU +course** — `gpu`, `code-gpu`, `Course-CV`, `Course-DL`, `Course-LLM`, and +`Course-PhySim` all set `amd.com/gpu: "1"` in `custom.resources.requirements` +and share the same `_configure_spawner()` path; `cpu`, `code-cpu`, and `none` +request no GPU. The count is admin-configurable but only as an integer number +of whole cards. + +Consequence: + +``` +concurrent GPU notebooks = total physical GPUs in the cluster +GPUs needed = peak concurrent GPU users +``` + +- An APU AIPC (e.g. Strix Halo 8060S) = **1 iGPU = 1 concurrent GPU user**. +- A workstation/server holds **N dGPUs = N concurrent GPU users**. +- When all GPUs are busy, extra GPU spawns stay `Pending` until one frees. + +### RAM/CPU dimension = per-machine spec + +CPU notebooks are **best-effort** in the default chart (`cpu: "0"`, +`memory: "0Gi"`), so they pack densely and the binding constraint is **RAM**. +Standard formulas +([TLJH](https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html), +[CDSS](https://cdss.berkeley.edu/choosing-right-jupyterhub-infrastructure)): + +``` +RAM per machine = (concurrent users on that machine × max memory per user) + overhead +vCPU per machine = (concurrent users on that machine × CPU per user) + 20% +``` + +Note: the spawner derives a CPU limit of `cpu × 1.25` and a memory limit of +`memory × 1.5` when not explicitly set, so if you raise the per-course +`requirements` the effective ceiling is a bit higher than the request. + +## Per-notebook resource config (typical) + +Web-sourced typical per-user values; tune with Prometheus once running. z2jh's +default guarantee is 1G RAM, and a conservative classroom starting point is +0.5 CPU + 2GB +([z2jh user resources](https://z2jh.jupyter.org/en/stable/jupyterhub/customizing/user-resources.html)). + +| Course / use | Memory per user | CPU per user | GPU | VRAM note | +| --- | --- | --- | --- | --- | +| Entry / light Python (generic `cpu`, code-server) | 2 GB (limit higher) | 0.5 vCPU | none | — | +| Computer Vision (`Course-CV`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | mid VRAM ok | +| Deep Learning (`Course-DL`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | needs decent VRAM; enlarge `/dev/shm` for PyTorch DataLoader | +| LLM from scratch (`Course-LLM`) | 16 GB+ | 2+ vCPU | 1 whole GPU | **large VRAM** — exclude 4GB iGPUs | +| Physics Sim / Genesis (`Course-PhySim`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | mid/large VRAM | + +DL frameworks try to grab most VRAM; with whole-GPU allocation that is fine +(one user per card), but it also means you cannot pack two GPU users onto one +card. + +## Accelerator catalog and VRAM tiers + +From `runtime/values.yaml` (`custom.accelerators`). The VRAM column is the key +chip-selection driver: + +| Accelerator key | Chip | VRAM | CU | `amd.com/gpu.product-name` | Good for | +| --- | --- | --- | --- | --- | --- | +| `phx` | Radeon 780M (Phoenix iGPU) | 4 GB shared | 12 | `AMD_Radeon_780M_Graphics` | light CPU/GPU only; NOT LLM | +| `strix` | Radeon 890M (Strix iGPU) | 4 GB shared | 16 | `AMD_Radeon_890M_Graphics` | light CPU/GPU only; NOT LLM | +| `strix-halo` | Radeon 8060S (Strix Halo iGPU) | 64 GB unified | 40 | `AMD_Radeon_8060S_Graphics` | CV/DL/LLM/PhySim | +| `9070xt` | Radeon RX 9070 XT | 16 GB GDDR6 | 64 | `AMD_Radeon_RX_9070_XT` | CV/DL; mid LLM | +| `r9700` | Radeon AI PRO R9700 | 32 GB GDDR6 | 64 | `AMD_Radeon_AI_PRO_R9700` | CV/DL/LLM; multi-card workstation/server | + +`phx` also sets `HSA_OVERRIDE_GFX_VERSION: 11.0.0`. If a fleet normalizes a +product name differently, the `nodeSelector` string must be changed to match +the real node label. + +## Researching current AMD hardware + +Always confirm against current AMD product pages; silicon refreshes often. + +1. **Search by form factor and capability**, not tier name: + - Ryzen AI APU mini-PCs / laptops (the AIPC, demo-like experience). + - Radeon workstation dGPUs (e.g. AI PRO class) for single- or multi-card boxes. + - Multi-GPU workstations / rack servers when concurrency is high. +2. **ROCm gate.** Only recommend chips with confirmed ROCm support; otherwise + the GPU notebooks will not run. +3. **Map to a chart key.** Fit the chip to an existing accelerator key + (`phx`/`strix`/`strix-halo`/`9070xt`/`r9700`) and the expected + `amd.com/gpu.product-name`. If it is a brand-new product with no key yet, + tell the user it needs a `configure-aup-learning-cloud-courses` accelerator + entry (and possibly a new image) before deployment. +4. **AIPC vs workstation vs server:** prefer many single-GPU AIPCs for small + labs and the closest match to the demo; switch to multi-GPU chassis when the + GPU count makes cabling/power/management of many boxes impractical. + +## Topology and network decision + +| Topology | When | Network needs | +| --- | --- | --- | +| **Single-node** (`./auplc-installer`) | One box; replicate the demo; ≤ a handful of users sharing one GPU sequentially | Any network; `localhost:30890` | +| **PXE-diskless cluster** | Bare AIPCs that can netboot; small teaching lab; zero per-machine install | **One flat L2 subnet**; the user's existing DHCP/router stays (dnsmasq runs Proxy-DHCP and does NOT hand out leases); service machine needs a **static/reserved IP**; Secure Boot off; netboot in firmware | +| **SSH-preinstalled cluster** | Nodes already run Ubuntu, or the network is routed/multi-subnet, or netboot is not possible | Each node reachable over SSH; tolerates multiple subnets/routers | + +Networking gear rules of thumb: + +- **One flat subnet** is strongly preferred for PXE-diskless (Proxy-DHCP is + broadcast/L2-bound). Multiple routers/subnets break it unless they share a + broadcast domain or you add DHCP relay — in that case prefer SSH-preinstalled. +- **Switch ports ≈ number of nodes + 1 uplink.** A typical consumer router has + ~4 LAN ports; beyond that, add a managed switch (1GbE is fine for a teaching + lab; NFS traffic benefits from 2.5/10GbE on larger clusters). +- **Static IP:** reserve one for the service/control machine (PXE/NFS/k3s + server / API endpoint all use it). Other nodes can be DHCP. +- Keep `k3s_version` and `pxe_k3s_version` in sync (agents must not be newer + than the server) — relevant when handing off to `deploy-aup-learning-cloud`. + +### Sample IP plan (single flat subnet) + +| Item | Value (example) | +| --- | --- | +| Subnet / CIDR | `192.168.1.0/24` | +| Gateway (existing router) | `192.168.1.1` | +| DHCP pool (existing) | `192.168.1.100-199` | +| Service machine (static) | `192.168.1.10` | +| Agents | DHCP from the existing pool (PXE) or static outside it (SSH) | +| Hub access | `http://192.168.1.10:30890` (NodePort) | + +## Sizing procedure + +1. Total users → **peak concurrent** (×0.4-0.6, or ×1.0 for a scheduled class). +2. Split peak into **GPU sessions** and **CPU-only sessions**. +3. **GPU count = peak concurrent GPU users.** Convert to machines by chassis: + AIPC = 1 GPU/box; workstation/server = N GPUs/box. +4. **RAM check** each machine against the CPU/GPU sessions it will host using + the RAM formula; bump per-machine memory or add a box if short. +5. **Chip tier** from per-course VRAM needs (LLM → 64GB Strix Halo or 32GB + R9700; light → smaller is fine). +6. **+1 control/service node** (or co-locate on a GPU node for a tiny lab, with + a stated SPOF caveat). +7. **Research current models** that satisfy 3-5 and are ROCm-supported; produce + the BOM. + +## Worked examples + +### Example A — 30 students, LLM course, one scheduled class slot + +- Concurrency: whole class on together → peak ≈ **30**, all GPU, all need large + VRAM. +- GPUs needed = 30. LLM ⇒ Strix Halo (64GB) or R9700 (32GB). +- **Option 1 (AIPC):** 30× Strix Halo AIPC (1 GPU each) + 1 control node ≈ + **31 machines**, one flat subnet, a 48-port switch. +- **Option 2 (dense):** workstations/servers with 4× R9700 each → ~8 GPU boxes + + 1 control node ≈ **9 machines**; fewer boxes to cable/power/manage, higher + per-box cost. +- Present both; let the user trade box count vs per-box cost. + +### Example B — 60 students, mixed CV/DL, self-paced + +- Concurrency ≈ 50% → peak ≈ **30** active; assume ~20 GPU + ~10 CPU at peak. +- GPUs needed = 20 (CV/DL ⇒ 16-32GB VRAM ok: 9070xt/R9700, or Strix Halo). +- CPU-only 10 sessions pack onto a few nodes; RAM = 10 × ~4GB + overhead ≈ a + single 64GB node handles them, or fold onto GPU nodes. +- ~20 GPU boxes (AIPC) **or** ~5 boxes × 4 cards + 1 control node. + +### Example C — small demo replica + +- 1 box, sequential single-GPU use. Use **single-node** `./auplc-installer` on + one Strix Halo AIPC. No switch/router changes. Hand off to + `install-aup-learning-cloud-single-node`. + +## BOM template + +``` +AUP Learning Cloud — recommended bill of materials + +Requirements assumed: + Courses : <e.g. LLM, DL> + Total students : <N> Peak concurrent: <M> (assumption: <ratio/scheduled>) + Peak GPU sessions : <G> Peak CPU sessions: <C> + +Compute: + <qty> × <AMD machine model> (<chip>, <VRAM>, <GPUs/box>) → <total GPUs> + 1 × control/service node (<model or "co-located">) + +Networking: + 1 × <managed switch, port count> (≈ nodes + uplink) + reuse existing router/DHCP; reserve 1 static IP for the service node + <cabling> + +Topology : <single-node | PXE-diskless | SSH-preinstalled> +Storage : <local-path (single box) | NFS on service node | dedicated NFS> + +Notes / assumptions: + - GPU is whole-card per user (no sharing): concurrent GPU users = total GPUs. + - <SPOF / backup caveats> +Next step : <install-aup-learning-cloud-single-node | deploy-aup-learning-cloud> +``` + +## Interview question bank + +Requirements: + +- Which courses/toolkits (CV / DL / LLM / PhySim / generic)? +- Total students; one scheduled class at a time, or self-paced? +- Best guess at peak concurrent users; how many of those need a GPU? +- Do notebooks need to persist across reboots? Rough per-user disk? +- Budget band? Internet access or air-gapped? + +Network: + +- How many routers? How many subnets/CIDRs and what IP ranges? +- Static IP available for one machine, or DHCP only? +- Managed switch? How many free ports? +- Can the machines network-boot (PXE), or will each get an OS install? +- Any VLANs/firewalls between the machines? + +## Handoff + +| After the plan is agreed | Use skill | +| --- | --- | +| Install on one box / demo replica | `install-aup-learning-cloud-single-node` | +| Build the multi-node cluster (PXE or SSH) | `deploy-aup-learning-cloud` | +| Enable the chosen courses / add an accelerator entry for a new chip | `configure-aup-learning-cloud-courses` | +| Build/publish custom course images | `build-aup-learning-cloud-images` | + +## Out of scope + +Running any install/deploy command, buying hardware, production HA/TLS/ingress +hardening, monitoring, and authoring images or course catalogs — this skill +stops at the recommendation/BOM and hands off. diff --git a/skills/plan-aup-learning-cloud-deployment/skill-card.md b/skills/plan-aup-learning-cloud-deployment/skill-card.md new file mode 100644 index 00000000..e35e1e4c --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Recommends hardware sizing, cluster topology, a network/IP plan, and a buyer-facing bill of materials for a prospective AUP Learning Cloud adopter who saw the demo and wants to deploy locally. + +## Owner + +AMD Research diff --git a/skills/troubleshoot-aup-learning-cloud/SKILL.md b/skills/troubleshoot-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..5ab217db --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/SKILL.md @@ -0,0 +1,93 @@ +--- +name: troubleshoot-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Diagnoses a broken AUP Learning Cloud + deployment against a known list of + causes: PXE/netboot failures, agent nodes not joining, GPU notebooks stuck + Pending or ROCm labels missing, NFS/PVC storage provisioning failures, and + login/authentication problems. Use when the user reports that AUPLC is broken, + a node won't join, a pod is Pending / CrashLoopBackOff / ImagePullBackOff, the + GPU isn't scheduling, storage won't bind, PXE agents won't boot, the Hub login + 404s, or asks to debug/diagnose/figure out why something failed. Evidence-first + and read-only: gather state, identify the cause, then hand off the fix to the + matching deploy/install/configure/upgrade skill. Do not use to perform a fresh + install or a routine config change when nothing is actually failing. +--- + +# Troubleshoot AUP Learning Cloud + +Find the root cause of a failing deployment from runtime evidence, name it, and +point at the fix — without thrashing. Gather state first, match the symptom to +a known cause, change one thing, re-check. The full symptom → cause → checks +matrices live in **[reference.md](reference.md)**. + +## Prerequisites + +- Access to the cluster (`kubectl`, the right `KUBECONFIG`) and/or the service + machine (for PXE/host issues). +- A checkout of `aup-learning-cloud` for config cross-checks. +- The deploy skill's `scripts/detect_cluster.sh` is a fast way to snapshot + nodes, GPU labels, storage classes, and the device plugin/labeller state. + +## Method (don't thrash) + +1. **Scope it.** Which layer is failing — netboot, node join, GPU scheduling, + storage, or auth? One layer at a time. +2. **Gather evidence before acting.** + + ```bash + kubectl get nodes -o wide + kubectl get pods -A | grep -Ev 'Running|Completed' + kubectl describe pod -n jupyterhub <pod> # Events explain Pending/ImagePull + scripts/detect_cluster.sh # from the deploy skill + ``` + +3. **Match to a cause** using the [reference.md](reference.md) matrices. +4. **Change one thing**, then re-check the same evidence. Do not stack + speculative changes. After ~4 failed attempts with no new evidence, stop and + report what you observed and the most likely next step. +5. **Hand off the fix** to the right skill (below) rather than improvising. + +## Where each fix lives + +| Failing layer | Fix with | +| --- | --- | +| PXE rootfs vars / rebuild, agent netboot, NFS rootfs, k3s token publish | deploy-aup-learning-cloud | +| Single-node install / GPU detect / `localhost:30890` | install-aup-learning-cloud-single-node | +| `nodeSelector` ↔ GPU label, course/team/quota, auth mode | configure-aup-learning-cloud-courses | +| Image tag / `ImagePullBackOff` from a missing build | build-aup-learning-cloud-images | +| Version mismatch after a bump, chart rollback | upgrade-aup-learning-cloud | + +## First checks by layer + +- **Netboot:** `systemctl status dnsmasq nfs-kernel-server apache2`, + `journalctl -u dnsmasq`, firmware boot order + Secure Boot, TFTP files in + `/srv/tftp`. +- **Node join:** `systemctl status k3s-agent`, `journalctl -u k3s-agent`, + hostname/`api_endpoint`/token, `curl http://<SERVICE_IP>:8080/k3s/token`. +- **GPU:** `kubectl get ds -A | grep amd`, + `kubectl describe node <n> | grep amd.com/gpu`, then compare to + `custom.accelerators.*.nodeSelector`. +- **Storage:** `kubectl get pvc -A`, provisioner logs, `showmount -e <NFS>`, + `/etc/exports`. +- **Auth:** Hub logs (`kubectl logs -n jupyterhub deploy/hub`), `custom.authMode` + (avoid `dummy`, whose login 404s), GitHub OAuth callback URL. + +## Safety + +Evidence-first and read-only by default. Stop and get explicit confirmation +before any state change, especially: + +- `kubectl delete node <name>` (clears a stale node object — debugging only). +- `helm uninstall`, `helm rollback`, or recreating any PVC (data loss). +- `pb-k3s-reset.yml` (whole cluster or `--limit <node>`). +- Rebuilding the PXE rootfs under running agents (`pxe_rootfs_force_rebuild`). + +Never commit changes or write secrets (k3s token, OAuth secrets, SSH keys) into +tracked files while debugging. + +## Reference + +Full symptom → cause → first-checks matrices for netboot, node join, GPU, +storage, auth, and kubeconfig, plus the reset/escape hatches: +[reference.md](reference.md). diff --git a/skills/troubleshoot-aup-learning-cloud/reference.md b/skills/troubleshoot-aup-learning-cloud/reference.md new file mode 100644 index 00000000..cbdee482 --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/reference.md @@ -0,0 +1,75 @@ +# Troubleshoot AUP Learning Cloud — Reference + +Symptom → cause → first-checks matrices by layer, plus the escape hatches. +Method and safety gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Multi-Node + 3-node mini-cluster troubleshooting sections: + <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- The deploy skill's reference troubleshooting table (PXE/agent detail). + +## PXE / netboot + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Playbook fails immediately on an assert | A required PXE var is empty | `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, ≥1 SSH key | +| Agent never shows the PXE menu | Firmware boot order, netboot disabled, Proxy-DHCP not reaching client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | +| Agent gets an IP but can't load boot files | TFTP blocked, missing files, Secure Boot on | `/srv/tftp`, firewall, Secure Boot disabled, dnsmasq logs | +| Agent has no network during netboot | NIC lacks an in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | +| Agent kernel boots but can't mount rootfs | NFS export / subnet ACL / wrong `pxe_controller_ip` | `showmount -e <SERVICE_IP>`, `/etc/exports`, rootfs kernel args | +| Agent waits for the k3s token | Token not published / apache ACL blocks subnet | `curl http://<SERVICE_IP>:8080/k3s/token`, apache config | +| Agent joins once but fails after reboot | Missing local k3s persistence / lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | + +## Node join (SSH topology) + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Agent node does not join | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent -n 100`, `/etc/hosts`, `ping <server>` | +| Agent fails to join with a version error | Agent k3s newer than server | Align `pxe_k3s_version`/agent version with server `k3s_version` | + +## GPU scheduling + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| GPU notebook stays Pending | `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub <pod>` (Events), node labels | +| `amd.com/gpu` labels missing | Device plugin / labeller not running | `kubectl get ds -A | grep amdgpu`, `kubectl describe node | grep amd.com/gpu` | +| Label exists but selector doesn't match | Product-name normalized differently per fleet | Compare real `amd.com/gpu.product-name` to `custom.accelerators.*.nodeSelector` | +| GPU pod runs but ROCm errors | Wrong gfx image or missing `HSA_OVERRIDE_GFX_VERSION` (Phoenix) | Image gfx target, accelerator `env` | + +## Storage + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| PVC stays Pending | StorageClass name mismatch or provisioner can't mount | `kubectl get storageclass`, `kubectl get pvc -A`, provisioner logs | +| NFS provisioner crashing | Wrong `nfs.server`/`nfs.path` or export ACL | `kubectl logs -n nfs-provisioner deploy/nfs-subdir-external-provisioner`, `showmount -e <NFS>`, `/etc/exports` | +| Notebook data not persisting | Using `local-path` on multi-node, or wrong storageClass | `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` = `nfs-client` | + +## Authentication / login + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Login page 404s | `custom.authMode: dummy` | Use `auto-login` (single machine) or a real OAuth mode | +| GitHub login loops/fails | OAuth callback URL or org/team config | `hub.config.GitHubOAuthenticator`, `custom.githubOrgName`, callback URL matches host | +| User sees no courses | Team mapping empty for their group | `custom.teams.mapping`, group membership in Admin console | +| Can't reach admin console | Wrong admin user | `custom.adminUser`, `/hub/admin` | + +## kubeconfig / access + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `permission denied` on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | +| `localhost:30890` refused (single-node) | Proxy down / NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | + +## Escape hatches (gated — confirm with the user) + +```bash +kubectl delete node <name> # clear a stale node object (debug only) +helm history jupyterhub -n jupyterhub # then: helm rollback jupyterhub <rev> +cd deploy/ansible +sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster (DESTRUCTIVE) +sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node> # single node +``` + +After a reset, redeploy with deploy-aup-learning-cloud (multi-node) or +install-aup-learning-cloud-single-node. diff --git a/skills/troubleshoot-aup-learning-cloud/skill-card.md b/skills/troubleshoot-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..9d2d71a9 --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Diagnose AUP Learning Cloud failures — netboot, node join, GPU scheduling, storage, and auth — from runtime evidence, for operators. + +## Owner + +AMD Research diff --git a/skills/upgrade-aup-learning-cloud/SKILL.md b/skills/upgrade-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..b83b6f4a --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/SKILL.md @@ -0,0 +1,81 @@ +--- +name: upgrade-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Upgrades a running AUP Learning Cloud + deployment: the JupyterHub Helm + release/chart and values, and the underlying k3s cluster. Use when the user + wants to upgrade, update, bump, or roll out a new version of AUPLC, the Hub + image, the chart, or k3s on an already-installed cluster; mentions helm + upgrade, ./auplc-installer rt upgrade / rt reinstall, pb-k3s-upgrade, + bumping k3s_version / pxe_k3s_version, or applying a values change to a live + Hub. Covers both single-node (installer) and multi-node (Ansible + Helm) + paths, and the safe ordering of cluster vs chart upgrades. Do not use for the + first install (install-/deploy-aup-learning-cloud), for building images + (build-aup-learning-cloud-images), or for routine course edits + (configure-aup-learning-cloud-courses) unless a version bump is involved. +--- + +# Upgrade AUP Learning Cloud + +Move a live deployment to new versions without losing user data: apply chart / +values / image changes, and (separately, more carefully) upgrade k3s. Two +independent axes — **the Hub (Helm)** and **the cluster (k3s)** — upgraded in a +safe order. Commands per topology and the rollback notes are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A running cluster and a checkout of `aup-learning-cloud` matching (or ahead + of) what is deployed. +- `helm` + `kubectl` (multi-node) or `./auplc-installer` (single-node). +- Know what is changing: values only, Hub image tag, chart version, and/or k3s + version. Each has a different, least-disruptive path. + +## Decide the smallest sufficient action + +| Change | Path | +| --- | --- | +| values.yaml / overlay only | `helm upgrade` (multi) or `./auplc-installer rt upgrade` (single) | +| New Hub/notebook image tag | bump `custom.resources.images`, then the same upgrade; single-node image swap: `rt reinstall` | +| Chart bump | `helm upgrade --install` with the new chart | +| k3s version | Ansible `pb-k3s-upgrade.yml` (multi) — separate, gated step | + +Prefer the narrowest path. A values/image change does **not** require a k3s +upgrade. + +## Workflow + +1. **Snapshot state.** `kubectl get nodes -o wide`, `helm list -n jupyterhub`, + `kubectl get pods -n jupyterhub`. Note the current chart + k3s versions and + that nothing is already broken. +2. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed before any apply. +3. **Upgrade the Hub (Helm).** Apply the chart/values change; watch the + rollout. This restarts the Hub pod (brief login blip); running user servers + are generally unaffected. +4. **Upgrade k3s only if needed** (gated — see Safety). Multi-node uses + `pb-k3s-upgrade.yml`. **Keep `pxe_k3s_version` (PXE rootfs) in sync with the + server `k3s_version`** — agents must not be newer than the server. +5. **Verify end to end.** Nodes `Ready`, no `CrashLoopBackOff`/`ImagePullBackOff`, + the Hub loads, an existing user can log in, and a fresh spawn (CPU then GPU) + works. + +## Safety + +Stop and get explicit confirmation before: + +- **A k3s upgrade** — it restarts the kubelet/control plane and can disrupt + running pods; do it in a maintenance window, server before agents. +- **`pb-k3s-reset.yml`** (whole cluster or `--limit <node>`) — destructive. +- **`helm uninstall`** or any change that recreates the Hub DB PVC — data loss. +- **A Hub image tag bump during a live class** — schedule the restart. + +Never commit changes, and never bump `pxe_k3s_version` above the server +`k3s_version`. If a chart upgrade misbehaves, `helm rollback jupyterhub <rev>` +(see reference) before experimenting further. + +## Reference + +Per-topology commands (single-node installer, multi-node Helm, k3s playbooks), +version-pin locations, `helm history`/`rollback`, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/upgrade-aup-learning-cloud/reference.md b/skills/upgrade-aup-learning-cloud/reference.md new file mode 100644 index 00000000..c07d96cb --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/reference.md @@ -0,0 +1,96 @@ +# Upgrade AUP Learning Cloud — Reference + +Per-topology upgrade commands, version-pin locations, rollback, and +troubleshooting. Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Multi-Node "Apply Later Configuration Changes" + upgrade playbooks: + <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- `scripts/helm_upgrade.bash` and `./auplc-installer help` (`rt`, `dev`). + +## Version-pin locations + +| Pin | File | +| --- | --- | +| k3s server version | `deploy/ansible/inventory.yml` → `k3s_version` | +| PXE agent rootfs k3s version | `deploy/ansible/playbooks/pb-pxe-controller.yml` → `pxe_k3s_version` | +| Hub image tag | `custom.resources.images` (values overlay) + `hub.image.tag` | +| Chart | `runtime/chart/Chart.yaml` | + +Keep `pxe_k3s_version == k3s_version`. The deploy skill's +`scripts/validate.py` cross-checks this. + +## Hub (Helm) upgrade — values / image / chart + +Single-node (installer): + +```bash +./auplc-installer rt upgrade # values change on a running runtime +./auplc-installer rt reinstall # container image change +./auplc-installer dev upgrade # dev overlay (student=admin, pullPolicy=Never) +``` + +Multi-node / manual: + +```bash +# pre-flight render +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +helm upgrade --install jupyterhub ./runtime/chart \ + -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +``` + +(`scripts/helm_upgrade.bash` runs the bare +`helm upgrade jupyterhub runtime/chart -n jupyterhub --values runtime/values.yaml`.) + +## k3s upgrade (multi-node, gated) + +```bash +cd deploy/ansible +# bump k3s_version in inventory.yml first (and pxe_k3s_version to match) +sudo ansible-playbook playbooks/pb-k3s-upgrade.yml +kubectl get nodes -o wide # versions advance, nodes stay Ready +``` + +Upgrade the server first, then agents. For PXE diskless agents, bump +`pxe_k3s_version` and rebuild the rootfs (deploy skill) so netbooted agents +match. + +## Install / refresh Helm itself + +```bash +wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz +cd /tmp && tar -zxvf helm.tar.gz && sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm +# or: ./auplc-installer install-tools # helm + k9s +``` + +## Rollback + +```bash +helm history jupyterhub -n jupyterhub +helm rollback jupyterhub <REVISION> -n jupyterhub +kubectl rollout status -n jupyterhub deploy/hub +``` + +k3s has no one-command rollback; pin back the version in inventory and re-run +the upgrade playbook, or restore from a node/etcd snapshot if you keep one. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Hub pod `CrashLoopBackOff` after upgrade | Bad values / incompatible chart | `kubectl logs -n jupyterhub deploy/hub`, `helm rollback` | +| `ImagePullBackOff` after image bump | Tag not pushed or wrong registry | `kubectl describe pod -n jupyterhub`, confirm the pushed tag | +| Agent fails to rejoin after k3s bump | Agent newer than server / rootfs not rebuilt | Align `pxe_k3s_version`, rebuild rootfs, `journalctl -u k3s-agent` | +| Quota CronJobs missing after upgrade | `custom.quota.refreshRules` changed | `kubectl get cronjob -n jupyterhub` | +| PVC lost / Hub DB reset | PVC recreated by an upgrade | Never delete the Hub DB PVC; restore from backup | + +## Out of scope + +First-time install/deploy, image authoring, and HA/external-DB migrations +(treat those as explicit operator projects). This skill upgrades an existing +deployment in place. diff --git a/skills/upgrade-aup-learning-cloud/skill-card.md b/skills/upgrade-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..dba47fb7 --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Upgrade a running AUP Learning Cloud deployment — the JupyterHub chart/values and the k3s cluster — safely, for operators. + +## Owner + +AMD Research diff --git a/templates/skill-template/SKILL.md b/templates/skill-template/SKILL.md new file mode 100644 index 00000000..60d3b2ce --- /dev/null +++ b/templates/skill-template/SKILL.md @@ -0,0 +1,41 @@ +--- +name: skill-template +description: >- + One- to three-sentence routing description in the third person. State WHAT + this skill produces and WHEN an agent should use it, and list the trigger + words a user is likely to say (product names, file names, commands, error + messages). Keep under 1024 characters. Add negative triggers if the + boundary is easily crossed (e.g. "Do not use for the single-node installer + flow"). Replace this entire block when you copy the template. +--- + +# Skill title + +One paragraph: what this skill does and the single, measurable outcome it +drives toward. + +## Prerequisites + +- List the tools, access, and state the agent must have before starting + (e.g. `kubectl` + `helm` on the operator machine, SSH access, a checkout of + `aup-learning-cloud`). + +## Workflow + +Describe the ordered steps. Use exact commands for fragile operations and +plain instructions for steps with acceptable variation. Keep the body under +500 lines; move long reference material into a sibling `reference.md` and link +to it one level deep. + +1. Step one. +2. Step two. + +## Safety + +Enumerate the risky or irreversible actions that REQUIRE explicit user +confirmation before running. Never commit, push, or write real secrets into +tracked files. + +## Reference + +Link to sibling files such as [reference.md](reference.md). diff --git a/templates/skill-template/reference.md b/templates/skill-template/reference.md new file mode 100644 index 00000000..a2352954 --- /dev/null +++ b/templates/skill-template/reference.md @@ -0,0 +1,19 @@ +# <Skill title> — Reference + +Long-form material that does not belong in `SKILL.md`: full command sequences, +field-by-field config guides, lookup tables, and a troubleshooting table. The +agent loads this only when `SKILL.md` links to it, so keep `SKILL.md` lean and +push the detail here. + +Add a table of contents once this file grows past ~100 lines so the agent can +see the full scope when it previews the top. + +## Section one + +Replace this with real reference content. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| ... | ... | ... | diff --git a/templates/skill-template/skill-card.md b/templates/skill-template/skill-card.md new file mode 100644 index 00000000..36e53cb2 --- /dev/null +++ b/templates/skill-template/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +<one sentence: what the skill does, for whom> + +## Owner + +AMD Research From 88690a4386f5e82b7e2d5313d328da7974540b13 Mon Sep 17 00:00:00 2001 From: KerwinTsaiii <kerwtsai@amd.com> Date: Fri, 17 Jul 2026 17:51:45 +0800 Subject: [PATCH 024/180] fix: satisfy repo lint for integrated skill scripts - gen_configs.py: drop quoted `-> None` annotation (ruff UP037) - gen_configs.py, validate.py: apply ruff formatter style - verify_monitoring.sh: use if/then/else instead of `A && B || C` (SC2015) - hub-api-env.sh: annotate intentional source/exec guard (SC2317) Co-authored-by: Cursor <cursoragent@cursor.com> --- .../scripts/gen_configs.py | 33 +++++++-------- .../scripts/validate.py | 42 +++++++++++-------- .../scripts/hub-api-env.sh | 3 ++ .../scripts/verify_monitoring.sh | 24 +++++++---- 4 files changed, 58 insertions(+), 44 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index e480c936..7c83061e 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -33,6 +33,7 @@ Exit codes: 0 on success; 1 on a spec/validation error; 2 on a usage error. """ + from __future__ import annotations import argparse @@ -78,12 +79,11 @@ "storage": {"class": "nfs-client"}, "proxy": {"node_port": 30890}, "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", - "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, } -def die(msg: str, code: int = 1) -> "None": +def die(msg: str, code: int = 1) -> None: print(f"gen_configs: {msg}", file=sys.stderr) raise SystemExit(code) @@ -134,8 +134,7 @@ def render_inventory(spec: dict, token: str) -> str: " ansible_user: root", f" k3s_version: {k3s_version}", f" token: {yaml_quote(token)}", - " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host']" - " | default(groups['server'][0]) }}\"", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", ] if topo == "pxe-diskless": lines += [ @@ -204,8 +203,10 @@ def render_values(spec: dict) -> str: for key, cfg in accel.items(): product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) if not product: - die(f"accelerator '{key}' has no product_name and no known default; " - "add accelerators.<key>.product_name from `kubectl describe node`") + die( + f"accelerator '{key}' has no product_name and no known default; " + "add accelerators.<key>.product_name from `kubectl describe node`" + ) lines += [ f" {key}:", " nodeSelector:", @@ -246,16 +247,12 @@ def write_file(path: Path, content: str, force: bool, secret: bool = False) -> N def main(argv=None) -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") - ap.add_argument("--out-dir", default="generated", - help="directory to write artifacts into (default: ./generated)") - ap.add_argument("--token-file", - help="read the k3s token from this file instead of generating one") + ap.add_argument("--out-dir", default="generated", help="directory to write artifacts into (default: ./generated)") + ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") ap.add_argument("--force", action="store_true", help="overwrite existing files") - ap.add_argument("--print-schema", action="store_true", - help="print an example cluster-spec and exit") + ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") args = ap.parse_args(argv) if args.print_schema: @@ -290,8 +287,10 @@ def main(argv=None) -> int: write_file(out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), args.force) write_file(out / "values-basic-example.yaml", render_values(spec), args.force) - print("\nNext: review the files, then copy them into your aup-learning-cloud " - "checkout. Never commit inventory.yml -- it holds the k3s token.") + print( + "\nNext: review the files, then copy them into your aup-learning-cloud " + "checkout. Never commit inventory.yml -- it holds the k3s token." + ) return 0 diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 9e073d27..89a33932 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -28,6 +28,7 @@ Exit codes: 0 if every check passed (warnings allowed); 1 if any check failed; 2 on a usage error. """ + from __future__ import annotations import argparse @@ -80,7 +81,7 @@ def list_nonempty(text: str, key: str) -> bool: if not m: return False indent = len(m.group(1)) - tail = text[m.end():].splitlines() + tail = text[m.end() :].splitlines() for line in tail: if not line.strip(): continue @@ -139,8 +140,10 @@ def check_version_sync(repo: Path) -> None: if agent_ver == server_ver: ok(f"k3s_version == pxe_k3s_version ({server_ver})") else: - fail(f"version mismatch: inventory k3s_version={server_ver} but " - f"pxe_k3s_version={agent_ver}. Agents must not be newer than the server.") + fail( + f"version mismatch: inventory k3s_version={server_ver} but " + f"pxe_k3s_version={agent_ver}. Agents must not be newer than the server." + ) def collect_values_text(repo: Path, values: list[str]) -> str: @@ -156,26 +159,25 @@ def collect_values_text(repo: Path, values: list[str]) -> str: def check_accelerator_labels(values_text: str, cluster: dict | None) -> None: - declared = sorted(set(re.findall( - r"amd\.com/gpu\.product-name\s*:\s*[\"']?([A-Za-z0-9_]+)[\"']?", values_text))) + declared = sorted(set(re.findall(r"amd\.com/gpu\.product-name\s*:\s*[\"']?([A-Za-z0-9_]+)[\"']?", values_text))) if not declared: warn("no amd.com/gpu.product-name nodeSelector found in the values overlay") return if cluster is None: - warn("no --cluster snapshot; cannot confirm nodeSelector labels match real " - f"nodes. Declared: {', '.join(declared)}") + warn( + "no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}" + ) return real = set(cluster.get("gpu_product_names", [])) if not real: - warn("cluster snapshot reports no GPU product labels yet (device plugin / " - "labeller not ready?)") + warn("cluster snapshot reports no GPU product labels yet (device plugin / labeller not ready?)") return for d in declared: if d in real: ok(f"nodeSelector '{d}' matches a real node label") else: - fail(f"nodeSelector '{d}' matches no node label. Real labels: " - f"{', '.join(sorted(real))}") + fail(f"nodeSelector '{d}' matches no node label. Real labels: {', '.join(sorted(real))}") def check_helm(repo: Path, values: list[str]) -> None: @@ -187,7 +189,7 @@ def check_helm(repo: Path, values: list[str]) -> None: warn(f"chart not found at {CHART}; skipped dry-run") return cmd = ["helm", "template", "jupyterhub", str(chart)] - for rel in (values or ["runtime/values.yaml"]): + for rel in values or ["runtime/values.yaml"]: p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) if p.exists(): cmd += ["-f", str(p)] @@ -200,11 +202,11 @@ def check_helm(repo: Path, values: list[str]) -> None: def main(argv=None) -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--repo", required=True, help="path to the aup-learning-cloud checkout") - ap.add_argument("--values", action="append", default=[], - help="values file (repeatable); defaults to runtime/values.yaml") + ap.add_argument( + "--values", action="append", default=[], help="values file (repeatable); defaults to runtime/values.yaml" + ) ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -231,8 +233,12 @@ def main(argv=None) -> int: check_helm(repo, args.values) if args.json: - print(json.dumps({"passed": passed, "warnings": warnings, "errors": errors, - "status": "ok" if not errors else "error"}, indent=2)) + print( + json.dumps( + {"passed": passed, "warnings": warnings, "errors": errors, "status": "ok" if not errors else "error"}, + indent=2, + ) + ) else: for m in passed: print(f"[ OK ] {m}") diff --git a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh index 7894e08e..fe5fb6d3 100644 --- a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +++ b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh @@ -23,6 +23,9 @@ if [ -z "$_auplc_token" ]; then echo "hub-api-env: could not read api-token from secret 'jupyterhub-admin-credentials'" >&2 echo " - is custom.adminUser.enabled: true and the Hub deployed?" >&2 echo " - is your kube context/namespace ('$_auplc_ns') correct?" >&2 + # This file is meant to be sourced; `return` exits the caller's shell. The + # `exit 1` fallback only runs if the file is executed directly. + # shellcheck disable=SC2317 return 1 2>/dev/null || exit 1 fi diff --git a/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh index e27e428f..5d426ecf 100755 --- a/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh +++ b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh @@ -26,9 +26,11 @@ warn() { printf ' [WARN] %s\n' "$1"; rc=1; } echo "Checking monitoring objects (mon ns=$MON_NS, hub ns=$HUB_NS)..." -kubectl -n "$MON_NS" get servicemonitor hub-metrics >/dev/null 2>&1 \ - && pass "ServiceMonitor hub-metrics present" \ - || warn "ServiceMonitor hub-metrics missing (serviceMonitor.enabled?)" +if kubectl -n "$MON_NS" get servicemonitor hub-metrics >/dev/null 2>&1; then + pass "ServiceMonitor hub-metrics present" +else + warn "ServiceMonitor hub-metrics missing (serviceMonitor.enabled?)" +fi if kubectl -n "$MON_NS" get secret 2>/dev/null | grep -q 'metrics-token'; then pass "metrics token secret present" @@ -36,13 +38,17 @@ else warn "metrics token secret missing (authorization.secret.create?)" fi -kubectl -n "$MON_NS" get configmap grafana-dashboard-aup-hub >/dev/null 2>&1 \ - && pass "Grafana dashboard ConfigMap present" \ - || warn "Grafana dashboard ConfigMap missing (grafana.dashboard.enabled?)" +if kubectl -n "$MON_NS" get configmap grafana-dashboard-aup-hub >/dev/null 2>&1; then + pass "Grafana dashboard ConfigMap present" +else + warn "Grafana dashboard ConfigMap missing (grafana.dashboard.enabled?)" +fi -kubectl -n "$HUB_NS" get networkpolicy hub-metrics >/dev/null 2>&1 \ - && pass "metrics NetworkPolicy present" \ - || warn "metrics NetworkPolicy missing (hubMetrics.enabled?)" +if kubectl -n "$HUB_NS" get networkpolicy hub-metrics >/dev/null 2>&1; then + pass "metrics NetworkPolicy present" +else + warn "metrics NetworkPolicy missing (hubMetrics.enabled?)" +fi echo "Checking the live Prometheus target..." if ! kubectl -n "$MON_NS" get svc "$PROM_SVC" >/dev/null 2>&1; then From c737944dac2b911fd6a89a81b831e2c9a449b100 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:51:20 +0800 Subject: [PATCH 025/180] fix(skills): harden deployment config helpers --- .../scripts/gen_configs.py | 135 ++- .../scripts/validate.py | 205 +++- tests/skills/test_deploy_scripts.py | 1009 +++++++++++++++++ 3 files changed, 1305 insertions(+), 44 deletions(-) create mode 100644 tests/skills/test_deploy_scripts.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index 7c83061e..e0d91ee8 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -8,9 +8,9 @@ 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the SSH topology, empty for PXE). - 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: the ``vars:`` values - to merge into - deploy/ansible/playbooks/pb-pxe-controller.yml. + 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to + pb-pxe-controller.yml with + ``-e @<absolute-path>``. 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, storage class, proxy NodePort, authMode. @@ -39,8 +39,12 @@ import argparse import base64 import json +import os import secrets +import shutil import sys +import tempfile +from contextlib import suppress from pathlib import Path HEADER_HASH = ( @@ -57,6 +61,7 @@ "strix-halo": "AMD_Radeon_8060S_Graphics", "9070xt": "AMD_Radeon_RX_9070_XT", "r9700": "AMD_Radeon_AI_PRO_R9700", + "9600gre": "AMD_Radeon_RX_9600_GRE", } SCHEMA = { @@ -106,6 +111,31 @@ def yaml_quote(s: str) -> str: return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' +def validate_accelerators(spec: dict) -> None: + if "accelerators" not in spec: + return + accelerators = spec["accelerators"] + if not isinstance(accelerators, dict): + die("spec.accelerators must be a mapping") + unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) + if len(unsupported) == 1: + die(f"unsupported accelerator key '{unsupported[0]}'") + if unsupported: + die(f"unsupported accelerator keys: {', '.join(unsupported)}") + for key, config in accelerators.items(): + if not isinstance(config, dict): + die(f"accelerators.{key} must be a mapping") + + +def validate_config_shapes(spec: dict) -> None: + if not isinstance(spec, dict): + die("spec must be a mapping") + validate_accelerators(spec) + for key in ("network", "pxe", "storage", "proxy", "images"): + if key in spec and not isinstance(spec[key], dict): + die(f"spec.{key} must be a mapping") + + def render_inventory(spec: dict, token: str) -> str: topo = spec["topology"] server = spec["server"] @@ -160,8 +190,8 @@ def render_pxe_vars(spec: dict) -> str: k3s_version = spec["k3s_version"] lines = [ HEADER_HASH, - "# Merge these into the vars: block of", - "# deploy/ansible/playbooks/pb-pxe-controller.yml", + "# Pass this file to pb-pxe-controller.yml with", + "# ansible-playbook ... -e @<absolute-path-to-this-file>", "# pxe_k3s_version is pinned to k3s_version so agents are never newer", "# than the server.", "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", @@ -212,8 +242,12 @@ def render_values(spec: dict) -> str: " nodeSelector:", f" amd.com/gpu.product-name: {yaml_quote(product)}", ] - if images: + if accel or images: lines.append(" resources:") + if accel: + lines += [" metadata:", " gpu:", " acceleratorKeys:"] + lines.extend(f" - {yaml_quote(key)}" for key in accel) + if images: lines.append(" images:") for k, v in images.items(): lines.append(f" {k}: {yaml_quote(v)}") @@ -235,15 +269,77 @@ def render_values(spec: dict) -> str: return "\n".join(lines) + "\n" -def write_file(path: Path, content: str, force: bool, secret: bool = False) -> None: - if path.exists() and not force: - die(f"refusing to overwrite existing {path} (use --force)", 1) +def preflight_destinations(paths: list[Path], force: bool) -> None: + if force: + return + for path in paths: + if os.path.lexists(path): + die(f"refusing to overwrite existing {path} (use --force)", 1) + + +def stage_file(path: Path, content: str, mode: int) -> Path: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - # Tighten perms on files that carry the cluster token. - if secret: - path.chmod(0o600) - print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as staged_file: + staged_file.write(content) + staged_file.flush() + os.fsync(staged_file.fileno()) + except OSError: + with suppress(OSError): + os.close(fd) + Path(staged_path).unlink(missing_ok=True) + raise + return Path(staged_path) + + +def remove_destination(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def backup_destination(path: Path) -> tuple[Path, Path]: + backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) + backup_path = backup_dir / path.name + os.replace(path, backup_path) + return backup_dir, backup_path + + +def publish_artifacts(artifacts: list[tuple[Path, str, int, bool]], force: bool) -> None: + staged: list[tuple[Path, Path, bool]] = [] + published: list[Path] = [] + backups: list[tuple[Path, Path, Path]] = [] + try: + for path, content, mode, secret in artifacts: + staged.append((path, stage_file(path, content, mode), secret)) + for path, staged_path, secret in staged: + if force and os.path.lexists(path): + backup_dir, backup_path = backup_destination(path) + backups.append((path, backup_dir, backup_path)) + if force: + os.replace(staged_path, path) + else: + os.link(staged_path, path) + os.unlink(staged_path) + published.append(path) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + except OSError as exc: + for path in reversed(published): + remove_destination(path) + for path, backup_dir, backup_path in reversed(backups): + remove_destination(path) + os.replace(backup_path, path) + backup_dir.rmdir() + die(f"could not publish generated artifacts: {exc}") + else: + for _, backup_dir, _ in backups: + shutil.rmtree(backup_dir) + finally: + for _, staged_path, _ in staged: + staged_path.unlink(missing_ok=True) def main(argv=None) -> int: @@ -267,12 +363,15 @@ def main(argv=None) -> int: except json.JSONDecodeError as exc: die(f"spec is not valid JSON: {exc}") + if not isinstance(spec, dict): + die("spec must be a mapping") topo = spec.get("topology") if topo not in ("pxe-diskless", "ssh-preinstalled"): die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") require(spec, "k3s_version") require(spec, "server.name") require(spec, "server.ip") + validate_config_shapes(spec) if args.token_file: token = Path(args.token_file).read_text(encoding="utf-8").strip() @@ -282,10 +381,12 @@ def main(argv=None) -> int: token = gen_token() out = Path(args.out_dir) - write_file(out / "inventory.yml", render_inventory(spec, token), args.force, secret=True) + artifacts = [(out / "inventory.yml", render_inventory(spec, token), 0o600, True)] if topo == "pxe-diskless": - write_file(out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), args.force) - write_file(out / "values-basic-example.yaml", render_values(spec), args.force) + artifacts.append((out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), 0o600, False)) + artifacts.append((out / "values-basic-example.yaml", render_values(spec), 0o644, False)) + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) print( "\nNext: review the files, then copy them into your aup-learning-cloud " diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 89a33932..8408f020 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -5,13 +5,13 @@ Catches the mistakes that otherwise surface only after a long playbook or a failed spawn: - * required PXE vars empty (interface / subnet / controller_ip / dns / - k3s_server_ips / at least one authorized key); - * the k3s server version and the PXE agent rootfs version disagree - (agents must not be newer than the server); - * a custom.accelerators.*.nodeSelector that names a GPU product label no - node actually reports (the #1 cause of GPU notebooks stuck Pending) -- - checked against detect_cluster.sh output when supplied; + * required PXE vars empty (PXE topology only: interface / subnet / + controller_ip / dns / k3s_server_ips / at least one authorized key); + * the k3s server version and the PXE agent rootfs version disagree (PXE + topology only; agents must not be newer than the server); + * nodeSelectors for the accelerators actually referenced by effective + custom.resources.metadata.*.acceleratorKeys, checked against + detect_cluster.sh output when supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -20,8 +20,9 @@ inspect something. Usage: - validate.py --repo ~/aup-learning-cloud + validate.py --repo ~/aup-learning-cloud --topology pxe-diskless validate.py --repo ~/aup-learning-cloud \ + --topology ssh-preinstalled \ --values runtime/values.yaml --values runtime/values-basic-example.yaml \ --cluster cluster.json --helm-dry-run @@ -69,6 +70,10 @@ def scalar(text: str, key: str) -> str | None: return val or None +def key_occurrences(text: str, key: str) -> int: + return len(re.findall(rf"^\s*{re.escape(key)}\s*:", text, re.MULTILINE)) + + def list_nonempty(text: str, key: str) -> bool: """True if `key:` is a YAML list with at least one item, or an inline non-empty flow list (``[...]`` with content).""" @@ -93,10 +98,14 @@ def list_nonempty(text: str, key: str) -> bool: return False -def check_pxe_vars(repo: Path) -> None: - pb = repo / PXE_PLAYBOOK +def pxe_vars_path(repo: Path, configured_path: str | None) -> Path: + return Path(configured_path).expanduser() if configured_path else repo / PXE_PLAYBOOK + + +def check_pxe_vars(repo: Path, configured_path: str | None = None) -> None: + pb = pxe_vars_path(repo, configured_path) if not pb.exists(): - warn(f"{PXE_PLAYBOOK} not found; skipping PXE checks (SSH topology?)") + fail(f"PXE vars file not found: {pb}") return text = pb.read_text(encoding="utf-8") required_scalars = { @@ -105,6 +114,10 @@ def check_pxe_vars(repo: Path) -> None: "pxe_controller_ip": "service host IP", "pxe_dns_servers": "rootfs DNS servers", } + safety_keys = [*required_scalars, "pxe_k3s_server_ips", "pxe_rootfs_authorized_keys", "pxe_k3s_version"] + for key in safety_keys: + if key_occurrences(text, key) > 1: + fail(f"duplicate PXE key '{key}' in {pb}") for key, what in required_scalars.items(): if scalar(text, key): ok(f"PXE var {key} is set") @@ -120,13 +133,17 @@ def check_pxe_vars(repo: Path) -> None: fail("PXE var pxe_rootfs_authorized_keys is empty (rootfs would be unreachable)") -def check_version_sync(repo: Path) -> None: +def check_version_sync(repo: Path, configured_path: str | None = None) -> None: inv = repo / INVENTORY - pb = repo / PXE_PLAYBOOK + pb = pxe_vars_path(repo, configured_path) if not inv.exists(): warn(f"{INVENTORY} not found; skipping k3s version sync check") return - server_ver = scalar(inv.read_text(encoding="utf-8"), "k3s_version") + inventory_text = inv.read_text(encoding="utf-8") + if key_occurrences(inventory_text, "k3s_version") > 1: + fail(f"duplicate inventory key 'k3s_version' in {inv}") + return + server_ver = scalar(inventory_text, "k3s_version") if not server_ver: warn("k3s_version not found in inventory.yml") return @@ -146,22 +163,137 @@ def check_version_sync(repo: Path) -> None: ) -def collect_values_text(repo: Path, values: list[str]) -> str: +def yaml_scalar(value: str) -> str: + return value.strip().strip('"').strip("'") + + +def yaml_optional_scalar(value: str) -> str: + scalar_value = yaml_scalar(value) + return "" if scalar_value in {"", "null", "~"} else scalar_value + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_inline_list(value: str) -> list[str]: + items = value.strip()[1:-1].strip() + if not items: + return [] + return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] + + +def is_relevant_flow_path(path: tuple[str, ...]) -> bool: + return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} + + +def unsupported_yaml_syntax(value: str) -> bool: + return value.startswith(("&", "*", "!", "|", ">")) + + +def parse_values_file(text: str) -> tuple[dict[str, str | None], dict[str, list[str]], list[str]]: + """Extract the deploy-relevant mappings from a fixed-shape values YAML file. + + The helpers deliberately remain stdlib-only. This scanner handles the + mapping/list shapes used by values overlays, rather than pretending to be a + general YAML parser. + """ + accelerators: dict[str, str | None] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + + if stripped.startswith("- "): + if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": + metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) + continue + + product_label_match = re.fullmatch( + r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped + ) + if product_label_match: + if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": + value = product_label_match.group(1).strip() + if unsupported_yaml_syntax(value): + parse_errors.append( + f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" + ) + else: + accelerators[path[2]] = yaml_optional_scalar(value) + continue + + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + candidate_path = path + (key,) + if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") + if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + if path == ("custom", "accelerators"): + accelerators.setdefault(key, None) + if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": + resource_key = path[3] + if unsupported_yaml_syntax(value): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + elif value.startswith("[") and value.endswith("]"): + metadata[resource_key] = parse_inline_list(value) + elif not value or value in {"null", "~"}: + metadata[resource_key] = [] + else: + parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") + stack.append((indent, key)) + return accelerators, metadata, parse_errors + + +def collect_effective_values(repo: Path, values: list[str]) -> tuple[dict[str, str], dict[str, list[str]], list[str]]: paths = values or ["runtime/values.yaml"] - chunks = [] + accelerators: dict[str, str] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] for rel in paths: p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) if p.exists(): - chunks.append(p.read_text(encoding="utf-8")) + parsed_accelerators, parsed_metadata, file_errors = parse_values_file(p.read_text(encoding="utf-8")) + for key, selector in parsed_accelerators.items(): + if selector is not None or key not in accelerators: + accelerators[key] = selector + metadata.update(parsed_metadata) + parse_errors.extend(file_errors) else: - warn(f"values file not found: {rel}") - return "\n".join(chunks) + fail(f"values file not found: {rel}") + return accelerators, metadata, parse_errors -def check_accelerator_labels(values_text: str, cluster: dict | None) -> None: - declared = sorted(set(re.findall(r"amd\.com/gpu\.product-name\s*:\s*[\"']?([A-Za-z0-9_]+)[\"']?", values_text))) +def check_accelerator_labels( + accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None +) -> None: + active_keys = sorted({key for keys in metadata.values() for key in keys}) + if not active_keys: + warn("no acceleratorKeys found in effective custom.resources.metadata") + return + declared: list[str] = [] + for key in active_keys: + if key not in accelerators: + fail(f"active accelerator '{key}' is not defined under custom.accelerators") + elif not accelerators[key]: + fail(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") + else: + declared.append(accelerators[key]) if not declared: - warn("no amd.com/gpu.product-name nodeSelector found in the values overlay") return if cluster is None: warn( @@ -171,7 +303,7 @@ def check_accelerator_labels(values_text: str, cluster: dict | None) -> None: return real = set(cluster.get("gpu_product_names", [])) if not real: - warn("cluster snapshot reports no GPU product labels yet (device plugin / labeller not ready?)") + fail("cluster snapshot has no GPU product labels for active accelerators") return for d in declared: if d in real: @@ -202,11 +334,25 @@ def check_helm(repo: Path, values: list[str]) -> None: def main(argv=None) -> int: + global errors, passed, warnings + errors = [] + warnings = [] + passed = [] ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--repo", required=True, help="path to the aup-learning-cloud checkout") + ap.add_argument( + "--topology", + choices=("pxe-diskless", "ssh-preinstalled"), + default="pxe-diskless", + help="deployment topology (default: pxe-diskless)", + ) ap.add_argument( "--values", action="append", default=[], help="values file (repeatable); defaults to runtime/values.yaml" ) + ap.add_argument( + "--pxe-vars", + help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", + ) ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -225,10 +371,15 @@ def main(argv=None) -> int: print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) return 2 - check_pxe_vars(repo) - check_version_sync(repo) - values_text = collect_values_text(repo, args.values) - check_accelerator_labels(values_text, cluster) + if args.topology == "pxe-diskless": + check_pxe_vars(repo, args.pxe_vars) + check_version_sync(repo, args.pxe_vars) + else: + ok("skipped PXE checks for ssh-preinstalled topology") + accelerators, metadata, parse_errors = collect_effective_values(repo, args.values) + for message in parse_errors: + fail(message) + check_accelerator_labels(accelerators, metadata, cluster) if args.helm_dry_run: check_helm(repo, args.values) diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py new file mode 100644 index 00000000..7e5b6f2c --- /dev/null +++ b/tests/skills/test_deploy_scripts.py @@ -0,0 +1,1009 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI regression tests for deploy-skill helper scripts.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import subprocess +import sys +from contextlib import redirect_stdout +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" +VALIDATE = DEPLOY_SCRIPTS / "validate.py" +GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" + + +def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(script), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def write_file(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def write_cluster(repo: Path, labels: list[str]) -> Path: + return write_file(repo / "cluster.json", json.dumps({"gpu_product_names": labels})) + + +def load_validate_module(): + spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_generator_module(): + spec = importlib.util.spec_from_file_location("deploy_generator", GEN_CONFIGS) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_ssh_topology_skips_pxe_checks_and_version_sync(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + write_file( + repo / "deploy/ansible/playbooks/pb-pxe-controller.yml", + """pxe_network_interface: "" +pxe_subnet: "" +pxe_controller_ip: "" +pxe_dns_servers: "" +pxe_k3s_server_ips: [] +pxe_rootfs_authorized_keys: [] +pxe_k3s_version: v1.33.0+k3s1 +""", + ) + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled") + + assert result.returncode == 0, result.stdout + result.stderr + assert "skipped PXE checks for ssh-preinstalled topology" in result.stdout + assert "[FAIL] PXE var" not in result.stdout + assert "version mismatch" not in result.stdout + + +def test_validator_checks_only_effective_active_accelerators_in_values_order(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + phx: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_780M_Graphics + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: + - phx +""", + ) + overlay = write_file( + repo / "runtime/values-strix-halo.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: + - strix-halo +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_8060S_Graphics" in result.stdout + assert "AMD_Radeon_780M_Graphics" not in result.stdout + + +def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + overlay = write_file( + repo / "runtime/values-overlay.yaml", + """custom: + accelerators: + strix-halo: + displayName: "Renamed Strix Halo" +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_8060S_Graphics" in result.stdout + + +def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + 9070xt: + nodeSelector: + "amd.com/gpu.product-name": "AMD_Radeon_RX_9070_XT" + resources: + metadata: + gpu: + acceleratorKeys: [9070xt] +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_RX_9070_XT"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_RX_9070_XT" in result.stdout + + +def test_validator_rejects_relevant_non_empty_flow_mappings(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: {9070xt: {nodeSelector: {amd.com/gpu.product-name: AMD_Radeon_RX_9070_XT}}} + resources: + metadata: + gpu: {acceleratorKeys: [9070xt]} +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping" in result.stdout + + +def test_validator_rejects_flow_style_custom_resources_wrapper(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: {metadata: {gpu: {acceleratorKeys: [strix-halo]}}} +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping at custom.resources" in result.stdout + + +def test_validator_rejects_fully_flow_style_custom_wrapper(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: {accelerators: {strix-halo: {nodeSelector: {amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics}}}, resources: {metadata: {gpu: {acceleratorKeys: [strix-halo]}}}} +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping at custom" in result.stdout + + +def test_validator_rejects_parent_aliases_and_scalar_accelerator_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + alias_values = write_file(repo / "alias.yaml", "defaults: {}\ncustom: *defaults\n") + scalar_keys = write_file( + repo / "scalar-keys.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: strix-halo +""", + ) + + alias_result = run_script( + VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(alias_values) + ) + scalar_result = run_script( + VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(scalar_keys) + ) + + assert alias_result.returncode == 1 + assert "unsupported YAML syntax at custom" in alias_result.stdout + assert scalar_result.returncode == 1 + assert "acceleratorKeys must be a list" in scalar_result.stdout + + +def test_validator_fails_for_missing_explicit_and_default_values_files(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + repo.mkdir() + explicit_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(repo / "missing.yaml"), + ) + default_result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled") + + assert explicit_result.returncode == 1 + assert default_result.returncode == 1 + assert "values file not found" in explicit_result.stdout + assert "values file not found" in default_result.stdout + + +def test_validator_rejects_duplicate_pxe_and_inventory_safety_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\nk3s_version: v1.33.0+k3s1\n") + vars_file = write_file( + repo / "pxe-vars.yml", + """pxe_network_interface: enp1s0 +pxe_network_interface: "" +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: + - 192.168.1.10 +pxe_rootfs_authorized_keys: + - ssh-ed25519 AAAA test@example +pxe_k3s_version: v1.32.3+k3s1 +pxe_k3s_version: v1.33.0+k3s1 +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--pxe-vars", + str(vars_file), + ) + + assert result.returncode == 1 + assert "duplicate PXE key 'pxe_network_interface'" in result.stdout + assert "duplicate PXE key 'pxe_k3s_version'" in result.stdout + assert "duplicate inventory key 'k3s_version'" in result.stdout + + +def test_validator_fails_empty_supplied_cluster_for_active_accelerators(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + cluster = write_file(repo / "cluster.json", "{}") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 1 + assert "cluster snapshot has no GPU product labels" in result.stdout + + +def test_validator_rejects_unsupported_yaml_syntax_at_relevant_values(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + for index, value in enumerate(("&keys [strix-halo]", "*keys", "!list [strix-halo]", "|")): + overlay = write_file( + repo / f"unsupported-keys-{index}.yaml", + f"""custom: + resources: + metadata: + gpu: + acceleratorKeys: {value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + ) + assert result.returncode == 1 + assert "unsupported YAML syntax" in result.stdout + + +def test_validator_rejects_unsupported_yaml_syntax_at_product_selector(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: &label AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported YAML syntax at custom.accelerators.strix-halo.nodeSelector" in result.stdout + + +def test_validator_uses_generated_pxe_vars_file_when_requested(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + spec_path = write_file(repo / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + generated = repo / "generated" + generation = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(generated)) + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + write_file(repo / "deploy/ansible/playbooks/pb-pxe-controller.yml", "pxe_k3s_version: v1.33.0+k3s1\n") + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--pxe-vars", + str(generated / "pb-pxe-controller.vars.yml"), + ) + + assert generation.returncode == 0, generation.stdout + generation.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert "k3s_version == pxe_k3s_version" in result.stdout + + +def test_validator_preserves_explicit_selector_and_accelerator_key_clears(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + selector_clear = write_file( + repo / "selector-clear.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: null +""", + ) + keys_clear = write_file( + repo / "keys-clear.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: ~ +""", + ) + + selector_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(selector_clear), + ) + keys_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(keys_clear), + ) + + assert selector_result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in selector_result.stdout + assert keys_result.returncode == 0, keys_result.stdout + keys_result.stderr + assert "no acceleratorKeys found" in keys_result.stdout + + +def test_validator_honors_every_supported_explicit_clear_syntax(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + for index, clear_value in enumerate(('""', "null", "~")): + selector_overlay = write_file( + repo / f"selector-clear-{index}.yaml", + f"""custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: {clear_value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(selector_overlay), + ) + assert result.returncode == 1 + assert "has no amd.com/gpu.product-name nodeSelector" in result.stdout + + for index, clear_value in enumerate(("null", "~", "[]")): + keys_overlay = write_file( + repo / f"keys-clear-{index}.yaml", + f"""custom: + resources: + metadata: + gpu: + acceleratorKeys: {clear_value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(keys_overlay), + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "no acceleratorKeys found" in result.stdout + + +def test_validator_main_resets_report_state_between_invocations(tmp_path: Path) -> None: + module = load_validate_module() + failed_repo = tmp_path / "failed" + success_repo = tmp_path / "success" + failed_values = write_file( + failed_repo / "runtime/values.yaml", + """custom: + accelerators: {} + resources: + metadata: + gpu: + acceleratorKeys: [missing] +""", + ) + success_values = write_file(success_repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + with redirect_stdout(io.StringIO()): + first = module.main( + ["--repo", str(failed_repo), "--topology", "ssh-preinstalled", "--values", str(failed_values)] + ) + second = module.main( + ["--repo", str(success_repo), "--topology", "ssh-preinstalled", "--values", str(success_values)] + ) + + assert first == 1 + assert second == 0 + + +def test_validator_requires_product_labels_under_active_accelerator_node_selectors(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + env: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout + + +def test_validator_ignores_accelerators_and_metadata_outside_custom_resources(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +other: + accelerators: + typo-gpu: + nodeSelector: + amd.com/gpu.product-name: AMD_Typo_GPU + metadata: + gpu: + acceleratorKeys: [typo-gpu] +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "typo-gpu" not in result.stdout + + +def test_validator_fails_when_an_active_accelerator_key_is_missing(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: {} + resources: + metadata: + gpu: + acceleratorKeys: + - typo-gpu +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'typo-gpu' is not defined under custom.accelerators" in result.stdout + + +def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: {} + resources: + metadata: + gpu: + acceleratorKeys: + - strix-halo +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout + + +def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": {"typo-gpu": {"product_name": "AMD_Typo_GPU"}}, + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "unsupported accelerator key 'typo-gpu'" in result.stderr + assert not out_dir.exists() + + +def test_generator_retains_known_accelerator_product_name_overrides(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": {"strix-halo": {"product_name": "AMD_Custom_8060S"}}, + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stdout + result.stderr + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert 'amd.com/gpu.product-name: "AMD_Custom_8060S"' in values + + +def test_generator_rejects_a_non_mapping_accelerators_field_before_writing_artifacts(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": [], + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "spec.accelerators must be a mapping" in result.stderr + assert not out_dir.exists() + + +def generator_spec(topology: str = "ssh-preinstalled", accelerators: object | None = None) -> dict[str, object]: + spec: dict[str, object] = { + "topology": topology, + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + } + if accelerators is not None: + spec["accelerators"] = accelerators + if topology == "pxe-diskless": + spec["network"] = {"interface": "enp1s0", "subnet": "192.168.1.0/24"} + spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"]} + return spec + + +def test_generator_validates_all_pxe_requirements_before_writing(tmp_path: Path) -> None: + spec = generator_spec("pxe-diskless") + spec["pxe"] = {"authorized_keys": []} + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "pxe.authorized_keys must contain at least one SSH public key" in result.stderr + assert not out_dir.exists() + + +def test_generator_rejects_non_mapping_known_accelerator_config_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec(accelerators={"9070xt": []}))) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "accelerators.9070xt must be a mapping" in result.stderr + assert not out_dir.exists() + + +def test_generator_preflights_second_destination_collisions_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + write_file(out_dir / "pb-pxe-controller.vars.yml", "existing\n") + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert not (out_dir / "inventory.yml").exists() + + +def test_generator_preflights_third_destination_collisions_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + write_file(out_dir / "values-basic-example.yaml", "existing\n") + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "pb-pxe-controller.vars.yml").exists() + + +def test_generator_refuses_dangling_symlink_destinations_without_partial_artifacts(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec())) + out_dir = tmp_path / "generated" + dangling_target = tmp_path / "missing-target" + out_dir.mkdir() + (out_dir / "inventory.yml").symlink_to(dangling_target) + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert (out_dir / "inventory.yml").is_symlink() + assert not dangling_target.exists() + assert not (out_dir / "values-basic-example.yaml").exists() + + +def test_generator_publishes_secret_and_public_artifacts_with_expected_modes(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stdout + result.stderr + assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 + + +def test_generator_force_replaces_symlink_entry_without_following_target(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec())) + out_dir = tmp_path / "generated" + target = write_file(tmp_path / "target-values.yaml", "keep-this-target\n") + out_dir.mkdir() + (out_dir / "values-basic-example.yaml").symlink_to(target) + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + + published = out_dir / "values-basic-example.yaml" + assert result.returncode == 0, result.stdout + result.stderr + assert not published.is_symlink() + assert target.read_text(encoding="utf-8") == "keep-this-target\n" + assert "Helm overlay generated" in published.read_text(encoding="utf-8") + + +def test_generator_force_failure_restores_all_original_destination_types( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = load_generator_module() + inventory = write_file(tmp_path / "inventory.yml", "old inventory\n") + pxe_vars = tmp_path / "pb-pxe-controller.vars.yml" + pxe_vars.mkdir() + write_file(pxe_vars / "legacy", "old directory\n") + values_target = write_file(tmp_path / "values-target.yml", "old symlink target\n") + values = tmp_path / "values-basic-example.yaml" + values.symlink_to(values_target) + artifacts = [ + (inventory, "new inventory\n", 0o600, True), + (pxe_vars, "new pxe vars\n", 0o600, False), + (values, "new values\n", 0o644, False), + ] + original_replace = module.os.replace + + def fail_late_replace(source, destination): + if Path(destination).name == "values-basic-example.yaml" and ".backup." not in Path(source).name: + raise OSError("injected late publish failure") + return original_replace(source, destination) + + monkeypatch.setattr(module.os, "replace", fail_late_replace) + + with pytest.raises(SystemExit): + module.publish_artifacts(artifacts, force=True) + + assert inventory.read_text(encoding="utf-8") == "old inventory\n" + assert pxe_vars.is_dir() + assert (pxe_vars / "legacy").read_text(encoding="utf-8") == "old directory\n" + assert values.is_symlink() + assert values_target.read_text(encoding="utf-8") == "old symlink target\n" + + +def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base_values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + 9070xt: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_RX_9070_XT + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + spec_path = write_file( + repo / "spec.json", + json.dumps(generator_spec(accelerators={"9070xt": {"product_name": "AMD_Radeon_RX_9070_XT"}})), + ) + generated = repo / "generated" + generation = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(generated)) + cluster = write_cluster(repo, ["AMD_Radeon_RX_9070_XT"]) + + validation = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base_values), + "--values", + str(generated / "values-basic-example.yaml"), + "--cluster", + str(cluster), + ) + + assert generation.returncode == 0, generation.stdout + generation.stderr + assert validation.returncode == 0, validation.stdout + validation.stderr + assert "AMD_Radeon_RX_9070_XT" in validation.stdout + assert "AMD_Radeon_8060S_Graphics" not in validation.stdout + + +def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: + result = run_script(GEN_CONFIGS, "--print-schema", cwd=ROOT) + + assert result.returncode == 0, result.stdout + result.stderr + assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout From 3f66ef25ebf67fbee7db1df9d79bb81aab58f559 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:52:35 +0800 Subject: [PATCH 026/180] docs(skills): correct deployment helper workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 80 +++++++++++++++---- skills/deploy-aup-learning-cloud/reference.md | 47 ++++++++--- .../scripts/README.md | 47 +++++++++-- 3 files changed, 141 insertions(+), 33 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 92522f1c..6946dca6 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -35,6 +35,25 @@ table live in **[reference.md](reference.md)**. - The user supplies the physical hardware. **No site values (IPs, subnet, SSH keys, tokens) ship in the repo** — this skill generates them. +## Helper script paths + +Resolve the deploy helpers before running the commands below. From any directory +in an AUP Learning Cloud checkout: + +```bash +REPO_ROOT="$(git rev-parse --show-toplevel)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +``` + +When this skill is installed as a plugin rather than used from a checkout, set +`DEPLOY_SKILL_DIR` to the absolute directory containing the loaded `SKILL.md`, +then derive the helpers from that directory: + +```bash +DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" +DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" +``` + ## Phase 1 — Interview Work through this in order. **The deployment-method choice (1a) is a hard gate: @@ -65,8 +84,9 @@ Collect, and confirm back to the user, before touching anything: - *SSH path only:* also the hostname + IP of every agent node, and confirm passwordless root SSH already reaches each one. 3. **GPU — do not ask the user to name the model.** Let the tooling find it: the - detectors report the GPUs (`detect_hardware.sh` in Phase 2) and the real ROCm - `amd.com/gpu.product-name` label (`detect_cluster.sh` in Phase 5). Then + detectors report the GPUs (`$DEPLOY_SCRIPTS/detect_hardware.sh` in Phase 2) + and the real ROCm `amd.com/gpu.product-name` label + (`$DEPLOY_SCRIPTS/detect_cluster.sh` in Phase 5). Then **confirm the detected GPU → accelerator-key mapping with the user** before it goes into the values file. 4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers @@ -83,7 +103,7 @@ On the service machine, run the bundled detector and cross-check its JSON against the Phase 1 answers: ```bash -scripts/detect_hardware.sh # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] +"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] ``` It reports the default-route NIC, the service-machine IP + subnet CIDR, the @@ -96,21 +116,22 @@ name them, so surface the detected list and confirm it with the user. ## Phase 3 — Generate config -Drive `scripts/gen_configs.py` rather than hand-writing YAML — it keeps the +Drive `$DEPLOY_SCRIPTS/gen_configs.py` rather than hand-writing YAML — it keeps the three artifacts consistent, mints the k3s token locally with a CSPRNG (never printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. ```bash -scripts/gen_configs.py --print-schema > spec.json # fill from Phase 1 + 2 -scripts/gen_configs.py --spec spec.json --out-dir ./generated +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # fill from Phase 1 + 2 +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" ``` It writes, into `--out-dir`: 1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for PXE; listed for SSH) plus the `pxe_controller` group for PXE. -2. `pb-pxe-controller.vars.yml` — PXE path only: the `vars:` to merge into - `deploy/ansible/playbooks/pb-pxe-controller.yml` (`pxe_network_interface`, +2. `pb-pxe-controller.vars.yml` — PXE path only: extra vars passed to + `deploy/ansible/playbooks/pb-pxe-controller.yml` with `-e @<absolute-path>` (`pxe_network_interface`, `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). @@ -118,22 +139,46 @@ It writes, into `--out-dir`: to real GPU labels in Phase 5), `custom.resources.images`, the storage class (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). -Review the artifacts, then copy them into the `aup-learning-cloud` checkout. +Review the artifacts, install the inventory and runtime overlay into the +checkout, and keep the PXE vars in the generated directory. **Never commit `inventory.yml` — it holds the token.** Field-by-field guidance is in [reference.md](reference.md). +Map the generated artifacts into the checkout before Phase 5 validation: + +```bash +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +# PXE only: keep this generated secret in place and use its absolute path. +PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" +chmod 0600 "$PXE_VARS" +``` + +The generated `gpu.acceleratorKeys` activates the selected accelerators only +for the generic GPU resource. Wire selected accelerators into course resources +separately with `configure-aup-learning-cloud-courses`. + ## Phase 4 — Execute (with confirmation gates) Run the install in order. **Pause for explicit user confirmation before each risky/irreversible step** (see Safety). The PXE path is, in brief: 1. Install host packages on the service machine. -2. `pb-pxe-controller.yml` to build the PXE/NFS rootfs, then verify the +2. Run `pb-pxe-controller.yml -e @"$PXE_VARS"` to build the PXE/NFS rootfs, + then verify the controller (dnsmasq, NFS, apache2, TFTP boot files). 3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. 4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. 5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. +Run the PXE controller step with the generated vars file: + +```bash +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" +``` + The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against the inventory instead. Full commands for both paths are in [reference.md](reference.md). @@ -143,7 +188,7 @@ the inventory instead. Full commands for both paths are in [reference.md](refere cluster state: ```bash - scripts/detect_cluster.sh > cluster.json # nodes[], gpu_product_names[], storage_classes[] +"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # nodes[], gpu_product_names[], storage_classes[] ``` Confirm the detected GPU → accelerator-key mapping with the user, then patch @@ -152,10 +197,15 @@ the inventory instead. Full commands for both paths are in [reference.md](refere pre-flight (exits non-zero on any mismatch): ```bash - scripts/validate.py --repo ~/aup-learning-cloud \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --cluster cluster.json --helm-dry-run - ``` +# Set this to the topology selected in Phase 1a. +DEPLOY_TOPOLOGY=pxe-diskless +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run +``` + +For the PXE path, the validator and Ansible receive the same generated vars +file. Omit `--pxe-vars "$PXE_VARS"` for the SSH path. 2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` (storage class `nfs-client`). diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index c032738f..e67faf02 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -21,6 +21,10 @@ workflow and confirmation gates are in [SKILL.md](SKILL.md). Treat the live docs as the source of truth for version pins; this file condenses the opinionated path. +The helper commands are resolved through `DEPLOY_SCRIPTS` as defined in +[SKILL.md](SKILL.md#helper-script-paths), not through a checkout-root +`scripts/` directory. + The two topology sections below are the two branches of the Phase 1a gate in [SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → [PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node @@ -86,12 +90,21 @@ pxe_controller: ansible_user: root ``` -### Step 3 — Configure the PXE controller playbook +### Step 3 — Prepare the generated PXE controller vars + +The network, controller, server-IP, and SSH-key values are empty by default and +the role asserts on them. `$DEPLOY_SCRIPTS/gen_configs.py` writes these values +to `generated/pb-pxe-controller.vars.yml`. Keep that file at mode `0600`; it can +contain `pxe_rootfs_password`. Resolve its absolute path for the Ansible and +validator commands instead of copying or merging it into the playbook: + +```bash +PXE_VARS="$(realpath ./generated/pb-pxe-controller.vars.yml)" +chmod 0600 "$PXE_VARS" +test "$(stat -c '%a' "$PXE_VARS")" = 600 +``` -Edit the `vars:` block in `deploy/ansible/playbooks/pb-pxe-controller.yml`. The -network, controller, server-IP, and SSH-key values are empty by default and the -role asserts on them. `scripts/gen_configs.py` emits this exact block as -`pb-pxe-controller.vars.yml` — generate it and merge, or hand-edit: +Review the generated values before the first run: ```yaml pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) @@ -118,8 +131,15 @@ unless discovery flagged a need. ### Step 4 — Run the PXE controller playbook ```bash -cd ~/aup-learning-cloud/deploy/ansible -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml +cd ~/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +PXE_VARS="$(realpath "$REPO_ROOT/generated/pb-pxe-controller.vars.yml")" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" \ + --topology pxe-diskless --pxe-vars "$PXE_VARS" \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" ``` ### Step 5 — Verify the controller @@ -285,9 +305,15 @@ kubectl get storageclass ## Step 12 — Configure JupyterHub values +The generated `runtime/values-basic-example.yaml` is the canonical deployment +overlay. Review and keep it when Phase 3 generated one. Only when no generated +overlay exists, start a manual overlay from the example: + ```bash cd ~/aup-learning-cloud/runtime -cp values-multi-nodes.yaml.example values-basic-example.yaml +if [ ! -e values-basic-example.yaml ]; then + cp values-multi-nodes.yaml.example values-basic-example.yaml +fi ``` Minimum edits (see the [field guide](#valuesyaml-field-guide)): @@ -363,14 +389,15 @@ differently per fleet. | `AMD_Radeon_8060S_Graphics` | `strix-halo` | | `AMD_Radeon_RX_9070_XT` | `9070xt` | | `AMD_Radeon_AI_PRO_R9700` | `r9700` | +| `AMD_Radeon_RX_9600_GRE` | `9600gre` | If your labeller reports a different product name, update the matching `custom.accelerators.*.nodeSelector` entry to that exact string. ## values.yaml field guide -Sections to review in `values-basic-example.yaml` (from -`values-multi-nodes.yaml.example`): +Sections to review in the generated `values-basic-example.yaml`, or in the +manual `values-multi-nodes.yaml.example` copy when generation was not used: | Field | Purpose | | --- | --- | diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 5d571b4d..4e212dcc 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -11,27 +11,58 @@ report and uses exit codes the agent can branch on. | `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | | `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | | `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | -| `validate.py` | Before each `ansible-playbook` / `helm` run | Checks required PXE vars are non-empty, `k3s_version == pxe_k3s_version`, that each `nodeSelector` GPU label matches a real node (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | +| `validate.py` | Before each `ansible-playbook` / `helm` run | For `pxe-diskless`, checks required PXE vars and `k3s_version == pxe_k3s_version`; for both topologies, checks GPU labels only for active resource `acceleratorKeys` (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | ## Quick reference +From any directory in a checkout, resolve helpers with: + +```bash +REPO_ROOT="$(git rev-parse --show-toplevel)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +``` + +For an installed plugin, set `DEPLOY_SKILL_DIR` to the absolute directory +containing the loaded `SKILL.md`, then use: + +```bash +DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" +DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" +``` + ```bash # Phase 2 — discover the host -./detect_hardware.sh # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] +"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] # Phase 3 — generate config from a spec -./gen_configs.py --print-schema > spec.json # then edit spec.json -./gen_configs.py --spec spec.json --out-dir ./generated +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # then edit spec.json +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +# PXE only: keep the generated secret in place and resolve its absolute path. +PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" +chmod 0600 "$PXE_VARS" +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" # Phase 5 — after k3s + device plugin are up -./detect_cluster.sh > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] +"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] -# Before running playbooks / helm -./validate.py --repo ~/aup-learning-cloud \ +# Before running playbooks / helm (set to the selected topology) +DEPLOY_TOPOLOGY=pxe-diskless +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --cluster cluster.json --helm-dry-run + --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run ``` +Omit `--pxe-vars "$PXE_VARS"` for `ssh-preinstalled`. For `pxe-diskless`, the +validator and Ansible must receive the same generated file. + +Generated `gpu.acceleratorKeys` wires the selected accelerators to the generic +GPU resource. Use `configure-aup-learning-cloud-courses` to wire course +resources separately. + ## Conventions - **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON From f6df6fe63125d56387d9a885e321e57c0eec9f00 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:53:19 +0800 Subject: [PATCH 027/180] docs(skills): align cross-skill helper paths --- skills/troubleshoot-aup-learning-cloud/SKILL.md | 11 +++++++++-- skills/upgrade-aup-learning-cloud/reference.md | 8 +++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/skills/troubleshoot-aup-learning-cloud/SKILL.md b/skills/troubleshoot-aup-learning-cloud/SKILL.md index 5ab217db..8bc51964 100644 --- a/skills/troubleshoot-aup-learning-cloud/SKILL.md +++ b/skills/troubleshoot-aup-learning-cloud/SKILL.md @@ -26,9 +26,16 @@ matrices live in **[reference.md](reference.md)**. - Access to the cluster (`kubectl`, the right `KUBECONFIG`) and/or the service machine (for PXE/host issues). - A checkout of `aup-learning-cloud` for config cross-checks. -- The deploy skill's `scripts/detect_cluster.sh` is a fast way to snapshot +- The deploy skill's `$DEPLOY_SCRIPTS/detect_cluster.sh` is a fast way to snapshot nodes, GPU labels, storage classes, and the device plugin/labeller state. +From any checkout directory, define +`REPO_ROOT="$(git rev-parse --show-toplevel)"` and +`DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts"`. For an +installed plugin, define `DEPLOY_SKILL_DIR` as the absolute directory containing +the loaded deploy skill's `SKILL.md`, then set +`DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts"`. + ## Method (don't thrash) 1. **Scope it.** Which layer is failing — netboot, node join, GPU scheduling, @@ -39,7 +46,7 @@ matrices live in **[reference.md](reference.md)**. kubectl get nodes -o wide kubectl get pods -A | grep -Ev 'Running|Completed' kubectl describe pod -n jupyterhub <pod> # Events explain Pending/ImagePull - scripts/detect_cluster.sh # from the deploy skill + "$DEPLOY_SCRIPTS/detect_cluster.sh" # from the deploy skill ``` 3. **Match to a cause** using the [reference.md](reference.md) matrices. diff --git a/skills/upgrade-aup-learning-cloud/reference.md b/skills/upgrade-aup-learning-cloud/reference.md index c07d96cb..09ae4eb0 100644 --- a/skills/upgrade-aup-learning-cloud/reference.md +++ b/skills/upgrade-aup-learning-cloud/reference.md @@ -19,7 +19,13 @@ troubleshooting. Workflow and gates are in [SKILL.md](SKILL.md). | Chart | `runtime/chart/Chart.yaml` | Keep `pxe_k3s_version == k3s_version`. The deploy skill's -`scripts/validate.py` cross-checks this. +`$DEPLOY_SCRIPTS/validate.py` cross-checks this when invoked with +`--topology pxe-diskless`. From a checkout, resolve that helper with +`REPO_ROOT="$(git rev-parse --show-toplevel)"` and +`DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts"`; from +an installed plugin, define `DEPLOY_SKILL_DIR` as the absolute directory +containing the loaded deploy skill's `SKILL.md`, then set +`DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts"`. ## Hub (Helm) upgrade — values / image / chart From d79894ff31e1bd709ac75914f804aa15db0c7205 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:54:07 +0800 Subject: [PATCH 028/180] docs(skills): align accelerator planning guidance --- skills/plan-aup-learning-cloud-deployment/SKILL.md | 10 ++++++---- skills/plan-aup-learning-cloud-deployment/reference.md | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/skills/plan-aup-learning-cloud-deployment/SKILL.md b/skills/plan-aup-learning-cloud-deployment/SKILL.md index 37e42259..1989cc54 100644 --- a/skills/plan-aup-learning-cloud-deployment/SKILL.md +++ b/skills/plan-aup-learning-cloud-deployment/SKILL.md @@ -73,10 +73,12 @@ the requirements: servers**. Compare by **compute (CU/TFLOPs) and VRAM**, not marketing tier. 2. **Gate every candidate on ROCm support** — if a chip is not ROCm-supported it cannot run the GPU notebooks. -3. **Map the chip to a chart accelerator key** (`phx`, `strix`, `strix-halo`, - `9070xt`, `r9700`, or the generic `rdna4`) and the expected - `amd.com/gpu.product-name` node label. A brand-new chip with no existing key - is a flag to raise with the user. +3. **Map the chip to an existing chart accelerator key** (`phx`, `strix`, + `strix-halo`, `9070xt`, `r9700`, or `9600gre`) and the expected + `amd.com/gpu.product-name` node label. `rdna4` is an installer detection + fallback, not an existing chart accelerator key accepted by + `gen_configs.py`. A new chart key requires + `configure-aup-learning-cloud-courses` work before it can be generated. 4. Prefer **multi-GPU chassis** (workstation/server) when peak concurrent GPU users is high enough that many single-GPU AIPCs become impractical to cable, power, and manage. Keep AIPCs for small labs and the demo-like experience. diff --git a/skills/plan-aup-learning-cloud-deployment/reference.md b/skills/plan-aup-learning-cloud-deployment/reference.md index fe2cb597..f0e840cf 100644 --- a/skills/plan-aup-learning-cloud-deployment/reference.md +++ b/skills/plan-aup-learning-cloud-deployment/reference.md @@ -128,6 +128,7 @@ chip-selection driver: | `strix-halo` | Radeon 8060S (Strix Halo iGPU) | 64 GB unified | 40 | `AMD_Radeon_8060S_Graphics` | CV/DL/LLM/PhySim | | `9070xt` | Radeon RX 9070 XT | 16 GB GDDR6 | 64 | `AMD_Radeon_RX_9070_XT` | CV/DL; mid LLM | | `r9700` | Radeon AI PRO R9700 | 32 GB GDDR6 | 64 | `AMD_Radeon_AI_PRO_R9700` | CV/DL/LLM; multi-card workstation/server | +| `9600gre` | Radeon RX 9600 GRE | 12 GB GDDR6 | 32 | `AMD_Radeon_RX_9600_GRE` | CV/DL; light to mid LLM | `phx` also sets `HSA_OVERRIDE_GFX_VERSION: 11.0.0`. If a fleet normalizes a product name differently, the `nodeSelector` string must be changed to match @@ -144,7 +145,7 @@ Always confirm against current AMD product pages; silicon refreshes often. 2. **ROCm gate.** Only recommend chips with confirmed ROCm support; otherwise the GPU notebooks will not run. 3. **Map to a chart key.** Fit the chip to an existing accelerator key - (`phx`/`strix`/`strix-halo`/`9070xt`/`r9700`) and the expected + (`phx`/`strix`/`strix-halo`/`9070xt`/`r9700`/`9600gre`) and the expected `amd.com/gpu.product-name`. If it is a brand-new product with no key yet, tell the user it needs a `configure-aup-learning-cloud-courses` accelerator entry (and possibly a new image) before deployment. From 91b0a34660ddc9958be9f0012db2a63419a17aa2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:55:31 +0800 Subject: [PATCH 029/180] docs(skills): align OEM kernel guidance --- skills/install-aup-learning-cloud-single-node/SKILL.md | 2 +- skills/install-aup-learning-cloud-single-node/reference.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md index 70f730b3..575141db 100644 --- a/skills/install-aup-learning-cloud-single-node/SKILL.md +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -33,7 +33,7 @@ table, offline flow, and troubleshooting are in **[reference.md](reference.md)** - **Ubuntu 24.04**. Docker installed and usable without `sudo` (`docker run hello-world` as the user). - **Ryzen AI APU only:** the ROCm OEM kernel - (`sudo apt install linux-image-6.14.0-1018-oem`) and a reboot. Radeon dGPU + (`sudo apt install linux-oem-6.14`) and a reboot. Radeon dGPU boxes typically use the stock kernel — confirm against ROCm docs. - For the interactive TUI: `python3-questionary` + `python3-prompt-toolkit` (apt), or `pip install questionary prompt_toolkit` in a venv. The diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 8faf677e..514da9f9 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -16,7 +16,7 @@ flags and version pins; this file condenses the opinionated path. ```bash # Ryzen AI APU only: ROCm OEM kernel (reboot afterwards) -sudo apt update && sudo apt install linux-image-6.14.0-1018-oem +sudo apt update && sudo apt install linux-oem-6.14 # Docker (rootless usage) curl -fsSL https://get.docker.com | sh From 9763c50081a374af96e7efe636ec54b3ee79ed1e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:56:01 +0800 Subject: [PATCH 030/180] docs(skills): correct quota API guidance --- .../manage-aup-learning-cloud-users/SKILL.md | 12 +++++----- .../reference.md | 22 ++++++++++--------- .../scripts/hub-api-env.sh | 6 ++--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/skills/manage-aup-learning-cloud-users/SKILL.md b/skills/manage-aup-learning-cloud-users/SKILL.md index 7da07854..6de97b3d 100644 --- a/skills/manage-aup-learning-cloud-users/SKILL.md +++ b/skills/manage-aup-learning-cloud-users/SKILL.md @@ -28,7 +28,7 @@ The two built-in scripts are the primary automation surface: - `scripts/generate_users_template.py` creates CSV/Excel rosters with the columns `manage_users.py` expects. - `scripts/manage_users.py` performs API-backed user/admin/password work and - quota commands that also exec into the Hub pod. + quota commands. Exact command variants, file formats, env setup, and the quota field guide are in **[reference.md](reference.md)**. @@ -43,8 +43,9 @@ in **[reference.md](reference.md)**. - `manage_users.py` requires `JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN` for every subcommand. The bundled `scripts/hub-api-env.sh` derives both from the `jupyterhub-admin-credentials` secret and checks reachability. -- Quota subcommands also require `kubectl` access to the Hub namespace because - they call `kubectl exec deployment/hub` after the API-token preflight. +- Quota subcommands use the Hub admin API. `kubectl` is only needed to bootstrap + an API token from `jupyterhub-admin-credentials` or inspect scheduled quota + refresh CronJobs. - Native-user creation/password reset requires `authMode: multi` (or another mode with native accounts). Password actions never apply to GitHub identities. @@ -138,8 +139,9 @@ refresh). Quota **rates and enable/disable knobs** (`custom.quota.*`, - **`set-admin` grants full platform control** — confirm the target list. - **Quota refresh rules apply broadly.** A global Refresh Quota or a broad `refreshRules` filter touches many users; confirm before applying. -- CLI quota commands run `kubectl exec` into `deployment/hub`; they need both a - valid API token for the script preflight and a healthy kube context/namespace. +- CLI quota commands call the Hub admin API; they need a valid API token and a + reachable Hub, not `kubectl` access. Use `kubectl` only for the secret + bootstrap or scheduled-refresh CronJob inspection described above. ## Reference diff --git a/skills/manage-aup-learning-cloud-users/reference.md b/skills/manage-aup-learning-cloud-users/reference.md index 929f7315..ea12f92f 100644 --- a/skills/manage-aup-learning-cloud-users/reference.md +++ b/skills/manage-aup-learning-cloud-users/reference.md @@ -41,8 +41,9 @@ source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh HUB_URL="https://hub.example.com" source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh ``` -CLI **quota** commands also use `kubectl exec` into `deployment/hub`, so they -need a working kube context + namespace in addition to the API token preflight. +CLI **quota** commands call the Hub admin API, so they need a valid API token +and a reachable Hub. `kubectl` is only needed to bootstrap the token from the +secret above or inspect scheduled quota refresh CronJobs. ## Python dependencies @@ -102,11 +103,12 @@ python scripts/manage_users.py add-quota --file users.csv --amount 100 python scripts/manage_users.py list-quota ``` -Every command accepts `--url` and `--token`, but prefer exported env vars so -tokens do not appear in shell history: +Every command accepts `--url` and `--token`, but export the environment instead +so tokens do not appear in shell history or process arguments. Use a read-only +CLI command to confirm reachability: ```bash -python scripts/manage_users.py --url "$JUPYTERHUB_URL" --token "$JUPYTERHUB_TOKEN" list +python scripts/manage_users.py list ``` ### Command behavior notes @@ -140,9 +142,9 @@ python scripts/manage_users.py --url "$JUPYTERHUB_URL" --token "$JUPYTERHUB_TOKE users, usage trends, resource distribution, top users, live sessions, pending spawns. -Admin quota API endpoints used by the UI: `GET/POST /admin/api/quota/`, -`POST /admin/api/quota/batch`, `POST /admin/api/quota/refresh`, -`GET /api/quota/rates`, `GET /api/quota/me`. +Admin quota API endpoints used by the UI: `GET/POST /hub/admin/api/quota/`, +`POST /hub/admin/api/quota/batch`, `POST /hub/admin/api/quota/refresh`, +`GET /hub/api/quota/rates`, `GET /hub/api/quota/me`. ## Scheduled quota refresh (`refreshRules`) @@ -226,9 +228,9 @@ cd runtime && helm upgrade --install jupyterhub ./chart \ | Symptom | Likely cause | First checks | | --- | --- | --- | -| Script cannot connect to the Hub | `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` wrong | Confirm both; `curl -H "Authorization: token $JUPYTERHUB_TOKEN" $JUPYTERHUB_URL/hub/api/` | +| Script cannot connect to the Hub | `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` wrong | Re-source `hub-api-env.sh`, then run `python scripts/manage_users.py list` | | Password reset fails | Target is a GitHub user, weak password, or session lacks perms | Native users only; meet the strength policy | -| Quota command passes API check but fails later | CLI uses `kubectl exec` into `deployment/hub` | Check kube context, namespace, and `kubectl -n jupyterhub get deploy/hub` | +| Quota command fails | Hub admin API rejects the token or is unreachable | Re-source the API environment and run `python scripts/manage_users.py list` before retrying quota work | | No api-token secret | `custom.adminUser.enabled: false` | Enable admin bootstrap, re-apply | | Group membership can't be edited | System-managed or GitHub-synced group | Only manual/editable groups accept edits | | Refresh rule didn't run | Rule disabled or absent from the applied values | `kubectl … get cronjobs -l …quota-refresh`; re-apply | diff --git a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh index fe5fb6d3..49b8f3a7 100644 --- a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +++ b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh @@ -34,9 +34,9 @@ export JUPYTERHUB_TOKEN="$_auplc_token" # Probe the API (non-fatal: token may still be valid behind an auth proxy). if command -v curl >/dev/null 2>&1; then - _auplc_code="$(curl -s -o /dev/null -w '%{http_code}' \ - -H "Authorization: token ${JUPYTERHUB_TOKEN}" \ - "${JUPYTERHUB_URL%/}/hub/api/" 2>/dev/null)" + _auplc_code="$(printf 'header = "Authorization: token %s"\n' "$JUPYTERHUB_TOKEN" | \ + curl --config - -s -o /dev/null -w '%{http_code}' \ + "${JUPYTERHUB_URL%/}/hub/api/" 2>/dev/null)" case "$_auplc_code" in 200) echo "hub-api-env: OK — $JUPYTERHUB_URL/hub/api/ reachable (200)" ;; *) echo "hub-api-env: WARNING — $JUPYTERHUB_URL/hub/api/ returned '$_auplc_code'; check HUB_URL/network" >&2 ;; From 5e1a0e60fda62e1617a484124108e1570662dc97 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:56:39 +0800 Subject: [PATCH 031/180] docs(skills): move plugin governance docs --- {docs => plugin-docs}/adding-a-skill.md | 6 +++--- {docs => plugin-docs}/skill-cards.md | 0 {docs => plugin-docs}/skill-categories.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename {docs => plugin-docs}/adding-a-skill.md (91%) rename {docs => plugin-docs}/skill-cards.md (100%) rename {docs => plugin-docs}/skill-categories.md (98%) diff --git a/docs/adding-a-skill.md b/plugin-docs/adding-a-skill.md similarity index 91% rename from docs/adding-a-skill.md rename to plugin-docs/adding-a-skill.md index 231f6aa5..8b7854ce 100644 --- a/docs/adding-a-skill.md +++ b/plugin-docs/adding-a-skill.md @@ -40,7 +40,7 @@ Fill in the `## Description` and `## Owner` sections. See The repo ships as a single bundled plugin (`source: "./"`), so the plugin manifests do **not** need a per-skill entry — dropping the folder under `skills/` is enough for every install method to pick it up. Add a row to the -catalog table in the [README](../README.md) **under the skill's group section**, +catalog table in the [skills README](../README-SKILL.md) **under the skill's group section**, list it under that group in [skill-categories.md](skill-categories.md) so people can discover it, then keep the Cursor manifests in sync: @@ -55,7 +55,7 @@ can discover it, then keep the Cursor manifests in sync: ``` CI runs the same validation on every pull request via -`.github/workflows/validate.yml`, fanning out one job per skill so a single +`.github/workflows/validate-skills.yml`, fanning out one job per skill so a single broken skill is easy to spot. ## Ideas for future skills @@ -63,7 +63,7 @@ broken skill is easy to spot. The catalog now covers install, deploy, configure (courses), build, upgrade, and troubleshoot, plus auth, user/quota management, monitoring, network/storage exposure, per-user repo cloning, and course authoring (see the -[README](../README.md)). Natural next additions, each following the same +[skills README](../README-SKILL.md)). Natural next additions, each following the same procedure: | Skill | Outcome | diff --git a/docs/skill-cards.md b/plugin-docs/skill-cards.md similarity index 100% rename from docs/skill-cards.md rename to plugin-docs/skill-cards.md diff --git a/docs/skill-categories.md b/plugin-docs/skill-categories.md similarity index 98% rename from docs/skill-categories.md rename to plugin-docs/skill-categories.md index b9799819..73755d5f 100644 --- a/docs/skill-categories.md +++ b/plugin-docs/skill-categories.md @@ -76,5 +76,5 @@ Tasks often cross a boundary; hand off rather than stretch a skill: Assign exactly one group, prepend the group's `Group:` tag to the new skill's `description`, add its row under that group's table in the -[README](../README.md), and list it here. See +[skills README](../README-SKILL.md), and list it here. See [adding-a-skill.md](adding-a-skill.md) for the full procedure. From e4dd899759f13c581c17998bc9467fece4d6c282 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:57:14 +0800 Subject: [PATCH 032/180] docs(skills): update plugin documentation links --- .github/scripts/validate_skills.py | 56 +++++++++--------------------- README-SKILL.md | 8 ++--- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py index 85384665..11a953cb 100755 --- a/.github/scripts/validate_skills.py +++ b/.github/scripts/validate_skills.py @@ -5,7 +5,7 @@ # /// """Validate auplc-skills against the standardized Agent Skills format. -Enforces the rules documented in CONTRIBUTING.md: +Enforces the repository's skill format and governance requirements: - SKILL.md exists at the skill root - YAML frontmatter is parseable @@ -31,7 +31,7 @@ uv run .github/scripts/validate_skills.py --marketplace-only # manifest only The `--list` / `--skill` options let CI validate each skill in its own job -(see .github/workflows/validate.yml) so a single bad skill doesn't mask the +(see .github/workflows/validate-skills.yml) so a single bad skill doesn't mask the status of the others. Exits non-zero if any validated skill (or the marketplace check) fails. @@ -53,7 +53,7 @@ CLAUDE_MARKETPLACE = REPO_ROOT / ".claude-plugin" / "marketplace.json" CLAUDE_PLUGIN = REPO_ROOT / ".claude-plugin" / "plugin.json" -# Limits from CONTRIBUTING.md and the standardized Agent Skills format. +# Limits from the standardized Agent Skills format and repository policy. MAX_NAME_LEN = 64 MAX_DESCRIPTION_LEN = 1024 MAX_BODY_LINES = 500 @@ -65,7 +65,7 @@ ) RESERVED_NAME_SUBSTRINGS = ("anthropic", "claude") -# Per-skill governance card (see docs/skill-cards.md). Each section must be a +# Per-skill governance card (see plugin-docs/skill-cards.md). Each section must be a # top-level `##` heading followed by some non-empty body text. CARD_FILENAME = "skill-card.md" REQUIRED_CARD_SECTIONS = ("Description", "Owner") @@ -90,8 +90,7 @@ def validate_skill(skill_dir: Path) -> SkillReport: match = FRONTMATTER_RE.match(text) if match is None: report.errors.append( - "SKILL.md must start with a `---` YAML frontmatter block " - "followed by `---` on its own line." + "SKILL.md must start with a `---` YAML frontmatter block followed by `---` on its own line." ) return report @@ -102,10 +101,7 @@ def validate_skill(skill_dir: Path) -> SkillReport: return report if not isinstance(frontmatter, dict): - report.errors.append( - "YAML frontmatter must be a mapping with at least `name` " - "and `description`." - ) + report.errors.append("YAML frontmatter must be a mapping with at least `name` and `description`.") return report _validate_name(frontmatter.get("name"), skill_dir.name, report) @@ -121,34 +117,24 @@ def _validate_name(name: object, dir_name: str, report: SkillReport) -> None: return if len(name) > MAX_NAME_LEN: - report.errors.append( - f"`name` length {len(name)} exceeds {MAX_NAME_LEN} characters." - ) + report.errors.append(f"`name` length {len(name)} exceeds {MAX_NAME_LEN} characters.") if not NAME_RE.match(name): report.errors.append( - f"`name` `{name}` must be lowercase-with-hyphens " - "(letters, digits, single hyphens between segments)." + f"`name` `{name}` must be lowercase-with-hyphens (letters, digits, single hyphens between segments)." ) for sub in RESERVED_NAME_SUBSTRINGS: if sub in name.lower(): report.errors.append(f"`name` may not contain `{sub}`.") if name != dir_name: - report.errors.append( - f"`name` `{name}` must match the skill directory name `{dir_name}`." - ) + report.errors.append(f"`name` `{name}` must match the skill directory name `{dir_name}`.") def _validate_description(description: object, report: SkillReport) -> None: if not isinstance(description, str) or not description: - report.errors.append( - "Frontmatter `description` is missing or not a non-empty string." - ) + report.errors.append("Frontmatter `description` is missing or not a non-empty string.") return if len(description) > MAX_DESCRIPTION_LEN: - report.errors.append( - f"`description` length {len(description)} exceeds " - f"{MAX_DESCRIPTION_LEN} characters." - ) + report.errors.append(f"`description` length {len(description)} exceeds {MAX_DESCRIPTION_LEN} characters.") def _validate_body(body: str, report: SkillReport) -> None: @@ -172,7 +158,7 @@ def _validate_card(skill_dir: Path, report: SkillReport) -> None: card = skill_dir / CARD_FILENAME if not card.exists(): report.errors.append( - f"Missing {CARD_FILENAME} (governance card). See docs/skill-cards.md; " + f"Missing {CARD_FILENAME} (governance card). See plugin-docs/skill-cards.md; " "it needs `## Description` and `## Owner` sections." ) return @@ -212,9 +198,7 @@ def discover_skills(root: Path) -> list[Path]: """List skill directories under `root`, ignoring dotfiles.""" if not root.exists(): return [] - return sorted( - p for p in root.iterdir() if p.is_dir() and not p.name.startswith(".") - ) + return sorted(p for p in root.iterdir() if p.is_dir() and not p.name.startswith(".")) def validate_claude_marketplace() -> list[str]: @@ -224,7 +208,7 @@ def validate_claude_marketplace() -> list[str]: repo), so the marketplace must list exactly one plugin and a matching `.claude-plugin/plugin.json` must exist. The marketplace's human-readable `description` is intentionally allowed to differ from the SKILL.md - descriptions (per CONTRIBUTING.md), so its text is not cross-checked. + descriptions, so its text is not cross-checked. """ errors: list[str] = [] @@ -241,10 +225,7 @@ def validate_claude_marketplace() -> list[str]: plugins = data.get("plugins") if isinstance(data, dict) else None if not isinstance(plugins, list): - return [ - f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: top-level `plugins` " - "array is missing." - ] + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: top-level `plugins` array is missing."] if len(plugins) != 1: return [ f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: expected exactly one " @@ -353,9 +334,7 @@ def run(skills_dir: Path) -> int: def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( "--skills-dir", type=Path, @@ -371,8 +350,7 @@ def main(argv: list[str] | None = None) -> int: group.add_argument( "--skill", metavar="NAME", - help="Validate only the named skill directory (skips the marketplace " - "cross-check, which is repo-wide).", + help="Validate only the named skill directory (skips the marketplace cross-check, which is repo-wide).", ) group.add_argument( "--marketplace-only", diff --git a/README-SKILL.md b/README-SKILL.md index e9eb4a03..a854d736 100644 --- a/README-SKILL.md +++ b/README-SKILL.md @@ -18,7 +18,7 @@ The skills are organized into three groups so an agent can start in the right place for a task. It is still **one bundled plugin** — installing it brings every skill at once; the groups are a routing aid (each skill's `description` also carries its `Group:` tag). See -[docs/skill-categories.md](docs/skill-categories.md) for the full taxonomy and +[plugin-docs/skill-categories.md](plugin-docs/skill-categories.md) for the full taxonomy and routing guidance. ### Plan and deploy AUP Learning Cloud @@ -30,7 +30,7 @@ images. | --- | --- | --- | | [`plan-aup-learning-cloud-deployment`](skills/plan-aup-learning-cloud-deployment/SKILL.md) | Size a new deployment for a prospective adopter: interview course/headcount needs and the network, research current AMD silicon, then recommend how many AIPCs/workstations/servers and routers/switches to buy, the topology, an IP plan, and a buyer-facing bill of materials. | in-repo | | [`install-aup-learning-cloud-single-node`](skills/install-aup-learning-cloud-single-node/SKILL.md) | Install on a single AMD GPU/APU box with the `./auplc-installer` flow: prerequisites, GPU/courses/image flags, gated install, verify at `localhost:30890`. | in-repo | -| [`deploy-aup-learning-cloud`](skills/deploy-aup-learning-cloud/SKILL.md) | Deploy end to end on a multi-AIPC PXE/k3s cluster: interview the operator, generate the Ansible inventory + PXE vars + Helm values (helper scripts), then drive the install with confirmation gates at risky steps. | in-repo | +| [`deploy-aup-learning-cloud`](skills/deploy-aup-learning-cloud/SKILL.md) | Deploy end to end on a multi-AIPC PXE-diskless or SSH-preinstalled k3s cluster: interview the operator, generate the Ansible inventory + PXE vars + Helm values (helper scripts), then drive the install with confirmation gates at risky steps. | in-repo | | [`build-aup-learning-cloud-images`](skills/build-aup-learning-cloud-images/SKILL.md) | Build and publish the Hub and notebook/course Docker images with `img build`, incl. GPU-target tagging and registry push. | in-repo | ### Maintain AUP Learning Cloud @@ -201,7 +201,7 @@ before touching any machine. ``` skills/ # All skills the agent can load templates/skill-template # Starting point for a new skill -docs/ # Authoring + governance docs +plugin-docs/ # Plugin authoring + governance docs .claude-plugin/ # Claude marketplace + bundled-plugin manifest (hand-maintained) .cursor-plugin/ # Cursor marketplace + plugin manifest (generated) plugin-metadata.json # Vendor-neutral identity/discovery metadata @@ -212,7 +212,7 @@ plugin-metadata.json # Vendor-neutral identity/discovery metadata ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for authoring conventions and -[docs/adding-a-skill.md](docs/adding-a-skill.md) for the step-by-step procedure +[plugin-docs/adding-a-skill.md](plugin-docs/adding-a-skill.md) for the step-by-step procedure to add a new skill. Run the same checks CI runs before opening a PR: ```bash From f7c05a35201c0b96dbe613a3df9654476161889b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:58:01 +0800 Subject: [PATCH 033/180] chore(skills): expand local validation --- .github/scripts/check.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh index 588fe900..7d0cc3f8 100755 --- a/.github/scripts/check.sh +++ b/.github/scripts/check.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Validate every SKILL.md and that generated plugin manifests are up to date. +# Validate skills, version metadata, skill tests, and generated plugin manifests. # # Usage: -# ./.github/scripts/check.sh Validate every skill and check manifests. +# ./.github/scripts/check.sh Run every skill-package validation. # ./.github/scripts/check.sh -h|--help Print this help. # # Requires `uv` (https://github.com/astral-sh/uv). @@ -19,6 +19,8 @@ usage() { case "${1:-}" in "") uv run .github/scripts/validate_skills.py + uv run python scripts/check_skills_version.py + uv run --extra test pytest tests/skills uv run .github/scripts/generate_cursor_marketplace.py --check ;; -h|--help) From 73feaf480cf40271f29ae78e3364f9866ab9dd4a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:58:37 +0800 Subject: [PATCH 034/180] ci(skills): enforce plugin validation --- .github/workflows/validate-skills.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml index c1b2f4cd..70ba5487 100644 --- a/.github/workflows/validate-skills.yml +++ b/.github/workflows/validate-skills.yml @@ -10,6 +10,12 @@ on: - ".cursor-plugin/**" - ".github/scripts/**" - ".github/workflows/validate-skills.yml" + - "pyproject.toml" + - "plugin-metadata.json" + - "scripts/check_skills_version.py" + - "tests/skills/**" + - "README-SKILL.md" + - "plugin-docs/**" pull_request: paths: - "skills/**" @@ -18,6 +24,12 @@ on: - ".cursor-plugin/**" - ".github/scripts/**" - ".github/workflows/validate-skills.yml" + - "pyproject.toml" + - "plugin-metadata.json" + - "scripts/check_skills_version.py" + - "tests/skills/**" + - "README-SKILL.md" + - "plugin-docs/**" workflow_dispatch: # Least privilege: these jobs only read the repo to validate skills/manifests. @@ -65,10 +77,10 @@ jobs: - name: Validate skill run: uv run .github/scripts/validate_skills.py --skill "${{ matrix.skill }}" - # Repo-wide checks that aren't tied to a single skill: the generated plugin - # manifests. + # Repo-wide checks that aren't tied to a single skill: version sync, public + # skill CLI tests, and generated plugin manifests. validate-manifests: - name: Validate plugin manifests + name: Validate plugin metadata and manifests runs-on: ubuntu-latest steps: - name: Check out repository @@ -80,6 +92,12 @@ jobs: - name: Validate marketplace manifest run: uv run .github/scripts/validate_skills.py --marketplace-only + - name: Validate skill version sync + run: uv run python scripts/check_skills_version.py + + - name: Run skill tests + run: uv run --extra test pytest tests/skills + - name: Validate generated Cursor manifest run: uv run .github/scripts/generate_cursor_marketplace.py --check From 51dcb90a7c9c9dbc04a857e3f53865ab4efb6d64 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:02:33 +0800 Subject: [PATCH 035/180] chore(skills): synchronize plugin metadata version --- .claude-plugin/marketplace.json | 3 +- .claude-plugin/plugin.json | 4 +- .cursor-plugin/marketplace.json | 3 +- .cursor-plugin/plugin.json | 2 +- .../scripts/generate_cursor_marketplace.py | 24 ++----- plugin-metadata.json | 4 +- pyproject.toml | 2 +- tests/skills/test_check_skills_version.py | 69 +++++++++++++++++++ 8 files changed, 83 insertions(+), 28 deletions(-) create mode 100644 tests/skills/test_check_skills_version.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 87b46405..2da6b952 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,8 +5,7 @@ }, "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", "metadata": { - "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", - "version": "0.1.0" + "version": "0.1.1" }, "plugins": [ { diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 1ecc51b4..49a42888 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,12 +1,12 @@ { "name": "auplc", "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", - "version": "0.1.0", + "version": "0.1.1", "author": { "name": "AMD Research" }, "homepage": "https://github.com/AMDResearch/aup-learning-cloud", - "repository": "https://github.com/AMDResearch/auplc-skills", + "repository": "https://github.com/AMDResearch/aup-learning-cloud", "keywords": [ "aup-learning-cloud", "auplc", diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 87b46405..2da6b952 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -5,8 +5,7 @@ }, "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", "metadata": { - "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", - "version": "0.1.0" + "version": "0.1.1" }, "plugins": [ { diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 3325c0ca..cfac9ec2 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "auplc", - "version": "0.1.0", + "version": "0.1.1", "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", "author": { "name": "AMD Research" diff --git a/.github/scripts/generate_cursor_marketplace.py b/.github/scripts/generate_cursor_marketplace.py index b08e0bc0..a2721b0d 100755 --- a/.github/scripts/generate_cursor_marketplace.py +++ b/.github/scripts/generate_cursor_marketplace.py @@ -56,9 +56,7 @@ def load_json(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) -def check_identity_consistency( - metadata: dict, claude: dict, claude_plugin: dict -) -> list[str]: +def check_identity_consistency(metadata: dict, claude: dict, claude_plugin: dict) -> list[str]: """Return error strings if the Claude manifests' identity has drifted from the canonical `plugin-metadata.json`.""" errors: list[str] = [] @@ -72,11 +70,9 @@ def check_identity_consistency( f".claude-plugin/marketplace.json `name` ({claude.get('name')!r}) " f"must match plugin-metadata.json `name` ({name!r})." ) - if claude.get("description") != description: - errors.append( - ".claude-plugin/marketplace.json `description` must match " - "plugin-metadata.json `description`." - ) + claude_description = claude.get("description") + if claude_description != description: + errors.append(".claude-plugin/marketplace.json `description` must match plugin-metadata.json `description`.") claude_version = (claude.get("metadata") or {}).get("version") if claude_version != version: errors.append( @@ -88,10 +84,7 @@ def check_identity_consistency( # The single bundled plugin entry's name must match the plugin manifest. plugins = claude.get("plugins") if not isinstance(plugins, list) or len(plugins) != 1: - errors.append( - ".claude-plugin/marketplace.json must list exactly one bundled " - "plugin (source `./`)." - ) + errors.append(".claude-plugin/marketplace.json must list exactly one bundled plugin (source `./`).") else: entry_name = plugins[0].get("name") if entry_name != claude_plugin.get("name"): @@ -119,7 +112,6 @@ def build_cursor_marketplace(metadata: dict, claude: dict) -> dict: "owner": {"name": owner_name} if owner_name else {}, "description": metadata["description"], "metadata": { - "description": metadata["description"], "version": metadata["version"], }, "plugins": claude.get("plugins", []), @@ -180,11 +172,7 @@ def main(argv: list[str] | None = None) -> int: CURSOR_PLUGIN: render_json(build_cursor_plugin(metadata, claude_plugin)), } - stale = [ - path - for path, content in targets.items() - if not write_or_check(path, content, check=args.check) - ] + stale = [path for path, content in targets.items() if not write_or_check(path, content, check=args.check)] if args.check: if stale: diff --git a/plugin-metadata.json b/plugin-metadata.json index f562b4e5..28935c85 100644 --- a/plugin-metadata.json +++ b/plugin-metadata.json @@ -1,12 +1,12 @@ { "name": "auplc-skills", "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", - "version": "0.1.0", + "version": "0.1.1", "author": { "name": "AMD Research" }, "homepage": "https://github.com/AMDResearch/aup-learning-cloud", - "repository": "https://github.com/AMDResearch/auplc-skills", + "repository": "https://github.com/AMDResearch/aup-learning-cloud", "license": "MIT", "keywords": [ "aup-learning-cloud", diff --git a/pyproject.toml b/pyproject.toml index 190d05ee..fe43c64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ [project] name = "aup-learning-cloud" -version = "0.1.0" +version = "0.1.1" description = "AUP Learning Cloud - JupyterHub deployment for AI education" requires-python = ">=3.10" diff --git a/tests/skills/test_check_skills_version.py b/tests/skills/test_check_skills_version.py new file mode 100644 index 00000000..05aec6f9 --- /dev/null +++ b/tests/skills/test_check_skills_version.py @@ -0,0 +1,69 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI regression tests for the skill-version checker.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts" / "check_skills_version.py" +CURSOR_GENERATOR = ROOT / ".github" / "scripts" / "generate_cursor_marketplace.py" + + +def write_json(path: Path, data: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_version_checker_fails_for_a_mismatched_manifest_version(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + checker = repo / "scripts" / CHECKER.name + checker.parent.mkdir(parents=True) + shutil.copy2(CHECKER, checker) + + (repo / "pyproject.toml").write_text('[project]\nname = "fixture"\nversion = "1.2.3"\n', encoding="utf-8") + for relative_path, data in { + ".claude-plugin/marketplace.json": {"metadata": {"version": "1.2.3"}}, + ".cursor-plugin/marketplace.json": {"metadata": {"version": "1.2.3"}}, + ".claude-plugin/plugin.json": {"version": "1.2.3"}, + ".cursor-plugin/plugin.json": {"version": "1.2.3"}, + "plugin-metadata.json": {"version": "0.0.0"}, + }.items(): + write_json(repo / relative_path, data) + + result = subprocess.run( + [sys.executable, str(checker)], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 1 + assert "version check failed:" in result.stderr + assert "plugin-metadata.json: version = 0.0.0, expected 1.2.3" in result.stderr + + +def test_marketplace_uses_root_description_and_metadata_version() -> None: + marketplace = json.loads((ROOT / ".claude-plugin" / "marketplace.json").read_text(encoding="utf-8")) + metadata = json.loads((ROOT / "plugin-metadata.json").read_text(encoding="utf-8")) + + assert marketplace["description"] == metadata["description"] + assert "description" not in marketplace["metadata"] + assert marketplace["metadata"]["version"] == metadata["version"] + assert marketplace["plugins"][0]["description"] + + result = subprocess.run( + [sys.executable, str(CURSOR_GENERATOR), "--check"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr From 44b5e16313c244592f02666f89c651c5057235e7 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Mon, 20 Jul 2026 19:10:24 +0100 Subject: [PATCH 036/180] feat(spawn): add auto-select accelerator option Let users skip manual GPU selection by choosing Auto (Best Available), which queries K8s node availability at spawn time and picks the accelerator with the most free GPUs (cheapest quota rate as tiebreaker). Backend: move _configure_spawner from options_from_form into start() so the async K8s query can resolve auto before pod configuration. Add _resolve_auto_accelerator with graceful fallback to cheapest rate. Frontend: synthesize an Auto radio option when a resource has 2+ accelerators. Show quota cost as a range and check affordability against the worst-case rate. --- runtime/hub/core/spawner/kubernetes.py | 82 ++++++++++++++++++- runtime/hub/frontend/apps/spawn/src/App.tsx | 47 ++++++++--- .../apps/spawn/src/components/CourseCard.tsx | 14 +++- 3 files changed, 129 insertions(+), 14 deletions(-) diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index e9d57fbd..6c759a49 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -299,9 +299,6 @@ def options_from_form(self, formdata) -> dict[str, Any]: if resource_type not in self.resource_images: raise RuntimeError(f"Unknown Resource: {resource_type}") - # Configure spawner based on selections - self._configure_spawner(resource_type, gpu_selection) - self.log.debug( f"User selected resource: {resource_type} with GPU: {gpu_selection} for {runtime_minutes} minutes" ) @@ -765,6 +762,75 @@ def _reset_per_spawn_state(self) -> None: self._has_git_init_container = False + async def _resolve_auto_accelerator(self, resource_type: str, eligible_keys: list[str]) -> str: + """Pick the best available accelerator from eligible_keys. + + Strategy: query K8s for GPU availability on nodes matching each + accelerator's nodeSelector, prefer nodes with free GPUs, break ties + by cheapest quotaRate. + """ + if not eligible_keys: + raise RuntimeError(f"No eligible accelerators for auto-selection on resource '{resource_type}'") + + if len(eligible_keys) == 1: + return eligible_keys[0] + + try: + from kubernetes_asyncio import client as k8s_client + from kubernetes_asyncio.client import ApiClient + + async with ApiClient() as api_client: + v1 = k8s_client.CoreV1Api(api_client) + nodes = await v1.list_node() + pods = await v1.list_pod_for_all_namespaces(field_selector="status.phase=Running") + + node_labels = { + node.metadata.name: (node.metadata.labels or {}, node.status.allocatable or {}) + for node in nodes.items + } + + used_gpus: dict[str, int] = {} + for pod in pods.items: + if not pod.spec.node_name: + continue + for container in pod.spec.containers or []: + requests = (container.resources.requests or {}) if container.resources else {} + gpu_req = int(requests.get("amd.com/gpu", 0)) + if gpu_req > 0: + used_gpus[pod.spec.node_name] = used_gpus.get(pod.spec.node_name, 0) + gpu_req + + availability = [] + for key in eligible_keys: + selector = self.node_selector_mapping.get(key, {}) + if not selector: + continue + + free = 0 + for node_name, (labels, allocatable) in node_labels.items(): + if all(labels.get(k) == v for k, v in selector.items()): + total = int(allocatable.get("amd.com/gpu", 0)) + free += max(0, total - used_gpus.get(node_name, 0)) + + rate = self.quota_rates.get(key, 99) + availability.append((key, free, rate)) + + if not availability: + self.log.warning("Auto-select found no matching accelerators, using first eligible key") + return eligible_keys[0] + + availability.sort(key=lambda x: (-x[1], x[2])) + chosen = availability[0] + self.log.info( + f"Auto-select candidates: {[(k, f'free={f}', f'rate={r}') for k, f, r in availability]} -> {chosen[0]}" + ) + return chosen[0] + + except Exception as e: + self.log.warning(f"Auto-accelerator K8s query failed, falling back to cheapest: {e}") + rated = [(k, self.quota_rates.get(k, 99)) for k in eligible_keys] + rated.sort(key=lambda x: x[1]) + return rated[0][0] + def _configure_spawner(self, resource_type: str, gpu_selection: str | None = None) -> None: """Configure the spawner based on the resource type and GPU selection.""" @@ -904,6 +970,16 @@ async def start(self): gpu_selection = self.user_options.get("gpu_selection", None) username = self.user.name.lower() + # Resolve "auto" accelerator selection before configuring the spawner + if gpu_selection == "auto": + metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None + eligible = list(metadata.acceleratorKeys) if metadata and metadata.acceleratorKeys else [] + gpu_selection = await self._resolve_auto_accelerator(resource_type, eligible) + self.user_options["gpu_selection"] = gpu_selection + self.log.info(f"Auto-selected accelerator '{gpu_selection}' for resource '{resource_type}'") + + self._configure_spawner(resource_type, gpu_selection) + # Determine accelerator type for quota calculation accelerator_type = gpu_selection if gpu_selection else "cpu" diff --git a/runtime/hub/frontend/apps/spawn/src/App.tsx b/runtime/hub/frontend/apps/spawn/src/App.tsx index ec2a5c98..fe4e9565 100644 --- a/runtime/hub/frontend/apps/spawn/src/App.tsx +++ b/runtime/hub/frontend/apps/spawn/src/App.tsx @@ -185,7 +185,19 @@ function App() { const availableAccelerators = useMemo(() => { if (!selectedResource?.metadata?.acceleratorKeys) return []; - return accelerators.filter(acc => selectedResource.metadata?.acceleratorKeys?.includes(acc.key)); + const real = accelerators.filter(acc => selectedResource.metadata?.acceleratorKeys?.includes(acc.key)); + if (real.length <= 1) return real; + const rates = real.map(a => a.quotaRate); + const minRate = Math.min(...rates); + const maxRate = Math.max(...rates); + const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; + const autoOption: Accelerator = { + key: 'auto', + displayName: 'Auto (Best Available)', + description: `Automatically selected based on availability. Rate: ${rateDesc}`, + quotaRate: minRate, + }; + return [autoOption, ...real]; }, [selectedResource, accelerators]); const selectedAccelerator = useMemo(() => { @@ -217,17 +229,26 @@ function App() { return `${spawnBase}?${params.toString()}`; }, [normalizedRepoUrl, repoBranch, repoUrlError, allowGitClone, selectedResource, selectedAccelerator]); - const { cost, canAfford, insufficientQuota, maxRuntime } = useMemo(() => { + const { cost, costMax, isAutoAccelerator, canAfford, insufficientQuota, maxRuntime } = useMemo(() => { + const isAuto = selectedAccelerator?.key === 'auto'; const rate = selectedAccelerator?.quotaRate ?? quota?.rates?.cpu ?? 1; const calculatedCost = quota?.enabled ? rate * runtime : 0; + let maxCost = calculatedCost; + if (isAuto && quota?.enabled) { + const realAccelerators = availableAccelerators.filter(a => a.key !== 'auto'); + const maxRate = Math.max(...realAccelerators.map(a => a.quotaRate)); + maxCost = maxRate * runtime; + } const balance = quota?.balance ?? 0; return { cost: calculatedCost, - canAfford: quota?.unlimited || balance >= calculatedCost, + costMax: maxCost, + isAutoAccelerator: isAuto, + canAfford: quota?.unlimited || balance >= maxCost, insufficientQuota: quota?.enabled && !quota?.unlimited && balance < 10, maxRuntime: quota?.enabled && !quota?.unlimited ? Math.min(240, Math.floor(balance / rate)) : 240, }; - }, [quota, selectedAccelerator?.quotaRate, runtime]); + }, [quota, selectedAccelerator?.quotaRate, selectedAccelerator?.key, runtime, availableAccelerators]); const canStart = selectedResource && canAfford && !repoUrlError && !repoValidating; const toggleFavorite = useCallback((key: string) => { @@ -575,12 +596,18 @@ function App() { </div> {quota?.enabled && !quota?.unlimited && ( <div className="sidebar-quota-preview"> - Est. cost: <strong style={{ color: canAfford ? '#2e7d32' : '#c62828' }}>{cost}</strong> - {' · '}Remaining: <strong style={{ color: canAfford ? '#2e7d32' : '#c62828' }}>{(quota?.balance ?? 0) - cost}</strong> + Est. cost: <strong style={{ color: canAfford ? '#2e7d32' : '#c62828' }}> + {isAutoAccelerator && cost !== costMax ? `${cost}–${costMax}` : cost} + </strong> + {' · '}Remaining: <strong style={{ color: canAfford ? '#2e7d32' : '#c62828' }}> + {(quota?.balance ?? 0) - (isAutoAccelerator ? costMax : cost)} + </strong> <span className="quota-rate-tip" title={ - `Rate: ${selectedAccelerator?.quotaRate ?? quota?.rates?.cpu ?? 1} credits/min` + - (selectedAccelerator ? ` (${selectedAccelerator.displayName})` : ' (CPU)') + - `\nCost = rate × ${runtime} min = ${cost} credits` + isAutoAccelerator + ? `Rate: varies by GPU assigned\nCost = ${cost}–${costMax} credits` + : `Rate: ${selectedAccelerator?.quotaRate ?? quota?.rates?.cpu ?? 1} credits/min` + + (selectedAccelerator ? ` (${selectedAccelerator.displayName})` : ' (CPU)') + + `\nCost = rate × ${runtime} min = ${cost} credits` }>?</span> </div> )} @@ -590,7 +617,7 @@ function App() { {/* Quota warning */} {quota?.enabled && !quota?.unlimited && !canAfford && selectedResource && ( <div className="sidebar-quota-warning"> - <strong>Insufficient Quota</strong> — You need {cost} credits but only have {quota?.balance ?? 0}. Reduce runtime or contact an administrator. + <strong>Insufficient Quota</strong> — You need {isAutoAccelerator && cost !== costMax ? `up to ${costMax}` : cost} credits but only have {quota?.balance ?? 0}. Reduce runtime or contact an administrator. </div> )} diff --git a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx index fd03de81..6388564e 100644 --- a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx +++ b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx @@ -86,7 +86,19 @@ export const CourseCard = memo(function CourseCard({ if (!acceleratorKeys || acceleratorKeys.length === 0) { return []; } - return accelerators.filter(acc => acceleratorKeys.includes(acc.key)); + const real = accelerators.filter(acc => acceleratorKeys.includes(acc.key)); + if (real.length <= 1) return real; + const rates = real.map(a => a.quotaRate); + const minRate = Math.min(...rates); + const maxRate = Math.max(...rates); + const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; + const autoOption: Accelerator = { + key: 'auto', + displayName: 'Auto (Best Available)', + description: `Automatically selected based on availability. Rate: ${rateDesc}`, + quotaRate: minRate, + }; + return [autoOption, ...real]; }, [acceleratorKeys, accelerators]); // Memoize resource tag to avoid recalculation From 9c0aca35f3acfa6ece701c485103447ff14cb44c Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Wed, 22 Jul 2026 13:13:35 +0100 Subject: [PATCH 037/180] Add button to show password --- runtime/hub/frontend/templates/login.html | 38 +++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/runtime/hub/frontend/templates/login.html b/runtime/hub/frontend/templates/login.html index 66edba21..b598f2b4 100755 --- a/runtime/hub/frontend/templates/login.html +++ b/runtime/hub/frontend/templates/login.html @@ -24,6 +24,20 @@ {% extends "page.html" %} +{% macro password_field(field_id="password_input") %} +<div> + <label for="{{ field_id }}" class="login-field-label block text-sm font-medium mb-1">Password</label> + <div class="relative"> + <input id="{{ field_id }}" type="password" autocomplete="current-password" name="password" + class="login-input block w-full pl-3 pr-10 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> + <button type="button" class="password-toggle absolute inset-y-0 right-0 flex items-center pr-3 text-gray-400 hover:text-gray-600" aria-label="Show password"> + <svg class="eye-open w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z"/></svg> + <svg class="eye-closed w-5 h-5 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13.875 18.825A10.05 10.05 0 0112 19c-4.478 0-8.268-2.943-9.543-7a9.97 9.97 0 011.563-3.029m5.858.908a3 3 0 114.243 4.243M9.878 9.878l4.242 4.242M9.88 9.88l-3.29-3.29m7.532 7.532l3.29 3.29M3 3l3.59 3.59m0 0A9.953 9.953 0 0112 5c4.478 0 8.268 2.943 9.543 7a10.025 10.025 0 01-4.132 5.411m0 0L21 21"/></svg> + </button> + </div> +</div> +{% endmacro %} + {% if announcement_login is string %} {% set announcement = announcement_login %} {% endif %} @@ -151,11 +165,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> </div> - <div> - <label for="password_input" class="login-field-label block text-sm font-medium mb-1">Password</label> - <input id="password_input" type="password" autocomplete="current-password" name="password" - class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> - </div> + {{ password_field() }} <div class="mt-6"> <button id="login_submit" type="submit" @@ -221,11 +231,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> </div> - <div> - <label for="password_input" class="login-field-label block text-sm font-medium mb-1">Password</label> - <input id="password_input" type="password" autocomplete="current-password" name="password" - class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> - </div> + {{ password_field() }} <div class="mt-6"> <button id="login_submit" type="submit" @@ -261,5 +267,17 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L form.find('.feedback-container button').attr('disabled', true); form.find('.feedback-container>*').toggleClass('hidden'); }); + + // password show/hide toggle + document.querySelectorAll('.password-toggle').forEach(function(btn) { + btn.addEventListener('click', function() { + var input = btn.parentElement.querySelector('input'); + var show = input.type === 'password'; + input.type = show ? 'text' : 'password'; + btn.querySelector('.eye-open').classList.toggle('hidden', show); + btn.querySelector('.eye-closed').classList.toggle('hidden', !show); + btn.setAttribute('aria-label', show ? 'Hide password' : 'Show password'); + }); + }); </script> {% endblock script %} From 8b13dbfcf684b1f87321697bac46d4eead491cdc Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Mon, 27 Jul 2026 15:31:10 +0100 Subject: [PATCH 038/180] Fix login issues for native users --- runtime/hub/core/jupyterhub_config.py | 4 +++- runtime/hub/frontend/templates/login.html | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime/hub/core/jupyterhub_config.py b/runtime/hub/core/jupyterhub_config.py index 0a2bd1f9..ccc75c2e 100644 --- a/runtime/hub/core/jupyterhub_config.py +++ b/runtime/hub/core/jupyterhub_config.py @@ -151,7 +151,9 @@ def _camel_case(s: str) -> str: # Inject platform identity into every Jinja template context so that # {{ powered_by }} is available in all Hub-rendered pages. -c.JupyterHub.template_vars = {"powered_by": "AUP Learning Cloud"} +if not isinstance(c.JupyterHub.template_vars, dict): + c.JupyterHub.template_vars = {} +c.JupyterHub.template_vars.setdefault("powered_by", "AUP Learning Cloud") # Database configuration db_type = z2jh.get_config("hub.db.type") diff --git a/runtime/hub/frontend/templates/login.html b/runtime/hub/frontend/templates/login.html index b598f2b4..50377583 100755 --- a/runtime/hub/frontend/templates/login.html +++ b/runtime/hub/frontend/templates/login.html @@ -42,6 +42,7 @@ {% set announcement = announcement_login %} {% endif %} {% set github_helper_text = login_github_helper_text|default("", true)|trim %} +{% set auth_mode = authenticator_mode|default('multi') %} {% block login_widget %} {% endblock login_widget %} @@ -105,7 +106,7 @@ </svg> <h2 class="text-3xl md:text-4xl font-bold text-white mb-4">{{ platform_name or 'AUP Learning Cloud' }}</h2> <p class="text-blue-100 mb-8">Experience the next generation of AI acceleration with AMD ROCm™.</p> - {% if login_service and not authenticator_mode.startswith('multi') %} + {% if login_service and auth_mode != 'multi' %} <a role="button" class='inline-block bg-white hover:bg-gray-100 text-black font-medium py-2 px-4 rounded transition duration-300' href='{{ authenticator_login_url | safe }}'> From c8cebedcb8f5669e214fb14457113e335e75fd5d Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 039/180] feat(installer): add guarded GPU access policy --- auplc_installer/gpu_access.py | 367 +++++++++++++++++ tests/installer/test_gpu_access.py | 627 +++++++++++++++++++++++++++++ 2 files changed, 994 insertions(+) create mode 100644 auplc_installer/gpu_access.py create mode 100644 tests/installer/test_gpu_access.py diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py new file mode 100644 index 00000000..7c46991a --- /dev/null +++ b/auplc_installer/gpu_access.py @@ -0,0 +1,367 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Single-node AMD GPU device-access source of truth. + +The host's existing ``render`` group is authoritative. Its numeric GID is +persisted here so installer reruns and runtime-only commands cannot silently +select a different permission model. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from auplc_installer.util import InstallerError, run, run_capture + +GPU_ACCESS_STATE_VERSION = 1 +MAX_RENDER_GID = (2**32) - 2 +GPU_ACCESS_STATE_PATH = Path("/var/lib/auplc/gpu-access.json") +GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") +LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") +LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +LEGACY_ROCM_DEVICES_RULES_PATH = Path("/etc/udev/rules.d/70-rocm-devices.rules") +LEGACY_KFD_RULES = 'KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n' +LEGACY_AMDGPU_RULES = ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_AMDGPU_PXE_RULES = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n' +LEGACY_ROCM_DEVICES_RULES = ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_RULE_CONTENTS: dict[Path, frozenset[str]] = { + LEGACY_KFD_RULES_PATH: frozenset((LEGACY_KFD_RULES,)), + LEGACY_AMDGPU_RULES_PATH: frozenset((LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_PXE_RULES)), + LEGACY_ROCM_DEVICES_RULES_PATH: frozenset((LEGACY_ROCM_DEVICES_RULES,)), +} +UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." +CANONICAL_UDEV_RULES = ( + f"{UDEV_MANAGED_MARKER}\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' +) +_FSYNC_PATH_SCRIPT = ( + "import os\n" + "import sys\n" + "fd = os.open(sys.argv[1], os.O_RDONLY)\n" + "try:\n" + " os.fsync(fd)\n" + "finally:\n" + " os.close(fd)\n" +) +_VERIFY_DEVICE_ACCESS_SCRIPT = ( + "import os, pathlib, stat, sys\n" + "gid = int(sys.argv[1])\n" + "paths = [pathlib.Path('/dev/kfd')]\n" + "for node in pathlib.Path('/sys/class/drm').glob('renderD*'):\n" + " driver = node / 'device' / 'driver'\n" + " if driver.exists() and driver.resolve().name == 'amdgpu': paths.append(pathlib.Path('/dev/dri') / node.name)\n" + "if len(paths) == 1: raise SystemExit('no AMD renderD device found')\n" + "for path in paths:\n" + " data = path.lstat()\n" + " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or data.st_gid != gid or stat.S_IMODE(data.st_mode) != 0o660: raise SystemExit(f'bad GPU device access: {path}')\n" +) + + +@dataclass(frozen=True) +class GpuAccessState: + """Versioned, immutable record of the host render-group GID.""" + + render_gid: int + version: int = GPU_ACCESS_STATE_VERSION + + def __post_init__(self) -> None: + if self.version != GPU_ACCESS_STATE_VERSION: + raise InstallerError(f"Unsupported GPU access state version: {self.version!r}") + _validate_render_gid(self.render_gid) + + +class GpuAccessHost(Protocol): + """Privileged host-operation seam for GPU access provisioning.""" + + def get_group_entry(self, group_name: str) -> str: + """Return the NSS group record for ``group_name``.""" + + def read_text(self, path: Path) -> str | None: + """Return a privileged file's text, or ``None`` when it is absent.""" + + def write_state_atomically(self, path: Path, text: str) -> None: + """Atomically replace a state file with same-directory persistence.""" + + def write_udev_rule(self, path: Path, text: str) -> None: + """Write a managed udev rule after reconciliation has authorized it.""" + + def reload_udev_rules(self) -> None: + """Reload host udev rules.""" + + def trigger_udev(self) -> None: + """Apply reloaded udev rules to current devices.""" + + def settle_udev(self) -> None: + """Wait until triggered udev events finish before inode verification.""" + + def remove_udev_rule(self, path: Path) -> None: + """Remove an explicitly recognized legacy udev rule.""" + + def verify_device_access(self, render_gid: int) -> None: + """Verify the relevant GPU device inodes use the requested access contract.""" + + def is_symlink(self, path: Path) -> bool: + """Return whether ``path`` is a symlink without following it.""" + + def is_regular_file(self, path: Path) -> bool: + """Return whether an existing ``path`` is a regular file.""" + + def path_exists(self, path: Path) -> bool: + """Return whether ``path`` exists after a separate symlink check.""" + + def is_directory(self, path: Path) -> bool: + """Return whether an existing ``path`` is a directory.""" + + +class SystemGpuAccessHost: + """Production host adapter using the installer's sudo-aware command helpers.""" + + def get_group_entry(self, group_name: str) -> str: + result = run_capture(["getent", "group", group_name], check=False) + if result.returncode != 0: + return "" + return result.stdout or "" + + def read_text(self, path: Path) -> str | None: + exists = run(["test", "-e", str(path)], sudo=True, check=False) + if exists.returncode != 0: + return None + result = run_capture(["cat", str(path)], sudo=True) + return result.stdout or "" + + def write_state_atomically(self, path: Path, text: str) -> None: + """Durably replace state with a same-directory temporary file.""" + self._write_text_atomically(path, text) + + def write_udev_rule(self, path: Path, text: str) -> None: + self._write_text_atomically(path, text) + + def _write_text_atomically(self, path: Path, text: str) -> None: + """Durably replace ``path`` after atomically renaming a temporary file.""" + _validate_parent_chain(self, path.parent) + run(["mkdir", "-p", str(path.parent)], sudo=True) + temporary_result = run_capture( + ["mktemp", str(path.parent / f".{path.name}.XXXXXX")], + sudo=True, + ) + temporary_path = (temporary_result.stdout or "").strip() + if not temporary_path: + raise InstallerError(f"Could not create temporary GPU access state beside {path}") + + try: + run(["tee", temporary_path], sudo=True, input_text=text) + run(["chmod", "0644", temporary_path], sudo=True) + self._fsync_path(temporary_path) + run(["mv", "-f", temporary_path, str(path)], sudo=True) + self._fsync_path(str(path.parent)) + except BaseException: + run(["rm", "-f", temporary_path], sudo=True, check=False) + raise + + def _fsync_path(self, path: str) -> None: + run(["python3", "-c", _FSYNC_PATH_SCRIPT, path], sudo=True) + + def reload_udev_rules(self) -> None: + run(["udevadm", "control", "--reload-rules"], sudo=True) + + def trigger_udev(self) -> None: + run(["udevadm", "trigger"], sudo=True) + + def settle_udev(self) -> None: + run(["udevadm", "settle"], sudo=True) + + def remove_udev_rule(self, path: Path) -> None: + run(["rm", "-f", str(path)], sudo=True) + + def verify_device_access(self, render_gid: int) -> None: + run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT, str(render_gid)], sudo=True) + + def is_symlink(self, path: Path) -> bool: + return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 + + def is_regular_file(self, path: Path) -> bool: + return run(["test", "-f", str(path)], sudo=True, check=False).returncode == 0 + + def path_exists(self, path: Path) -> bool: + return run(["test", "-e", str(path)], sudo=True, check=False).returncode == 0 + + def is_directory(self, path: Path) -> bool: + return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 + + +def serialize_gpu_access_state(state: GpuAccessState) -> str: + """Return the canonical on-disk JSON representation for ``state``.""" + return ( + json.dumps( + {"renderGid": state.render_gid, "version": state.version}, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + + +def parse_gpu_access_state(text: str) -> GpuAccessState: + """Parse strict versioned GPU access state, failing closed on bad input.""" + try: + payload = json.loads(text) + except (TypeError, json.JSONDecodeError) as exc: + raise InstallerError("Malformed GPU access state") from exc + + if not isinstance(payload, dict) or set(payload) != {"renderGid", "version"}: + raise InstallerError("Malformed GPU access state") + + version = payload["version"] + render_gid = payload["renderGid"] + if type(version) is not int or version != GPU_ACCESS_STATE_VERSION: + raise InstallerError("Unsupported GPU access state version") + _validate_render_gid(render_gid) + return GpuAccessState(render_gid=render_gid, version=version) + + +def resolve_render_gid(getent_output: str) -> int: + """Parse the numeric GID from one ``getent group render`` record.""" + if not isinstance(getent_output, str): + raise InstallerError("Could not resolve the host render group") + + lines = getent_output.splitlines() + if len(lines) != 1: + raise InstallerError("Could not resolve the host render group") + + fields = lines[0].split(":") + if len(fields) != 4 or fields[0] != "render": + raise InstallerError("Could not resolve the host render group") + + raw_gid = fields[2] + if not raw_gid.isascii() or not raw_gid.isdecimal(): + raise InstallerError("Could not resolve the host render group") + + render_gid = int(raw_gid) + _validate_render_gid(render_gid) + return render_gid + + +def render_udev_rules() -> str: + """Return the canonical, least-privilege AMD GPU udev rules.""" + return CANONICAL_UDEV_RULES + + +def provision_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: + """Create or reuse immutable state and reconcile the managed udev rule. + + When state is absent, adopt the current host ``render`` GID only after the + udev rule has been applied and verified. Existing state must match the host + group before any mutation occurs. + """ + return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) + + +def load_existing_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: + """Reconcile runtime GPU access, adopting missing state for pre-change installs. + + Runtime, upgrade, and reinstall paths reuse persisted state when present. + For an installation created before GPU access state existed, this performs a + one-time host ``render`` GID adoption after udev verification. A persisted + GID that differs from the current host group remains a hard failure. + """ + return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) + + +def _reconcile_gpu_access(host: GpuAccessHost) -> GpuAccessState: + _validate_parent_chain(host, GPU_ACCESS_STATE_PATH.parent) + _validate_parent_chain(host, GPU_ACCESS_RULES_PATH.parent) + state_text = _read_regular_text(host, GPU_ACCESS_STATE_PATH) + host_gid = resolve_render_gid(host.get_group_entry("render")) + + if state_text is None: + state = GpuAccessState(render_gid=host_gid) + persist_state = True + else: + state = parse_gpu_access_state(state_text) + if state.render_gid != host_gid: + raise InstallerError( + f"Persisted render GID does not match the current host render group ({state.render_gid} != {host_gid})" + ) + persist_state = False + + legacy_paths = _legacy_rules_to_remove(host) + existing_rule = _read_regular_text(host, GPU_ACCESS_RULES_PATH) + rewrite_rule = _should_rewrite_udev_rule(existing_rule) + + for path in legacy_paths: + host.remove_udev_rule(path) + if rewrite_rule: + host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) + host.reload_udev_rules() + host.trigger_udev() + host.settle_udev() + host.verify_device_access(state.render_gid) + if persist_state: + host.write_state_atomically(GPU_ACCESS_STATE_PATH, serialize_gpu_access_state(state)) + + return state + + +def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: + if host.is_symlink(path): + raise InstallerError(f"Refusing symlinked GPU access file: {path}") + if not host.path_exists(path): + return None + if not host.is_regular_file(path): + raise InstallerError(f"Refusing non-regular GPU access file: {path}") + return host.read_text(path) + + +def _validate_parent_chain(host: GpuAccessHost, parent: Path) -> None: + components = [*reversed(parent.parents), parent] + for index, component in enumerate(components): + if host.is_symlink(component): + raise InstallerError(f"Refusing symlinked GPU access directory: {component}") + if not host.path_exists(component): + if index != len(components) - 1: + raise InstallerError(f"Missing parent GPU access directory: {component}") + return + if not host.is_directory(component): + raise InstallerError(f"Refusing non-directory GPU access parent: {component}") + + +def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: + removals: list[Path] = [] + for path, expected_contents in LEGACY_RULE_CONTENTS.items(): + content = _read_regular_text(host, path) + if content is None: + continue + if content not in expected_contents: + raise InstallerError(f"Refusing to remove unexpected legacy GPU udev rule: {path}") + removals.append(path) + return removals + + +def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: + if existing_rule is None: + return True + if existing_rule == render_udev_rules(): + return False + if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: + raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") + return True + + +def _validate_render_gid(render_gid: object) -> None: + if type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID: + raise InstallerError(f"Invalid render group GID: {render_gid!r}") diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py new file mode 100644 index 00000000..e742c219 --- /dev/null +++ b/tests/installer/test_gpu_access.py @@ -0,0 +1,627 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for the single-node AMD GPU access source of truth.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + GPU_ACCESS_RULES_PATH, + GPU_ACCESS_STATE_PATH, + LEGACY_AMDGPU_PXE_RULES, + LEGACY_AMDGPU_RULES, + LEGACY_AMDGPU_RULES_PATH, + LEGACY_KFD_RULES, + LEGACY_KFD_RULES_PATH, + LEGACY_ROCM_DEVICES_RULES, + LEGACY_ROCM_DEVICES_RULES_PATH, + MAX_RENDER_GID, + GpuAccessState, + SystemGpuAccessHost, + load_existing_gpu_access, + parse_gpu_access_state, + provision_gpu_access, + render_udev_rules, + resolve_render_gid, + serialize_gpu_access_state, +) +from auplc_installer.util import InstallerError + + +class FakeGpuAccessHost: + """In-memory adapter for the installer host-operation seam.""" + + def __init__(self, *, getent_output: str, files: dict[Path, str] | None = None) -> None: + self.getent_output = getent_output + self.files = dict(files or {}) + self.calls: list[str] = [] + self.symlinks: set[Path] = set() + self.nonregular_files: set[Path] = set() + self.directories = { + Path("/"), + Path("/etc"), + Path("/etc/udev"), + Path("/etc/udev/rules.d"), + Path("/var"), + Path("/var/lib"), + Path("/var/lib/auplc"), + } + + def get_group_entry(self, group_name: str) -> str: + self.calls.append(f"get-group:{group_name}") + return self.getent_output + + def read_text(self, path: Path) -> str | None: + self.calls.append(f"read:{path}") + return self.files.get(path) + + def write_state_atomically(self, path: Path, text: str) -> None: + self.calls.append(f"write-state:{path}") + self.files[path] = text + + def write_udev_rule(self, path: Path, text: str) -> None: + self.calls.append(f"write-rule:{path}") + self.files[path] = text + + def remove_udev_rule(self, path: Path) -> None: + self.calls.append(f"remove-rule:{path}") + self.files.pop(path, None) + + def reload_udev_rules(self) -> None: + self.calls.append("reload-udev") + + def trigger_udev(self) -> None: + self.calls.append("trigger-udev") + + def settle_udev(self) -> None: + self.calls.append("settle-udev") + + def verify_device_access(self, render_gid: int) -> None: + self.calls.append(f"verify-devices:{render_gid}") + + def is_symlink(self, path: Path) -> bool: + return path in self.symlinks + + def is_regular_file(self, path: Path) -> bool: + return path in self.files + + def path_exists(self, path: Path) -> bool: + return path in self.files or path in self.symlinks or path in self.nonregular_files or path in self.directories + + def is_directory(self, path: Path) -> bool: + return path in self.directories + + +def test_gpu_access_state_round_trips_as_versioned_json() -> None: + state = GpuAccessState(render_gid=993) + + serialized = serialize_gpu_access_state(state) + + assert serialized == '{"renderGid":993,"version":1}\n' + assert parse_gpu_access_state(serialized) == state + + +@pytest.mark.parametrize( + "state_text", + [ + "not json", + '{"renderGid":993,"version":2}', + '{"renderGid":0,"version":1}', + f'{{"renderGid":{MAX_RENDER_GID + 1},"version":1}}', + '{"renderGid":true,"version":1}', + '{"renderGid":993,"unexpected":true,"version":1}', + ], +) +def test_parse_gpu_access_state_rejects_malformed_or_unsupported_state(state_text: str) -> None: + with pytest.raises(RuntimeError): + parse_gpu_access_state(state_text) + + +def test_resolve_render_gid_reads_the_numeric_getent_field() -> None: + assert resolve_render_gid("render:x:993:student\n") == 993 + + +@pytest.mark.parametrize( + "getent_output", + [ + "", + "video:x:44:student\n", + "render:x:0:student\n", + "render:x:not-a-number:student\n", + f"render:x:{MAX_RENDER_GID + 1}:student\n", + "render:x:993:student\nrender:x:994:student\n", + ], +) +def test_resolve_render_gid_rejects_missing_or_invalid_group_records(getent_output: str) -> None: + with pytest.raises(RuntimeError): + resolve_render_gid(getent_output) + + +def test_render_udev_rules_is_the_canonical_least_privilege_policy() -> None: + rules = render_udev_rules() + + assert rules == ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + ) + assert "card" not in rules + assert "0666" not in rules + assert "chmod" not in rules + + +def test_device_verification_uses_lstat_and_requires_character_devices() -> None: + assert "path.lstat()" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + assert "stat.S_ISCHR(data.st_mode)" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + + +@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d"), Path("/var/lib/auplc")]) +def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.symlinks.add(unsafe_parent) + + with pytest.raises(InstallerError, match="symlinked GPU access directory"): + provision_gpu_access(host) + + assert not any(call.startswith(("read:", "write-", "remove-rule:")) for call in host.calls) + + +def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.nonregular_files.add(GPU_ACCESS_RULES_PATH) + + with pytest.raises(InstallerError, match="non-regular GPU access file"): + provision_gpu_access(host) + + assert f"read:{GPU_ACCESS_RULES_PATH}" not in host.calls + assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls + + +def test_provision_adopts_host_render_gid_and_installs_canonical_rule() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-4:] == [ + "trigger-udev", + "settle-udev", + "verify-devices:993", + f"write-state:{GPU_ACCESS_STATE_PATH}", + ] + + +def test_provision_migrates_exact_legacy_rules_then_verifies_before_persisting_state() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_KFD_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n'), + LEGACY_AMDGPU_RULES_PATH: ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_KFD_RULES == ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n') + assert LEGACY_AMDGPU_RULES == ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + assert LEGACY_KFD_RULES_PATH not in host.files + assert LEGACY_AMDGPU_RULES_PATH not in host.files + assert host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") < host.calls.index( + f"write-rule:{GPU_ACCESS_RULES_PATH}" + ) + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_provision_migrates_exact_legacy_rocm_devices_rule() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_ROCM_DEVICES_RULES_PATH: ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_ROCM_DEVICES_RULES == ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + assert LEGACY_ROCM_DEVICES_RULES_PATH not in host.files + + +def test_provision_migrates_exact_legacy_pxe_rule_at_amdgpu_path() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_AMDGPU_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n'), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_AMDGPU_PXE_RULES == ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n') + assert LEGACY_AMDGPU_RULES_PATH not in host.files + + +def test_near_legacy_pxe_rule_fails_closed_without_removal() -> None: + near_variant = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' + host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_AMDGPU_RULES_PATH: near_variant}) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[LEGACY_AMDGPU_RULES_PATH] == near_variant + + +def test_modified_legacy_rocm_devices_rule_fails_closed_without_removal() -> None: + modified = ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_ROCM_DEVICES_RULES_PATH: modified}) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[LEGACY_ROCM_DEVICES_RULES_PATH] == modified + + +def test_provision_reapplies_and_verifies_matching_immutable_state() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', + GPU_ACCESS_RULES_PATH: render_udev_rules(), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert not any(call.startswith("write-") for call in host.calls) + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices:993"] + + +def test_provision_fails_before_mutation_when_persisted_gid_differs_from_host() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:994:student\n", + files={GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n'}, + ) + + with pytest.raises(RuntimeError, match="does not match"): + provision_gpu_access(host) + + assert not any(call.startswith("write-") for call in host.calls) + assert "reload-udev" not in host.calls + assert "trigger-udev" not in host.calls + + +def test_provision_fails_before_writing_state_when_rule_is_unmanaged() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}, + ) + + with pytest.raises(RuntimeError, match="unmanaged"): + provision_gpu_access(host) + + assert GPU_ACCESS_STATE_PATH not in host.files + assert not any(call.startswith("write-") for call in host.calls) + + +def test_load_existing_gpu_access_adopts_missing_state_after_verification() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + state = load_existing_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_managed_rule_is_reconciled_and_reloaded_when_content_changes() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', + GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", + }, + ) + + state = load_existing_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-5:] == [ + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + "trigger-udev", + "settle-udev", + "verify-devices:993", + ] + + +def test_system_adapter_persists_state_with_a_same_directory_temporary_file(monkeypatch) -> None: + commands: list[list[str]] = [] + capture_commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + return SimpleNamespace(returncode=0) + + def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: + capture_commands.append(command) + return SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n") + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) + + SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + + assert capture_commands == [["mktemp", "/var/lib/auplc/.gpu-access.json.XXXXXX"]] + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/var/lib/auplc"], + ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], + ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], + ["mv", "-f", "/var/lib/auplc/.gpu-access.json.temporary", "/var/lib/auplc/gpu-access.json"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc"], + ] + + +def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: + commands: list[list[str]] = [] + capture_commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + return SimpleNamespace(returncode=0) + + def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: + capture_commands.append(command) + return SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n") + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) + + SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") + + assert capture_commands == [["mktemp", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.XXXXXX"]] + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/etc/udev/rules.d"], + ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + [ + "mv", + "-f", + "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", + "/etc/udev/rules.d/70-auplc-gpu-access.rules", + ], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d"], + ] + + +def test_system_adapter_removes_temporary_file_when_durable_write_fails(monkeypatch) -> None: + commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + if command == ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"]: + raise InstallerError("fsync failed") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr( + gpu_access, + "run_capture", + lambda command, **kwargs: SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n"), + ) + + with pytest.raises(InstallerError, match="fsync failed"): + SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/var/lib/auplc"], + ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], + ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], + ["rm", "-f", "/var/lib/auplc/.gpu-access.json.temporary"], + ] + + +@pytest.mark.parametrize( + ("failing_method", "expected_calls"), + [ + ( + "write_udev_rule", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + ], + ), + ( + "reload_udev_rules", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + ], + ), + ( + "trigger_udev", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + "trigger-udev", + ], + ), + ], +) +def test_first_install_does_not_persist_state_until_udev_reconciliation_succeeds( + monkeypatch, + failing_method: str, + expected_calls: list[str], +) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + original_method = getattr(host, failing_method) + + def fail_after_recording(*args: object) -> None: + original_method(*args) + raise InstallerError(f"{failing_method} failed") + + monkeypatch.setattr(host, failing_method, fail_after_recording) + + with pytest.raises(InstallerError, match=f"{failing_method} failed"): + provision_gpu_access(host) + + assert host.calls[-len(expected_calls) :] == expected_calls + assert GPU_ACCESS_STATE_PATH not in host.files + + +def test_failed_udev_reconciliation_never_rewrites_existing_state(monkeypatch) -> None: + original_state = '{"renderGid":993,"version":1}\n' + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: original_state, + GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", + }, + ) + + def fail_reload() -> None: + host.calls.append("reload-udev") + raise InstallerError("reload failed") + + monkeypatch.setattr(host, "reload_udev_rules", fail_reload) + + with pytest.raises(InstallerError, match="reload failed"): + provision_gpu_access(host) + + assert host.files[GPU_ACCESS_STATE_PATH] == original_state + assert not any(call.startswith("write-state:") for call in host.calls) + assert "trigger-udev" not in host.calls + + +def test_failed_reload_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_reload() -> None: + host.calls.append("reload-udev") + raise InstallerError("reload failed") + + monkeypatch.setattr(host, "reload_udev_rules", fail_reload) + with pytest.raises(InstallerError, match="reload failed"): + provision_gpu_access(host) + assert GPU_ACCESS_STATE_PATH not in host.files + + monkeypatch.setattr(host, "reload_udev_rules", FakeGpuAccessHost.reload_udev_rules.__get__(host)) + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.calls[-2:] == ["verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_failed_settle_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_settle() -> None: + host.calls.append("settle-udev") + raise InstallerError("settle failed") + + monkeypatch.setattr(host, "settle_udev", fail_settle) + with pytest.raises(InstallerError, match="settle failed"): + provision_gpu_access(host) + assert GPU_ACCESS_STATE_PATH not in host.files + assert "verify-devices:993" not in host.calls + + monkeypatch.setattr(host, "settle_udev", FakeGpuAccessHost.settle_udev.__get__(host)) + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_failed_inode_verification_does_not_adopt_state(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_verification(render_gid: int) -> None: + host.calls.append(f"verify-devices:{render_gid}") + raise InstallerError("device ownership mismatch") + + monkeypatch.setattr(host, "verify_device_access", fail_verification) + + with pytest.raises(InstallerError, match="ownership mismatch"): + provision_gpu_access(host) + + assert host.calls[-1] == "verify-devices:993" + assert GPU_ACCESS_STATE_PATH not in host.files + + +@pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) +def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.symlinks.add(path) + + with pytest.raises(InstallerError, match="symlinked"): + provision_gpu_access(host) + + assert GPU_ACCESS_STATE_PATH not in host.files + + +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_KFD_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\n'), + (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", GROUP="render", MODE="0660"\n'), + ], +) +def test_one_line_legacy_variants_fail_closed_without_removal(path: Path, content: str) -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={path: content}, + ) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[path] == content + assert GPU_ACCESS_STATE_PATH not in host.files From c6701a7d5fbac0cc2c7b5d2d7d57335aece1ca2a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 040/180] feat(installer): classify local GPU hardware --- auplc_installer/gpu_hardware.py | 63 +++++++++++++++++ tests/installer/test_gpu_hardware.py | 102 +++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 auplc_installer/gpu_hardware.py create mode 100644 tests/installer/test_gpu_hardware.py diff --git a/auplc_installer/gpu_hardware.py b/auplc_installer/gpu_hardware.py new file mode 100644 index 00000000..0b292eb7 --- /dev/null +++ b/auplc_installer/gpu_hardware.py @@ -0,0 +1,63 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Read-only local AMD GPU hardware classification from Linux PCI sysfs.""" + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import Final + +PCI_DEVICES_ROOT: Final = Path("/sys/bus/pci/devices") +AMD_PCI_VENDOR: Final = "0x1002" +DISPLAY_CLASS_PREFIX: Final = "0x03" +_HEX_DIGITS: Final = frozenset("0123456789abcdef") + + +class GpuHardware(Enum): + """The local host's AMD display-hardware eligibility.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +def classify_gpu_hardware(pci_devices_root: Path = PCI_DEVICES_ROOT) -> GpuHardware: + """Classify local hardware using complete PCI vendor and class evidence.""" + try: + devices = tuple(pci_devices_root.iterdir()) + except OSError: + return GpuHardware.UNKNOWN + + if not devices: + return GpuHardware.UNKNOWN + + scan_is_complete = True + for device in devices: + vendor = _read_pci_attribute(device / "vendor") + pci_class = _read_pci_attribute(device / "class") + if vendor is None or pci_class is None or not _has_valid_pci_attributes(vendor, pci_class): + scan_is_complete = False + continue + if vendor == AMD_PCI_VENDOR and pci_class.startswith(DISPLAY_CLASS_PREFIX): + return GpuHardware.GPU + + return GpuHardware.CPU if scan_is_complete else GpuHardware.UNKNOWN + + +def _read_pci_attribute(path: Path) -> str | None: + try: + value = path.read_text(encoding="ascii").strip().lower() + except (OSError, UnicodeDecodeError): + return None + return value or None + + +def _has_valid_pci_attributes(vendor: str, pci_class: str) -> bool: + return _is_pci_hex(vendor, digits=4) and _is_pci_hex(pci_class, digits=6) + + +def _is_pci_hex(value: str, *, digits: int) -> bool: + return ( + len(value) == digits + 2 and value.startswith("0x") and all(character in _HEX_DIGITS for character in value[2:]) + ) diff --git a/tests/installer/test_gpu_hardware.py b/tests/installer/test_gpu_hardware.py new file mode 100644 index 00000000..6a455e26 --- /dev/null +++ b/tests/installer/test_gpu_hardware.py @@ -0,0 +1,102 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for local AMD GPU hardware classification from PCI sysfs evidence.""" + +from __future__ import annotations + +from pathlib import Path + +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware + + +def test_classify_gpu_hardware_returns_gpu_for_amd_display_controller(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + device = pci_devices / "0000:03:00.0" + device.mkdir(parents=True) + (device / "vendor").write_text("0x1002\n", encoding="ascii") + (device / "class").write_text("0x030200\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU + + +def test_classify_gpu_hardware_returns_cpu_for_complete_scan_without_amd_display(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + intel_display = pci_devices / "0000:00:02.0" + intel_display.mkdir(parents=True) + (intel_display / "vendor").write_text("0x8086\n", encoding="ascii") + (intel_display / "class").write_text("0x030000\n", encoding="ascii") + amd_audio = pci_devices / "0000:03:00.1" + amd_audio.mkdir() + (amd_audio / "vendor").write_text("0x1002\n", encoding="ascii") + (amd_audio / "class").write_text("0x040300\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.CPU + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_missing(tmp_path: Path) -> None: + hardware = classify_gpu_hardware(tmp_path / "missing") + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_empty(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + pci_devices.mkdir() + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_incomplete_pci_evidence(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + missing_vendor = pci_devices / "0000:00:02.0" + missing_vendor.mkdir(parents=True) + (missing_vendor / "class").write_text("0x030000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_malformed_pci_evidence(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + malformed_vendor = pci_devices / "0000:00:02.0" + malformed_vendor.mkdir(parents=True) + (malformed_vendor / "vendor").write_text("0xZZZZ\n", encoding="ascii") + (malformed_vendor / "class").write_text("0x030000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_unreadable_pci_attribute(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + unreadable_class = pci_devices / "0000:00:02.0" + unreadable_class.mkdir(parents=True) + (unreadable_class / "vendor").write_text("0x8086\n", encoding="ascii") + (unreadable_class / "class").mkdir() + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_prefers_positive_amd_evidence_over_incomplete_sibling(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + incomplete_device = pci_devices / "0000:00:02.0" + incomplete_device.mkdir(parents=True) + (incomplete_device / "vendor").write_text("0x8086\n", encoding="ascii") + gpu_device = pci_devices / "0000:03:00.0" + gpu_device.mkdir() + (gpu_device / "vendor").write_text("0x1002\n", encoding="ascii") + (gpu_device / "class").write_text("0x038000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU From 3e2bf6b98527d90fb0f3ddb899bd1c4b45725c01 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 041/180] feat(installer): wire GPU access into workflows --- auplc_installer/cli.py | 52 +++++- tests/installer/test_cli_gpu_access.py | 227 +++++++++++++++++++++++++ tests/installer/test_cli_helpers.py | 1 + 3 files changed, 271 insertions(+), 9 deletions(-) create mode 100644 tests/installer/test_cli_gpu_access.py diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f1aea4a7..f2e9b538 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -13,8 +13,9 @@ import contextlib import sys import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path +from typing import NoReturn from auplc_installer import __version__ from auplc_installer.catalog import parse_selection_spec @@ -22,6 +23,8 @@ detect_and_configure_gpu, refine_gpu_config_from_node_labels, ) +from auplc_installer.gpu_access import GpuAccessState, load_existing_gpu_access, provision_gpu_access +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware from auplc_installer.helm import ( deploy_runtime, dev_quick_rollout, @@ -319,6 +322,22 @@ def cmd_install(state: InstallerState, *, pull: bool) -> None: keepalive.stop() +def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: + raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") + + +def _render_gid_for_local_hardware(reconcile_gpu_access: Callable[[], GpuAccessState]) -> int | None: + match classify_gpu_hardware(): + case GpuHardware.GPU: + return reconcile_gpu_access().render_gid + case GpuHardware.CPU: + return None + case GpuHardware.UNKNOWN: + raise InstallerError("Could not determine local AMD GPU hardware; refusing to modify installer state") + case unreachable: + _raise_unreachable_gpu_hardware(unreachable) + + def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: """Body of ``cmd_install`` after sudo session has been primed.""" # Pre-compute the image-stage label so the user knows up-front which path @@ -330,13 +349,16 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: else: image_stage_label = "Pulling external images & building custom images" - total = 8 + total = 9 with stage("Detecting GPU", idx=1, total=total): detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) + + with stage("Provisioning GPU device access", idx=2, total=total): + render_gid = _render_gid_for_local_hardware(provision_gpu_access) paths = state.runtime_paths() - with stage("Generating values overlay (initial)", idx=2, total=total): + with stage("Generating values overlay (initial)", idx=3, total=total): # First pass: use local detection so image pulls / builds get the # right GPU_TARGET. Overlay is regenerated again below from # labeller-published labels. @@ -346,13 +368,14 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) - with stage("Installing helm + k9s", idx=3, total=total): + with stage("Installing helm + k9s", idx=4, total=total): install_tools(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) - with stage("Installing K3s (single-node)", idx=4, total=total): + with stage("Installing K3s (single-node)", idx=5, total=total): install_k3s_single_node( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, @@ -360,7 +383,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: mirror_prefix=state.mirror_prefix, ) - with stage(image_stage_label, idx=5, total=total): + with stage(image_stage_label, idx=6, total=total): if state.offline_mode and state.bundle_dir is not None: load_offline_images(state.bundle_dir) elif pull: @@ -397,13 +420,13 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: k3s_images_dir=state.k3s_images_dir, ) - with stage("Deploying ROCm GPU device plugin + node labeller", idx=6, total=total): + with stage("Deploying ROCm GPU device plugin + node labeller", idx=7, total=total): deploy_rocm_gpu_device_plugin( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, ) - with stage("Refreshing values overlay from node labels", idx=7, total=total): + with stage("Refreshing values overlay from node labels", idx=8, total=total): refine_gpu_config_from_node_labels(state.gpu) generate_values_overlay( state.gpu, @@ -411,10 +434,11 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) - with stage("Deploying JupyterHub runtime (helm install + wait)", idx=8, total=total): + with stage("Deploying JupyterHub runtime (helm install + wait)", idx=9, total=total): deploy_runtime(paths) _print_success_banner() @@ -582,6 +606,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -591,12 +616,14 @@ def cmd_dev_deploy(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths, dev=True) def cmd_dev_upgrade(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -607,12 +634,14 @@ def cmd_dev_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths, dev=True) def cmd_dev_reinstall(state: InstallerState) -> None: + _render_gid_for_local_hardware(load_existing_gpu_access) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -623,6 +652,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -632,12 +662,14 @@ def cmd_rt_install(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths) def cmd_rt_upgrade(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -648,6 +680,7 @@ def cmd_rt_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths) @@ -678,6 +711,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: + _render_gid_for_local_hardware(load_existing_gpu_access) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py new file mode 100644 index 00000000..ede70311 --- /dev/null +++ b/tests/installer/test_cli_gpu_access.py @@ -0,0 +1,227 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""GPU access sequencing tests for installer command orchestration.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import contextmanager +from pathlib import Path + +import pytest + +from auplc_installer import cli +from auplc_installer.gpu_access import GpuAccessState +from auplc_installer.gpu_hardware import GpuHardware +from auplc_installer.helm import RuntimePaths +from auplc_installer.state import InstallerState + + +@pytest.mark.parametrize( + ("hardware", "expected_render_gid", "expected_provision_count"), + [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], +) +def test_full_install_gates_gpu_access_without_skipping_later_gpu_flow( + monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_provision_count: int +) -> None: + events: list[object] = [] + stages: list[tuple[str, int, int]] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + @contextmanager + def fake_stage(label: str, *, idx: int, total: int): + stages.append((label, idx, total)) + yield + + def fake_overlay(*args: object, **kwargs: object) -> Path: + events.append(("overlay", kwargs["render_gid"])) + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "stage", fake_stage) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: events.append("provision") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) + monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) + monkeypatch.setattr(cli, "pull_custom_images", lambda **kwargs: events.append("custom-images")) + monkeypatch.setattr(cli, "pull_external_images", lambda **kwargs: events.append("external-images")) + monkeypatch.setattr(cli, "deploy_rocm_gpu_device_plugin", lambda **kwargs: events.append("device-plugin")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("runtime")) + monkeypatch.setattr(cli, "_print_success_banner", lambda: events.append("success")) + + cli._cmd_install_inner(state, pull=True) + + assert events.count("provision") == expected_provision_count + if expected_provision_count: + assert events.index("provision") < events.index("device-plugin") + assert [event for event in events if isinstance(event, tuple)] == [ + ("overlay", expected_render_gid), + ("overlay", expected_render_gid), + ] + assert stages == [ + ("Detecting GPU", 1, 9), + ("Provisioning GPU device access", 2, 9), + ("Generating values overlay (initial)", 3, 9), + ("Installing helm + k9s", 4, 9), + ("Installing K3s (single-node)", 5, 9), + ("Pulling custom + external images", 6, 9), + ("Deploying ROCm GPU device plugin + node labeller", 7, 9), + ("Refreshing values overlay from node labels", 8, 9), + ("Deploying JupyterHub runtime (helm install + wait)", 9, 9), + ] + + +@pytest.mark.parametrize( + ("hardware", "expected_render_gid", "expected_load_count"), + [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], +) +def test_runtime_upgrade_gates_existing_gpu_access_without_provisioning( + monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_load_count: int +) -> None: + events: list[object] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + def fake_overlay(*args: object, **kwargs: object) -> Path: + events.append(("overlay", kwargs["render_gid"])) + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr( + cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + ) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) + + cli.cmd_rt_upgrade(state) + + assert events.count("load") == expected_load_count + assert events[-5:] == ["detect", "refine", "preserve-courses", ("overlay", expected_render_gid), "upgrade-runtime"] + + +@pytest.mark.parametrize( + ("command", "expected_events"), + [ + (cli.cmd_dev_deploy, ("detect", "refine", "overlay:None", "deploy-runtime")), + (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + (cli.cmd_rt_install, ("detect", "refine", "overlay:None", "deploy-runtime")), + (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + ], +) +def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( + monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] +) -> None: + events: list[str] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) + monkeypatch.setattr(cli, "load_existing_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not load"))) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) + monkeypatch.setattr( + cli, + "generate_values_overlay", + lambda *args, **kwargs: events.append(f"overlay:{kwargs['render_gid']}") or paths.overlay_path, + ) + monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) + monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) + + command(state) + + assert events == list(expected_events) + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [ + (cli.cmd_dev_reinstall, "cmd_dev_deploy"), + (cli.cmd_rt_reinstall, "cmd_rt_install"), + ], +) +@pytest.mark.parametrize( + ("hardware", "expected_access_events"), + [(GpuHardware.GPU, ["load"]), (GpuHardware.CPU, [])], +) +def test_reinstall_gates_existing_gpu_access_before_removing_runtime( + monkeypatch, + reinstall: Callable[[InstallerState], None], + delegate_name: str, + hardware: GpuHardware, + expected_access_events: list[str], +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr( + cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + reinstall(state) + + assert events == [*expected_access_events, "remove-runtime", "sleep", "delegate"] + + +def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeypatch) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr( + cli, + "provision_gpu_access", + lambda: (_ for _ in ()).throw(AssertionError("must not provision")), + ) + + with pytest.raises(RuntimeError, match="hardware"): + cli._cmd_install_inner(state, pull=True) + + assert events == ["detect"] + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [ + (cli.cmd_dev_reinstall, "cmd_dev_deploy"), + (cli.cmd_rt_reinstall, "cmd_rt_install"), + ], +) +def test_unknown_hardware_blocks_reinstall_before_runtime_removal( + monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr( + cli, + "load_existing_gpu_access", + lambda: (_ for _ in ()).throw(AssertionError("must not load")), + ) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + with pytest.raises(RuntimeError, match="hardware"): + reinstall(state) + + assert events == [] diff --git a/tests/installer/test_cli_helpers.py b/tests/installer/test_cli_helpers.py index bc33cffa..2cc863e8 100644 --- a/tests/installer/test_cli_helpers.py +++ b/tests/installer/test_cli_helpers.py @@ -41,6 +41,7 @@ def _write_overlay(path: Path, courses: CourseSelection) -> None: image_tag="v1.0", courses=courses, offline_mode=False, + render_gid=993, overlay_path=path, ) From 0ea997876d7c7659020fa62eae3a6c2e8191164e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 042/180] feat(installer): emit resolved render GID overlays --- auplc_installer/overlay.py | 8 +++++ tests/installer/test_overlay.py | 34 ++++++++++++++++++++ tests/installer/test_values_gpu_overrides.py | 9 ++++++ 3 files changed, 51 insertions(+) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 5815f391..202b5c6c 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -47,6 +47,7 @@ def emit_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, + render_gid: int | None, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" buf = StringIO() @@ -66,6 +67,11 @@ def emit_overlay( buf.write(f"# Env selection : {courses.description()}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") + buf.write(" gpuAccess:\n") + if render_gid is None: + buf.write(" renderGid: null\n") + else: + buf.write(f" renderGid: {render_gid}\n") # --- accelerators --- any_accel_emitted = False @@ -164,6 +170,7 @@ def generate_values_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, + render_gid: int | None, overlay_path: Path, ) -> Path: """Render the overlay and write it to ``overlay_path``. Returns the path.""" @@ -175,6 +182,7 @@ def generate_values_overlay( image_tag=image_tag, courses=courses, offline_mode=offline_mode, + render_gid=render_gid, ) overlay_path.write_text(text, encoding="utf-8") return overlay_path diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index b028cdd9..c93e4fc7 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -27,6 +27,7 @@ ) from auplc_installer.gpu import GpuConfig, SkuEntry, append_product from auplc_installer.overlay import ( + GPU_RESOURCE_KEYS, emit_overlay, generate_values_overlay, try_load_courses_from_overlay, @@ -45,6 +46,7 @@ def _render( courses: CourseSelection, offline_mode: bool = False, image_tag: str = "v1.0", + render_gid: int | None = 993, ) -> tuple[str, dict]: text = emit_overlay( cfg, @@ -52,6 +54,7 @@ def _render( image_tag=image_tag, courses=courses, offline_mode=offline_mode, + render_gid=render_gid, ) return text, yaml.safe_load(text) @@ -93,6 +96,7 @@ def _write_and_read_back(courses: CourseSelection) -> CourseSelection | None: image_tag="v1.0", courses=courses, offline_mode=False, + render_gid=993, overlay_path=path, ) return try_load_courses_from_overlay(path) @@ -106,6 +110,36 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] +def test_overlay_emits_explicit_gpu_access_gid_without_global_pod_groups() -> None: + text = emit_overlay( + _strix_halo_cfg(), + image_registry="ghcr.io/amdresearch", + image_tag="v1.0", + courses=CourseSelection.default(), + offline_mode=False, + render_gid=993, + ) + parsed = yaml.safe_load(text) + + assert parsed["custom"]["gpuAccess"]["renderGid"] == 993 + assert "supplementalGroups" not in text + + +def test_overlay_emits_null_render_gid_without_removing_gpu_resources() -> None: + _, parsed = _render( + _strix_halo_cfg(), + courses=CourseSelection.default(), + render_gid=None, + ) + + custom = parsed["custom"] + assert custom["gpuAccess"]["renderGid"] is None + assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) + assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) + assert "teams" not in custom + assert "profiles" not in custom + + def test_resource_images_use_primary_tag() -> None: _, parsed = _render(_strix_halo_cfg(), courses=CourseSelection.default()) images = parsed["custom"]["resources"]["images"] diff --git a/tests/installer/test_values_gpu_overrides.py b/tests/installer/test_values_gpu_overrides.py index 7f6ef6ad..18de940f 100644 --- a/tests/installer/test_values_gpu_overrides.py +++ b/tests/installer/test_values_gpu_overrides.py @@ -69,3 +69,12 @@ def test_default_values_route_gpu_resources_to_supported_image_tags() -> None: assert overrides[accelerator_key]["image"] == ( f"ghcr.io/amdresearch/{image_name}:latest-{gpu_target}" ), values_file + + +def test_default_values_use_fs_gid_without_overriding_pod_security_context() -> None: + for values_file in VALUES_FILES: + values = _load_values(values_file) + singleuser = values["singleuser"] + + assert singleuser["fsGid"] == 100, values_file + assert "securityContext" not in singleuser.get("extraPodConfig", {}), values_file From 50ecc64d59dc847f08e3e058c8d9bc127017b8cf Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 043/180] feat(chart): model GPU render GID --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 14 ++++++++++++++ runtime/chart/values.yaml | 5 +++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index ef7efffc..f53227d3 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"gpuAccess":{"type":"object","additionalProperties":false,"properties":{"renderGid":{"type":["integer","null"],"minimum":1,"maximum":4294967294}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 22ff5fec..3eab8688 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3195,6 +3195,20 @@ properties: Enable auto-admin creation on first install. Credentials will be stored in `jupyterhub-admin-credentials` secret. + gpuAccess: + type: object + additionalProperties: false + description: | + Host group access settings for GPU-enabled user pods. + properties: + renderGid: + type: [integer, "null"] + minimum: 1 + maximum: 4294967294 + description: | + Numeric GID of the host render group. GPU resources receive this + as a supplemental group; CPU resources do not. + notifications: type: object additionalProperties: false diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index a548691f..e888f22f 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -50,6 +50,11 @@ custom: # Define these in runtime/values.yaml, not here accelerators: {} + # Host render-group access for GPU user pods. The installer overlay sets this + # to the detected host render GID when GPU access is provisioned. + gpuAccess: + renderGid: null + # Resource images, requirements, and metadata # Define these in runtime/values.yaml, not here resources: From 24325361f7ee84398ea1e3dfe38dbf26be37260f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 044/180] fix(runtime): separate storage and GPU groups --- runtime/values-multi-nodes.yaml.example | 12 +++++++----- runtime/values.yaml | 14 ++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 3db3cffa..63c48382 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -139,6 +139,11 @@ custom: defaultPersistence: true allowPersistenceChoice: false + # Generated deployment overlays resolve this from corroborated host evidence. + # Keep null in the base example; do not choose a fleet GID manually here. + gpuAccess: + renderGid: null + # -------------------------------------------------------------------------- # Accelerator Configuration # -------------------------------------------------------------------------- @@ -576,11 +581,8 @@ monitoring: enabled: false singleuser: - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 + # Must match the storage ownership group used by the shared volume. + fsGid: 100 storage: dynamic: storageClass: nfs-client diff --git a/runtime/values.yaml b/runtime/values.yaml index da79243a..7bce7308 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -61,6 +61,10 @@ custom: adminUser: enabled: false + # The installer overlay supplies the host render GID for GPU user pods. + gpuAccess: + renderGid: null + # ============================================================================ # Notifications # ============================================================================ @@ -668,14 +672,8 @@ monitoring: enabled: false singleuser: - # Security context for user pods to access GPU devices - # supplementalGroups grants container access to host's render group (GID 993) - # This allows non-root users to access /dev/kfd and /dev/dri devices - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 # render group for ROCm GPU access + # Preserve storage volume ownership without replacing KubeSpawner's security context. + fsGid: 100 storage: dynamic: From cff8981ff602a08c6d6ab164e7a95da889ef6ee4 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 045/180] feat(hub): enforce GPU render group policy --- runtime/hub/core/config.py | 31 ++++ runtime/hub/core/spawner/kubernetes.py | 101 ++++++++-- runtime/hub/tests/test_spawner_gpu_access.py | 182 +++++++++++++++++++ 3 files changed, 303 insertions(+), 11 deletions(-) create mode 100644 runtime/hub/tests/test_spawner_gpu_access.py diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 3925bbf0..28e24ded 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -45,6 +45,8 @@ import yaml from pydantic import BaseModel, Field, field_validator +MAX_RENDER_GID = (2**32) - 2 + # ============================================================================= # YAML Configuration Models # ============================================================================= @@ -85,6 +87,25 @@ class QuotaSettings(BaseModel): model_config = {"extra": "allow"} +class GpuAccessSettings(BaseModel): + """Host group access settings for GPU-enabled user pods.""" + + renderGid: int | None = None + + @field_validator("renderGid", mode="before") + @classmethod + def validate_render_gid(cls, value: Any) -> int | None: + """Require a native positive integer GID when GPU access is configured.""" + + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_RENDER_GID: + raise ValueError(f"custom.gpuAccess.renderGid must be an integer between 1 and {MAX_RENDER_GID}") + return value + + model_config = {"extra": "allow"} + + class AcceleratorOverride(BaseModel): """Per-accelerator overrides for a resource (image and/or env).""" @@ -211,6 +232,7 @@ class ParsedConfig(BaseModel): accelerators: dict[str, AcceleratorConfig] = Field(default_factory=dict) teams: TeamsConfig = Field(default_factory=TeamsConfig) quota: QuotaSettings = Field(default_factory=QuotaSettings) + gpuAccess: GpuAccessSettings = Field(default_factory=GpuAccessSettings) gitClone: GitCloneSettings = Field(default_factory=GitCloneSettings) hub: HubNetworkSettings = Field(default_factory=HubNetworkSettings) notebook: NotebookNetworkSettings = Field(default_factory=NotebookNetworkSettings) @@ -226,6 +248,7 @@ def from_dicts( accelerators: dict | None = None, teams: dict | None = None, quota: dict | None = None, + gpu_access: dict | None = None, git_clone: dict | None = None, hub: dict | None = None, notebook: dict | None = None, @@ -243,6 +266,8 @@ def from_dicts( raw_config["teams"] = teams if quota: raw_config["quota"] = quota + if gpu_access is not None: + raw_config["gpuAccess"] = gpu_access if git_clone: raw_config["gitClone"] = git_clone if hub: @@ -334,6 +359,7 @@ def init(cls, config_path: str | Path) -> HubConfig: accelerators=raw_config.get("accelerators"), teams=raw_config.get("teams"), quota=raw_config.get("quota"), + gpu_access=raw_config.get("gpuAccess"), git_clone=raw_config.get("gitClone"), hub=raw_config.get("hub"), notebook=raw_config.get("notebook"), @@ -411,6 +437,11 @@ def quota(self) -> QuotaSettings: """Get quota configuration.""" return self._config.quota + @property + def gpu_access(self) -> GpuAccessSettings: + """Get GPU pod access configuration.""" + return self._config.gpuAccess + @property def git_clone(self) -> GitCloneSettings: """Get git clone configuration.""" diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index e9d57fbd..a99b7fff 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -40,6 +40,7 @@ from kubespawner import KubeSpawner from tornado import web +from core.config import MAX_RENDER_GID from core.metrics import ( pod_failure_total, repo_clone_failed_total, @@ -93,6 +94,7 @@ class RemoteLabKubeSpawner(KubeSpawner): auth_mode: str = "auto-login" single_node_mode: bool = False quota_enabled: bool | None = False + render_gid: int | None = None # Resource configuration (set from config) resource_images: dict[str, str] = {} @@ -154,6 +156,7 @@ def configure_from_config(cls, config: HubConfig) -> None: cls.default_quota = config.quota.defaultQuota cls.minimum_quota_to_start = config.quota.minimumToStart cls.quota_enabled = config.quota.enabled + cls.render_gid = config.gpu_access.renderGid # Extract git clone settings (single source of truth: GitCloneSettings) git_config = config.git_clone @@ -169,8 +172,8 @@ def configure_from_config(cls, config: HubConfig) -> None: # Extract code-server link protection settings cls.code_server_extra_trusted_domains = list(config.code_server.extraTrustedDomains) - async def get_user_resources(self) -> list[str]: - """Get available resources for the user based on their JupyterHub group memberships. + def _resolve_user_resources(self) -> list[str]: + """Resolve available resources for the current user from server-side policy. For auto-login/dummy modes, returns all configured resources. For all other users, resolves resources from JupyterHub groups @@ -195,6 +198,43 @@ async def get_user_resources(self) -> list[str]: self.log.debug(f"User '{username}' resolved resources: {available_resources}") return available_resources + async def get_user_resources(self) -> list[str]: + """Get available resources for the user based on their JupyterHub group memberships.""" + return self._resolve_user_resources() + + def _resolve_accelerator_selection(self, resource_type: str, gpu_selection: Any) -> str | None: + """Validate or default the accelerator selection for a resource.""" + requirements = self.resource_requirements[resource_type] + if gpu_selection is None: + selected_accelerator = "" + elif isinstance(gpu_selection, str): + selected_accelerator = gpu_selection.strip() + else: + raise RuntimeError("Accelerator selection must be a string") + + if "amd.com/gpu" not in requirements: + if selected_accelerator: + raise RuntimeError(f"CPU resource '{resource_type}' does not allow GPU selection") + return None + + resource_metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None + allowed_accelerators = list(getattr(resource_metadata, "acceleratorKeys", []) or []) + if not allowed_accelerators: + raise RuntimeError(f"GPU resource '{resource_type}' has no authorized accelerators configured") + + if not selected_accelerator: + if len(allowed_accelerators) == 1: + selected_accelerator = allowed_accelerators[0] + else: + raise RuntimeError(f"GPU resource '{resource_type}' requires selecting an accelerator") + + if selected_accelerator not in allowed_accelerators: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not authorized for resource '{resource_type}'") + if selected_accelerator not in self.accelerator_options: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not configured") + + return selected_accelerator + async def options_form(self, _) -> str: """Generate the HTML form for resource selection. @@ -291,13 +331,17 @@ def options_from_form(self, formdata) -> dict[str, Any]: resource_type = resource_type_list[0] options["resource_type"] = resource_type - # Parse GPU selection if available - gpu_selection = formdata.get(f"gpu_selection_{resource_type}", [None])[0] - options["gpu_selection"] = gpu_selection - # Validate resource type if resource_type not in self.resource_images: raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + + gpu_selection = self._resolve_accelerator_selection( + resource_type, + formdata.get(f"gpu_selection_{resource_type}", [None])[0], + ) + options["gpu_selection"] = gpu_selection # Configure spawner based on selections self._configure_spawner(resource_type, gpu_selection) @@ -758,6 +802,7 @@ def _reset_per_spawn_state(self) -> None: "init_containers": copy.deepcopy(self.init_containers), "extra_container_config": copy.deepcopy(self.extra_container_config), "environment": copy.deepcopy(self.environment), + "supplemental_gids": copy.deepcopy(self.supplemental_gids), } for key, value in self._resource_baseline_state.items(): @@ -765,6 +810,28 @@ def _reset_per_spawn_state(self) -> None: self._has_git_init_container = False + def _add_gpu_render_gid(self) -> None: + """Add the configured host render group to a GPU resource's pod.""" + if self.render_gid is None: + raise RuntimeError( + "GPU resource requires custom.gpuAccess.renderGid. " + "Set it to the numeric GID of the host render group before spawning GPU resources." + ) + if ( + isinstance(self.render_gid, bool) + or not isinstance(self.render_gid, int) + or not 1 <= self.render_gid <= MAX_RENDER_GID + ): + raise RuntimeError( + "GPU resource requires a valid custom.gpuAccess.renderGid. " + f"Set it to an integer between 1 and {MAX_RENDER_GID} before spawning GPU resources." + ) + + supplemental_gids = list(self.supplemental_gids or []) + if self.render_gid not in supplemental_gids: + supplemental_gids.append(self.render_gid) + self.supplemental_gids = supplemental_gids + def _configure_spawner(self, resource_type: str, gpu_selection: str | None = None) -> None: """Configure the spawner based on the resource type and GPU selection.""" @@ -829,6 +896,7 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non if "amd.com/gpu" in requirements: self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} + self._add_gpu_render_gid() elif "amd.com/npu" in requirements: self.log.debug("NPU DEVICE PLUGIN are removed, amd.com/npu is no more needed") @@ -895,13 +963,25 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non async def start(self): """Start the spawner and schedule automatic shutdown.""" + runtime_minutes = self.user_options.get("runtime_minutes", 20) + resource_type = self.user_options.get("resource_type", "cpu") + if resource_type not in self.resource_images: + raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + gpu_selection = self._resolve_accelerator_selection( + resource_type, + self.user_options.get("gpu_selection"), + ) + self.user_options["gpu_selection"] = gpu_selection + self._configure_spawner(resource_type, gpu_selection) + # Ensure pod fails immediately (not retried) when an init container fails. # JupyterHub manages pod lifecycle; Kubernetes should not silently restart pods. - self.extra_pod_config = {"restartPolicy": "Never"} + extra_pod_config = copy.deepcopy(self.extra_pod_config or {}) + extra_pod_config["restartPolicy"] = "Never" + self.extra_pod_config = extra_pod_config - runtime_minutes = self.user_options.get("runtime_minutes", 20) - resource_type = self.user_options.get("resource_type", "cpu") - gpu_selection = self.user_options.get("gpu_selection", None) username = self.user.name.lower() # Determine accelerator type for quota calculation @@ -1099,7 +1179,6 @@ async def start(self): if hasattr(self, "_spawn_start_timestamp"): duration = time.time() - self._spawn_start_timestamp spawn_duration_seconds.observe(duration) - accelerator_type = self.user_options.get("gpu_selection") or "cpu" # active session count is derived from quota manager, not inc/dec except Exception: pass diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py new file mode 100644 index 00000000..edacf4c2 --- /dev/null +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -0,0 +1,182 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import copy +import importlib.util +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" + +if "core" not in sys.modules: + core_module = types.ModuleType("core") + core_module.__path__ = [str(CORE)] + sys.modules["core"] = core_module + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class DummyMetric: + def labels(self, **_kwargs): + return self + + def inc(self): + pass + + def observe(self, _value): + pass + + +class TestKubeSpawner: + def get_pod_manifest(self): + manifest = {"spec": copy.deepcopy(self.extra_pod_config or {})} + security_context = manifest["spec"].setdefault("securityContext", {}) + if self.fs_gid is not None: + security_context["fsGroup"] = self.fs_gid + if self.supplemental_gids: + security_context["supplementalGroups"] = list(self.supplemental_gids) + return manifest + + +def load_spawner_module(): + metrics_module = types.ModuleType("core.metrics") + for metric_name in ( + "pod_failure_total", + "repo_clone_failed_total", + "session_runtime_minutes", + "spawn_duration_seconds", + "spawn_failed_total", + "spawn_gpu_total", + ): + setattr(metrics_module, metric_name, DummyMetric()) + + jupyterhub_module = types.ModuleType("jupyterhub") + jupyterhub_module.__path__ = [] + user_module = types.ModuleType("jupyterhub.user") + user_module.User = type("User", (), {}) + kubespawner_module = types.ModuleType("kubespawner") + kubespawner_module.KubeSpawner = TestKubeSpawner + tornado_module = types.ModuleType("tornado") + web_module = types.ModuleType("tornado.web") + web_module.HTTPError = type("HTTPError", (Exception,), {}) + + with patch.dict( + sys.modules, + { + "core.metrics": metrics_module, + "jupyterhub": jupyterhub_module, + "jupyterhub.user": user_module, + "kubespawner": kubespawner_module, + "tornado": tornado_module, + "tornado.web": web_module, + }, + ): + return load_module("gpu_access_test_spawner", CORE / "spawner" / "kubernetes.py") + + +config = load_module("core.config", CORE / "config.py") +kubernetes = load_spawner_module() +RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner + + +class DummyLog: + def debug(self, _message): + pass + + +class ResourceMetadata: + acceleratorKeys = ["gpu-a"] + acceleratorOverrides = None + allowGitClone = False + defaultPath = None + env = {} + launchMode = None + + +class HubConfig: + def get_resource_metadata(self, _resource_type): + return ResourceMetadata() + + +def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = None): + spawner = object.__new__(RemoteLabKubeSpawner) + spawner._hub_config = HubConfig() + spawner.render_gid = render_gid + spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image"} + spawner.resource_requirements = { + "cpu": {"cpu": "1", "memory": "1Gi"}, + "gpu": {"cpu": "1", "memory": "1Gi", "amd.com/gpu": "1"}, + } + spawner.accelerator_options = {"gpu-a": {}} + spawner.node_selector_mapping = {} + spawner.environment_mapping = {} + spawner.cmd = [] + spawner.args = [] + spawner.default_url = "" + spawner.node_affinity_required = [] + spawner.extra_resource_guarantees = {} + spawner.extra_resource_limits = {} + spawner.init_containers = [] + spawner.extra_container_config = {} + spawner.environment = {} + spawner.fs_gid = 100 + spawner.supplemental_gids = list(supplemental_gids or []) + spawner.extra_pod_config = {} + spawner.log = DummyLog() + spawner._resolve_user_resources = lambda: ["cpu", "gpu"] + return spawner + + +def test_gpu_render_gid_is_injected_only_for_gpu_pods(): + spawner = make_spawner(render_gid=993) + + spawner._configure_spawner("gpu", "gpu-a") + gpu_manifest = spawner.get_pod_manifest() + spawner._configure_spawner("cpu") + cpu_manifest = spawner.get_pod_manifest() + + assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [993]} + assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100} + + +def test_gpu_render_gid_preserves_existing_supplemental_groups(): + spawner = make_spawner(render_gid=993, supplemental_gids=[1234]) + + spawner._configure_spawner("gpu", "gpu-a") + + assert spawner.supplemental_gids == [1234, 993] + + +def test_gpu_spawn_requires_a_host_render_gid(): + spawner = make_spawner(render_gid=None) + + with pytest.raises(RuntimeError, match=r"custom\.gpuAccess\.renderGid"): + spawner._configure_spawner("gpu", "gpu-a") + + +def test_gpu_access_config_validates_render_gid(): + assert config.GpuAccessSettings(renderGid=993).renderGid == 993 + assert config.ParsedConfig.from_dicts(gpu_access={"renderGid": 993}).gpuAccess.renderGid == 993 + with pytest.raises(ValidationError, match="renderGid"): + config.GpuAccessSettings(renderGid=True) + + +def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): + spawner = make_spawner(render_gid=993) + spawner._resolve_user_resources = lambda: ["cpu"] + spawner._configure_spawner = lambda *_args: pytest.fail("unauthorized resource configured the spawner") + + with pytest.raises(RuntimeError, match="not authorized"): + spawner.options_from_form({"runtime": ["20"], "resource_type": ["gpu"], "gpu_selection_gpu": ["gpu-a"]}) From 3489075c6b4df2f3b4d6eb41e96290d0cf59df43 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 046/180] fix(images): remove embedded GPU permission policy --- dockerfiles/Base/Dockerfile.rocm | 17 +----------- tests/scripts/test_gpu_image_permissions.py | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 tests/scripts/test_gpu_image_permissions.py diff --git a/dockerfiles/Base/Dockerfile.rocm b/dockerfiles/Base/Dockerfile.rocm index 43cf8c25..3c2267a6 100644 --- a/dockerfiles/Base/Dockerfile.rocm +++ b/dockerfiles/Base/Dockerfile.rocm @@ -226,14 +226,6 @@ RUN if getent passwd 1000 > /dev/null; then \ RUN useradd -m -s /bin/bash -N -u $NB_UID -g $NB_GID $NB_USER && \ echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -# Add jovyan to video and render groups for ROCm access -RUN if getent group render; then \ - groupmod -g 992 render; \ - else \ - groupadd -g 992 render; \ - fi -RUN usermod -aG video,render ${NB_USER} - # Create necessary Jupyter directories with correct permissions RUN mkdir -p /home/$NB_USER/.jupyter && \ mkdir -p /home/$NB_USER/.local/share/jupyter/runtime && \ @@ -250,15 +242,8 @@ RUN echo '#!/bin/bash' > /home/$NB_USER/start-jupyter.sh && \ # Verify the file exists (will fail build if not) ls -la /home/$NB_USER/start-jupyter.sh -# Set proper permissions for ROCm devices -RUN mkdir -p /etc/udev/rules.d && \ - echo 'SUBSYSTEM=="kfd", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-kfd.rules && \ - echo 'SUBSYSTEM=="dri", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-dri.rules - -# Create entrypoint script to set permissions and start services +# Create entrypoint script to start services RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'chmod 666 /dev/kfd 2>/dev/null || true' >> /entrypoint.sh && \ - echo 'chmod 666 /dev/dri/renderD* 2>/dev/null || true' >> /entrypoint.sh && \ echo 'export USER=jovyan' >> /entrypoint.sh && \ echo 'export SHELL=/bin/bash' >> /entrypoint.sh && \ echo 'exec python3 -m jupyterhub.singleuser --ip=0.0.0.0 --port=8888 "$@"' >> /entrypoint.sh && \ diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py new file mode 100644 index 00000000..ce5f094a --- /dev/null +++ b/tests/scripts/test_gpu_image_permissions.py @@ -0,0 +1,30 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DOCKERFILE = ROOT / "dockerfiles" / "Base" / "Dockerfile.rocm" + + +def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + forbidden_patterns = ( + r"groupmod\s+-g\s+992\s+render", + r"groupadd\s+-g\s+992\s+render", + r"usermod\s+-aG\s+video,render\s+\$\{NB_USER\}", + r"\brender\b", + r"/etc/udev", + r"chmod\s+666\b", + ) + for pattern in forbidden_patterns: + assert re.search(pattern, dockerfile) is None, pattern + + assert "echo 'export USER=jovyan' >> /entrypoint.sh" in dockerfile + assert "echo 'export SHELL=/bin/bash' >> /entrypoint.sh" in dockerfile + assert 'CMD ["/bin/bash", "/entrypoint.sh"]' in dockerfile + assert "USER $NB_UID" in dockerfile + assert "WORKDIR /home/jovyan" in dockerfile From 5176884387ba4cfe4943e460569dedcff1139374 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 047/180] feat(ansible): define canonical GPU access contract --- deploy/ansible/filter_plugins/auplc_json.py | 41 ++++++++++++++ .../roles/gpu_access/defaults/main.yml | 13 +++++ .../ansible/roles/gpu_access/tasks/main.yml | 14 +++++ .../roles/gpu_access/tasks/validate.yml | 53 +++++++++++++++++++ .../templates/70-auplc-gpu-access.rules.j2 | 3 ++ .../gpu_access/templates/gpu-access.json.j2 | 1 + 6 files changed, 125 insertions(+) create mode 100644 deploy/ansible/filter_plugins/auplc_json.py create mode 100644 deploy/ansible/roles/gpu_access/defaults/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/validate.yml create mode 100644 deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 create mode 100644 deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 diff --git a/deploy/ansible/filter_plugins/auplc_json.py b/deploy/ansible/filter_plugins/auplc_json.py new file mode 100644 index 00000000..080de6af --- /dev/null +++ b/deploy/ansible/filter_plugins/auplc_json.py @@ -0,0 +1,41 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Strict JSON filters used by AUP Learning Cloud Ansible roles.""" + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import TypeAlias + +from ansible.errors import AnsibleFilterError + +JSONValue: TypeAlias = None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] + + +@dataclass(frozen=True, slots=True) +class DuplicateJsonKeyError(ValueError): + key: str + + def __str__(self) -> str: + return f"Duplicate JSON object key: {self.key!r}" + + +def _reject_duplicate_keys(pairs: list[tuple[str, JSONValue]]) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, value in pairs: + if key in result: + raise DuplicateJsonKeyError(key) + result[key] = value + return result + + +def auplc_from_json_strict(value: str) -> JSONValue: + try: + return json.loads(value, object_pairs_hook=_reject_duplicate_keys) + except (TypeError, DuplicateJsonKeyError, json.JSONDecodeError): + raise AnsibleFilterError("Invalid JSON value") from None + + +class FilterModule: + def filters(self) -> dict[str, Callable[[str], JSONValue]]: + return {"auplc_from_json_strict": auplc_from_json_strict} diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml new file mode 100644 index 00000000..74e40ed3 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -0,0 +1,13 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +# Set this explicitly in cluster inventory or PXE extra vars. The role never +# assumes a site-specific GID. +auplc_render_gid: null +auplc_normalize_render_gid: false +auplc_gpu_access_enabled: false +# Set for a PXE rootfs. Leave empty to configure the live host. +auplc_rootfs_path: "" +# Rootfs adapters must explicitly constrain their writable target below this +# canonical directory. Live hosts leave this empty. +auplc_rootfs_allowed_root: "" diff --git a/deploy/ansible/roles/gpu_access/tasks/main.yml b/deploy/ansible/roles/gpu_access/tasks/main.yml new file mode 100644 index 00000000..4826393b --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/main.yml @@ -0,0 +1,14 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration + ansible.builtin.import_tasks: validate.yml + when: auplc_gpu_access_enabled | bool + +- name: Preflight GPU access target + ansible.builtin.import_tasks: preflight.yml + when: auplc_gpu_access_enabled | bool + +- name: Apply GPU access configuration + ansible.builtin.import_tasks: apply.yml + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/roles/gpu_access/tasks/validate.yml b/deploy/ansible/roles/gpu_access/tasks/validate.yml new file mode 100644 index 00000000..3e32ee7a --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/validate.yml @@ -0,0 +1,53 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate desired render GID + ansible.builtin.assert: + that: + - auplc_render_gid is not none + - auplc_render_gid is integer + - auplc_render_gid >= 1 + - auplc_render_gid <= 4294967294 + fail_msg: auplc_render_gid must be an explicit integer between 1 and 4294967294. + +- name: Validate GPU access rootfs path syntax + ansible.builtin.assert: + that: + - auplc_rootfs_path is string + - auplc_rootfs_path == '' or auplc_rootfs_path is match('^/') + - auplc_rootfs_path != '/' + - "'..' not in auplc_rootfs_path.split('/')" + - auplc_rootfs_path == '' or auplc_rootfs_allowed_root | length > 0 + fail_msg: auplc_rootfs_path must be a non-root absolute path without traversal and with an allowed root. + +- name: Canonicalize GPU access rootfs path + ansible.builtin.command: + argv: + - realpath + - --canonicalize-missing + - "{{ auplc_rootfs_path }}" + register: _auplc_canonical_rootfs + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Canonicalize allowed GPU access rootfs parent + ansible.builtin.command: + argv: + - realpath + - --canonicalize-existing + - "{{ auplc_rootfs_allowed_root }}" + register: _auplc_canonical_allowed_root + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Constrain canonical GPU access rootfs path + ansible.builtin.assert: + that: + - _auplc_canonical_rootfs.stdout != '/' + - _auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/') + fail_msg: GPU access rootfs escapes auplc_rootfs_allowed_root. + when: auplc_rootfs_path | length > 0 + +- name: Record canonical GPU access target root + ansible.builtin.set_fact: + _auplc_target_root: "{{ _auplc_canonical_rootfs.stdout if auplc_rootfs_path | length > 0 else '' }}" diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 new file mode 100644 index 00000000..c75ec1e2 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 @@ -0,0 +1,3 @@ +# Managed by auplc-installer: AMD GPU device access. +KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" +SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" diff --git a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 new file mode 100644 index 00000000..89b9110a --- /dev/null +++ b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 @@ -0,0 +1 @@ +{"renderGid":{{ auplc_render_gid | int }},"version":1} From 59110290bd5671e6575bd058819d95c0be8cc1f1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 048/180] feat(ansible): apply verified GPU device policy --- .../roles/gpu_access/handlers/main.yml | 17 ++ .../ansible/roles/gpu_access/tasks/apply.yml | 190 ++++++++++++++++ .../roles/gpu_access/tasks/preflight.yml | 206 ++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 deploy/ansible/roles/gpu_access/handlers/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/apply.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/preflight.yml diff --git a/deploy/ansible/roles/gpu_access/handlers/main.yml b/deploy/ansible/roles/gpu_access/handlers/main.yml new file mode 100644 index 00000000..6afe9532 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/handlers/main.yml @@ -0,0 +1,17 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Reload udev rules + ansible.builtin.command: + argv: + - udevadm + - control + - --reload-rules + when: auplc_rootfs_path | length == 0 + +- name: Trigger udev rules + ansible.builtin.command: + argv: + - udevadm + - trigger + when: auplc_rootfs_path | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml new file mode 100644 index 00000000..1def00de --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -0,0 +1,190 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Inspect recognized project-owned legacy GPU rules for apply + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_apply_legacy_gpu_rule_stats + +- name: Reject legacy GPU rule symlinks and non-regular files before apply + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules for apply + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_apply_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content before apply + ansible.builtin.assert: + that: + - (item.content | b64decode) in item.item.item.contents + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Remove recognized project-owned legacy GPU rules + ansible.builtin.file: + path: "{{ item.item.item.path }}" + state: absent + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Normalize live render GID + ansible.builtin.command: + argv: [groupmod, -g, "{{ auplc_render_gid | string }}", render] + when: + - _auplc_target_root | length == 0 + - (_auplc_current_render_gid | int) != (auplc_render_gid | int) + - auplc_normalize_render_gid | bool + +- name: Normalize rootfs render GID + ansible.builtin.command: + argv: [chroot, "{{ _auplc_target_root }}", groupmod, -g, "{{ auplc_render_gid | string }}", render] + when: + - _auplc_target_root | length > 0 + - (_auplc_current_render_gid | int) != (auplc_render_gid | int) + - auplc_normalize_render_gid | bool + +- name: Verify target render GID + ansible.builtin.command: + argv: >- + {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} + register: _auplc_verified_render_group + changed_when: false + +- name: Require verified render GID + ansible.builtin.assert: + that: + - _auplc_verified_render_group.stdout_lines | length == 1 + - _auplc_verified_render_group.stdout.split(':')[0] == 'render' + - (_auplc_verified_render_group.stdout.split(':')[2] | int) == (auplc_render_gid | int) + fail_msg: Target render group did not resolve to auplc_render_gid. + +- name: Create target udev rules directory + ansible.builtin.file: + path: "{{ _auplc_target_root }}/etc/udev/rules.d" + state: directory + owner: root + group: root + mode: "0755" + +- name: Install canonical AMD GPU udev rules + ansible.builtin.template: + src: 70-auplc-gpu-access.rules.j2 + dest: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + owner: root + group: root + mode: "0644" + +- name: Reload live udev rules on every apply + ansible.builtin.command: + argv: [udevadm, control, --reload-rules] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Trigger live udev rules on every apply + ansible.builtin.command: + argv: [udevadm, trigger] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Settle live udev events before inode verification + ansible.builtin.command: + argv: [udevadm, settle] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Inspect /dev/kfd after live reconciliation + ansible.builtin.stat: + path: /dev/kfd + follow: false + register: _auplc_kfd + when: _auplc_target_root | length == 0 + +- name: Verify /dev/kfd ownership and mode + ansible.builtin.assert: + that: + - _auplc_kfd.stat.exists + - _auplc_kfd.stat.ischr + - _auplc_kfd.stat.uid == 0 + - _auplc_kfd.stat.gid == (auplc_render_gid | int) + - _auplc_kfd.stat.mode == '0660' + fail_msg: /dev/kfd is not root:render with mode 0660 after reconciliation. + when: _auplc_target_root | length == 0 + +- name: Find live DRM render nodes + ansible.builtin.find: + paths: /dev/dri + patterns: renderD* + file_type: any + recurse: false + register: _auplc_render_nodes + when: _auplc_target_root | length == 0 + +- name: Resolve live DRM render node driver symlinks + ansible.builtin.command: + argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] + loop: "{{ _auplc_render_nodes.files }}" + register: _auplc_render_node_drivers + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Select AMD live DRM render nodes + ansible.builtin.set_fact: + _auplc_amd_render_nodes: >- + {{ (_auplc_amd_render_nodes | default([])) + + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' else []) }} + loop: "{{ _auplc_render_node_drivers.results }}" + when: _auplc_target_root | length == 0 + +- name: Require AMD live DRM render nodes + ansible.builtin.assert: + that: _auplc_amd_render_nodes | length > 0 + fail_msg: No AMD renderD node was available for GPU access verification. + when: _auplc_target_root | length == 0 + +- name: Inspect AMD live DRM render nodes + ansible.builtin.stat: + path: "{{ item }}" + follow: false + loop: "{{ _auplc_amd_render_nodes }}" + register: _auplc_amd_render_node_stats + when: _auplc_target_root | length == 0 + +- name: Verify AMD render node ownership and mode + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.ischr + - item.stat.uid == 0 + - item.stat.gid == (auplc_render_gid | int) + - item.stat.mode == '0660' + fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0660." + loop: "{{ _auplc_amd_render_node_stats.results }}" + when: _auplc_target_root | length == 0 + +- name: Create target GPU access state directory + ansible.builtin.file: + path: "{{ _auplc_target_root }}/var/lib/auplc" + state: directory + owner: root + group: root + mode: "0755" + +- name: Persist target GPU access state + ansible.builtin.template: + src: gpu-access.json.j2 + dest: "{{ _auplc_target_root }}/var/lib/auplc/gpu-access.json" + owner: root + group: root + mode: "0644" diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml new file mode 100644 index 00000000..fcb4808b --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before target preflight + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs target + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs directory + ansible.builtin.assert: + that: + - _auplc_rootfs.stat.isdir + - not _auplc_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect canonical GPU access destination parents + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + - /var + - /var/lib + - /var/lib/auplc + register: _auplc_destination_parent_stats + +- name: Reject unsafe canonical GPU access destination parents + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isdir and not item.stat.islnk) + fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" + loop: "{{ _auplc_destination_parent_stats.results }}" + +- name: Inspect canonical GPU access destinations + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-auplc-gpu-access.rules + - /var/lib/auplc/gpu-access.json + register: _auplc_destination_stats + +- name: Reject unsafe canonical GPU access destinations + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unsafe canonical GPU access destination: {{ item.item }}" + loop: "{{ _auplc_destination_stats.results }}" + +- name: Read existing canonical GPU access destinations + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ item.item }}" + loop: "{{ _auplc_destination_stats.results }}" + when: item.stat.exists + register: _auplc_existing_destinations + +- name: Define canonical GPU access rule content + ansible.builtin.set_fact: + _auplc_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + +- name: Reject unmanaged canonical GPU access rule + ansible.builtin.assert: + that: (item.content | b64decode) == _auplc_canonical_rule + fail_msg: "Unmanaged canonical GPU access rule: {{ item.item.item }}" + loop: "{{ _auplc_existing_destinations.results }}" + when: + - not item.skipped | default(false) + - item.item.item.endswith('70-auplc-gpu-access.rules') + +- name: Parse existing canonical GPU access state + ansible.builtin.set_fact: + _auplc_existing_state: "{{ item.content | b64decode | auplc_from_json_strict }}" + loop: "{{ _auplc_existing_destinations.results }}" + when: + - not item.skipped | default(false) + - item.item.item.endswith('gpu-access.json') + +- name: Define recognized project-owned legacy GPU rules + ansible.builtin.set_fact: + _auplc_legacy_gpu_rules: + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" + contents: + - "KERNEL==\"kfd\", MODE=\"0666\"\nSUBSYSTEM==\"drm\", KERNEL==\"renderD*\", MODE=\"0666\"\n" + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" + contents: + - | + # ROCm device permissions + # Grant render group access to AMD GPU devices + # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules + KERNEL=="kfd", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + - "KERNEL==\"kfd\", MODE=\"0666\"\nKERNEL==\"renderD[0-9]*\", MODE=\"0666\"\n" + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" + contents: + - | + # ROCm device permissions + # Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group + SUBSYSTEM=="kfd", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + +- name: Inspect recognized project-owned legacy GPU rules + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_legacy_gpu_rule_stats + +- name: Reject legacy GPU rule symlinks and non-regular files + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content + ansible.builtin.assert: + that: + - (item.content | b64decode) in item.item.item.contents + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Read target render group + ansible.builtin.command: + argv: >- + {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} + register: _auplc_render_group + changed_when: false + failed_when: false + +- name: Require target render group + ansible.builtin.assert: + that: + - _auplc_render_group.rc == 0 + - _auplc_render_group.stdout_lines | length == 1 + - _auplc_render_group.stdout.split(':') | length == 4 + - _auplc_render_group.stdout.split(':')[0] == 'render' + - _auplc_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _auplc_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Target has no valid render group; this role never creates groups. + +- name: Record target render GID + ansible.builtin.set_fact: + _auplc_current_render_gid: "{{ _auplc_render_group.stdout.split(':')[2] | int }}" + +- name: Reject invalid canonical GPU access state except Interrupted normalization retry + ansible.builtin.assert: + that: + - _auplc_existing_state is mapping + - _auplc_existing_state.keys() | list | sort == ['renderGid', 'version'] + - _auplc_existing_state.version is integer + - _auplc_existing_state.version == 1 + - _auplc_existing_state.renderGid is integer + - _auplc_existing_state.renderGid >= 1 + - _auplc_existing_state.renderGid <= 4294967294 + - >- + _auplc_existing_state.renderGid == auplc_render_gid or + ((auplc_normalize_render_gid | bool) and + (_auplc_existing_state.renderGid == _auplc_current_render_gid or + _auplc_current_render_gid == auplc_render_gid)) + fail_msg: Invalid canonical GPU access state. + when: _auplc_existing_state is defined + +- name: List target groups for desired GID collision + ansible.builtin.command: + argv: >- + {{ ['getent', 'group'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group'] }} + register: _auplc_all_groups + changed_when: false + failed_when: false + +- name: Reject desired GID collision + ansible.builtin.assert: + that: + - _auplc_all_groups.rc == 0 + - >- + _auplc_all_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (auplc_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: auplc_render_gid is already assigned to another target group. + +- name: Reject render GID mismatch without normalization + ansible.builtin.assert: + that: + - _auplc_current_render_gid == auplc_render_gid or (auplc_normalize_render_gid | bool) + fail_msg: Target render GID differs from auplc_render_gid and normalization is disabled. From f22777a3551f9db75f7b64c301a1878dac768771 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 049/180] feat(ansible): integrate unified GPU access role --- deploy/ansible/playbooks/pb-rocm.yml | 25 +++++++++++++- deploy/ansible/playbooks/pb-udev.yml | 23 +++++++++++-- deploy/ansible/roles/rocm/tasks/main.yml | 22 ------------ deploy/ansible/roles/udev/main.yml | 44 ------------------------ 4 files changed, 45 insertions(+), 69 deletions(-) delete mode 100644 deploy/ansible/roles/udev/main.yml diff --git a/deploy/ansible/playbooks/pb-rocm.yml b/deploy/ansible/playbooks/pb-rocm.yml index 504a7b71..6788bf3d 100644 --- a/deploy/ansible/playbooks/pb-rocm.yml +++ b/deploy/ansible/playbooks/pb-rocm.yml @@ -19,6 +19,29 @@ - name: Install AMD GPU driver for ROCm 7.13.0 hosts: all + any_errors_fatal: true become: yes + pre_tasks: + - name: Assert explicit GPU access enablement + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - auplc_gpu_access_enabled is boolean + fail_msg: >- + Set auplc_gpu_access_enabled to true or false for every host in the + inventory before running pb-rocm.yml. + + - name: Preflight enabled GPU access hosts before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: auplc_gpu_access_enabled | bool roles: - - rocm + - role: rocm + when: auplc_gpu_access_enabled | bool + tasks: + - name: Apply GPU access after ROCm installation + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/playbooks/pb-udev.yml b/deploy/ansible/playbooks/pb-udev.yml index 508b7b42..7e77eb80 100644 --- a/deploy/ansible/playbooks/pb-udev.yml +++ b/deploy/ansible/playbooks/pb-udev.yml @@ -19,7 +19,26 @@ - name: Configure ROCm udev rules hosts: all + any_errors_fatal: true become: yes - roles: - - udev-rocm + pre_tasks: + - name: Assert explicit GPU access enablement + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - auplc_gpu_access_enabled is boolean + fail_msg: >- + Set auplc_gpu_access_enabled to true or false for every host in the + inventory before running pb-udev.yml. + - name: Preflight enabled GPU access hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: auplc_gpu_access_enabled | bool + tasks: + - name: Apply GPU access on enabled hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/roles/rocm/tasks/main.yml b/deploy/ansible/roles/rocm/tasks/main.yml index 525a3a18..d0353f1b 100644 --- a/deploy/ansible/roles/rocm/tasks/main.yml +++ b/deploy/ansible/roles/rocm/tasks/main.yml @@ -53,25 +53,3 @@ apt: name: amdgpu-dkms state: present - -- name: Ensure render group exists with consistent GID - group: - name: render - gid: 993 - state: present - -- name: Set udev rules for ROCm devices with correct permissions - copy: - content: | - # ROCm device permissions - # Grant render group access to AMD GPU devices - # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules - KERNEL=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" - dest: /etc/udev/rules.d/70-amdgpu.rules - mode: '0644' - register: udev_rules_changed - -- name: Reload udev rules if changed - shell: udevadm control --reload-rules && udevadm trigger - when: udev_rules_changed.changed diff --git a/deploy/ansible/roles/udev/main.yml b/deploy/ansible/roles/udev/main.yml deleted file mode 100644 index 235fe25e..00000000 --- a/deploy/ansible/roles/udev/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - ---- -- name: Create /etc/udev/rules.d/70-kfd.rules - copy: - dest: /etc/udev/rules.d/70-kfd.rules - content: | - KERNEL=="kfd", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666" - owner: root - group: root - mode: '0644' - -- name: Reload udev rules - command: udevadm control --reload-rules - -- name: Trigger udev rules - command: udevadm trigger - -- name: Reboot the system (optional) - reboot: - msg: "Rebooting to apply udev rule changes" - pre_reboot_delay: 5 - reboot_timeout: 300 - post_reboot_delay: 30 - when: udev_rocm_reboot_enabled - From 132917e34c09c92fe85a0d6c0e98d43871ee2ea7 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 050/180] feat(deploy): add validated config generation primitives --- .../scripts/config_common.py | 57 +++++ .../scripts/config_generation.py | 209 ++++++++++++++++++ .../scripts/config_rendering.py | 163 ++++++++++++++ 3 files changed, 429 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_common.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_generation.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_rendering.py diff --git a/skills/deploy-aup-learning-cloud/scripts/config_common.py b/skills/deploy-aup-learning-cloud/scripts/config_common.py new file mode 100644 index 00000000..edc3f67e --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_common.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Shared schema constants and scalar rendering helpers.""" + +from __future__ import annotations + +import json +import sys + +HEADER_HASH = ( + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" + "# Generated by auplc-skills gen_configs.py -- review before use.\n" +) + +DEFAULT_ACCEL_LABELS = { + "phx": "AMD_Radeon_780M_Graphics", + "strix": "AMD_Radeon_890M_Graphics", + "strix-halo": "AMD_Radeon_8060S_Graphics", + "9070xt": "AMD_Radeon_RX_9070_XT", + "r9700": "AMD_Radeon_AI_PRO_R9700", + "9600gre": "AMD_Radeon_RX_9600_GRE", +} + + +class DuplicateJsonKeyError(ValueError): + pass + + +def _unique_json_object(pairs): + document = {} + for key, value in pairs: + if key in document: + raise DuplicateJsonKeyError(f"duplicate JSON key '{key}'") + document[key] = value + return document + + +def strict_json_loads(raw: str): + return json.loads(raw, object_pairs_hook=_unique_json_object) + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def require(spec: dict, path: str): + cur = spec + for part in path.split("."): + if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): + die(f"spec is missing required field '{path}'") + cur = cur[part] + return cur + + +def yaml_quote(value: str) -> str: + return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py new file mode 100644 index 00000000..5fdcf84a --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Validate cluster specifications and render deploy configuration artifacts.""" + +from __future__ import annotations + +import ipaddress +import re + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values + +__all__ = [ + "DEFAULT_ACCEL_LABELS", + "HEADER_HASH", + "ResolvedGpuPolicy", + "SCHEMA", + "die", + "render_inventory", + "render_pxe_vars", + "render_values", + "require", + "validate_accelerators", + "validate_config_shapes", + "validate_yaml_scalar", + "validate_spec", + "yaml_quote", +] + +SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + "diskless_agents_have_amd_gpus": True, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} + +HOSTNAME_PATTERN = re.compile( + r"(?=.{1,253}\Z)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*\Z" +) +K3S_VERSION_PATTERN = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+\Z") +IMAGE_KEY_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_-]*\Z") + + +def validate_accelerators(spec: dict) -> None: + if "accelerators" not in spec: + return + accelerators = spec["accelerators"] + if not isinstance(accelerators, dict): + die("spec.accelerators must be a mapping") + unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) + if len(unsupported) == 1: + die(f"unsupported accelerator key '{unsupported[0]}'") + if unsupported: + die(f"unsupported accelerator keys: {', '.join(unsupported)}") + for key, config in accelerators.items(): + if not isinstance(config, dict): + die(f"accelerators.{key} must be a mapping") + + +def validate_config_shapes(spec: dict) -> None: + if not isinstance(spec, dict): + die("spec must be a mapping") + validate_accelerators(spec) + for key in ("server", "network", "pxe", "storage", "proxy", "images"): + if key in spec and not isinstance(spec[key], dict): + die(f"spec.{key} must be a mapping") + if "agents" in spec and not isinstance(spec["agents"], list): + die("spec.agents must be a list") + + +def _safe_text(value, path: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + die(f"{path} must be a non-empty string" if not allow_empty else f"{path} must be a string") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + die(f"{path} must not contain control characters") + return value + + +def validate_yaml_scalar(value, path: str, *, allow_empty: bool = False) -> str: + return _safe_text(value, path, allow_empty=allow_empty) + + +def _safe_hostname(value, path: str) -> str: + hostname = _safe_text(value, path) + if not HOSTNAME_PATTERN.fullmatch(hostname): + die(f"{path} must be a safe hostname") + return hostname + + +def _safe_ip(value, path: str) -> str: + address = _safe_text(value, path) + try: + ipaddress.ip_address(address) + except ValueError: + die(f"{path} must be a valid IP address") + return address + + +def _safe_port(value, path: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + die(f"{path} must be an integer between {minimum} and {maximum}") + return value + + +def _validate_server(server: dict, path: str) -> str: + if set(server) != {"name", "ip"}: + die(f"{path} must contain exactly name and ip") + name = _safe_hostname(server["name"], f"{path}.name") + _safe_ip(server["ip"], f"{path}.ip") + return name + + +def _validate_agents(spec: dict, server_name: str) -> None: + agents = spec.get("agents", []) + if not isinstance(agents, list): + die("spec.agents must be a list") + names = {server_name} + for index, agent in enumerate(agents): + path = f"spec.agents[{index}]" + if not isinstance(agent, dict): + die(f"{path} must be a mapping") + name = _validate_server(agent, path) + if name in names: + die("server and agent names must be unique") + names.add(name) + + +def _validate_rendered_options(spec: dict) -> None: + if "auth_mode" in spec: + _safe_text(spec["auth_mode"], "spec.auth_mode") + if "storage" in spec and "class" in spec["storage"]: + _safe_text(spec["storage"]["class"], "spec.storage.class") + if "proxy" in spec and "node_port" in spec["proxy"]: + _safe_port(spec["proxy"]["node_port"], "spec.proxy.node_port", 30000, 32767) + if "images" in spec: + for key, value in spec["images"].items(): + if not isinstance(key, str) or not IMAGE_KEY_PATTERN.fullmatch(key): + die("spec.images key must be a safe identifier") + _safe_text(value, f"spec.images.{key}") + if "accelerators" in spec: + for key, config in spec["accelerators"].items(): + if "product_name" in config: + _safe_text(config["product_name"], f"spec.accelerators.{key}.product_name") + + +def _validate_pxe(spec: dict) -> None: + pxe = require(spec, "pxe") + keys = pxe.get("authorized_keys") + if not isinstance(keys, list) or not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + for index, key in enumerate(keys): + _safe_text(key, f"spec.pxe.authorized_keys[{index}]") + if "rootfs_password" in pxe: + _safe_text(pxe["rootfs_password"], "spec.pxe.rootfs_password", allow_empty=True) + if "web_port" in pxe: + _safe_port(pxe["web_port"], "spec.pxe.web_port", 1, 65535) + if type(pxe.get("diskless_agents_have_amd_gpus")) is not bool: + die("spec.pxe.diskless_agents_have_amd_gpus must be a boolean") + network = require(spec, "network") + _safe_text(require(spec, "network.interface"), "spec.network.interface") + subnet = _safe_text(require(spec, "network.subnet"), "spec.network.subnet") + try: + ipaddress.ip_network(subnet, strict=True) + except ValueError: + die("spec.network.subnet must be a valid network CIDR") + if "gateway" in network: + _safe_ip(network["gateway"], "spec.network.gateway") + if "dns_servers" in network: + for index, address in enumerate(_safe_text(network["dns_servers"], "spec.network.dns_servers").split(",")): + _safe_ip(address.strip(), f"spec.network.dns_servers[{index}]") + + +def validate_spec(spec: dict) -> str: + if not isinstance(spec, dict): + die("spec must be a mapping") + topo = spec.get("topology") + if topo not in ("pxe-diskless", "ssh-preinstalled"): + die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") + validate_config_shapes(spec) + k3s_version = _safe_text(require(spec, "k3s_version"), "spec.k3s_version") + if not K3S_VERSION_PATTERN.fullmatch(k3s_version): + die("spec.k3s_version must be a safe k3s version") + server_name = _validate_server(require(spec, "server"), "spec.server") + _validate_agents(spec, server_name) + _validate_rendered_options(spec) + if "render_gid" in spec: + die("spec.render_gid is no longer accepted; GPU policy is discovered automatically") + if "gpu_access" in spec: + die("spec.gpu_access is no longer accepted; GPU policy is discovered automatically") + if topo == "pxe-diskless": + _validate_pxe(spec) + return topo diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py new file mode 100644 index 00000000..2ac59a86 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Render deployment artifacts from resolved configuration values.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from gpu_access_resolution import FleetResolution, HostStatus + + +@dataclass(frozen=True, slots=True) +class ResolvedGpuPolicy: + host_gpu_enabled: dict[str, bool] + render_gid: int | None + pxe_gpu_enabled: bool + + +def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str: + topo = spec["topology"] + server = spec["server"] + k3s_version = spec["k3s_version"] + render_gid = resolution.render_gid + host_gpu_enabled = {host.target.name: host.status is HostStatus.GPU for host in resolution.hosts} + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[server['name']] else 'false'}", + " agent:", + ] + if topo == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines.append(f" {agent['name']}:") + lines.append(f" ansible_host: {yaml_quote(agent['ip'])}") + lines.append( + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[agent['name']] else 'false'}" + ) + else: + lines.append(" hosts: {}") + lines += [ + " vars:", + " ansible_port: 22", + " ansible_user: root", + f" k3s_version: {yaml_quote(k3s_version)}", + f" auplc_render_gid: {'null' if render_gid is None else render_gid}", + f" token: {yaml_quote(token)}", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", + ] + if topo == "pxe-diskless": + lines += [ + "", + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + ] + return "\n".join(lines) + "\n" + + +def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: str | None = None) -> str: + net = require(spec, "network") + pxe = spec.get("pxe", {}) + keys = pxe.get("authorized_keys", []) + if not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + server_ip = spec["server"]["ip"] + k3s_version = spec["k3s_version"] + render_gid = policy.render_gid + lines = [ + HEADER_HASH, + "# Pass this file to pb-pxe-controller.yml with", + "# ansible-playbook ... -e @<absolute-path-to-this-file>", + "# pxe_k3s_version is pinned to k3s_version so agents are never newer", + "# than the server.", + "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", + f"pxe_network_interface: {yaml_quote(net['interface'])}", + f"pxe_subnet: {yaml_quote(net['subnet'])}", + f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", + f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", + f"pxe_controller_ip: {yaml_quote(server_ip)}", + "pxe_k3s_server_ips:", + f" - {yaml_quote(server_ip)}", + f"pxe_k3s_version: {yaml_quote(k3s_version)}", + f"auplc_render_gid: {'null' if render_gid is None else render_gid}", + f"pxe_gpu_access_enabled: {'true' if policy.pxe_gpu_enabled else 'false'}", + f"pxe_web_port: {int(pxe.get('web_port', 8080))}", + f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", + "pxe_rootfs_authorized_keys:", + ] + for key in keys: + lines.append(f" - {yaml_quote(key)}") + if finalizer_context is not None: + lines.append(f"pxe_finalizer_context: {yaml_quote(finalizer_context)}") + return "\n".join(lines) + "\n" + + +def render_values(spec: dict, resolution: FleetResolution) -> str: + accel = spec.get("accelerators") or {} + storage_class = (spec.get("storage") or {}).get("class", "nfs-client") + node_port = (spec.get("proxy") or {}).get("node_port", 30890) + auth_mode = spec.get("auth_mode", "auto-login") + images = spec.get("images") or {} + render_gid = resolution.render_gid + lines = [ + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", + "# Helm overlay generated by auplc-skills gen_configs.py.", + "# Layer this on top of runtime/values.yaml:", + "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", + "# --create-namespace -f runtime/values.yaml -f <this file>", + "custom:", + f" authMode: {yaml_quote(auth_mode)}", + " gpuAccess:", + f" renderGid: {'null' if render_gid is None else render_gid}", + ] + if accel: + lines.append(" accelerators:") + for key, config in accel.items(): + product = (config or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) + if not product: + die( + f"accelerator '{key}' has no product_name and no known default; " + "add accelerators.<key>.product_name from `kubectl describe node`" + ) + lines += [ + f" {key}:", + " nodeSelector:", + f" amd.com/gpu.product-name: {yaml_quote(product)}", + ] + if accel or images: + lines.append(" resources:") + if accel: + lines += [" metadata:", " gpu:", " acceleratorKeys:"] + lines.extend(f" - {yaml_quote(key)}" for key in accel) + if images: + lines.append(" images:") + for key, value in images.items(): + lines.append(f" {key}: {yaml_quote(value)}") + lines += [ + "hub:", + " db:", + " pvc:", + f" storageClassName: {yaml_quote(storage_class)}", + "singleuser:", + " storage:", + " dynamic:", + f" storageClass: {yaml_quote(storage_class)}", + "proxy:", + " service:", + " type: NodePort", + " nodePorts:", + f" http: {int(node_port)}", + ] + return "\n".join(lines) + "\n" From e1e838249ffb71278b16feec853c8737f7b3d1a1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 051/180] feat(deploy): publish generated artifacts atomically --- .../scripts/artifact_store.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/artifact_store.py diff --git a/skills/deploy-aup-learning-cloud/scripts/artifact_store.py b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py new file mode 100644 index 00000000..5b9de061 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Stage and atomically publish generated deployment artifacts.""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +from contextlib import suppress +from pathlib import Path + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def preflight_destinations(paths: list[Path], force: bool) -> None: + if force: + return + for path in paths: + if os.path.lexists(path): + die(f"refusing to overwrite existing {path} (use --force)", 1) + + +def stage_file(path: Path, content: str, mode: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as staged_file: + staged_file.write(content) + staged_file.flush() + os.fsync(staged_file.fileno()) + except OSError: + with suppress(OSError): + os.close(fd) + Path(staged_path).unlink(missing_ok=True) + raise + return Path(staged_path) + + +def remove_destination(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def backup_destination(path: Path) -> tuple[Path, Path]: + backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) + backup_path = backup_dir / path.name + os.replace(path, backup_path) + return backup_dir, backup_path + + +def _fsync_parent(path: Path) -> None: + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + +def publish_artifacts( + artifacts: list[tuple[Path, str, int, bool]], force: bool, remove_paths: tuple[Path, ...] = () +) -> None: + staged: list[tuple[Path, Path, bool]] = [] + published: list[Path] = [] + backups: list[tuple[Path, Path, Path]] = [] + replacement_paths = tuple(path for path, _, _, _ in artifacts) + try: + for path, content, mode, secret in artifacts: + staged.append((path, stage_file(path, content, mode), secret)) + if force: + for path in (*replacement_paths, *(path for path in remove_paths if path not in replacement_paths)): + if os.path.lexists(path): + backup_dir, backup_path = backup_destination(path) + backups.append((path, backup_dir, backup_path)) + _fsync_parent(path) + for path, staged_path, secret in staged: + if force: + os.replace(staged_path, path) + else: + os.link(staged_path, path) + published.append(path) + if not force: + os.unlink(staged_path) + _fsync_parent(path) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + except OSError as exc: + for path in reversed(published): + remove_destination(path) + _fsync_parent(path) + for path, backup_dir, backup_path in reversed(backups): + remove_destination(path) + os.replace(backup_path, path) + _fsync_parent(path) + backup_dir.rmdir() + die(f"could not publish generated artifacts: {exc}") + else: + for _, backup_dir, _ in backups: + shutil.rmtree(backup_dir) + finally: + for _, staged_path, _ in staged: + staged_path.unlink(missing_ok=True) From 1333e152cfa3a95a5195730316c5056c878d6cba Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 052/180] feat(deploy): resolve unanimous fleet GPU policy --- .../scripts/gpu_access_resolution.py | 314 ++++++++++++++ .../scripts/gpu_resolution_manifest.py | 68 +++ tests/skills/test_gpu_access_resolution.py | 408 ++++++++++++++++++ 3 files changed, 790 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py create mode 100644 tests/skills/test_gpu_access_resolution.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py new file mode 100644 index 00000000..b618b2f8 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py @@ -0,0 +1,314 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Parse read-only host evidence and resolve a safe fleet GPU-access policy.""" + +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Final + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_manifest import ResolutionManifest, build_resolution_manifest + +EVIDENCE_VERSION: Final = 2 +MAX_RENDER_GID: Final = 4_294_967_294 +CANONICAL_RULE: Final = ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' +) +BDF_PATTERN: Final = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") + + +class HostStatus(str, Enum): + """Classify one inventory host from mutually corroborated discovery probes.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +class FleetStatus(str, Enum): + """Describe whether fleet evidence yields a publication-safe GPU policy.""" + + GPU_RESOLVED = "gpu_resolved" + CPU_ONLY = "cpu_only" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class EvidenceParseError(ValueError): + """Raised when discovery JSON does not match the fixed evidence schema.""" + + field: str + + def __str__(self) -> str: + return f"Malformed GPU-access discovery evidence at {self.field}" + + +@dataclass(frozen=True, slots=True) +class InventoryTarget: + name: str + + +@dataclass(frozen=True, slots=True) +class CommandEvidence: + rc: int + stdout: str + + +@dataclass(frozen=True, slots=True) +class FileEvidence: + stat_success: bool + content_success: bool + exists: bool + regular: bool + symlink: bool + content: str + + +@dataclass(frozen=True, slots=True) +class LegacyRuleEvidence: + kfd: FileEvidence + amdgpu: FileEvidence + rocm_devices: FileEvidence + + +@dataclass(frozen=True, slots=True) +class HostEvidence: + target: InventoryTarget + reachable: bool + lspci: CommandEvidence + sysfs: CommandEvidence + render_group: CommandEvidence + groups: CommandEvidence + state: FileEvidence + rule: FileEvidence + legacy_rules: LegacyRuleEvidence + + +@dataclass(frozen=True, slots=True) +class HostResolution: + target: InventoryTarget + status: HostStatus + render_gid: int | None + reason: str | None + + +@dataclass(frozen=True, slots=True) +class FleetResolution: + status: FleetStatus + hosts: tuple[HostResolution, ...] + render_gid: int | None + reason: str | None + + +def parse_fleet_evidence(raw: str) -> tuple[HostEvidence, ...]: + """Parse the exact JSON emitted by the GPU-access discovery playbook.""" + try: + document = strict_json_loads(raw) + except DuplicateJsonKeyError as error: + raise EvidenceParseError(field=str(error)) from error + except (TypeError, json.JSONDecodeError) as error: + raise EvidenceParseError(field="document") from error + _require_mapping(document, "document") + if set(document) != {"version", "hosts"}: + raise EvidenceParseError(field="document") + if type(document["version"]) is not int or document["version"] != EVIDENCE_VERSION: + raise EvidenceParseError(field="version") + if type(document["hosts"]) is not list: + raise EvidenceParseError(field="hosts") + return tuple(_parse_host(item, f"hosts[{index}]") for index, item in enumerate(document["hosts"])) + + +def resolve_fleet(expected_targets: tuple[InventoryTarget, ...], evidence: tuple[HostEvidence, ...]) -> FleetResolution: + """Resolve a fleet only when complete evidence proves one safe policy.""" + resolutions = tuple(_resolve_host(host) for host in evidence) + expected_names = tuple(target.name for target in expected_targets) + actual_names = tuple(host.target.name for host in evidence) + if len(set(expected_names)) != len(expected_names) or len(set(actual_names)) != len(actual_names): + return _blocked(resolutions, "duplicate host") + if set(expected_names) != set(actual_names): + return _blocked(resolutions, "incomplete host coverage") + if any(host.status is HostStatus.UNKNOWN for host in resolutions): + return _blocked(resolutions, "unknown host evidence") + gpu_hosts = tuple(host for host in resolutions if host.status is HostStatus.GPU) + if not gpu_hosts: + return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None, None) + gids = {host.render_gid for host in gpu_hosts} + if len(gids) != 1: + return _blocked(resolutions, "GPU render GIDs disagree") + return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, next(iter(gids)), None) + + +def resolution_manifest(resolution: FleetResolution) -> ResolutionManifest: + """Build the public serialized manifest for a resolved fleet.""" + return build_resolution_manifest( + version=1, + status=resolution.status.value, + render_gid=resolution.render_gid, + hosts={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, + ) + + +def _parse_host(raw, field: str) -> HostEvidence: + _require_mapping(raw, field) + required = {"host", "reachable", "lspci", "sysfs", "render_group", "groups", "state", "rule", "legacy_rules"} + if set(raw) != required or type(raw["host"]) is not str or not raw["host"]: + raise EvidenceParseError(field=field) + if type(raw["reachable"]) is not bool: + raise EvidenceParseError(field=f"{field}.reachable") + return HostEvidence( + target=InventoryTarget(name=raw["host"]), + reachable=raw["reachable"], + lspci=_parse_command(raw["lspci"], f"{field}.lspci"), + sysfs=_parse_command(raw["sysfs"], f"{field}.sysfs"), + render_group=_parse_command(raw["render_group"], f"{field}.render_group"), + groups=_parse_command(raw["groups"], f"{field}.groups"), + state=_parse_file(raw["state"], f"{field}.state"), + rule=_parse_file(raw["rule"], f"{field}.rule"), + legacy_rules=_parse_legacy_rules(raw["legacy_rules"], f"{field}.legacy_rules"), + ) + + +def _parse_command(raw, field: str) -> CommandEvidence: + _require_mapping(raw, field) + if set(raw) != {"rc", "stdout"} or type(raw["rc"]) is not int or type(raw["stdout"]) is not str: + raise EvidenceParseError(field=field) + return CommandEvidence(rc=raw["rc"], stdout=raw["stdout"]) + + +def _parse_file(raw, field: str) -> FileEvidence: + _require_mapping(raw, field) + required = {"stat_success", "content_success", "exists", "regular", "symlink", "content"} + if set(raw) != required or any( + type(raw[key]) is not bool for key in ("stat_success", "content_success", "exists", "regular", "symlink") + ): + raise EvidenceParseError(field=field) + if type(raw["content"]) is not str: + raise EvidenceParseError(field=f"{field}.content") + return FileEvidence(**raw) + + +def _parse_legacy_rules(raw, field: str) -> LegacyRuleEvidence: + _require_mapping(raw, field) + if set(raw) != {"kfd", "amdgpu", "rocm_devices"}: + raise EvidenceParseError(field=field) + return LegacyRuleEvidence( + kfd=_parse_file(raw["kfd"], f"{field}.kfd"), + amdgpu=_parse_file(raw["amdgpu"], f"{field}.amdgpu"), + rocm_devices=_parse_file(raw["rocm_devices"], f"{field}.rocm_devices"), + ) + + +def _require_mapping(value, field: str) -> None: + if type(value) is not dict: + raise EvidenceParseError(field=field) + + +def _resolve_host(evidence: HostEvidence) -> HostResolution: + if not evidence.reachable or evidence.lspci.rc != 0 or evidence.sysfs.rc != 0: + return _unknown(evidence, "GPU discovery probe failed") + if not _file_probes_succeeded(evidence): + return _unknown(evidence, "GPU access file probe failed") + lspci_bdfs = _bdfs(evidence.lspci.stdout) + sysfs_bdfs = _bdfs(evidence.sysfs.stdout) + if lspci_bdfs is None or sysfs_bdfs is None or lspci_bdfs != sysfs_bdfs: + return _unknown(evidence, "AMD GPU BDF probes disagree") + if not lspci_bdfs: + if evidence.state.exists or evidence.rule.exists or _legacy_rule_exists(evidence.legacy_rules): + return _unknown(evidence, "CPU host retains GPU access contract") + return HostResolution(evidence.target, HostStatus.CPU, None, None) + render_gid = _render_gid(evidence) + if render_gid is None or not _safe_gpu_files(evidence, render_gid): + return _unknown(evidence, "GPU access contract is unsafe") + return HostResolution(evidence.target, HostStatus.GPU, render_gid, None) + + +def _bdfs(stdout: str) -> frozenset[str] | None: + bdfs = frozenset(line.split(maxsplit=1)[0] for line in stdout.splitlines()) + if all(BDF_PATTERN.fullmatch(bdf) for bdf in bdfs): + return bdfs + return None + + +def _render_gid(evidence: HostEvidence) -> int | None: + if evidence.render_group.rc != 0 or evidence.groups.rc != 0: + return None + record = _group_record(evidence.render_group.stdout) + if record is None or record[0] != "render": + return None + gid = record[1] + groups = tuple(_group_record(line) for line in evidence.groups.stdout.splitlines()) + if not groups or any(group is None for group in groups): + return None + if sum(group[0] == "render" and group[1] == gid for group in groups) != 1: + return None + if any(group[0] != "render" and group[1] == gid for group in groups): + return None + return gid + + +def _group_record(record: str) -> tuple[str, int] | None: + fields = record.split(":") + if len(fields) != 4 or not fields[0] or not fields[2].isascii() or not fields[2].isdecimal(): + return None + gid = int(fields[2]) + if 1 <= gid <= MAX_RENDER_GID: + return fields[0], gid + return None + + +def _safe_gpu_files(evidence: HostEvidence, render_gid: int) -> bool: + if not _safe_file(evidence.state) or not _safe_file(evidence.rule): + return False + if evidence.state.exists and _state_gid(evidence.state.content) != render_gid: + return False + return not evidence.rule.exists or evidence.rule.content == CANONICAL_RULE + + +def _safe_file(evidence: FileEvidence) -> bool: + if not evidence.stat_success or not evidence.content_success: + return False + if evidence.exists: + return evidence.regular and not evidence.symlink + return not evidence.regular and not evidence.symlink and not evidence.content + + +def _file_probes_succeeded(evidence: HostEvidence) -> bool: + return all( + file_evidence.stat_success and file_evidence.content_success + for file_evidence in ( + evidence.state, + evidence.rule, + evidence.legacy_rules.kfd, + evidence.legacy_rules.amdgpu, + evidence.legacy_rules.rocm_devices, + ) + ) + + +def _legacy_rule_exists(evidence: LegacyRuleEvidence) -> bool: + return any(file_evidence.exists for file_evidence in (evidence.kfd, evidence.amdgpu, evidence.rocm_devices)) + + +def _state_gid(raw: str) -> int | None: + try: + state = strict_json_loads(raw) + except (DuplicateJsonKeyError, TypeError, json.JSONDecodeError): + return None + if type(state) is not dict or set(state) != {"renderGid", "version"}: + return None + gid = state["renderGid"] + if type(gid) is not int or type(state["version"]) is not int or state["version"] != 1: + return None + return gid if 1 <= gid <= MAX_RENDER_GID else None + + +def _unknown(evidence: HostEvidence, reason: str) -> HostResolution: + return HostResolution(evidence.target, HostStatus.UNKNOWN, None, reason) + + +def _blocked(hosts: tuple[HostResolution, ...], reason: str) -> FleetResolution: + return FleetResolution(FleetStatus.BLOCKED, hosts, None, reason) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py new file mode 100644 index 00000000..60a59fd7 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py @@ -0,0 +1,68 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Typed GPU-resolution manifest schemas and primitive builders.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TypedDict + + +class ResolutionManifest(TypedDict): + """Serialized fleet GPU-resolution evidence.""" + + version: int + status: str + render_gid: int | None + hosts: dict[str, bool] + + +class PxeRootfsManifest(TypedDict): + """Serialized GPU policy applied to the PXE root filesystem.""" + + gpu_access_enabled: bool + render_gid: int | None + + +class PxeResolutionManifest(ResolutionManifest): + """Serialized fleet resolution with its PXE rootfs policy.""" + + pxe_rootfs: PxeRootfsManifest + + +def build_resolution_manifest( + *, + version: int, + status: str, + render_gid: int | None, + hosts: Mapping[str, bool], +) -> ResolutionManifest: + """Build a deterministic ordinary dictionary for fleet resolution.""" + return { + "version": version, + "status": status, + "render_gid": render_gid, + "hosts": {name: hosts[name] for name in sorted(hosts)}, + } + + +def build_pxe_resolution_manifest( + *, + version: int, + status: str, + render_gid: int | None, + hosts: Mapping[str, bool], + gpu_access_enabled: bool, + pxe_render_gid: int | None, +) -> PxeResolutionManifest: + """Build a PXE manifest without mutating a base fleet manifest.""" + return { + "version": version, + "status": status, + "render_gid": render_gid, + "hosts": {name: hosts[name] for name in sorted(hosts)}, + "pxe_rootfs": { + "gpu_access_enabled": gpu_access_enabled, + "render_gid": pxe_render_gid, + }, + } diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py new file mode 100644 index 00000000..ea853f00 --- /dev/null +++ b/tests/skills/test_gpu_access_resolution.py @@ -0,0 +1,408 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Behavior tests for fleet GPU-access discovery resolution.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +RESOLUTION = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_access_resolution.py" +MANIFEST = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_resolution_manifest.py" +GPU_BDF = "0000:03:00.0" + + +def load_resolution_module(): + sys.path.insert(0, str(RESOLUTION.parent)) + spec = importlib.util.spec_from_file_location("gpu_access_resolution", RESOLUTION) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + try: + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_manifest_module(): + spec = importlib.util.spec_from_file_location("gpu_resolution_manifest", MANIFEST) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def host_evidence( + host: str, + *, + lspci_bdfs: list[str] | None = None, + sysfs_bdfs: list[str] | None = None, + lspci_rc: int = 0, + sysfs_rc: int = 0, + reachable: bool = True, + render_gid: int = 993, + group_listing: str | None = None, + state: str | None = None, + rule: str | None = None, + state_stat_success: bool = True, + state_content_success: bool = True, + legacy_rules: dict[str, str | None] | None = None, +) -> dict: + lspci = "\n".join(lspci_bdfs or []) + sysfs = "\n".join(sysfs_bdfs if sysfs_bdfs is not None else lspci_bdfs or []) + return { + "host": host, + "reachable": reachable, + "lspci": {"rc": lspci_rc, "stdout": lspci}, + "sysfs": {"rc": sysfs_rc, "stdout": sysfs}, + "render_group": {"rc": 0, "stdout": f"render:x:{render_gid}:\n"}, + "groups": {"rc": 0, "stdout": group_listing or f"render:x:{render_gid}:\n"}, + "state": { + "stat_success": state_stat_success, + "content_success": state_content_success, + "exists": state is not None, + "regular": state is not None, + "symlink": False, + "content": state or "", + }, + "rule": { + "stat_success": True, + "content_success": True, + "exists": rule is not None, + "regular": rule is not None, + "symlink": False, + "content": rule or "", + }, + "legacy_rules": { + key: { + "stat_success": True, + "content_success": True, + "exists": content is not None, + "regular": content is not None, + "symlink": False, + "content": content or "", + } + for key, content in (legacy_rules or {}).items() + } + | { + key: { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + } + for key in ("kfd", "amdgpu", "rocm_devices") + if key not in (legacy_rules or {}) + }, + } + + +def evidence_document(*hosts: dict) -> str: + return json.dumps({"version": 2, "hosts": list(hosts)}) + + +def expected_targets(module, *names: str): + return tuple(module.InventoryTarget(name=name) for name in names) + + +def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: + module = load_resolution_module() + raw = evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF])) + + evidence = module.parse_fleet_evidence(raw) + + assert evidence[0].target == module.InventoryTarget(name="gpu-1") + assert evidence[0].lspci.stdout == GPU_BDF + assert evidence[0].state.exists is False + + +@pytest.mark.parametrize( + "replacement", + [ + {"version": True, "hosts": []}, + {"version": 1, "hosts": [], "unexpected": "field"}, + {"version": 1, "hosts": [{"host": "gpu-1"}]}, + {"version": 1, "hosts": [host_evidence("gpu-1", lspci_rc=True)]}, + ], +) +def test_parse_fleet_evidence_rejects_nonexact_or_boolean_integer_values(replacement: dict) -> None: + module = load_resolution_module() + + with pytest.raises(module.EvidenceParseError): + module.parse_fleet_evidence(json.dumps(replacement)) + + +def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: + module = load_resolution_module() + + with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): + module.parse_fleet_evidence('{"version":2,"version":2,"hosts":[]}') + + +def test_resolve_fleet_blocks_duplicate_persisted_state_render_gid() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state='{"renderGid":993,"renderGid":993,"version":1}') + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + + +@pytest.mark.parametrize( + "state", + [ + '{"renderGid":993,"version":1,"version":1}', + '{"renderGid":993,"version":1,"r\\u0065nderGid":993}', + '{"renderGid":993,"version":1,"v\\u0065rsion":1}', + ], + ids=["duplicate-version", "escaped-render-gid", "escaped-version"], +) +def test_resolve_fleet_blocks_semantic_duplicate_persisted_state_keys(state: str) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state=state))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + + +def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), evidence) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert resolution.render_gid == 993 + assert resolution.hosts[0].status is module.HostStatus.GPU + + +def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("cpu-1"))) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), evidence) + + assert resolution.status is module.FleetStatus.CPU_ONLY + assert resolution.render_gid is None + assert resolution.hosts[0].status is module.HostStatus.CPU + + +@pytest.mark.parametrize( + "evidence", + [ + host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"]), + host_evidence("host-1", lspci_bdfs=[GPU_BDF], lspci_rc=1), + host_evidence("host-1", lspci_bdfs=[GPU_BDF], reachable=False), + ], +) +def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(evidence)) + + resolution = module.resolve_fleet(expected_targets(module, "host-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize( + ("targets", "hosts"), + [ + (("gpu-1", "gpu-2"), ("gpu-1",)), + (("gpu-1",), ("gpu-1", "gpu-2")), + ], +) +def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( + targets: tuple[str, ...], hosts: tuple[str, ...] +) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(*(host_evidence(host, lspci_bdfs=[GPU_BDF]) for host in hosts)) + ) + + resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.render_gid is None + + +def test_resolve_fleet_blocks_render_gid_collisions() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence( + "gpu-1", + lspci_bdfs=[GPU_BDF], + group_listing="render:x:993:\nother:x:993:\n", + ) + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_blocks_cpu_hosts_with_persisted_gpu_access_contracts() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", state='{"renderGid":993,"version":1}')) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize("legacy_key", ["kfd", "amdgpu", "rocm_devices"]) +def test_resolve_fleet_blocks_cpu_hosts_with_any_legacy_gpu_access_rule(legacy_key: str) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", legacy_rules={legacy_key: 'KERNEL=="kfd", MODE="0666"\n'})) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_keeps_gpu_legacy_rule_admission_for_the_later_exact_migration() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], legacy_rules={"amdgpu": "legacy\n"})) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + + +def test_resolve_fleet_blocks_file_probe_failures_instead_of_treating_them_as_absence() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", state_stat_success=False, state_content_success=False)) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize( + "contracts", + [ + {"state": '{"renderGid":994,"version":1}'}, + {"rule": 'KERNEL=="kfd", MODE="0666"\n'}, + ], +) +def test_resolve_fleet_blocks_gpu_hosts_with_unsafe_persisted_contracts(contracts: dict) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], **contracts))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_requires_unanimous_gpu_render_gid() -> None: + module = load_resolution_module() + same_gid = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=993), + ) + ) + mixed_gid = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=994), + ) + ) + + resolved = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), same_gid) + blocked = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), mixed_gid) + + assert resolved.status is module.FleetStatus.GPU_RESOLVED + assert resolved.render_gid == 993 + assert blocked.status is module.FleetStatus.BLOCKED + assert blocked.render_gid is None + + +def test_resolution_manifest_preserves_explicit_host_booleans_and_unanimous_gid() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("cpu-1"), + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed) + manifest = module.resolution_manifest(resolution) + + assert manifest == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"cpu-1": False, "gpu-1": True}, + } + + +def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: + manifest = load_manifest_module().build_resolution_manifest( + version=1, + status="gpu_resolved", + render_gid=993, + hosts={"zeta": True, "alpha": False}, + ) + + assert type(manifest) is dict + assert list(manifest) == ["version", "status", "render_gid", "hosts"] + assert list(manifest["hosts"]) == ["alpha", "zeta"] + assert set(manifest) == {"version", "status", "render_gid", "hosts"} + + +def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: + module = load_manifest_module() + base = module.build_resolution_manifest( + version=1, + status="gpu_resolved", + render_gid=993, + hosts={"gpu-2": True, "gpu-1": True}, + ) + + manifest = module.build_pxe_resolution_manifest( + version=base["version"], + status=base["status"], + render_gid=base["render_gid"], + hosts=base["hosts"], + gpu_access_enabled=True, + pxe_render_gid=994, + ) + + assert base == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"gpu-1": True, "gpu-2": True}, + } + assert list(manifest) == ["version", "status", "render_gid", "hosts", "pxe_rootfs"] + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 994} + assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled", "render_gid"} From d41f301ae342fc9f3607022f7aaabd9c2642124c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 053/180] feat(deploy): discover fleet GPU access state --- .../playbooks/pb-gpu-access-discovery.yml | 386 ++++++++++++++++++ .../scripts/gpu_artifact_generation.py | 218 ++++++++++ tests/skills/test_gpu_artifact_generation.py | 308 ++++++++++++++ 3 files changed, 912 insertions(+) create mode 100644 deploy/ansible/playbooks/pb-gpu-access-discovery.yml create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py create mode 100644 tests/skills/test_gpu_artifact_generation.py diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml new file mode 100644 index 00000000..cd8c366a --- /dev/null +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -0,0 +1,386 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +--- +- name: Discover fleet GPU-access evidence + hosts: k3s_cluster + gather_facts: false + become: false + ignore_unreachable: true + vars: + _auplc_gpu_access_unknown_evidence: + reachable: false + lspci: + rc: 255 + stdout: "" + sysfs: + rc: 255 + stdout: "" + render_group: + rc: 255 + stdout: "" + groups: + rc: 255 + stdout: "" + state: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + rule: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + legacy_rules: + kfd: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + amdgpu: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + rocm_devices: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + pre_tasks: + - name: Require a safe local discovery evidence output path + ansible.builtin.assert: + that: + - gpu_access_discovery_output_path is defined + - gpu_access_discovery_output_path is string + - gpu_access_discovery_output_path is match('^/') + fail_msg: >- + Set gpu_access_discovery_output_path to an absolute controller-local + path before running discovery. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence parent + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path | dirname }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_parent + changed_when: false + + - name: Require safe local discovery evidence parent + ansible.builtin.assert: + that: + - _auplc_discovery_output_parent.stat.exists + - _auplc_discovery_output_parent.stat.isdir + - not _auplc_discovery_output_parent.stat.islnk + fail_msg: Discovery output parent must be an existing non-symlink directory. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence destination + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_destination + changed_when: false + + - name: Require safe local discovery evidence destination + ansible.builtin.assert: + that: + - >- + not _auplc_discovery_output_destination.stat.exists or + (_auplc_discovery_output_destination.stat.isreg and + not _auplc_discovery_output_destination.stat.islnk) + fail_msg: Discovery output destination must be absent or a regular non-symlink file. + delegate_to: localhost + run_once: true + changed_when: false + tasks: + - name: Discover AMD VGA display BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0300" + register: _auplc_discovery_lspci_vga + changed_when: false + failed_when: false + + - name: Discover AMD 3D display BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0302" + register: _auplc_discovery_lspci_3d + changed_when: false + failed_when: false + + - name: Discover AMD display-controller BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0380" + register: _auplc_discovery_lspci_display + changed_when: false + failed_when: false + + - name: Combine AMD display lspci evidence + ansible.builtin.set_fact: + _auplc_discovery_lspci: + rc: >- + {{ 0 if _auplc_discovery_lspci_vga.rc == 0 and + _auplc_discovery_lspci_3d.rc == 0 and + _auplc_discovery_lspci_display.rc == 0 else 1 }} + stdout: >- + {{ [_auplc_discovery_lspci_vga.stdout | default(''), + _auplc_discovery_lspci_3d.stdout | default(''), + _auplc_discovery_lspci_display.stdout | default('')] + | reject('equalto', '') | join('\n') }} + changed_when: false + + - name: Discover AMD display BDFs through sysfs + ansible.builtin.command: + argv: + - python3 + - -c + - >- + from pathlib import Path; devices = Path('/sys/bus/pci/devices'); + print('\n'.join(sorted(device.name for device in devices.iterdir() + if (device / 'vendor').read_text().strip() == '0x1002' and + (device / 'class').read_text().strip().startswith('0x03')))) + register: _auplc_discovery_sysfs + changed_when: false + failed_when: false + + - name: Read render group record + ansible.builtin.command: + argv: + - getent + - group + - render + register: _auplc_discovery_render_group + changed_when: false + failed_when: false + + - name: Read all group records for render GID collision detection + ansible.builtin.command: + argv: + - getent + - group + register: _auplc_discovery_groups + changed_when: false + failed_when: false + + - name: Inspect persisted GPU access state + ansible.builtin.stat: + path: /var/lib/auplc/gpu-access.json + follow: false + register: _auplc_discovery_state + changed_when: false + ignore_errors: true + + - name: Read persisted GPU access state + ansible.builtin.slurp: + src: /var/lib/auplc/gpu-access.json + register: _auplc_discovery_state_content + when: + - _auplc_discovery_state.stat.exists | default(false) + - _auplc_discovery_state.stat.isreg | default(false) + - not (_auplc_discovery_state.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect canonical GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-auplc-gpu-access.rules + follow: false + register: _auplc_discovery_rule + changed_when: false + ignore_errors: true + + - name: Read canonical GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-auplc-gpu-access.rules + register: _auplc_discovery_rule_content + when: + - _auplc_discovery_rule.stat.exists | default(false) + - _auplc_discovery_rule.stat.isreg | default(false) + - not (_auplc_discovery_rule.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy kfd GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-kfd.rules + follow: false + register: _auplc_discovery_legacy_kfd + changed_when: false + ignore_errors: true + + - name: Read legacy kfd GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-kfd.rules + register: _auplc_discovery_legacy_kfd_content + when: + - _auplc_discovery_legacy_kfd.stat.exists | default(false) + - _auplc_discovery_legacy_kfd.stat.isreg | default(false) + - not (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy amdgpu GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-amdgpu.rules + follow: false + register: _auplc_discovery_legacy_amdgpu + changed_when: false + ignore_errors: true + + - name: Read legacy amdgpu GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-amdgpu.rules + register: _auplc_discovery_legacy_amdgpu_content + when: + - _auplc_discovery_legacy_amdgpu.stat.exists | default(false) + - _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) + - not (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy ROCm devices GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-rocm-devices.rules + follow: false + register: _auplc_discovery_legacy_rocm_devices + changed_when: false + ignore_errors: true + + - name: Read legacy ROCm devices GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-rocm-devices.rules + register: _auplc_discovery_legacy_rocm_devices_content + when: + - _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) + - _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) + - not (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Record machine-readable GPU access discovery evidence + ansible.builtin.set_fact: + _auplc_gpu_access_discovery_evidence: + host: "{{ inventory_hostname }}" + reachable: true + lspci: + rc: "{{ _auplc_discovery_lspci.rc }}" + stdout: "{{ _auplc_discovery_lspci.stdout | default('') }}" + sysfs: + rc: "{{ _auplc_discovery_sysfs.rc }}" + stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" + render_group: + rc: "{{ _auplc_discovery_render_group.rc }}" + stdout: "{{ _auplc_discovery_render_group.stdout | default('') }}" + groups: + rc: "{{ _auplc_discovery_groups.rc }}" + stdout: "{{ _auplc_discovery_groups.stdout | default('') }}" + state: + stat_success: "{{ not (_auplc_discovery_state.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_state.failed | default(false)) and + (not (_auplc_discovery_state.stat.exists | default(false)) or + not (_auplc_discovery_state.stat.isreg | default(false)) or + (_auplc_discovery_state.stat.islnk | default(false)) or + not (_auplc_discovery_state_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_state.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_state.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_state.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_state_content.content | default('') | b64decode }}" + rule: + stat_success: "{{ not (_auplc_discovery_rule.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_rule.failed | default(false)) and + (not (_auplc_discovery_rule.stat.exists | default(false)) or + not (_auplc_discovery_rule.stat.isreg | default(false)) or + (_auplc_discovery_rule.stat.islnk | default(false)) or + not (_auplc_discovery_rule_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_rule.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_rule.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_rule.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_rule_content.content | default('') | b64decode }}" + legacy_rules: + kfd: + stat_success: "{{ not (_auplc_discovery_legacy_kfd.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_kfd.failed | default(false)) and + (not (_auplc_discovery_legacy_kfd.stat.exists | default(false)) or + not (_auplc_discovery_legacy_kfd.stat.isreg | default(false)) or + (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_kfd_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_kfd.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_kfd.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_kfd.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_kfd_content.content | default('') | b64decode }}" + amdgpu: + stat_success: "{{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) and + (not (_auplc_discovery_legacy_amdgpu.stat.exists | default(false)) or + not (_auplc_discovery_legacy_amdgpu.stat.isreg | default(false)) or + (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_amdgpu_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_amdgpu.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_amdgpu.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_amdgpu_content.content | default('') | b64decode }}" + rocm_devices: + stat_success: "{{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) and + (not (_auplc_discovery_legacy_rocm_devices.stat.exists | default(false)) or + not (_auplc_discovery_legacy_rocm_devices.stat.isreg | default(false)) or + (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_rocm_devices_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_rocm_devices.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_rocm_devices_content.content | default('') | b64decode }}" + changed_when: false + + - name: Write machine-readable GPU access discovery evidence locally + ansible.builtin.copy: + content: | + {"version":2,"hosts":[{% for discovery_host in ansible_play_hosts_all %} + {{ ( + hostvars[discovery_host]._auplc_gpu_access_discovery_evidence + | default( + _auplc_gpu_access_unknown_evidence | combine({'host': discovery_host}), + true + ) + | to_json + ) }}{% if not loop.last %},{% endif %} + {% endfor %}]} + dest: "{{ gpu_access_discovery_output_path }}" + mode: "0600" + delegate_to: localhost + run_once: true + changed_when: false diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py new file mode 100644 index 00000000..3c47e1d2 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Discover live GPU facts and prepare publication-safe resolved artifacts.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import NoReturn + +from artifact_store import publish_artifacts +from config_common import yaml_quote +from config_generation import HEADER_HASH +from gpu_access_resolution import ( + EvidenceParseError, + FleetResolution, + FleetStatus, + InventoryTarget, + parse_fleet_evidence, + resolution_manifest, + resolve_fleet, +) + +DISCOVERY_TIMEOUT_BASE_SECONDS = 30 +DISCOVERY_TIMEOUT_PER_TARGET_SECONDS = 15 +DISCOVERY_TIMEOUT_MAX_SECONDS = 300 +DISCOVERY_DIAGNOSTIC_MAX_CHARS = 1200 + + +def assert_never(value: FleetStatus) -> NoReturn: + raise AssertionError(f"unexpected fleet status: {value}") + + +@dataclass(frozen=True, slots=True) +class DiscoveryFailure(Exception): + reason: str + + def __str__(self) -> str: + return self.reason + + +@dataclass(frozen=True, slots=True) +class DiscoveryPaths: + inventory: Path + evidence: Path + + +@dataclass(frozen=True, slots=True) +class DiscoveryResult: + resolution: FleetResolution + + +def canonical_paths(out_dir: Path) -> tuple[Path, Path, Path]: + return ( + out_dir / "inventory.yml", + out_dir / "values-basic-example.yaml", + out_dir / "gpu-access-resolution.json", + ) + + +def discover_gpu_policy(spec: dict, out_dir: Path) -> DiscoveryResult: + targets = live_targets(spec) + paths = stage_private_discovery(spec, out_dir) + run_discovery(paths, len(targets)) + try: + evidence = parse_fleet_evidence(read_regular_file(paths.evidence)) + except EvidenceParseError as error: + raise DiscoveryFailure("GPU discovery evidence is malformed") from error + resolution = resolve_fleet(targets, evidence) + match resolution.status: + case FleetStatus.BLOCKED: + raise DiscoveryFailure(f"GPU discovery is blocked: {resolution.reason}") + case FleetStatus.GPU_RESOLVED | FleetStatus.CPU_ONLY: + pass + case unreachable: + assert_never(unreachable) + return DiscoveryResult(resolution=resolution) + + +def live_targets(spec: dict) -> tuple[InventoryTarget, ...]: + names = [spec["server"]["name"]] + if spec["topology"] == "ssh-preinstalled": + names.extend(agent["name"] for agent in spec.get("agents", [])) + if len(names) != len(set(names)): + raise DiscoveryFailure("live target names must be unique") + return tuple(InventoryTarget(name=name) for name in names) + + +def stage_private_discovery(spec: dict, out_dir: Path) -> DiscoveryPaths: + resolved_out_dir = out_dir.resolve() + paths = DiscoveryPaths( + inventory=resolved_out_dir / ".gpu-access-discovery.inventory.yml", + evidence=resolved_out_dir / ".gpu-access-discovery-evidence.json", + ) + publish_artifacts( + [ + (paths.inventory, render_discovery_inventory(spec), 0o600, False), + (paths.evidence, "", 0o600, False), + ], + force=True, + ) + return paths + + +def render_discovery_inventory(spec: dict) -> str: + server = spec["server"] + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " agent:", + ] + if spec["topology"] == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines += [f" {agent['name']}:", f" ansible_host: {yaml_quote(agent['ip'])}"] + else: + lines.append(" hosts: {}") + lines += [" vars:", " ansible_port: 22", " ansible_user: root"] + return "\n".join(lines) + "\n" + + +def discovery_timeout_seconds(target_count: int) -> int: + configured = os.environ.get("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS") + if configured is not None: + try: + timeout = int(configured) + except ValueError as error: + raise DiscoveryFailure("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be an integer") from error + if not DISCOVERY_TIMEOUT_BASE_SECONDS <= timeout <= DISCOVERY_TIMEOUT_MAX_SECONDS: + raise DiscoveryFailure( + f"AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be between {DISCOVERY_TIMEOUT_BASE_SECONDS} and " + f"{DISCOVERY_TIMEOUT_MAX_SECONDS}" + ) + return timeout + return min( + DISCOVERY_TIMEOUT_MAX_SECONDS, + DISCOVERY_TIMEOUT_BASE_SECONDS + (DISCOVERY_TIMEOUT_PER_TARGET_SECONDS * target_count), + ) + + +def _bounded_diagnostic(*values: str | bytes | None) -> str: + text = "\n".join(value.decode(errors="replace") if isinstance(value, bytes) else value or "" for value in values) + text = re.sub(r"(?i)\b(token|password|secret|private[_-]?key)\s*[:=]\s*\S+", r"\1=<redacted>", text) + lines = [line.strip() for line in text.splitlines() if line.strip()] + summary = " | ".join(lines[-8:]) + return summary[-DISCOVERY_DIAGNOSTIC_MAX_CHARS:] or "no Ansible diagnostics" + + +def run_discovery(paths: DiscoveryPaths, target_count: int) -> None: + playbook = Path(__file__).resolve().parents[3] / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" + argv = [ + "ansible-playbook", + "-i", + str(paths.inventory), + str(playbook), + "-e", + f"gpu_access_discovery_output_path={paths.evidence}", + ] + environment = os.environ.copy() + environment["ANSIBLE_CONFIG"] = str(playbook.parents[1] / "ansible.cfg") + environment["ANSIBLE_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_ARGS"] = "-o StrictHostKeyChecking=yes" + for key in ( + "ANSIBLE_SSH_COMMON_ARGS", + "ANSIBLE_SSH_EXTRA_ARGS", + "ANSIBLE_SCP_IF_SSH", + "ANSIBLE_SCP_EXTRA_ARGS", + "ANSIBLE_SFTP_EXTRA_ARGS", + ): + environment.pop(key, None) + timeout = discovery_timeout_seconds(target_count) + try: + result = subprocess.run( + argv, + capture_output=True, + check=False, + cwd=playbook.parents[1], + env=environment, + text=True, + timeout=timeout, + ) + except FileNotFoundError as error: + raise DiscoveryFailure("ansible-playbook is required for GPU discovery") from error + except subprocess.TimeoutExpired as error: + diagnostic = _bounded_diagnostic(error.stderr, error.stdout) + raise DiscoveryFailure(f"GPU discovery playbook timed out after {timeout}s: {diagnostic}") from error + if result.returncode != 0: + diagnostic = _bounded_diagnostic(result.stderr, result.stdout) + raise DiscoveryFailure(f"GPU discovery playbook failed with exit code {result.returncode}: {diagnostic}") + + +def read_regular_file(path: Path) -> str: + try: + mode = os.lstat(path).st_mode + except FileNotFoundError as error: + raise DiscoveryFailure("GPU discovery evidence was not written") from error + if not stat.S_ISREG(mode): + raise DiscoveryFailure("GPU discovery evidence must be a regular file") + try: + return path.read_text(encoding="utf-8") + except OSError as error: + raise DiscoveryFailure("GPU discovery evidence could not be read") from error + + +def manifest_content(result: DiscoveryResult) -> str: + document = resolution_manifest(result.resolution) + return json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py new file mode 100644 index 00000000..92924077 --- /dev/null +++ b/tests/skills/test_gpu_artifact_generation.py @@ -0,0 +1,308 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for automatic GPU artifact generation.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def evidence_host(name: str, *, gpu: bool = False, gid: int = 993, reachable: bool = True) -> dict: + bdf = "0000:03:00.0" if gpu else "" + return { + "host": name, + "reachable": reachable, + "lspci": {"rc": 0, "stdout": bdf}, + "sysfs": {"rc": 0, "stdout": bdf}, + "render_group": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, + "groups": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, + "state": { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + }, + "rule": { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + }, + "legacy_rules": { + key: { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + } + for key in ("kfd", "amdgpu", "rocm_devices") + }, + } + + +def ssh_spec() -> dict: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "agents": [{"name": "agent", "ip": "192.168.1.11"}], + } + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> Path: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +arguments = sys.argv[1:] +Path(os.environ["FAKE_ANSIBLE_RECORD"]).write_text(json.dumps(arguments), encoding="utf-8") +environment_record = os.environ.get("FAKE_ANSIBLE_ENV_RECORD") +if environment_record: + Path(environment_record).write_text( + json.dumps({key: os.environ.get(key) for key in ("ANSIBLE_CONFIG", "ANSIBLE_HOST_KEY_CHECKING", "ANSIBLE_SSH_ARGS", "ANSIBLE_SSH_COMMON_ARGS", "ANSIBLE_SSH_EXTRA_ARGS", "ANSIBLE_SSH_HOST_KEY_CHECKING", "ANSIBLE_SCP_IF_SSH", "ANSIBLE_SCP_EXTRA_ARGS", "ANSIBLE_SFTP_EXTRA_ARGS")}), + encoding="utf-8", + ) +output = next(value.split("=", 1)[1] for value in arguments if value.startswith("gpu_access_discovery_output_path=")) +Path(output).write_text(os.environ["FAKE_ANSIBLE_EVIDENCE"], encoding="utf-8") +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + record = tmp_path / "ansible-argv.json" + monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(record)) + monkeypatch.setenv("FAKE_ANSIBLE_EVIDENCE", json.dumps(document)) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + return record + + +def test_generator_forces_repository_host_key_checking_over_disabled_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, monkeypatch, {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]} + ) + environment_record = tmp_path / "ansible-environment.json" + monkeypatch.setenv("FAKE_ANSIBLE_ENV_RECORD", str(environment_record)) + monkeypatch.setenv("ANSIBLE_CONFIG", str(tmp_path / "disabled-ansible.cfg")) + monkeypatch.setenv("ANSIBLE_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SSH_COMMON_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SSH_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SCP_IF_SSH", "True") + monkeypatch.setenv("ANSIBLE_SCP_EXTRA_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SFTP_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 0, result.stderr + assert json.loads(environment_record.read_text(encoding="utf-8")) == { + "ANSIBLE_CONFIG": str(ROOT / "deploy" / "ansible" / "ansible.cfg"), + "ANSIBLE_HOST_KEY_CHECKING": "True", + "ANSIBLE_SSH_ARGS": "-o StrictHostKeyChecking=yes", + "ANSIBLE_SSH_COMMON_ARGS": None, + "ANSIBLE_SSH_EXTRA_ARGS": None, + "ANSIBLE_SSH_HOST_KEY_CHECKING": "True", + "ANSIBLE_SCP_IF_SSH": None, + "ANSIBLE_SCP_EXTRA_ARGS": None, + "ANSIBLE_SFTP_EXTRA_ARGS": None, + } + + +def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + "#!/bin/sh\nprintf '%s\\n' 'fatal: [server]: UNREACHABLE! token=do-not-disclose' >&2\nexit 2\n", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 1 + assert "exit code 2" in result.stderr + assert "fatal: [server]: UNREACHABLE!" in result.stderr + assert "do-not-disclose" not in result.stderr + assert "token=<redacted>" in result.stderr + + +def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def write_json(path: Path, document: dict) -> Path: + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record = write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 2, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, + ) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: 993" in inventory + assert inventory.count("auplc_gpu_access_enabled: true") == 1 + assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": False, "server": True}, + } + discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" + discovery_evidence = out_dir / ".gpu-access-discovery-evidence.json" + assert discovery_inventory.stat().st_mode & 0o777 == 0o600 + assert discovery_evidence.stat().st_mode & 0o777 == 0o600 + assert json.loads(record.read_text(encoding="utf-8")) == [ + "-i", + str(discovery_inventory), + str(ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml"), + "-e", + f"gpu_access_discovery_output_path={discovery_evidence}", + ] + + +def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]}, + ) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: null" in inventory + assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "renderGid: null" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest == { + "version": 1, + "status": "cpu_only", + "render_gid": None, + "hosts": {"agent": False, "server": False}, + } + + +@pytest.mark.parametrize("failure", ["missing", "nonzero"]) +def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + if failure == "nonzero": + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", str(fake_bin)) + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 1 + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + + +@pytest.mark.parametrize( + "document", + [ + {"version": 2, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, + { + "version": 2, + "hosts": [evidence_host("server", gpu=True, gid=993), evidence_host("agent", gpu=True, gid=994)], + }, + ], + ids=["unknown", "gid-disagreement"], +) +def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict +) -> None: + write_fake_ansible(tmp_path, monkeypatch, document) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + out_dir.mkdir() + inventory = out_dir / "inventory.yml" + values = out_dir / "values-basic-example.yaml" + manifest = out_dir / "gpu-access-resolution.json" + inventory.write_text("previous inventory\n", encoding="utf-8") + values.write_text("previous values\n", encoding="utf-8") + manifest.write_text("previous manifest\n", encoding="utf-8") + + result = run_generator(spec_path, out_dir, "--force") + + assert result.returncode == 1 + assert inventory.read_text(encoding="utf-8") == "previous inventory\n" + assert values.read_text(encoding="utf-8") == "previous values\n" + assert manifest.read_text(encoding="utf-8") == "previous manifest\n" + + +@pytest.mark.parametrize( + "field", + ["render_gid", "gpu_access"], +) +def test_generator_rejects_removed_public_gpu_fields_before_running_discovery( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + record = write_fake_ansible(tmp_path, monkeypatch, {"version": 2, "hosts": []}) + spec = ssh_spec() + spec[field] = 993 if field == "render_gid" else {"hosts": []} + spec_path = write_json(tmp_path / "spec.json", spec) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 1 + assert f"spec.{field} is no longer accepted" in result.stderr + assert not record.exists() From 0f5a1cb77e32585872d0fb1b2390217975b42d49 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 054/180] feat(deploy): stage atomic PXE GPU finalization --- .../scripts/pxe_finalization.py | 173 ++++++ .../scripts/pxe_finalization_support.py | 257 +++++++++ tests/skills/test_pxe_finalization.py | 516 ++++++++++++++++++ 3 files changed, 946 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py create mode 100644 tests/skills/test_pxe_finalization.py diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py new file mode 100644 index 00000000..cb9c7c9f --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Orchestrate transactional PXE configuration finalization.""" + +from __future__ import annotations + +import json +import os +import secrets +from pathlib import Path + +import pxe_finalization_support as _support +from artifact_store import preflight_destinations, publish_artifacts +from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values +from gpu_access_resolution import ( + FleetResolution, + FleetStatus, + HostStatus, + resolution_manifest, +) +from gpu_access_resolution import HostResolution as HostResolution +from gpu_resolution_manifest import build_pxe_resolution_manifest +from pxe_finalization_support import MAX_RENDER_GID as MAX_RENDER_GID +from pxe_finalization_support import ( + VERSION, + Artifact, + JsonDocument, +) +from pxe_finalization_support import FinalizationError as FinalizationError +from pxe_finalization_support import PxePaths as PxePaths +from pxe_finalization_support import paths as paths + +_artifact_attestations = _support.artifact_attestations +_completion = _support.completion +_controller_resolution = _support.controller_resolution +_exclusive_lock = _support.exclusive_lock +_final_resolution = _support.final_resolution +_generation_paths = _support.generation_paths +_read_artifact_attestation = _support.read_artifact_attestation +_read_document = _support.read_document +_spec_sha256 = _support.spec_sha256 +_target = _support.target +_valid_gid = _support.valid_gid +_validate = _support.validate +_verify_canonical_artifacts = _support.verify_canonical_artifacts + + +def stage_pending(spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool) -> PxePaths: + pending = paths(out_dir) + if controller.status is FleetStatus.BLOCKED: + raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + context: JsonDocument = { + "version": VERSION, + "generation": secrets.token_urlsafe(32), + "spec_sha256": _spec_sha256(spec), + "topology": "pxe-diskless", + "spec": spec, + "token": token, + "controller": resolution_manifest(controller), + } + bootstrap = render_pxe_vars(spec, _controller_policy(controller, True), str(pending.context)) + bootstrap += "\n".join( + [ + f"pxe_finalizer_handoff: {_yaml_quote(str(pending.handoff))}", + f"pxe_finalizer_generation: {_yaml_quote(context['generation'])}", + f"pxe_finalizer_spec_sha256: {_yaml_quote(context['spec_sha256'])}", + f"pxe_finalizer_script: {_yaml_quote(str(Path(__file__).with_name('gen_configs.py').resolve()))}", + "", + ] + ) + artifacts: list[Artifact] = [ + (pending.bootstrap_inventory, _render_bootstrap_inventory(spec), 0o600, True), + (pending.bootstrap_vars, bootstrap, 0o600, True), + (pending.context, json.dumps(context, sort_keys=True) + "\n", 0o600, True), + ] + if not force: + preflight_destinations(_generation_paths(pending), False) + publish_artifacts(artifacts, force, _generation_paths(pending)) + return pending + + +def publish_disabled_rootfs( + spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool +) -> None: + pending = paths(out_dir) + if controller.status is FleetStatus.BLOCKED: + raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + policy = _controller_policy(controller, False) + artifacts: list[Artifact] = [ + (pending.inventory, render_inventory(spec, token, controller), 0o600, True), + (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), + (pending.values, render_values(spec, controller), 0o644, False), + (pending.manifest, _manifest(controller, False, None), 0o644, False), + ] + if not force: + preflight_destinations(_generation_paths(pending), False) + publish_artifacts(artifacts, force, _generation_paths(pending)) + + +def finalize(out_dir: Path, context_path: Path, handoff_path: Path) -> None: + pending = paths(out_dir) + if context_path.resolve() != pending.context or handoff_path.resolve() != pending.handoff: + raise FinalizationError("PXE finalizer context and handoff paths must be the generated private paths") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + context = _read_document(pending.context, "PXE finalizer context") + handoff = _read_document(pending.handoff, "PXE finalizer handoff") + spec, controller, rootfs_gid = _validate(context, handoff) + resolution = _final_resolution(controller, rootfs_gid) + policy = _controller_policy(resolution, True) + artifacts: list[Artifact] = [ + (pending.inventory, render_inventory(spec, context["token"], resolution), 0o600, True), + (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), + (pending.values, render_values(spec, resolution), 0o644, False), + (pending.manifest, _manifest(resolution, True, rootfs_gid), 0o644, False), + ] + completion = _completion(context, handoff, _artifact_attestations(artifacts)) + if os.path.lexists(pending.completion): + if _read_document(pending.completion, "PXE finalizer completion") != completion: + raise FinalizationError("PXE finalizer completion does not match the supplied handoff") + _verify_canonical_artifacts(pending, completion["artifacts"]) + return + published: list[Artifact] = [ + *artifacts, + (pending.completion, json.dumps(completion, sort_keys=True) + "\n", 0o600, True), + ] + preflight_destinations([path for path, _, _, _ in published], False) + publish_artifacts(published, False) + + +def _controller_policy(resolution: FleetResolution, rootfs_enabled: bool) -> ResolvedGpuPolicy: + return ResolvedGpuPolicy( + host_gpu_enabled={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, + render_gid=resolution.render_gid, + pxe_gpu_enabled=rootfs_enabled, + ) + + +def _manifest(resolution: FleetResolution, rootfs_enabled: bool, rootfs_gid: int | None) -> str: + base = resolution_manifest(resolution) + document = build_pxe_resolution_manifest( + version=base["version"], + status=base["status"], + render_gid=base["render_gid"], + hosts=base["hosts"], + gpu_access_enabled=rootfs_enabled, + pxe_render_gid=rootfs_gid, + ) + return json.dumps(document, indent=2, sort_keys=True) + "\n" + + +def _render_bootstrap_inventory(spec: JsonDocument) -> str: + server = spec["server"] + return "\n".join( + [ + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + "", + ] + ) + + +def _yaml_quote(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py new file mode 100644 index 00000000..e94923f7 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Typed security and verification support for PXE finalization.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TypeAlias, TypedDict + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_access_resolution import FleetResolution, FleetStatus, HostResolution, HostStatus, InventoryTarget + +VERSION: Final = 1 +MAX_RENDER_GID: Final = 4_294_967_294 + +JsonScalar: TypeAlias = str | int | float | bool | None +JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] +JsonDocument: TypeAlias = dict[str, JsonValue] +Artifact: TypeAlias = tuple[Path, str, int, bool] + + +class ArtifactAttestation(TypedDict): + sha256: str + mode: int + owner_uid: int + + +ArtifactAttestations: TypeAlias = dict[str, ArtifactAttestation] + + +@dataclass(frozen=True, slots=True) +class FinalizationError(Exception): + reason: str + + def __str__(self) -> str: + return self.reason + + +@dataclass(frozen=True, slots=True) +class PxePaths: + bootstrap_inventory: Path + bootstrap_vars: Path + context: Path + handoff: Path + completion: Path + lock: Path + inventory: Path + pxe_vars: Path + values: Path + manifest: Path + + +def paths(out_dir: Path) -> PxePaths: + root = out_dir.resolve() + return PxePaths( + bootstrap_inventory=root / ".pxe-bootstrap.inventory.yml", + bootstrap_vars=root / ".pxe-bootstrap.vars.yml", + context=root / ".pxe-finalizer-context.json", + handoff=root / ".pxe-finalizer-handoff.json", + completion=root / ".pxe-finalizer-completion.json", + lock=root / ".pxe-finalizer.lock", + inventory=root / "inventory.yml", + pxe_vars=root / "pb-pxe-controller.vars.yml", + values=root / "values-basic-example.yaml", + manifest=root / "gpu-access-resolution.json", + ) + + +def spec_sha256(spec: JsonDocument) -> str: + encoded = json.dumps(spec, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def valid_gid(value: JsonValue) -> bool: + return type(value) is int and 1 <= value <= MAX_RENDER_GID + + +def read_document(path: Path, label: str) -> JsonDocument: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) + with os.fdopen(descriptor, encoding="utf-8") as source: + mode = os.fstat(source.fileno()).st_mode + if not stat.S_ISREG(mode): + raise FinalizationError(f"{label} must be a regular file") + document = strict_json_loads(source.read()) + except FinalizationError: + raise + except (DuplicateJsonKeyError, FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: + raise FinalizationError(f"{label} cannot be read") from error + if type(document) is not dict: + raise FinalizationError(f"{label} must be a JSON object") + return document + + +def validate(context: JsonDocument, handoff: JsonDocument) -> tuple[JsonDocument, FleetResolution, int]: + required_context = {"version", "generation", "spec_sha256", "topology", "spec", "token", "controller"} + required_handoff = {"version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid"} + if set(context) != required_context or set(handoff) != required_handoff: + raise FinalizationError("PXE finalizer context or handoff has an unexpected schema") + if type(context["version"]) is not int or type(handoff["version"]) is not int: + raise FinalizationError("PXE finalizer context or handoff version is invalid") + if context["version"] != VERSION or handoff["version"] != VERSION: + raise FinalizationError("PXE finalizer context or handoff version is unsupported") + if context["topology"] != "pxe-diskless" or handoff["topology"] != "pxe-diskless": + raise FinalizationError("PXE finalizer topology is invalid") + generation = context["generation"] + if type(generation) is not str or not generation or handoff["generation"] != generation: + raise FinalizationError("PXE finalizer generation does not match") + spec = context["spec"] + if type(spec) is not dict or spec_sha256(spec) != context["spec_sha256"]: + raise FinalizationError("PXE finalizer context spec does not match its digest") + if handoff["spec_sha256"] != context["spec_sha256"] or spec.get("topology") != "pxe-diskless": + raise FinalizationError("PXE finalizer handoff does not match its pending spec") + if "render_gid" in spec or "gpu_access" in spec: + raise FinalizationError("PXE finalizer context contains removed public GPU policy fields") + if type(context["token"]) is not str or not context["token"]: + raise FinalizationError("PXE finalizer context token is invalid") + pxe = spec.get("pxe") + if type(pxe) is not dict or pxe.get("diskless_agents_have_amd_gpus") is not True: + raise FinalizationError("PXE finalizer context is not for GPU-enabled diskless agents") + rootfs_gid = handoff["render_gid"] + if handoff["pxe_gpu_access_enabled"] is not True or not valid_gid(rootfs_gid): + raise FinalizationError("PXE finalizer handoff has no valid resolved rootfs GID") + controller = controller_resolution(spec, context["controller"]) + if controller.render_gid is not None and controller.render_gid != rootfs_gid: + raise FinalizationError("PXE rootfs render GID disagrees with the GPU-enabled controller") + return spec, controller, rootfs_gid + + +def controller_resolution(spec: JsonDocument, raw: JsonValue) -> FleetResolution: + if type(raw) is not dict or set(raw) != {"version", "status", "render_gid", "hosts"}: + raise FinalizationError("PXE finalizer context controller evidence is invalid") + server = spec.get("server") + name = server.get("name") if type(server) is dict else None + hosts = raw["hosts"] + if type(name) is not str or type(hosts) is not dict or set(hosts) != {name} or type(hosts[name]) is not bool: + raise FinalizationError("PXE finalizer context controller host is invalid") + enabled = hosts[name] + gid = raw["render_gid"] + if enabled and not valid_gid(gid): + raise FinalizationError("PXE finalizer context controller GID is invalid") + if not enabled and gid is not None: + raise FinalizationError("CPU-only PXE controller must not publish a render GID") + status = HostStatus.GPU if enabled else HostStatus.CPU + fleet_status = FleetStatus.GPU_RESOLVED if enabled else FleetStatus.CPU_ONLY + if type(raw["version"]) is not int or raw["version"] != VERSION or raw["status"] != fleet_status.value: + raise FinalizationError("PXE finalizer context controller status is invalid") + host = HostResolution(target=target(name), status=status, render_gid=gid, reason=None) + return FleetResolution(fleet_status, (host,), gid, None) + + +def target(name: str) -> InventoryTarget: + return InventoryTarget(name=name) + + +def final_resolution(controller: FleetResolution, rootfs_gid: int) -> FleetResolution: + return FleetResolution(FleetStatus.GPU_RESOLVED, controller.hosts, rootfs_gid, None) + + +def generation_paths(pending: PxePaths) -> tuple[Path, ...]: + return ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + pending.inventory, + pending.pxe_vars, + pending.values, + pending.manifest, + ) + + +def artifact_attestations(artifacts: list[Artifact]) -> ArtifactAttestations: + return { + path.name: {"sha256": hashlib.sha256(content.encode()).hexdigest(), "mode": mode, "owner_uid": os.geteuid()} + for path, content, mode, _ in artifacts + } + + +def verify_canonical_artifacts(pending: PxePaths, expected: JsonValue) -> None: + canonical = (pending.inventory, pending.pxe_vars, pending.values, pending.manifest) + if type(expected) is not dict or set(expected) != {path.name for path in canonical}: + raise FinalizationError("PXE finalizer completion artifacts are invalid") + for path in canonical: + attestation = expected[path.name] + if ( + type(attestation) is not dict + or set(attestation) != {"sha256", "mode", "owner_uid"} + or type(attestation["sha256"]) is not str + or type(attestation["mode"]) is not int + or type(attestation["owner_uid"]) is not int + or read_artifact_attestation(path) != attestation + ): + raise FinalizationError(f"PXE finalizer canonical artifact is missing or corrupted: {path.name}") + + +def read_artifact_attestation(path: Path) -> ArtifactAttestation: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as source: + artifact_stat = os.fstat(source.fileno()) + if not stat.S_ISREG(artifact_stat.st_mode): + raise FinalizationError("PXE finalizer canonical artifact must be a regular file") + digest = hashlib.sha256() + while chunk := source.read(65_536): + digest.update(chunk) + return { + "sha256": digest.hexdigest(), + "mode": stat.S_IMODE(artifact_stat.st_mode), + "owner_uid": artifact_stat.st_uid, + } + except FinalizationError: + raise + except (FileNotFoundError, OSError) as error: + raise FinalizationError("PXE finalizer canonical artifact cannot be read") from error + + +def completion(context: JsonDocument, handoff: JsonDocument, artifacts: ArtifactAttestations) -> JsonDocument: + return { + **{ + key: handoff[key] + for key in ("version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid") + }, + "artifacts": artifacts, + } + + +@contextmanager +def exclusive_lock(path: Path) -> Iterator[None]: + descriptor = -1 + locked = False + try: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600) + lock_stat = os.fstat(descriptor) + if not stat.S_ISREG(lock_stat.st_mode): + raise FinalizationError("PXE finalizer lock must be a regular file") + if lock_stat.st_uid != os.geteuid() or stat.S_IMODE(lock_stat.st_mode) != 0o600: + raise FinalizationError("PXE finalizer lock has unsafe owner or mode") + fcntl.flock(descriptor, fcntl.LOCK_EX) + locked = True + yield + except OSError as error: + raise FinalizationError("PXE finalizer lock cannot be opened") from error + finally: + if locked: + fcntl.flock(descriptor, fcntl.LOCK_UN) + if descriptor >= 0: + os.close(descriptor) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py new file mode 100644 index 00000000..effccb02 --- /dev/null +++ b/tests/skills/test_pxe_finalization.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import FrozenInstanceError +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" +PXE_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-pxe-controller.yml" + + +def pxe_spec(gpu_agents: bool) -> dict: + return { + "topology": "pxe-diskless", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "controller", "ip": "192.168.1.10"}, + "agents": [{"name": "diskless-agent", "ip": "192.168.1.11"}], + "network": {"interface": "enp1s0", "subnet": "192.168.1.0/24"}, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA test@example"], + "rootfs_password": "do-not-print-this-secret", + "diskless_agents_have_amd_gpus": gpu_agents, + }, + } + + +def write_json(path: Path, document: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, controller_gpu: bool = False) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + bdf = "0000:03:00.0" if controller_gpu else "" + fake_ansible.write_text( + f"""#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({{ + 'version': 2, + 'hosts': [{{ + 'host': 'controller', 'reachable': True, + 'lspci': {{'rc': 0, 'stdout': {bdf!r}}}, + 'sysfs': {{'rc': 0, 'stdout': {bdf!r}}}, + 'render_group': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, + 'groups': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, + 'state': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, + 'rule': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, + 'legacy_rules': {{key: {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}} for key in ('kfd', 'amdgpu', 'rocm_devices')}}, + }}], +}}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + +def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), *arguments], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def load_finalizer_module(): + scripts = GEN_CONFIGS.parent + sys.path.insert(0, str(scripts)) + try: + spec = spec_from_file_location("test_pxe_finalizer", scripts / "pxe_finalization.py") + assert spec is not None and spec.loader is not None + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def pending_handoff(out_dir: Path, *, render_gid: int = 995) -> tuple[Path, Path]: + context_path = out_dir / ".pxe-finalizer-context.json" + context = json.loads(context_path.read_text(encoding="utf-8")) + handoff_path = out_dir / ".pxe-finalizer-handoff.json" + write_json( + handoff_path, + { + "version": 1, + "generation": context["generation"], + "spec_sha256": context["spec_sha256"], + "topology": "pxe-diskless", + "pxe_gpu_access_enabled": True, + "render_gid": render_gid, + }, + ) + return context_path, handoff_path + + +def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: + return ( + out_dir / "inventory.yml", + out_dir / "pb-pxe-controller.vars.yml", + out_dir / "values-basic-example.yaml", + out_dir / "gpu-access-resolution.json", + out_dir / ".pxe-finalizer-completion.json", + ) + + +def cpu_controller(finalizer): + return finalizer.FleetResolution( + finalizer.FleetStatus.CPU_ONLY, + (finalizer.HostResolution(finalizer._target("controller"), finalizer.HostStatus.CPU, None, None),), + None, + None, + ) + + +def test_pxe_finalizer_preserves_moved_imports_as_immutable_support_types(tmp_path: Path) -> None: + finalizer = load_finalizer_module() + support = sys.modules["pxe_finalization_support"] + + assert finalizer.FinalizationError is support.FinalizationError + assert finalizer.PxePaths is support.PxePaths + assert finalizer.paths is support.paths + assert finalizer.VERSION == support.VERSION == 1 + assert finalizer.MAX_RENDER_GID == support.MAX_RENDER_GID == 4_294_967_294 + assert finalizer._read_document is support.read_document + assert finalizer._generation_paths is support.generation_paths + assert finalizer._artifact_attestations is support.artifact_attestations + assert finalizer._completion is support.completion + assert finalizer._verify_canonical_artifacts is support.verify_canonical_artifacts + assert finalizer._exclusive_lock is support.exclusive_lock + + error = finalizer.FinalizationError("immutable") + pending = finalizer.paths(tmp_path) + with pytest.raises(FrozenInstanceError): + error.reason = "changed" + with pytest.raises(FrozenInstanceError): + pending.context = tmp_path / "changed.json" + + +def test_pxe_gpu_agents_stage_only_private_bootstrap_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stderr + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + assert "pxe_controller:" in (out_dir / ".pxe-bootstrap.inventory.yml").read_text(encoding="utf-8") + bootstrap = (out_dir / ".pxe-bootstrap.vars.yml").read_text(encoding="utf-8") + assert "pxe_gpu_access_enabled: true" in bootstrap + assert "pxe_finalizer_context:" in bootstrap + assert (out_dir / ".pxe-finalizer-context.json").stat().st_mode & 0o777 == 0o600 + + +def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(False)) + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stderr + assert "pxe_gpu_access_enabled: false" in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: null" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False, "render_gid": None} + + +def test_pxe_disabled_rootfs_force_replaces_private_generation_state_under_the_generation_lock(tmp_path: Path) -> None: + finalizer = load_finalizer_module() + out_dir = tmp_path / "generated" + pending = finalizer.paths(out_dir) + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("stale\n", encoding="utf-8") + + finalizer.publish_disabled_rootfs(pxe_spec(False), "token", cpu_controller(finalizer), out_dir, True) + + assert all( + not path.exists() + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ) + ) + assert all(path.exists() for path in canonical_artifacts(out_dir)[:-1]) + + +def test_pxe_disabled_rootfs_force_restores_private_and_canonical_generation_when_publication_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + out_dir = tmp_path / "generated" + pending = finalizer.paths(out_dir) + finalizer.publish_disabled_rootfs(pxe_spec(False), "old-token", cpu_controller(finalizer), out_dir, False) + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ): + path.write_text(f"old {path.name}\n", encoding="utf-8") + tracked = ( + *canonical_artifacts(out_dir)[:-1], + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ) + before = {path.name: path.read_bytes() for path in tracked} + original_replace = artifact_store.os.replace + + def fail_values_replace(source, destination): + if Path(destination) == pending.values and ".backup." not in str(source): + raise OSError("injected disabled-rootfs publication failure") + return original_replace(source, destination) + + monkeypatch.setattr(artifact_store.os, "replace", fail_values_replace) + with pytest.raises(SystemExit): + finalizer.publish_disabled_rootfs(pxe_spec(False), "new-token", cpu_controller(finalizer), out_dir, True) + + assert {path.name: path.read_bytes() for path in tracked} == before + + +def test_pxe_finalizer_publishes_resolved_policy_idempotently_without_secret_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + pending = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + context, handoff = pending_handoff(out_dir) + + first = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + second = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert pending.returncode == 0, pending.stderr + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert "do-not-print-this-secret" not in first.stdout + first.stderr + second.stdout + second.stderr + assert "auplc_render_gid: 995" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_gpu_access_enabled: false" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "renderGid: 995" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 995} + completion = json.loads((out_dir / ".pxe-finalizer-completion.json").read_text(encoding="utf-8")) + assert completion["artifacts"]["inventory.yml"]["mode"] == 0o600 + assert completion["artifacts"]["inventory.yml"]["owner_uid"] == os.geteuid() + + +def test_pxe_finalizer_retry_rejects_canonical_mode_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + (out_dir / "inventory.yml").chmod(0o644) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + + +def test_pxe_pending_generation_rejects_existing_private_or_canonical_state_without_force( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite" in result.stderr + + +def test_pxe_forced_pending_generation_hides_prior_public_and_private_generation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + old_generation = json.loads(context.read_text(encoding="utf-8"))["generation"] + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + + assert result.returncode == 0, result.stderr + assert json.loads(context.read_text(encoding="utf-8"))["generation"] != old_generation + assert not handoff.exists() + assert all(not path.exists() for path in canonical_artifacts(out_dir)) + + +def test_pxe_forced_pending_generation_restores_prior_generation_if_staging_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + previous = {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + original_replace = artifact_store.os.replace + + def fail_new_bootstrap(source, destination): + if Path(destination) == out_dir / ".pxe-bootstrap.inventory.yml" and ".backup." not in str(source): + raise OSError("injected staging failure") + return original_replace(source, destination) + + monkeypatch.setattr(artifact_store.os, "replace", fail_new_bootstrap) + controller = finalizer._controller_resolution( + pxe_spec(True), json.loads(context.read_text(encoding="utf-8"))["controller"] + ) + with pytest.raises(SystemExit): + finalizer.stage_pending(pxe_spec(True), "replacement-token", controller, out_dir, True) + + assert {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} == previous + + +@pytest.mark.parametrize("document_name", (".pxe-finalizer-context.json", ".pxe-finalizer-handoff.json")) +def test_pxe_finalizer_rejects_duplicate_keys_in_private_documents( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document_name: str +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + document_path = out_dir / document_name + document_path.write_text( + '{"generation":"duplicate",' + document_path.read_text(encoding="utf-8")[1:], encoding="utf-8" + ) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert all(not path.exists() for path in canonical_artifacts(out_dir)) + + +@pytest.mark.parametrize("mutation", ("missing", "tampered")) +def test_pxe_finalizer_retry_rejects_missing_or_tampered_canonical_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + inventory = out_dir / "inventory.yml" + if mutation == "missing": + inventory.unlink() + else: + inventory.write_text("tampered\n", encoding="utf-8") + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + + +def test_pxe_finalizer_rejects_symlink_lock_without_touching_its_target( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + target = tmp_path / "lock-target" + target.write_text("unchanged\n", encoding="utf-8") + target.chmod(0o644) + lock = out_dir / ".pxe-finalizer.lock" + lock.unlink() + lock.symlink_to(target) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert target.read_text(encoding="utf-8") == "unchanged\n" + assert target.stat().st_mode & 0o777 == 0o644 + + +@pytest.mark.parametrize( + ("field", "value"), + [("generation", "stale"), ("topology", "ssh-preinstalled"), ("render_gid", None), ("version", True)], +) +def test_pxe_finalizer_rejects_invalid_handoffs_without_publishing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: str | int | None +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + document = json.loads(handoff.read_text(encoding="utf-8")) + document[field] = value + write_json(handoff, document) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + + +def test_pxe_finalizer_rolls_back_if_late_canonical_publication_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + original_link = artifact_store.os.link + + def fail_values_link(source, destination): + if Path(destination).name == "values-basic-example.yaml": + raise OSError("injected publication failure") + return original_link(source, destination) + + monkeypatch.setattr(artifact_store.os, "link", fail_values_link) + with pytest.raises(SystemExit): + finalizer.finalize(out_dir, context, handoff) + + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "pb-pxe-controller.vars.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + assert not (out_dir / ".pxe-finalizer-completion.json").exists() + + +def test_pxe_playbook_writes_and_finalizes_private_rootfs_handoff_locally() -> None: + playbook = PXE_PLAYBOOK.read_text(encoding="utf-8") + + assert "pxe_finalizer_handoff" in playbook + assert "pxe_finalizer_context" in playbook + assert "--finalize-pxe" in playbook + assert "delegate_to: localhost" in playbook + assert "run_once: true" in playbook + assert "become: false" in playbook + assert "argv:" in playbook From 27f36f1572a08205933565307515165efb1ac117 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 055/180] feat(deploy): generate and validate GPU artifacts --- .../scripts/gen_configs.py | 381 ++-------- .../scripts/gpu_resolution_parsing.py | 246 +++++++ .../scripts/gpu_resolution_validation.py | 125 ++++ .../scripts/validate.py | 198 ++--- .../scripts/values_resolution_parsing.py | 130 ++++ .../skills/test_config_generation_security.py | 95 +++ tests/skills/test_deploy_scripts.py | 692 +++++++++++++++++- 7 files changed, 1379 insertions(+), 488 deletions(-) create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py create mode 100644 tests/skills/test_config_generation_security.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index e0d91ee8..3a3bdb9f 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -2,8 +2,9 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. """Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. -Given a JSON cluster-spec (see ``--print-schema``), write the three files the -deploy skill needs, keeping them mutually consistent: +Given a JSON cluster-spec (see ``--print-schema``), discover the managed hosts' +GPU policy. SSH and PXE without GPU-enabled diskless agents immediately write +mutually consistent canonical deployment artifacts: 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the @@ -11,17 +12,26 @@ 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to pb-pxe-controller.yml with ``-e @<absolute-path>``. - 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, - storage class, proxy NodePort, authMode. + 3. ``values-basic-example.yaml`` -- Helm overlay: resolved render GID, + storage, proxy, and authentication. + 4. ``gpu-access-resolution.json`` -- Machine-readable resolved host policy. + +GPU-enabled PXE instead writes private ``.pxe-bootstrap.inventory.yml``, +``.pxe-bootstrap.vars.yml``, and ``.pxe-finalizer-context.json`` files while +canonical artifacts remain absent. The controller playbook publishes the +canonical artifacts only after it resolves the rootfs GID and succeeds. Design choices (deliberate): * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this runs on a bare operator machine. YAML is emitted from templates, not a serialiser -- the output is small, fixed-shape, and carries the copyright header. - * The k3s token is generated locally with ``secrets`` (CSPRNG) and written - ONLY into inventory.yml. It is never printed to stdout/stderr. Pass - ``--token-file`` to reuse an existing token instead of minting one. + * The k3s token is generated locally with ``secrets`` (CSPRNG). Immediate + canonical output writes it only into ``inventory.yml``. Pending GPU-enabled + PXE stores it only in private ``.pxe-finalizer-context.json`` until the + controller succeeds and finalization writes ``inventory.yml``. It is never + printed to stdout/stderr. Pass ``--token-file`` to reuse an existing token + instead of minting one. * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never be newer than the server (k3s refuses that). * Existing files are not overwritten unless ``--force`` is given. @@ -39,58 +49,22 @@ import argparse import base64 import json -import os import secrets -import shutil import sys -import tempfile -from contextlib import suppress from pathlib import Path -HEADER_HASH = ( - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" - "# Generated by auplc-skills gen_configs.py -- review before use.\n" +from artifact_store import preflight_destinations, publish_artifacts +from config_common import DuplicateJsonKeyError, strict_json_loads +from config_generation import ( + SCHEMA, + die, + render_inventory, + render_values, + validate_spec, + validate_yaml_scalar, ) - -# Default GPU product-name labels, keyed by the accelerator key used in -# runtime/values.yaml (custom.accelerators.<key>). Verified against the chart's -# values.yaml; override per fleet via spec["accelerators"][key]["product_name"]. -DEFAULT_ACCEL_LABELS = { - "phx": "AMD_Radeon_780M_Graphics", - "strix": "AMD_Radeon_890M_Graphics", - "strix-halo": "AMD_Radeon_8060S_Graphics", - "9070xt": "AMD_Radeon_RX_9070_XT", - "r9700": "AMD_Radeon_AI_PRO_R9700", - "9600gre": "AMD_Radeon_RX_9600_GRE", -} - -SCHEMA = { - "topology": "pxe-diskless | ssh-preinstalled", - "k3s_version": "v1.32.3+k3s1", - "server": {"name": "aipc1", "ip": "192.168.0.140"}, - "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], - "network": { - "interface": "enp1s0", - "subnet": "192.168.0.0/24", - "gateway": "192.168.0.1", - "dns_servers": "8.8.8.8,8.8.4.4", - }, - "pxe": { - "authorized_keys": ["ssh-ed25519 AAAA... you@host"], - "rootfs_password": "", - "web_port": 8080, - }, - "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, - "storage": {"class": "nfs-client"}, - "proxy": {"node_port": 30890}, - "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, -} - - -def die(msg: str, code: int = 1) -> None: - print(f"gen_configs: {msg}", file=sys.stderr) - raise SystemExit(code) +from gpu_artifact_generation import DiscoveryFailure, canonical_paths, discover_gpu_policy, manifest_content +from pxe_finalization import FinalizationError, finalize, publish_disabled_rootfs, stage_pending def gen_token() -> str: @@ -98,250 +72,6 @@ def gen_token() -> str: return base64.b64encode(secrets.token_bytes(64)).decode("ascii") -def require(spec: dict, path: str): - cur = spec - for part in path.split("."): - if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): - die(f"spec is missing required field '{path}'") - cur = cur[part] - return cur - - -def yaml_quote(s: str) -> str: - return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' - - -def validate_accelerators(spec: dict) -> None: - if "accelerators" not in spec: - return - accelerators = spec["accelerators"] - if not isinstance(accelerators, dict): - die("spec.accelerators must be a mapping") - unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) - if len(unsupported) == 1: - die(f"unsupported accelerator key '{unsupported[0]}'") - if unsupported: - die(f"unsupported accelerator keys: {', '.join(unsupported)}") - for key, config in accelerators.items(): - if not isinstance(config, dict): - die(f"accelerators.{key} must be a mapping") - - -def validate_config_shapes(spec: dict) -> None: - if not isinstance(spec, dict): - die("spec must be a mapping") - validate_accelerators(spec) - for key in ("network", "pxe", "storage", "proxy", "images"): - if key in spec and not isinstance(spec[key], dict): - die(f"spec.{key} must be a mapping") - - -def render_inventory(spec: dict, token: str) -> str: - topo = spec["topology"] - server = spec["server"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "k3s_cluster:", - " children:", - " server:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " agent:", - ] - if topo == "ssh-preinstalled" and spec.get("agents"): - lines.append(" hosts:") - for a in spec["agents"]: - lines.append(f" {a['name']}:") - lines.append(f" ansible_host: {a['ip']}") - else: - # PXE diskless agents auto-join by netboot; do NOT list them here. - lines.append(" hosts: {}") - lines += [ - " vars:", - " ansible_port: 22", - " ansible_user: root", - f" k3s_version: {k3s_version}", - f" token: {yaml_quote(token)}", - " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", - ] - if topo == "pxe-diskless": - lines += [ - "", - "pxe_controller:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " vars:", - " ansible_port: 22", - " ansible_user: root", - ] - return "\n".join(lines) + "\n" - - -def render_pxe_vars(spec: dict) -> str: - net = require(spec, "network") - pxe = spec.get("pxe", {}) - keys = pxe.get("authorized_keys", []) - if not keys: - die("pxe.authorized_keys must contain at least one SSH public key") - server_ip = spec["server"]["ip"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "# Pass this file to pb-pxe-controller.yml with", - "# ansible-playbook ... -e @<absolute-path-to-this-file>", - "# pxe_k3s_version is pinned to k3s_version so agents are never newer", - "# than the server.", - "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", - f"pxe_network_interface: {yaml_quote(net['interface'])}", - f"pxe_subnet: {yaml_quote(net['subnet'])}", - f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", - f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", - f"pxe_controller_ip: {yaml_quote(server_ip)}", - "pxe_k3s_server_ips:", - f" - {yaml_quote(server_ip)}", - f"pxe_k3s_version: {yaml_quote(k3s_version)}", - f"pxe_web_port: {int(pxe.get('web_port', 8080))}", - f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", - "pxe_rootfs_authorized_keys:", - ] - for k in keys: - lines.append(f" - {yaml_quote(k)}") - return "\n".join(lines) + "\n" - - -def render_values(spec: dict) -> str: - accel = spec.get("accelerators") or {} - storage_class = (spec.get("storage") or {}).get("class", "nfs-client") - node_port = (spec.get("proxy") or {}).get("node_port", 30890) - auth_mode = spec.get("auth_mode", "auto-login") - images = spec.get("images") or {} - - lines = [ - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", - "# Helm overlay generated by auplc-skills gen_configs.py.", - "# Layer this on top of runtime/values.yaml:", - "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", - "# --create-namespace -f runtime/values.yaml -f <this file>", - "custom:", - f" authMode: {yaml_quote(auth_mode)}", - ] - if accel: - lines.append(" accelerators:") - for key, cfg in accel.items(): - product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) - if not product: - die( - f"accelerator '{key}' has no product_name and no known default; " - "add accelerators.<key>.product_name from `kubectl describe node`" - ) - lines += [ - f" {key}:", - " nodeSelector:", - f" amd.com/gpu.product-name: {yaml_quote(product)}", - ] - if accel or images: - lines.append(" resources:") - if accel: - lines += [" metadata:", " gpu:", " acceleratorKeys:"] - lines.extend(f" - {yaml_quote(key)}" for key in accel) - if images: - lines.append(" images:") - for k, v in images.items(): - lines.append(f" {k}: {yaml_quote(v)}") - lines += [ - "hub:", - " db:", - " pvc:", - f" storageClassName: {yaml_quote(storage_class)}", - "singleuser:", - " storage:", - " dynamic:", - f" storageClass: {yaml_quote(storage_class)}", - "proxy:", - " service:", - " type: NodePort", - " nodePorts:", - f" http: {int(node_port)}", - ] - return "\n".join(lines) + "\n" - - -def preflight_destinations(paths: list[Path], force: bool) -> None: - if force: - return - for path in paths: - if os.path.lexists(path): - die(f"refusing to overwrite existing {path} (use --force)", 1) - - -def stage_file(path: Path, content: str, mode: int) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - try: - os.fchmod(fd, mode) - with os.fdopen(fd, "w", encoding="utf-8") as staged_file: - staged_file.write(content) - staged_file.flush() - os.fsync(staged_file.fileno()) - except OSError: - with suppress(OSError): - os.close(fd) - Path(staged_path).unlink(missing_ok=True) - raise - return Path(staged_path) - - -def remove_destination(path: Path) -> None: - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink(missing_ok=True) - - -def backup_destination(path: Path) -> tuple[Path, Path]: - backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) - backup_path = backup_dir / path.name - os.replace(path, backup_path) - return backup_dir, backup_path - - -def publish_artifacts(artifacts: list[tuple[Path, str, int, bool]], force: bool) -> None: - staged: list[tuple[Path, Path, bool]] = [] - published: list[Path] = [] - backups: list[tuple[Path, Path, Path]] = [] - try: - for path, content, mode, secret in artifacts: - staged.append((path, stage_file(path, content, mode), secret)) - for path, staged_path, secret in staged: - if force and os.path.lexists(path): - backup_dir, backup_path = backup_destination(path) - backups.append((path, backup_dir, backup_path)) - if force: - os.replace(staged_path, path) - else: - os.link(staged_path, path) - os.unlink(staged_path) - published.append(path) - print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) - except OSError as exc: - for path in reversed(published): - remove_destination(path) - for path, backup_dir, backup_path in reversed(backups): - remove_destination(path) - os.replace(backup_path, path) - backup_dir.rmdir() - die(f"could not publish generated artifacts: {exc}") - else: - for _, backup_dir, _ in backups: - shutil.rmtree(backup_dir) - finally: - for _, staged_path, _ in staged: - staged_path.unlink(missing_ok=True) - - def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") @@ -349,44 +79,61 @@ def main(argv=None) -> int: ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") ap.add_argument("--force", action="store_true", help="overwrite existing files") ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") + ap.add_argument("--finalize-pxe", action="store_true", help=argparse.SUPPRESS) + ap.add_argument("--context", help=argparse.SUPPRESS) + ap.add_argument("--handoff", help=argparse.SUPPRESS) args = ap.parse_args(argv) if args.print_schema: print(json.dumps(SCHEMA, indent=2)) return 0 + if args.finalize_pxe: + if args.spec or args.token_file or args.context is None or args.handoff is None: + die("--finalize-pxe requires --out-dir, --context, and --handoff", 2) + try: + finalize(Path(args.out_dir), Path(args.context), Path(args.handoff)) + except FinalizationError as error: + die(str(error)) + return 0 if not args.spec: die("--spec is required (or use --print-schema)", 2) raw = sys.stdin.read() if args.spec == "-" else Path(args.spec).read_text(encoding="utf-8") try: - spec = json.loads(raw) - except json.JSONDecodeError as exc: + spec = strict_json_loads(raw) + except (DuplicateJsonKeyError, json.JSONDecodeError) as exc: die(f"spec is not valid JSON: {exc}") - if not isinstance(spec, dict): - die("spec must be a mapping") - topo = spec.get("topology") - if topo not in ("pxe-diskless", "ssh-preinstalled"): - die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") - require(spec, "k3s_version") - require(spec, "server.name") - require(spec, "server.ip") - validate_config_shapes(spec) - + topo = validate_spec(spec) if args.token_file: token = Path(args.token_file).read_text(encoding="utf-8").strip() - if not token: - die("--token-file is empty") + validate_yaml_scalar(token, "--token-file") else: token = gen_token() out = Path(args.out_dir) - artifacts = [(out / "inventory.yml", render_inventory(spec, token), 0o600, True)] + try: + discovery = discover_gpu_policy(spec, out) + except DiscoveryFailure as error: + die(str(error)) if topo == "pxe-diskless": - artifacts.append((out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), 0o600, False)) - artifacts.append((out / "values-basic-example.yaml", render_values(spec), 0o644, False)) - preflight_destinations([path for path, _, _, _ in artifacts], args.force) - publish_artifacts(artifacts, args.force) + try: + if spec["pxe"]["diskless_agents_have_amd_gpus"]: + stage_pending(spec, token, discovery.resolution, out, args.force) + print("PXE GPU rootfs is pending finalization after pb-pxe-controller.yml resolves its render GID.") + else: + publish_disabled_rootfs(spec, token, discovery.resolution, out, args.force) + except FinalizationError as error: + die(str(error)) + else: + inventory, values, manifest = canonical_paths(out) + artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] + artifacts += [ + (values, render_values(spec, discovery.resolution), 0o644, False), + (manifest, manifest_content(discovery), 0o644, False), + ] + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) print( "\nNext: review the files, then copy them into your aup-learning-cloud " diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py new file mode 100644 index 00000000..7f14faad --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -0,0 +1,246 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import re +from dataclasses import dataclass +from pathlib import Path + +from config_common import DuplicateJsonKeyError, strict_json_loads + +MAX_RENDER_GID = 4_294_967_294 + + +@dataclass(frozen=True, slots=True) +class GpuInventory: + hosts: dict[str, bool] + render_gid: int | None + + +@dataclass(frozen=True, slots=True) +class GpuResolution: + status: str + hosts: dict[str, bool] + render_gid: int | None + pxe_rootfs_enabled: bool | None + pxe_rootfs_gid: int | None + + +@dataclass(frozen=True, slots=True) +class PxeGpuPolicy: + enabled: bool + render_gid: int | None + + +def configured_path(repo: Path, value: str) -> Path: + path = Path(value).expanduser() + return path if path.is_absolute() else repo / path + + +def parse_gpu_gid(value: str) -> int | None | str: + normalized = value.strip() + if normalized in {"null", "~"}: + return None + if normalized.isascii() and normalized.isdecimal(): + gid = int(normalized) + if 1 <= gid <= MAX_RENDER_GID: + return gid + return "invalid" + + +def parse_gpu_boolean(value: str) -> bool | None: + normalized = value.strip() + if normalized == "true": + return True + if normalized == "false": + return False + return None + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: + host_values: dict[str, list[str]] = {} + host_names: list[str] = [] + render_gids: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + if len(path) == 4 and path[:4] in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + }: + host_names.append(key) + host_values.setdefault(key, []) + elif ( + len(path) == 5 + and path[:4] + in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + } + and key == "auplc_gpu_access_enabled" + ): + host_values.setdefault(path[4], []).append(value) + elif path == ("k3s_cluster", "vars") and key == "auplc_render_gid": + render_gids.append(value) + stack.append((indent, key)) + + parse_errors: list[str] = [] + if not host_names: + parse_errors.append("inventory has no generated k3s server or agent hosts") + if len(set(host_names)) != len(host_names): + parse_errors.append("inventory has duplicate generated host names") + hosts: dict[str, bool] = {} + for host in host_names: + values = host_values[host] + if len(values) != 1: + parse_errors.append(f"inventory host '{host}' must define exactly one auplc_gpu_access_enabled") + continue + enabled = parse_gpu_boolean(values[0]) + if enabled is None: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + continue + hosts[host] = enabled + if len(render_gids) != 1: + parse_errors.append("inventory must define exactly one k3s_cluster.vars.auplc_render_gid") + return None, parse_errors + render_gid = parse_gpu_gid(render_gids[0]) + if render_gid == "invalid": + parse_errors.append("inventory has malformed auplc_render_gid") + return None, parse_errors + if parse_errors: + return None, parse_errors + return GpuInventory(hosts=hosts, render_gid=render_gid), parse_errors + + +def parse_values_gpu_gid(text: str) -> tuple[int | None, bool, list[str]]: + render_gids: list[str] = [] + stack: list[tuple[int, str]] = [] + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + if path == ("custom", "gpuAccess") and key == "renderGid": + render_gids.append(value) + stack.append((indent, key)) + if not render_gids: + return None, False, [] + if len(render_gids) != 1: + return None, True, ["custom.gpuAccess.renderGid is duplicated"] + render_gid = parse_gpu_gid(render_gids[0]) + if render_gid == "invalid": + return None, True, ["custom.gpuAccess.renderGid is malformed"] + return render_gid, True, [] + + +def collect_effective_gpu_gid(repo: Path, values: list[str]) -> tuple[int | None, list[str]]: + effective_gid: int | None = None + found = False + parse_errors: list[str] = [] + for rel in values or ["runtime/values.yaml"]: + path = configured_path(repo, rel) + if not path.exists(): + continue + render_gid, present, file_errors = parse_values_gpu_gid(path.read_text(encoding="utf-8")) + parse_errors.extend(f"{path}: {error}" for error in file_errors) + if present and not file_errors: + effective_gid = render_gid + found = True + if not found: + parse_errors.append("effective values have no custom.gpuAccess.renderGid") + return effective_gid, parse_errors + + +def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: + try: + document = strict_json_loads(text) + except DuplicateJsonKeyError as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + except (TypeError, ValueError) as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + if type(document) is not dict: + return None, ["GPU resolution manifest must be a JSON object"] + expected_keys = {"version", "status", "render_gid", "hosts"} + if topology == "pxe-diskless": + expected_keys.add("pxe_rootfs") + if set(document) != expected_keys: + return None, ["GPU resolution manifest has an unexpected schema"] + if type(document["version"]) is not int or document["version"] != 1: + return None, ["GPU resolution manifest version must be integer 1"] + status = document["status"] + if type(status) is not str or status not in {"cpu_only", "gpu_resolved"}: + return None, ["GPU resolution manifest status must be cpu_only or gpu_resolved"] + if type(document["hosts"]) is not dict or not document["hosts"]: + return None, ["GPU resolution manifest hosts must be a non-empty object"] + if any( + type(host) is not str or not host or type(enabled) is not bool for host, enabled in document["hosts"].items() + ): + return None, ["GPU resolution manifest hosts must map non-empty names to booleans"] + render_gid = document["render_gid"] + if render_gid is not None and (type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID): + return None, ["GPU resolution manifest render_gid must be an integer or null"] + if topology == "ssh-preinstalled": + return GpuResolution(status, document["hosts"], render_gid, None, None), [] + rootfs = document["pxe_rootfs"] + if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled", "render_gid"}: + return None, ["GPU resolution manifest pxe_rootfs has an unexpected schema"] + rootfs_enabled = rootfs["gpu_access_enabled"] + rootfs_gid = rootfs["render_gid"] + if type(rootfs_enabled) is not bool: + return None, ["GPU resolution manifest pxe_rootfs.gpu_access_enabled must be boolean"] + if rootfs_gid is not None and (type(rootfs_gid) is not int or not 1 <= rootfs_gid <= MAX_RENDER_GID): + return None, ["GPU resolution manifest pxe_rootfs.render_gid must be an integer or null"] + return GpuResolution(status, document["hosts"], render_gid, rootfs_enabled, rootfs_gid), [] + + +def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: + values: dict[str, list[str]] = {"auplc_render_gid": [], "pxe_gpu_access_enabled": []} + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip() or yaml_indent(line) != 0: + continue + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", line.strip()) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + if key in values: + values[key].append((mapping_match.group(2) or "").strip()) + parse_errors: list[str] = [] + for key, occurrences in values.items(): + if len(occurrences) != 1: + parse_errors.append(f"PXE vars must define exactly one {key}") + if parse_errors: + return None, parse_errors + render_gid = parse_gpu_gid(values["auplc_render_gid"][0]) + enabled = parse_gpu_boolean(values["pxe_gpu_access_enabled"][0]) + if render_gid == "invalid": + parse_errors.append("PXE vars have malformed auplc_render_gid") + if enabled is None: + parse_errors.append("PXE vars have malformed pxe_gpu_access_enabled") + if parse_errors: + return None, parse_errors + return PxeGpuPolicy(enabled=enabled, render_gid=render_gid), [] diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py new file mode 100644 index 00000000..1fd8c041 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -0,0 +1,125 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from dataclasses import dataclass +from pathlib import Path + +from gpu_resolution_parsing import ( + collect_effective_gpu_gid, + configured_path, + parse_gpu_inventory, + parse_gpu_resolution, + parse_pxe_gpu_policy, +) + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationRequest: + repo: Path + inventory_path: str + resolution_path: str + values: list[str] + topology: str + pxe_vars_path: Path + has_prior_errors: bool + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationResult: + errors: list[str] + passed: list[str] + + +@dataclass(frozen=True, slots=True) +class AcceleratorValidationResult: + errors: list[str] + warnings: list[str] + passed: list[str] + + +def check_accelerator_labels( + accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None +) -> AcceleratorValidationResult: + errors: list[str] = [] + warnings: list[str] = [] + passed: list[str] = [] + active_keys = sorted({key for keys in metadata.values() for key in keys}) + if not active_keys: + return AcceleratorValidationResult([], ["no acceleratorKeys found in effective custom.resources.metadata"], []) + declared: list[str] = [] + for key in active_keys: + if key not in accelerators: + errors.append(f"active accelerator '{key}' is not defined under custom.accelerators") + elif not accelerators[key]: + errors.append(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") + else: + declared.append(accelerators[key]) + if not declared: + return AcceleratorValidationResult(errors, warnings, passed) + if cluster is None: + warnings.append( + "no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + real = set(cluster.get("gpu_product_names", [])) + if not real: + errors.append("cluster snapshot has no GPU product labels for active accelerators") + return AcceleratorValidationResult(errors, warnings, passed) + for declared_label in declared: + if declared_label in real: + passed.append(f"nodeSelector '{declared_label}' matches a real node label") + else: + errors.append( + f"nodeSelector '{declared_label}' matches no node label. Real labels: {', '.join(sorted(real))}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + + +def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactValidationResult: + errors: list[str] = [] + inventory_file = configured_path(request.repo, request.inventory_path) + resolution_file = configured_path(request.repo, request.resolution_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"generated inventory not found: {inventory_file}"], []) + if not resolution_file.exists(): + return GpuArtifactValidationResult([f"GPU resolution manifest not found: {resolution_file}"], []) + inventory, inventory_errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + resolution, resolution_errors = parse_gpu_resolution(resolution_file.read_text(encoding="utf-8"), request.topology) + helm_gid, helm_errors = collect_effective_gpu_gid(request.repo, request.values) + errors.extend([*inventory_errors, *resolution_errors, *helm_errors]) + if inventory is None or resolution is None or errors: + return GpuArtifactValidationResult(errors, []) + if set(inventory.hosts) != set(resolution.hosts): + errors.append("inventory hosts do not exactly match GPU resolution manifest hosts") + for host, enabled in inventory.hosts.items(): + if resolution.hosts.get(host) != enabled: + errors.append(f"inventory host '{host}' GPU access boolean disagrees with the resolution manifest") + pxe_policy = None + if request.topology == "pxe-diskless": + if not request.pxe_vars_path.exists(): + return GpuArtifactValidationResult([*errors, f"PXE vars file not found: {request.pxe_vars_path}"], []) + pxe_policy, pxe_errors = parse_pxe_gpu_policy(request.pxe_vars_path.read_text(encoding="utf-8")) + errors.extend(pxe_errors) + if pxe_policy is None or pxe_errors: + return GpuArtifactValidationResult(errors, []) + if pxe_policy.enabled != resolution.pxe_rootfs_enabled: + errors.append("PXE pxe_gpu_access_enabled disagrees with GPU resolution manifest pxe_rootfs") + if resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is None: + errors.append("GPU-enabled PXE rootfs requires a numeric render GID") + if not resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is not None: + errors.append("GPU-disabled PXE rootfs requires a null render GID") + if resolution.pxe_rootfs_enabled and pxe_policy.render_gid != resolution.pxe_rootfs_gid: + errors.append("PXE auplc_render_gid disagrees with GPU resolution manifest pxe_rootfs render_gid") + gids = [inventory.render_gid, helm_gid, resolution.render_gid] + if pxe_policy is not None: + gids.append(pxe_policy.render_gid) + if len(set(gids)) != 1: + errors.append("inventory, Helm, PXE, and GPU resolution render GIDs disagree") + enabled_scope = any(resolution.hosts.values()) or resolution.pxe_rootfs_enabled is True + if resolution.status == "cpu_only": + if enabled_scope or resolution.render_gid is not None or any(gid is not None for gid in gids): + errors.append("cpu_only GPU resolution requires all host/rootfs booleans false and all render GIDs null") + elif not enabled_scope or resolution.render_gid is None: + errors.append("gpu_resolved GPU resolution requires an enabled scope and a numeric render GID") + passed = [] if request.has_prior_errors or errors else ["GPU access artifacts agree"] + return GpuArtifactValidationResult(errors, passed) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 8408f020..ccb5bb6b 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,6 +12,8 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; + * generated inventory, GPU-resolution manifest, Helm render GID, and PXE + rootfs policy agree when generated artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -20,9 +22,10 @@ inspect something. Usage: - validate.py --repo ~/aup-learning-cloud --topology pxe-diskless validate.py --repo ~/aup-learning-cloud \ --topology ssh-preinstalled \ + --inventory generated/inventory.yml \ + --gpu-resolution generated/gpu-access-resolution.json \ --values runtime/values.yaml --values runtime/values-basic-example.yaml \ --cluster cluster.json --helm-dry-run @@ -30,8 +33,6 @@ 2 on a usage error. """ -from __future__ import annotations - import argparse import json import re @@ -40,6 +41,10 @@ import sys from pathlib import Path +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_validation import GpuArtifactValidationRequest, check_accelerator_labels, check_gpu_artifacts +from values_resolution_parsing import collect_effective_values + PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" INVENTORY = "deploy/ansible/inventory.yml" CHART = "runtime/chart" @@ -163,155 +168,6 @@ def check_version_sync(repo: Path, configured_path: str | None = None) -> None: ) -def yaml_scalar(value: str) -> str: - return value.strip().strip('"').strip("'") - - -def yaml_optional_scalar(value: str) -> str: - scalar_value = yaml_scalar(value) - return "" if scalar_value in {"", "null", "~"} else scalar_value - - -def yaml_indent(line: str) -> int: - return len(line) - len(line.lstrip()) - - -def parse_inline_list(value: str) -> list[str]: - items = value.strip()[1:-1].strip() - if not items: - return [] - return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] - - -def is_relevant_flow_path(path: tuple[str, ...]) -> bool: - return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} - - -def unsupported_yaml_syntax(value: str) -> bool: - return value.startswith(("&", "*", "!", "|", ">")) - - -def parse_values_file(text: str) -> tuple[dict[str, str | None], dict[str, list[str]], list[str]]: - """Extract the deploy-relevant mappings from a fixed-shape values YAML file. - - The helpers deliberately remain stdlib-only. This scanner handles the - mapping/list shapes used by values overlays, rather than pretending to be a - general YAML parser. - """ - accelerators: dict[str, str | None] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - stack: list[tuple[int, str]] = [] - - for raw_line in text.splitlines(): - line = raw_line.split("#", 1)[0].rstrip() - if not line.strip(): - continue - indent = yaml_indent(line) - stripped = line.strip() - - while stack and indent <= stack[-1][0]: - stack.pop() - path = tuple(key for _, key in stack) - - if stripped.startswith("- "): - if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": - metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) - continue - - product_label_match = re.fullmatch( - r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped - ) - if product_label_match: - if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": - value = product_label_match.group(1).strip() - if unsupported_yaml_syntax(value): - parse_errors.append( - f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" - ) - else: - accelerators[path[2]] = yaml_optional_scalar(value) - continue - - mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) - if not mapping_match: - continue - key = mapping_match.group(1).strip("\"'") - value = (mapping_match.group(2) or "").strip() - candidate_path = path + (key,) - if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") - if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - if path == ("custom", "accelerators"): - accelerators.setdefault(key, None) - if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": - resource_key = path[3] - if unsupported_yaml_syntax(value): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - elif value.startswith("[") and value.endswith("]"): - metadata[resource_key] = parse_inline_list(value) - elif not value or value in {"null", "~"}: - metadata[resource_key] = [] - else: - parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") - stack.append((indent, key)) - return accelerators, metadata, parse_errors - - -def collect_effective_values(repo: Path, values: list[str]) -> tuple[dict[str, str], dict[str, list[str]], list[str]]: - paths = values or ["runtime/values.yaml"] - accelerators: dict[str, str] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - for rel in paths: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - parsed_accelerators, parsed_metadata, file_errors = parse_values_file(p.read_text(encoding="utf-8")) - for key, selector in parsed_accelerators.items(): - if selector is not None or key not in accelerators: - accelerators[key] = selector - metadata.update(parsed_metadata) - parse_errors.extend(file_errors) - else: - fail(f"values file not found: {rel}") - return accelerators, metadata, parse_errors - - -def check_accelerator_labels( - accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None -) -> None: - active_keys = sorted({key for keys in metadata.values() for key in keys}) - if not active_keys: - warn("no acceleratorKeys found in effective custom.resources.metadata") - return - declared: list[str] = [] - for key in active_keys: - if key not in accelerators: - fail(f"active accelerator '{key}' is not defined under custom.accelerators") - elif not accelerators[key]: - fail(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") - else: - declared.append(accelerators[key]) - if not declared: - return - if cluster is None: - warn( - "no --cluster snapshot; cannot confirm nodeSelector labels match real " - f"nodes. Declared: {', '.join(declared)}" - ) - return - real = set(cluster.get("gpu_product_names", [])) - if not real: - fail("cluster snapshot has no GPU product labels for active accelerators") - return - for d in declared: - if d in real: - ok(f"nodeSelector '{d}' matches a real node label") - else: - fail(f"nodeSelector '{d}' matches no node label. Real labels: {', '.join(sorted(real))}") - - def check_helm(repo: Path, values: list[str]) -> None: if not shutil.which("helm"): warn("helm not on PATH; skipped chart dry-run") @@ -353,6 +209,8 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) + ap.add_argument("--inventory", help="generated inventory.yml to cross-check with GPU resolution") + ap.add_argument("--gpu-resolution", help="generated gpu-access-resolution.json to cross-check") ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -366,8 +224,8 @@ def main(argv=None) -> int: cluster = None if args.cluster: try: - cluster = json.loads(Path(args.cluster).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: + cluster = strict_json_loads(Path(args.cluster).read_text(encoding="utf-8")) + except (DuplicateJsonKeyError, OSError, json.JSONDecodeError) as exc: print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) return 2 @@ -376,10 +234,36 @@ def main(argv=None) -> int: check_version_sync(repo, args.pxe_vars) else: ok("skipped PXE checks for ssh-preinstalled topology") - accelerators, metadata, parse_errors = collect_effective_values(repo, args.values) - for message in parse_errors: + values_result = collect_effective_values(repo, args.values) + for message in values_result.missing_files: + fail(message) + for message in values_result.parse_errors: + fail(message) + accelerator_result = check_accelerator_labels(values_result.accelerators, values_result.metadata, cluster) + for message in accelerator_result.errors: fail(message) - check_accelerator_labels(accelerators, metadata, cluster) + for message in accelerator_result.warnings: + warn(message) + for message in accelerator_result.passed: + ok(message) + if bool(args.inventory) != bool(args.gpu_resolution): + fail("--inventory and --gpu-resolution must be supplied together") + elif args.inventory and args.gpu_resolution: + artifact_result = check_gpu_artifacts( + GpuArtifactValidationRequest( + repo=repo, + inventory_path=args.inventory, + resolution_path=args.gpu_resolution, + values=args.values, + topology=args.topology, + pxe_vars_path=pxe_vars_path(repo, args.pxe_vars), + has_prior_errors=bool(errors), + ) + ) + for message in artifact_result.errors: + fail(message) + for message in artifact_result.passed: + ok(message) if args.helm_dry_run: check_helm(repo, args.values) diff --git a/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py new file mode 100644 index 00000000..9836162d --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py @@ -0,0 +1,130 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Fixed-shape parsing and overlay resolution for deploy values files.""" + +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True, slots=True) +class ValuesFileParseResult: + accelerators: dict[str, str | None] + metadata: dict[str, list[str]] + parse_errors: list[str] + + +@dataclass(frozen=True, slots=True) +class EffectiveValuesResult: + accelerators: dict[str, str] + metadata: dict[str, list[str]] + missing_files: list[str] + parse_errors: list[str] + + +def yaml_scalar(value: str) -> str: + return value.strip().strip('"').strip("'") + + +def yaml_optional_scalar(value: str) -> str: + scalar_value = yaml_scalar(value) + return "" if scalar_value in {"", "null", "~"} else scalar_value + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_inline_list(value: str) -> list[str]: + items = value.strip()[1:-1].strip() + if not items: + return [] + return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] + + +def is_relevant_flow_path(path: tuple[str, ...]) -> bool: + return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} + + +def unsupported_yaml_syntax(value: str) -> bool: + return value.startswith(("&", "*", "!", "|", ">")) + + +def parse_values_file(text: str) -> ValuesFileParseResult: + accelerators: dict[str, str | None] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + + if stripped.startswith("- "): + if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": + metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) + continue + + product_label_match = re.fullmatch( + r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped + ) + if product_label_match: + if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": + value = product_label_match.group(1).strip() + if unsupported_yaml_syntax(value): + parse_errors.append( + f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" + ) + else: + accelerators[path[2]] = yaml_optional_scalar(value) + continue + + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + candidate_path = path + (key,) + if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") + if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + if path == ("custom", "accelerators"): + accelerators.setdefault(key, None) + if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": + resource_key = path[3] + if unsupported_yaml_syntax(value): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + elif value.startswith("[") and value.endswith("]"): + metadata[resource_key] = parse_inline_list(value) + elif not value or value in {"null", "~"}: + metadata[resource_key] = [] + else: + parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") + stack.append((indent, key)) + return ValuesFileParseResult(accelerators, metadata, parse_errors) + + +def collect_effective_values(repo: Path, values: list[str]) -> EffectiveValuesResult: + accelerators: dict[str, str] = {} + metadata: dict[str, list[str]] = {} + missing_files: list[str] = [] + parse_errors: list[str] = [] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if not path.exists(): + missing_files.append(f"values file not found: {rel}") + continue + parsed = parse_values_file(path.read_text(encoding="utf-8")) + for key, selector in parsed.accelerators.items(): + if selector is not None or key not in accelerators: + accelerators[key] = selector + metadata.update(parsed.metadata) + parse_errors.extend(parsed.parse_errors) + return EffectiveValuesResult(accelerators, metadata, missing_files, parse_errors) diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py new file mode 100644 index 00000000..6ec39bf8 --- /dev/null +++ b/tests/skills/test_config_generation_security.py @@ -0,0 +1,95 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def safe_spec() -> dict[str, object]: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server-1", "ip": "192.168.1.10"}, + "agents": [{"name": "agent-1", "ip": "192.168.1.11"}], + "images": {"cpu": "registry.example/auplc:latest"}, + } + + +def load_config_generation_module(): + sys.path.insert(0, str(GEN_CONFIGS.parent)) + try: + import config_generation + + return config_generation + finally: + sys.path.pop(0) + + +@pytest.mark.parametrize( + ("path", "value", "message"), + [ + (("server", "name"), "server\n vars: {injected: true}", "spec.server.name"), + (("server", "ip"), "192.168.1.10\n injected: true", "spec.server.ip"), + (("k3s_version",), "v1.32.3+k3s1\n injected: true", "spec.k3s_version"), + (("agents",), [{"name": "server-1", "ip": "192.168.1.11"}], "unique"), + (("agents",), [{"name": "agent-1", "ip": "not-an-ip"}], "spec.agents[0].ip"), + (("images",), {"cpu\n injected": "registry.example/auplc:latest"}, "spec.images key"), + ], +) +def test_generator_rejects_unsafe_public_spec_scalars_before_discovery( + path: tuple[str, ...], value: object, message: str, capsys: pytest.CaptureFixture[str] +) -> None: + module = load_config_generation_module() + spec = safe_spec() + if len(path) == 1: + spec[path[0]] = value + else: + target = spec[path[0]] + assert isinstance(target, dict) + target[path[1]] = value + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert message in capsys.readouterr().err + + +def test_generator_rejects_an_invalid_k3s_version_before_discovery(capsys: pytest.CaptureFixture[str]) -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["k3s_version"] = "v1.32.3+k3s1 # comments are not accepted" + + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert "spec.k3s_version" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "raw", + [ + '{"topology":"ssh-preinstalled","topology":"pxe-diskless"}', + '{"topology":"pxe-diskless","k3s_version":"v1.32.3+k3s1","server":{"name":"server","ip":"192.168.1.10"},"network":{"interface":"eno1","subnet":"192.168.1.0/24"},"pxe":{"authorized_keys":["ssh-ed25519 AAA"],"diskless_agents_have_amd_gpus":true,"diskless_agents_have_amd_gpus":false}}', + ], +) +def test_generator_rejects_duplicate_public_policy_keys_before_discovery(tmp_path: Path, raw: str) -> None: + spec = tmp_path / "spec.json" + spec.write_text(raw, encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec), "--out-dir", str(tmp_path / "generated")], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 1 + assert "duplicate JSON key" in result.stderr diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 7e5b6f2c..0469f4fc 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -19,6 +19,33 @@ DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" VALIDATE = DEPLOY_SCRIPTS / "validate.py" GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" +CONFIG_GENERATION = DEPLOY_SCRIPTS / "config_generation.py" +ARTIFACT_STORE = DEPLOY_SCRIPTS / "artifact_store.py" +VALUES_RESOLUTION_PARSING = DEPLOY_SCRIPTS / "values_resolution_parsing.py" + +EXPECTED_GENERATOR_SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + "diskless_agents_have_amd_gpus": True, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -37,23 +64,113 @@ def write_file(path: Path, content: str) -> Path: return path +@pytest.fixture(autouse=True) +def fake_ansible_playbook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fake_bin = tmp_path / "fake-ansible" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + r"""#!/usr/bin/env python3 +import json +from pathlib import Path +import sys + +arguments = sys.argv[1:] +inventory = Path(arguments[arguments.index('-i') + 1]) +output = next(value.split('=', 1)[1] for value in arguments if value.startswith('gpu_access_discovery_output_path=')) +hosts = [line.strip()[:-1] for line in inventory.read_text(encoding='utf-8').splitlines() if line.startswith(' ') and line.rstrip().endswith(':')] +evidence = { + 'version': 2, + 'hosts': [{ + 'host': host, + 'reachable': True, + 'lspci': {'rc': 0, 'stdout': ''}, + 'sysfs': {'rc': 0, 'stdout': ''}, + 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'legacy_rules': { + key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} + for key in ('kfd', 'amdgpu', 'rocm_devices') + }, + } for host in hosts], +} +Path(output).write_text(json.dumps(evidence), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + def write_cluster(repo: Path, labels: list[str]) -> Path: return write_file(repo / "cluster.json", json.dumps({"gpu_product_names": labels})) -def load_validate_module(): - spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module +def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + ) + values = write_file( + repo / "generated/values-basic-example.yaml", + """custom: + gpuAccess: + renderGid: 993 + resources: + metadata: {} +""", + ) + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": False, "server": True}, + } + ), + ) + return inventory, values, resolution -def load_generator_module(): - spec = importlib.util.spec_from_file_location("deploy_generator", GEN_CONFIGS) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) +def load_validate_module(): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_deploy_module(module_name: str, script: Path): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(module_name, script) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + finally: + sys.path.pop(0) return module @@ -174,6 +291,43 @@ def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: assert "AMD_Radeon_8060S_Graphics" in result.stdout +def test_values_resolution_parser_preserves_overlay_precedence_and_error_categories(tmp_path: Path) -> None: + parser = load_deploy_module("values_resolution_parsing", VALUES_RESOLUTION_PARSING) + repo = tmp_path / "checkout" + base = write_file( + repo / "base.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + partial_overlay = write_file( + repo / "partial.yaml", + """custom: + accelerators: + strix-halo: + displayName: Renamed +""", + ) + invalid_overlay = write_file(repo / "invalid.yaml", "custom: *defaults\n") + + result = parser.collect_effective_values( + repo, + [str(base), str(partial_overlay), "missing.yaml", str(invalid_overlay)], + ) + + assert result.accelerators == {"strix-halo": "AMD_Radeon_8060S_Graphics"} + assert result.metadata == {"gpu": ["strix-halo"]} + assert result.missing_files == ["values file not found: missing.yaml"] + assert result.parse_errors == ["unsupported YAML syntax at custom"] + + def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -751,6 +905,345 @@ def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_ assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout +def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: false + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: null +""", + ) + values = write_file( + repo / "generated/values-basic-example.yaml", + """custom: + gpuAccess: + renderGid: null + resources: + metadata: {} +""", + ) + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "cpu_only", + "render_gid": None, + "hosts": {"agent": False, "server": False}, + } + ), + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access artifacts agree" in result.stdout + + +def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access artifacts agree" in result.stdout + + +@pytest.mark.parametrize( + ("resolution_content", "expected_error"), + [ + ("not JSON", "GPU resolution manifest is malformed"), + ( + '{"version":1,"status":"pending","render_gid":993,"hosts":{"agent":false,"server":true}}', + "GPU resolution manifest status must be cpu_only or gpu_resolved", + ), + ( + '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"server":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ( + '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"ser\\u0076er":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ], +) +def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( + tmp_path: Path, resolution_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + resolution.write_text(resolution_content, encoding="utf-8") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +@pytest.mark.parametrize( + ("inventory_content", "expected_error"), + [ + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' must define exactly one auplc_gpu_access_enabled", + ), + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: yes + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' has malformed auplc_gpu_access_enabled", + ), + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: true + auplc_gpu_access_enabled: false + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' must define exactly one auplc_gpu_access_enabled", + ), + ], +) +def test_validator_rejects_missing_malformed_or_duplicate_inventory_host_booleans( + tmp_path: Path, inventory_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + inventory.write_text(inventory_content, encoding="utf-8") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, _ = write_resolved_gpu_artifacts(repo) + missing_resolution = repo / "generated/missing-gpu-access-resolution.json" + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(missing_resolution), + ) + + assert result.returncode == 1 + assert "GPU resolution manifest not found" in result.stdout + + +def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + values.write_text( + """custom: + gpuAccess: + renderGid: 994 + resources: + metadata: {} +""", + encoding="utf-8", + ) + resolution.write_text( + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": True, "server": True}, + } + ), + encoding="utf-8", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert "inventory host 'agent' GPU access boolean disagrees" in result.stdout + assert "render GIDs disagree" in result.stdout + + +def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: false + agent: + hosts: {} + vars: + auplc_render_gid: 993 +""", + ) + values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n gpuAccess:\n renderGid: 993\n") + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"server": False}, + "pxe_rootfs": {"gpu_access_enabled": True, "render_gid": 993}, + } + ), + ) + pxe_vars = write_file( + repo / "generated/pb-pxe-controller.vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +auplc_render_gid: 994 +pxe_gpu_access_enabled: false +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe_gpu_access_enabled disagrees" in result.stdout + assert "PXE auplc_render_gid disagrees" in result.stdout + + def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: spec = write_file( tmp_path / "spec.json", @@ -824,13 +1317,13 @@ def generator_spec(topology: str = "ssh-preinstalled", accelerators: object | No spec["accelerators"] = accelerators if topology == "pxe-diskless": spec["network"] = {"interface": "enp1s0", "subnet": "192.168.1.0/24"} - spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"]} + spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"], "diskless_agents_have_amd_gpus": False} return spec def test_generator_validates_all_pxe_requirements_before_writing(tmp_path: Path) -> None: spec = generator_spec("pxe-diskless") - spec["pxe"] = {"authorized_keys": []} + spec["pxe"] = {"authorized_keys": [], "diskless_agents_have_amd_gpus": False} spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) out_dir = tmp_path / "generated" @@ -924,7 +1417,7 @@ def test_generator_force_replaces_symlink_entry_without_following_target(tmp_pat def test_generator_force_failure_restores_all_original_destination_types( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - module = load_generator_module() + module = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) inventory = write_file(tmp_path / "inventory.yml", "old inventory\n") pxe_vars = tmp_path / "pb-pxe-controller.vars.yml" pxe_vars.mkdir() @@ -956,6 +1449,58 @@ def fail_late_replace(source, destination): assert values_target.read_text(encoding="utf-8") == "old symlink target\n" +def test_artifact_store_rolls_back_destination_when_staged_unlink_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = load_deploy_module("deploy_artifact_store_unlink", ARTIFACT_STORE) + destination = tmp_path / "inventory.yml" + original_unlink = module.os.unlink + failed = False + + def fail_first_staged_unlink(path, *args, **kwargs): + nonlocal failed + if not failed and Path(path).name.startswith(".inventory.yml."): + failed = True + raise OSError("injected staged unlink failure") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(module.os, "unlink", fail_first_staged_unlink) + + with pytest.raises(SystemExit): + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) + + assert not destination.exists() + + +@pytest.mark.parametrize("force", (False, True)) +def test_artifact_store_rolls_back_destination_when_parent_fsync_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, force: bool +) -> None: + module = load_deploy_module(f"deploy_artifact_store_fsync_{force}", ARTIFACT_STORE) + destination = tmp_path / "inventory.yml" + if force: + destination.write_text("old inventory\n", encoding="utf-8") + original_fsync_parent = module._fsync_parent + calls = 0 + + def fail_after_publication(path): + nonlocal calls + calls += 1 + if calls == (2 if force else 1): + raise OSError("injected parent fsync failure") + return original_fsync_parent(path) + + monkeypatch.setattr(module, "_fsync_parent", fail_after_publication) + + with pytest.raises(SystemExit): + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=force) + + if force: + assert destination.read_text(encoding="utf-8") == "old inventory\n" + else: + assert not destination.exists() + + def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: repo = tmp_path / "checkout" base_values = write_file( @@ -1007,3 +1552,122 @@ def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: assert result.returncode == 0, result.stdout + result.stderr assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout + + +def test_generator_print_schema_is_byte_stable() -> None: + result = run_script(GEN_CONFIGS, "--print-schema") + + assert result.returncode == 0, result.stdout + result.stderr + assert result.stderr == "" + assert result.stdout == json.dumps(EXPECTED_GENERATOR_SCHEMA, indent=2) + "\n" + + +def test_generator_exits_with_usage_error_when_spec_is_omitted() -> None: + result = run_script(GEN_CONFIGS) + + assert result.returncode == 2 + assert result.stdout == "" + assert result.stderr == "gen_configs: --spec is required (or use --print-schema)\n" + + +def test_generator_replaces_colliding_artifacts_when_force_is_given(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + token_path = write_file(tmp_path / "token.txt", "characterization-token\n") + out_dir = tmp_path / "generated" + write_file(out_dir / "inventory.yml", "old inventory\n") + write_file(out_dir / "pb-pxe-controller.vars.yml", "old pxe vars\n") + write_file(out_dir / "values-basic-example.yaml", "old values\n") + + result = run_script( + GEN_CONFIGS, + "--spec", + str(spec_path), + "--out-dir", + str(out_dir), + "--token-file", + str(token_path), + "--force", + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "old inventory" not in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "old pxe vars" not in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "old values" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 + + +def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: + generation = load_deploy_module("deploy_config_generation", CONFIG_GENERATION) + artifacts = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) + + assert generation.SCHEMA == EXPECTED_GENERATOR_SCHEMA + assert generation.validate_spec(generator_spec()) == "ssh-preinstalled" + assert callable(generation.render_inventory) + assert callable(generation.render_pxe_vars) + assert callable(generation.render_values) + assert callable(artifacts.preflight_destinations) + assert callable(artifacts.publish_artifacts) + + +def test_generator_rejects_legacy_public_gpu_policy_fields_before_discovery(tmp_path: Path) -> None: + spec = generator_spec() + spec["render_gid"] = 1055 + spec["gpu_access"] = {"hosts": [], "pxe_rootfs_enabled": False} + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "spec.render_gid is no longer accepted" in result.stderr + assert not out_dir.exists() + + +def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + r"""#!/usr/bin/env python3 +import json +import pathlib +import sys +args = sys.argv[1:] +output = next(arg.split('=', 1)[1] for arg in args if arg.startswith('gpu_access_discovery_output_path=')) +def host(name, bdf): + return { + 'host': name, 'reachable': True, + 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, + 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'legacy_rules': {key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} for key in ('kfd', 'amdgpu', 'rocm_devices')}, + } +pathlib.Path(output).write_text(json.dumps({'version': 2, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + spec = generator_spec() + spec["agents"] = [{"name": "agent", "ip": "192.168.1.11"}] + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + result = subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir)], + capture_output=True, + check=False, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: 993" in inventory + assert inventory.count("auplc_gpu_access_enabled: true") == 1 + assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["hosts"] == {"agent": False, "server": True} From 09c10f0a6760e13b1819a9b1a93494812ef898fc Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 056/180] feat(ansible): admit and finalize PXE GPU rootfs --- .../ansible/playbooks/pb-pxe-controller.yml | 38 ++ .../roles/pxe_controller/defaults/main.yml | 3 + .../roles/pxe_controller/tasks/gpu_access.yml | 296 ++++++++++++ .../roles/pxe_controller/tasks/main.yml | 87 ++++ .../templates/chroot-setup.sh.j2 | 6 - tests/skills/test_gpu_access_role.py | 455 ++++++++++++++++++ 6 files changed, 879 insertions(+), 6 deletions(-) create mode 100644 deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml create mode 100644 tests/skills/test_gpu_access_role.py diff --git a/deploy/ansible/playbooks/pb-pxe-controller.yml b/deploy/ansible/playbooks/pb-pxe-controller.yml index 425b939e..250508cc 100644 --- a/deploy/ansible/playbooks/pb-pxe-controller.yml +++ b/deploy/ansible/playbooks/pb-pxe-controller.yml @@ -102,3 +102,41 @@ roles: - role: pxe_controller + + post_tasks: + - name: Write private PXE finalizer handoff from resolved rootfs facts + ansible.builtin.copy: + content: >- + {{ { + 'version': 1, + 'generation': pxe_finalizer_generation, + 'spec_sha256': pxe_finalizer_spec_sha256, + 'topology': 'pxe-diskless', + 'pxe_gpu_access_enabled': pxe_gpu_access_enabled | bool, + 'render_gid': _pxe_resolved_render_gid | default(none) + } | to_json }} + dest: "{{ pxe_finalizer_handoff }}" + mode: "0600" + delegate_to: localhost + run_once: true + become: false + no_log: true + when: pxe_finalizer_context is defined + + - name: Finalize generated PXE GPU policy from resolved rootfs facts + ansible.builtin.command: + argv: + - "{{ pxe_finalizer_script }}" + - --finalize-pxe + - --out-dir + - "{{ pxe_finalizer_context | dirname }}" + - --context + - "{{ pxe_finalizer_context }}" + - --handoff + - "{{ pxe_finalizer_handoff }}" + delegate_to: localhost + run_once: true + become: false + no_log: true + changed_when: false + when: pxe_finalizer_context is defined diff --git a/deploy/ansible/roles/pxe_controller/defaults/main.yml b/deploy/ansible/roles/pxe_controller/defaults/main.yml index e381140f..66ec003c 100644 --- a/deploy/ansible/roles/pxe_controller/defaults/main.yml +++ b/deploy/ansible/roles/pxe_controller/defaults/main.yml @@ -85,11 +85,14 @@ pxe_apt_mirror: "http://tw.archive.ubuntu.com/ubuntu" pxe_rootfs_force_rebuild: true # Run apt-get upgrade inside rootfs during chroot setup pxe_rootfs_upgrade: false +# Explicitly enable GPU access only for a rootfs intended for GPU workers. +pxe_gpu_access_enabled: false # ============================================================ # Paths # ============================================================ pxe_nfs_root: "/srv/nfs/rootfs" +pxe_nfs_allowed_root: "/srv/nfs" pxe_tftp_root: "/srv/tftp" pxe_web_root: "/var/www/html" diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml new file mode 100644 index 00000000..06b5d5f8 --- /dev/null +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -0,0 +1,296 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Record PXE GPU admission disposition + ansible.builtin.set_fact: + _pxe_rootfs_disposition: "{{ 'fresh' if _pxe_rootfs_rebuilt_this_run | bool else 'retained' }}" + _pxe_unanimous_live_render_gid: "{{ auplc_render_gid if auplc_render_gid is defined and auplc_render_gid is not none else none }}" + _pxe_resolved_render_gid: null + +- name: Assert PXE GPU admission phase + ansible.builtin.assert: + that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] + fail_msg: PXE GPU admission phase is invalid. + +- name: Validate optional unanimous live render GID + ansible.builtin.assert: + that: + - _pxe_unanimous_live_render_gid is integer + - _pxe_unanimous_live_render_gid >= 1 + - _pxe_unanimous_live_render_gid <= 4294967294 + fail_msg: auplc_render_gid must be an integer between 1 and 4294967294 when supplied for a PXE GPU rootfs. + when: + - pxe_gpu_access_enabled | bool + - _pxe_unanimous_live_render_gid is not none + +- name: Inspect PXE rootfs render group for GPU admission + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] + register: _pxe_admission_render_group + changed_when: false + failed_when: false + when: pxe_gpu_access_enabled | bool + +- name: Require fresh PXE render group lookup outcome + ansible.builtin.assert: + that: _pxe_admission_render_group.rc in [0, 2] + fail_msg: Unable to determine whether the fresh PXE rootfs has a render group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' + +- name: Require strict existing fresh PXE render group + ansible.builtin.assert: + that: + - _pxe_admission_render_group.stdout_lines | length == 1 + - _pxe_admission_render_group.stdout.split(':') | length == 4 + - _pxe_admission_render_group.stdout.split(':')[0] == 'render' + - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Fresh PXE rootfs render group is malformed. + when: >- + pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' and + _pxe_admission_render_group.rc == 0 + +- name: Require strict retained PXE render group + ansible.builtin.assert: + that: + - _pxe_admission_render_group.rc == 0 + - _pxe_admission_render_group.stdout_lines | length == 1 + - _pxe_admission_render_group.stdout.split(':') | length == 4 + - _pxe_admission_render_group.stdout.split(':')[0] == 'render' + - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Retained PXE rootfs must already have one valid render group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Record existing PXE render GID + ansible.builtin.set_fact: + _pxe_existing_render_gid: "{{ _pxe_admission_render_group.stdout.split(':')[2] | int }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_admission_render_group.rc == 0 + +- name: List fresh PXE rootfs groups before render GID creation + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group] + register: _pxe_fresh_groups + changed_when: false + failed_when: false + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + - _pxe_unanimous_live_render_gid is not none + +- name: Reject fresh PXE render GID collision + ansible.builtin.assert: + that: + - _pxe_fresh_groups.rc == 0 + - >- + _pxe_fresh_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (_pxe_unanimous_live_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: Fresh PXE rootfs render GID is already assigned to another group. + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + - _pxe_unanimous_live_render_gid is not none + +- name: Create missing fresh PXE render group + ansible.builtin.command: + argv: >- + {{ ['chroot', pxe_nfs_root, 'groupadd', '--system', '-g', (_pxe_unanimous_live_render_gid | string), 'render'] + if _pxe_unanimous_live_render_gid is not none + else ['chroot', pxe_nfs_root, 'groupadd', '--system', 'render'] }} + changed_when: true + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Read fresh PXE render group after creation + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] + register: _pxe_created_render_group + changed_when: false + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Resolve newly created fresh PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_created_render_group.stdout.split(':')[2] | int }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Resolve existing fresh PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is not none + +- name: Resolve retained PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + +- name: List retained PXE rootfs groups for render GID collision check + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group] + register: _pxe_retained_groups + changed_when: false + failed_when: false + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Reject retained PXE render GID collision + ansible.builtin.assert: + that: + - _pxe_retained_groups.rc == 0 + - >- + _pxe_retained_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (_pxe_resolved_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: Retained PXE rootfs render GID is already assigned to another group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE render GID matches unanimous live GID + ansible.builtin.assert: + that: _pxe_resolved_render_gid == _pxe_unanimous_live_render_gid + fail_msg: Retained PXE rootfs render GID differs from the supplied unanimous live render GID; rebuild or migrate it separately. + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - _pxe_unanimous_live_render_gid is not none + +- name: Inspect retained PXE canonical GPU access parents + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + - /var + - /var/lib + - /var/lib/auplc + register: _pxe_retained_canonical_gpu_parent_stats + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU access parents + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isdir + - not item.stat.islnk + fail_msg: "Retained PXE rootfs has an unsafe canonical GPU access parent: {{ item.item }}" + loop: "{{ _pxe_retained_canonical_gpu_parent_stats.results }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Inspect retained PXE GPU policy paths + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-kfd.rules + - /etc/udev/rules.d/70-amdgpu.rules + - /etc/udev/rules.d/70-rocm-devices.rules + - /etc/udev/rules.d/70-auplc-gpu-access.rules + - /var/lib/auplc/gpu-access.json + register: _pxe_retained_gpu_policy_stats + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE legacy GPU rules absent + ansible.builtin.assert: + that: not item.stat.exists + fail_msg: "Retained PXE rootfs has a legacy GPU rule requiring a separate migration: {{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU destinations + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isreg + - not item.stat.islnk + fail_msg: "Retained PXE rootfs requires an exact canonical GPU access destination: {{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Read retained PXE canonical GPU access destinations + ansible.builtin.slurp: + src: "{{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + register: _pxe_retained_canonical_gpu_destinations + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Define retained PXE canonical GPU rule + ansible.builtin.set_fact: + _pxe_retained_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU rule + ansible.builtin.assert: + that: (item.content | b64decode) == _pxe_retained_canonical_rule + fail_msg: "Retained PXE rootfs has a non-canonical GPU access rule: {{ item.item.item }}" + loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - item.item.item.endswith('70-auplc-gpu-access.rules') + +- name: Parse retained PXE canonical GPU state + ansible.builtin.set_fact: + _pxe_retained_canonical_state: "{{ item.content | b64decode | auplc_from_json_strict }}" + loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - item.item.item.endswith('gpu-access.json') + +- name: Require retained PXE canonical GPU state + ansible.builtin.assert: + that: + - _pxe_retained_canonical_state is mapping + - _pxe_retained_canonical_state.keys() | list | sort == ['renderGid', 'version'] + - _pxe_retained_canonical_state.version is integer + - _pxe_retained_canonical_state.version == 1 + - _pxe_retained_canonical_state.renderGid is integer + - _pxe_retained_canonical_state.renderGid == _pxe_resolved_render_gid + fail_msg: Retained PXE rootfs has a non-canonical GPU access state. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Preflight GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_render_gid: "{{ _pxe_resolved_render_gid }}" + auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' + +- name: Apply GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_render_gid: "{{ _pxe_resolved_render_gid }}" + auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' diff --git a/deploy/ansible/roles/pxe_controller/tasks/main.yml b/deploy/ansible/roles/pxe_controller/tasks/main.yml index b16e2af0..b5217f60 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/main.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/main.yml @@ -82,6 +82,86 @@ # 2. Build NFS rootfs with debootstrap # ========================================================== +- name: Validate PXE rootfs path syntax before lifecycle changes + ansible.builtin.assert: + that: + - pxe_nfs_root is string + - pxe_nfs_root is match('^/') + - pxe_nfs_root != '/' + - "'..' not in pxe_nfs_root.split('/')" + - pxe_nfs_allowed_root is string + - pxe_nfs_allowed_root is match('^/') + fail_msg: pxe_nfs_root and pxe_nfs_allowed_root must be absolute non-root paths without traversal. + +- name: Canonicalize PXE rootfs before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-missing, "{{ pxe_nfs_root }}"] + register: _pxe_canonical_nfs_root_result + changed_when: false + +- name: Canonicalize trusted PXE rootfs parent before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ pxe_nfs_allowed_root }}"] + register: _pxe_canonical_nfs_allowed_root + changed_when: false + +- name: Inspect PXE rootfs path before canonical lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}" + follow: false + register: _pxe_rootfs_lstat + +- name: Constrain canonical PXE rootfs before lifecycle changes + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or not _pxe_rootfs_lstat.stat.islnk + - _pxe_canonical_nfs_root_result.stdout != '/' + - _pxe_canonical_nfs_root_result.stdout == pxe_nfs_root + - _pxe_canonical_nfs_root_result.stdout.startswith(_pxe_canonical_nfs_allowed_root.stdout + '/') + fail_msg: pxe_nfs_root must be a non-symlink descendant of pxe_nfs_allowed_root. + +- name: Record canonical PXE rootfs for lifecycle operations + ansible.builtin.set_fact: + _pxe_canonical_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + pxe_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + +- name: Require existing PXE rootfs is a directory + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or _pxe_rootfs_lstat.stat.isdir + fail_msg: "pxe_nfs_root must be a directory when it already exists: {{ pxe_nfs_root }}" + +- name: Inspect PXE rootfs readiness before lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}/bin/bash" + follow: false + register: _pxe_rootfs_start + +- name: Record PXE rootfs state before lifecycle changes + ansible.builtin.set_fact: + _pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}" + _pxe_rootfs_rebuilt_this_run: >- + {{ (pxe_rootfs_force_rebuild | bool) or not (_pxe_rootfs_lstat.stat.exists | bool) }} + +- name: Require incomplete PXE rootfs force rebuild + ansible.builtin.assert: + that: + - >- + not (_pxe_rootfs_lstat.stat.exists | bool) or + (_pxe_rootfs_start.stat.exists | bool) or + (pxe_rootfs_force_rebuild | bool) + fail_msg: >- + Existing PXE rootfs is incomplete and must be rebuilt with + pxe_rootfs_force_rebuild=true; debootstrap will not modify it in place. + +- name: Admit retained PXE GPU rootfs read-only before lifecycle changes + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: retained-read-only + when: + - pxe_gpu_access_enabled | bool + - not (_pxe_rootfs_rebuilt_this_run | bool) + - name: Stop NFS before rootfs rebuild when: pxe_rootfs_force_rebuild | bool ansible.builtin.systemd: @@ -229,6 +309,13 @@ path: "{{ pxe_nfs_root }}/tmp/chroot-setup.sh" state: absent +- name: Re-preflight PXE GPU rootfs before TFTP + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: final + when: + - pxe_gpu_access_enabled | bool + # ========================================================== # 6. Copy kernel and initrd to TFTP # ========================================================== diff --git a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 index 50ea0867..c504a497 100644 --- a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 +++ b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 @@ -50,12 +50,6 @@ echo "PermitRootLogin yes" > /etc/ssh/sshd_config.d/allow-root.conf echo "PermitRootLogin prohibit-password" > /etc/ssh/sshd_config.d/allow-root.conf {% endif %} -# -- GPU udev rules (let containers access AMD GPUs) -- -tee /etc/udev/rules.d/70-amdgpu.rules << RULES -KERNEL=="kfd", MODE="0666" -KERNEL=="renderD[0-9]*", MODE="0666" -RULES - # -- Disable systemd-networkd (kernel ip=dhcp handles NFS root networking) -- rm -f /etc/netplan/*.yaml systemctl disable systemd-networkd 2>/dev/null || true diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py new file mode 100644 index 00000000..12f76505 --- /dev/null +++ b/tests/skills/test_gpu_access_role.py @@ -0,0 +1,455 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Canonical artifact tests for the multi-node GPU access role.""" + +from pathlib import Path + +import pytest +from ansible.errors import AnsibleFilterError +from jinja2 import Environment + +from deploy.ansible.filter_plugins.auplc_json import ( + DuplicateJsonKeyError, + _reject_duplicate_keys, + auplc_from_json_strict, +) + +ROOT = Path(__file__).resolve().parents[2] +ANSIBLE = ROOT / "deploy" / "ansible" +GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" +PXE_CONTROLLER_ROLE = ANSIBLE / "roles" / "pxe_controller" +PXE_GPU_ACCESS_TASKS = PXE_CONTROLLER_ROLE / "tasks" / "gpu_access.yml" + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_jinja_integer_test_rejects_boolean_and_float_state_values() -> None: + template = Environment().from_string("{% if value is integer %}integer{% else %}invalid{% endif %}") + + assert template.render(value=993) == "integer" + assert template.render(value=True) == "invalid" + assert template.render(value=993.0) == "invalid" + + +def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: + assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { + "renderGid": 993, + "version": 1, + } + + +def test_duplicate_json_key_error_preserves_typed_key() -> None: + with pytest.raises(DuplicateJsonKeyError) as error: + _reject_duplicate_keys([("version", 1), ("version", 2)]) + + assert error.value.key == "version" + assert str(error.value) == "Duplicate JSON object key: 'version'" + + +@pytest.mark.parametrize( + "value", + [ + '{"renderGid":1,"renderGid":993,"version":1}', + '{"renderGid":1,"render\\u0047id":993,"version":1}', + '{"renderGid":1,"version":1,"version":2}', + '{"outer":{"version":1,"version":2}}', + ], +) +def test_strict_json_filter_rejects_semantic_duplicate_keys(value: str) -> None: + with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): + auplc_from_json_strict(value) + + +@pytest.mark.parametrize("value", ["{", '{"renderGid":1,}']) +def test_strict_json_filter_rejects_malformed_json(value: str) -> None: + with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): + auplc_from_json_strict(value) + + +def test_gpu_access_role_renders_the_unified_render_gid_contract() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") + state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") + + assert "auplc_render_gid: null" in defaults + assert "auplc_normalize_render_gid: false" in defaults + assert 'auplc_rootfs_path: ""' in defaults + assert "getent" in tasks + assert "groupmod" in tasks + assert "auplc_normalize_render_gid" in tasks + assert "_auplc_all_groups" in tasks + assert "reject('match', '^render:')" in tasks + assert "_auplc_render_group.stdout.split(':')[2] | int <= 4294967294" in tasks + assert "notify:" not in tasks + assert "Reload live udev rules on every apply" in tasks + assert "Trigger live udev rules on every apply" in tasks + assert "ansible.builtin.group:" not in tasks + assert rules == ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + ) + assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + + +def test_gpu_access_role_is_wired_for_live_hosts_and_pxe_rootfs_without_legacy_udev_paths() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + rocm_tasks = read(ANSIBLE / "roles" / "rocm" / "tasks" / "main.yml") + pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + pxe_chroot = read(ANSIBLE / "roles" / "pxe_controller" / "templates" / "chroot-setup.sh.j2") + + assert "name: gpu_access" in rocm_playbook + assert "name: gpu_access" in udev_playbook + assert "udev-rocm" not in udev_playbook + assert "render:993" not in rocm_tasks + assert "70-amdgpu.rules" not in rocm_tasks + assert "include_tasks: gpu_access.yml" in pxe_tasks + assert "name: gpu_access" in pxe_gpu_tasks + assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in pxe_gpu_tasks + assert "0666" not in pxe_chroot + assert not (ANSIBLE / "roles" / "udev" / "main.yml").exists() + + +def test_gpu_access_live_host_playbooks_abort_all_hosts_on_preflight_failure() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + + assert "any_errors_fatal: true" in rocm_playbook + assert "any_errors_fatal: true" in udev_playbook + + +def test_gpu_access_role_migrates_only_recognized_legacy_rules_and_reconciles_live_devices() -> None: + tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + + assert "70-kfd.rules" in tasks + assert "70-amdgpu.rules" in tasks + assert "contents:" in tasks + assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in tasks + assert "70-rocm-devices.rules" in tasks + assert 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"' in tasks + assert "islnk" in tasks + assert "ansible.builtin.slurp" in tasks + assert "Define recognized project-owned legacy GPU rules" in tasks + assert "Unexpected legacy GPU rule content" in tasks + assert "udevadm" in tasks + assert "Verify /dev/kfd ownership and mode" in tasks + assert "Verify AMD render node ownership and mode" in tasks + assert "Settle live udev events before inode verification" in tasks + assert ( + tasks.index("Trigger live udev rules on every apply") + < tasks.index("Settle live udev events before inode verification") + < tasks.index("Inspect /dev/kfd after live reconciliation") + ) + assert tasks.index("Verify AMD render node ownership and mode") < tasks.index("Persist target GPU access state") + assert "/sys/class/drm" in tasks + assert "readlink" in tasks + assert "basename" in tasks + assert "DRIVER=amdgpu" not in tasks + assert "notify:" not in tasks + + +def test_gpu_access_role_validates_legacy_rules_before_render_gid_normalization() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + tasks = preflight + apply + + assert "Define recognized project-owned legacy GPU rules" in preflight + assert "Inspect recognized project-owned legacy GPU rules" in preflight + assert "Reject legacy GPU rule symlinks and non-regular files" in preflight + assert "Read recognized project-owned legacy GPU rules" in preflight + assert "Reject unexpected legacy GPU rule content" in preflight + assert "follow: false" in preflight + assert "contents:" in preflight + assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in preflight + assert "not item.skipped | default(false)" in preflight + assert "(item.content | b64decode) in item.item.item.contents" in preflight + assert preflight.index("Inspect recognized project-owned legacy GPU rules") < preflight.index( + "Reject legacy GPU rule symlinks and non-regular files" + ) + assert preflight.index("Reject legacy GPU rule symlinks and non-regular files") < preflight.index( + "Read recognized project-owned legacy GPU rules" + ) + assert preflight.index("Read recognized project-owned legacy GPU rules") < preflight.index( + "Reject unexpected legacy GPU rule content" + ) + assert tasks.index("Reject unexpected legacy GPU rule content") < tasks.index("Normalize live render GID") + assert "Remove recognized project-owned legacy GPU rules" in apply + assert "Inspect recognized project-owned legacy GPU rules for apply" in apply + assert "Reject legacy GPU rule symlinks and non-regular files before apply" in apply + assert "Read recognized project-owned legacy GPU rules for apply" in apply + assert "Reject unexpected legacy GPU rule content before apply" in apply + assert "register: _auplc_apply_legacy_gpu_rule_stats" in apply + assert "register: _auplc_apply_legacy_gpu_rule_contents" in apply + assert "_auplc_apply_legacy_gpu_rule_contents.results" in apply + assert "_auplc_legacy_gpu_rule_contents.results" not in apply + assert apply.index("Reject legacy GPU rule symlinks and non-regular files before apply") < apply.index( + "Read recognized project-owned legacy GPU rules for apply" + ) + assert apply.index("Read recognized project-owned legacy GPU rules for apply") < apply.index( + "Reject unexpected legacy GPU rule content before apply" + ) + assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( + "Remove recognized project-owned legacy GPU rules" + ) + assert apply.index("Remove recognized project-owned legacy GPU rules") < apply.index("Normalize live render GID") + + +def test_pxe_rootfs_lifecycle_uses_an_independent_trusted_parent() -> None: + defaults = read(ANSIBLE / "roles" / "pxe_controller" / "defaults" / "main.yml") + tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert 'pxe_nfs_allowed_root: "/srv/nfs"' in defaults + assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in gpu_tasks + assert "Constrain canonical PXE rootfs before lifecycle changes" in tasks + assert tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < tasks.index( + "Admit retained PXE GPU rootfs read-only before lifecycle changes" + ) + + +def test_pxe_rootfs_is_canonicalized_before_gpu_admission() -> None: + tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + + assert "Canonicalize PXE rootfs before lifecycle changes" in tasks + assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( + "Stop NFS before rootfs rebuild" + ) + assert "_pxe_canonical_nfs_root" in tasks + assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( + "Admit retained PXE GPU rootfs read-only before lifecycle changes" + ) + + +def test_gpu_access_preflight_refuses_unmanaged_canonical_destinations() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Inspect canonical GPU access destinations" in preflight + assert "70-auplc-gpu-access.rules" in preflight + assert "gpu-access.json" in preflight + assert "follow: false" in preflight + assert "Reject unmanaged canonical GPU access rule" in preflight + assert "Reject invalid canonical GPU access state" in preflight + assert "auplc_from_json_strict" in preflight + assert "| from_json" not in preflight + assert "renderGid" in preflight + assert "version" in preflight + assert 'src: "{{ _auplc_target_root }}{{ item.item }}"' in preflight + assert "Interrupted normalization retry" in preflight + assert "_auplc_current_render_gid == auplc_render_gid" in preflight + assert "_auplc_existing_state.version is integer" in preflight + assert "_auplc_existing_state.renderGid is integer" in preflight + assert "_auplc_existing_state.renderGid | int" not in preflight + + +def test_gpu_access_roles_use_strict_json_for_canonical_state_readers() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + pxe_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Parse existing canonical GPU access state" in preflight + assert "auplc_from_json_strict" in preflight + assert "auplc_from_json_strict" in pxe_tasks + assert "| from_json" not in preflight + assert "| from_json" not in pxe_tasks + + +def test_canonical_gpu_access_state_contract_is_exact_json() -> None: + state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") + + assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + + +def test_gpu_access_role_splits_safe_preflight_and_rootfs_apply() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") + pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "auplc_gpu_access_enabled: false" in defaults + assert "auplc_rootfs_allowed_root" in defaults + assert "realpath" in validation + assert "auplc_rootfs_path != '/'" in validation + assert "islnk" in preflight + assert "include_tasks: gpu_access.yml" in pxe_tasks + assert "tasks_from: validate" not in pxe_tasks + assert "Constrain canonical PXE rootfs before lifecycle changes" in pxe_tasks + assert pxe_tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < pxe_tasks.index( + "Stop NFS before rootfs rebuild" + ) + assert "tasks_from: preflight" in pxe_gpu_tasks + assert "tasks_from: apply" in pxe_gpu_tasks + assert "auplc_normalize_render_gid:" in pxe_gpu_tasks + assert "pxe_rootfs_force_rebuild | bool" in pxe_tasks + assert "pxe_gpu_access_normalize_render_gid | bool" not in pxe_tasks + assert "pxe_gpu_access_normalize_render_gid" not in read(PXE_CONTROLLER_ROLE / "defaults" / "main.yml") + + +def test_live_playbooks_preflight_gpu_hosts_before_mutating_roles() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + + assert "pre_tasks:" in rocm_playbook + assert "Assert explicit GPU access enablement" in rocm_playbook + assert "auplc_gpu_access_enabled is defined" in rocm_playbook + assert "auplc_gpu_access_enabled is boolean" in rocm_playbook + assert "default(false)" not in rocm_playbook + assert "tasks_from: preflight" in rocm_playbook + assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") + assert "- role: rocm" in rocm_playbook + assert rocm_playbook.count("auplc_gpu_access_enabled") >= 3 + assert "tasks_from: apply" in rocm_playbook + assert "auplc_gpu_access_enabled" in rocm_playbook + assert "pre_tasks:" in udev_playbook + assert "Assert explicit GPU access enablement" in udev_playbook + assert "auplc_gpu_access_enabled is defined" in udev_playbook + assert "auplc_gpu_access_enabled is boolean" in udev_playbook + assert "default(false)" not in udev_playbook + assert "tasks_from: preflight" in udev_playbook + assert "tasks_from: apply" in udev_playbook + + +def test_gpu_access_discovery_playbook_is_read_only_and_serializes_live_host_evidence() -> None: + playbook = read(ANSIBLE / "playbooks" / "pb-gpu-access-discovery.yml") + + assert "hosts: k3s_cluster" in playbook + assert "gather_facts: false" in playbook + assert "ignore_unreachable: true" in playbook + assert "ansible.builtin.command:" in playbook + assert "ansible.builtin.stat:" in playbook + assert "ansible.builtin.slurp:" in playbook + assert "ansible.builtin.shell:" not in playbook + assert "changed_when: false" in playbook + assert "lspci" in playbook + assert '"1002::0300"' in playbook + assert '"1002::0302"' in playbook + assert '"1002::0380"' in playbook + assert "getent" in playbook + assert "/sys/bus/pci/devices" in playbook + assert "gpu_access_discovery_output_path" in playbook + assert "delegate_to: localhost" in playbook + assert "ansible.builtin.copy:" in playbook + assert "to_json" in playbook + assert "stat_success" in playbook + assert "content_success" in playbook + assert "legacy_rules" in playbook + assert "/etc/udev/rules.d/70-kfd.rules" in playbook + assert "/etc/udev/rules.d/70-amdgpu.rules" in playbook + assert "/etc/udev/rules.d/70-rocm-devices.rules" in playbook + file_probes = playbook[ + playbook.index("Inspect persisted GPU access state") : playbook.index( + "Record machine-readable GPU access discovery evidence" + ) + ] + assert file_probes.count("ignore_errors: true") == 10 + assert "failed_when: false" not in file_probes + assert 'mode: "0600"' in playbook + assert "hosts: pxe_controller" not in playbook + + +def test_pxe_gpu_admission_resolves_fresh_rootfs_and_refuses_retained_migrations() -> None: + assert PXE_GPU_ACCESS_TASKS.exists() + + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Record PXE rootfs state before lifecycle changes" in main + assert "_pxe_rootfs_existed_at_start" in main + assert "_pxe_rootfs_rebuilt_this_run" in main + assert "include_tasks: gpu_access.yml" in main + assert main.index("Record PXE rootfs state before lifecycle changes") < main.index("Stop NFS before rootfs rebuild") + assert main.index("include_tasks: gpu_access.yml") < main.index("Find latest kernel in rootfs") + assert "tasks_from: validate" not in main + assert "tasks_from: preflight" not in main + assert "tasks_from: apply" not in main + + assert "_pxe_rootfs_disposition" in tasks + assert "_pxe_unanimous_live_render_gid" in tasks + assert "_pxe_resolved_render_gid" in tasks + assert "fresh" in tasks + assert "retained" in tasks + assert "groupadd" in tasks + assert "--system" in tasks + assert "groupmod" not in tasks + assert "getent" in tasks + assert 'auplc_render_gid: "{{ _pxe_resolved_render_gid }}"' in tasks + assert "auplc_normalize_render_gid: \"{{ _pxe_rootfs_disposition == 'fresh' }}\"" in tasks + assert "Require retained PXE legacy GPU rules absent" in tasks + assert "Require retained PXE render GID matches unanimous live GID" in tasks + assert "tasks_from: preflight" in tasks + assert "tasks_from: apply" in tasks + assert "render:993" not in tasks + assert "lspci" not in tasks + assert "pxe_gpu_access_normalize_render_gid" not in tasks + + +def test_pxe_gpu_admission_preflights_retained_rootfs_before_lifecycle_mutation() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + retained_admission = "Admit retained PXE GPU rootfs read-only before lifecycle changes" + final_admission = "Re-preflight PXE GPU rootfs before TFTP" + + assert main.count("include_tasks: gpu_access.yml") == 2 + assert main.index("Record PXE rootfs state before lifecycle changes") < main.index(retained_admission) + assert main.index(retained_admission) < main.index("Stop NFS before rootfs rebuild") + assert main.index("Remove chroot setup script") < main.index(final_admission) + assert main.index(final_admission) < main.index("Find latest kernel in rootfs") + + retained_branch = main[main.index(retained_admission) : main.index("Stop NFS before rootfs rebuild")] + final_branch = main[main.index(final_admission) : main.index("Find latest kernel in rootfs")] + + assert "pxe_gpu_access_enabled | bool" in retained_branch + assert "not (_pxe_rootfs_rebuilt_this_run | bool)" in retained_branch + assert "pxe_gpu_access_enabled | bool" in final_branch + assert "pxe_gpu_admission_phase: final" in final_branch + + +def test_pxe_rootfs_disposition_uses_initial_root_path_and_rejects_partial_retained_trees() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + assert "Require existing PXE rootfs is a directory" in main + assert "Require incomplete PXE rootfs force rebuild" in main + assert '_pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}"' in main + assert "not (_pxe_rootfs_lstat.stat.exists | bool)" in main + assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("Stop NFS before rootfs rebuild") + assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("- name: Build NFS rootfs") + + +def test_pxe_retained_admission_is_read_only_until_post_chroot_repreflight_and_apply() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main + assert "Re-preflight PXE GPU rootfs before TFTP" in main + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( + "Stop NFS before rootfs rebuild" + ) + assert main.index("Remove chroot setup script") < main.index("Re-preflight PXE GPU rootfs before TFTP") + assert "pxe_gpu_admission_phase: retained-read-only" in main + assert "pxe_gpu_admission_phase: final" in main + assert "Require retained PXE canonical GPU rule" in tasks + assert "Require retained PXE canonical GPU state" in tasks + assert "Apply GPU access after final PXE re-preflight" in tasks + retained_read_only = tasks[: tasks.index("Preflight GPU access after final PXE re-preflight")] + assert "tasks_from: apply" not in retained_read_only + assert "pxe_gpu_admission_phase == 'final'" in tasks + + +def test_pxe_retained_admission_checks_canonical_parent_chain_before_lifecycle_or_chroot_mutation() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Inspect retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU access parents" in tasks + for parent in ("/etc", "/etc/udev", "/etc/udev/rules.d", "/var", "/var/lib", "/var/lib/auplc"): + assert parent in tasks + assert "item.stat.exists" in tasks + assert "item.stat.isdir" in tasks + assert "not item.stat.islnk" in tasks + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( + "Execute chroot setup" + ) From a45d2e8d7f2c65770bf15d8cff96775a9c1477ba Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 057/180] docs(deploy): document unified GPU permission flow --- deploy/README.md | 123 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index c1a8e843..d2a1a72f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,18 +53,115 @@ sudo ./auplc-installer install ### Multi-Node Cluster +Generate the spec, fill in the normal network and node details, then let the +generator discover GPU hosts and their shared `render` GID. The SSH flow asks +for no GPU host list and no GID. A PXE spec asks one extra GPU question: +`pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless +agents' hardware is not inferred from the controller. + +#### SSH-preinstalled + ```bash -# 1. Configure Ansible inventory -cd ansible -vim inventory.yml - -# 2. Run playbooks -sudo ansible-playbook playbooks/pb-base.yml -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# 3. Deploy JupyterHub -cd ../../runtime -cp values-multi-nodes.yaml.example values-multi-nodes.yaml -vim values-multi-nodes.yaml -helm upgrade --install jupyterhub ./chart -n jupyterhub --create-namespace -f values-multi-nodes.yaml +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose ssh-preinstalled and fill the node/network fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml ``` + +Generation runs read-only Ansible discovery against every managed host. It +cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class +records under `/sys/bus/pci/devices`; it does not require the devices to be +attached to `amdgpu` before ROCm installation. It checks +the `render` group and existing GPU access files, and publishes only when every +GPU host agrees on one GID. CPU-only fleets publish `null` for the generated +inventory and Helm render GID. GPU policy details in generated files are +internal outputs, not fields to maintain by hand. + +Configure notebook storage ownership with `singleuser.fsGid: 100`. Never set +storage `fsGroup` through `extraPodConfig.securityContext`, because that Pod +security-context override can replace the GPU resource's generated +`supplementalGroups`. + +#### PXE-diskless + +After setting `topology` to `pxe-diskless`, fill the PXE network fields and set +only `pxe.diskless_agents_have_amd_gpus` for GPU policy. When it is `true`, the +first generation is pending and creates private bootstrap files instead of +canonical deployment files. + +```bash +cd "$REPO_ROOT" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/.pxe-bootstrap.inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/.pxe-bootstrap.vars.yml" + +# pb-pxe-controller finalizes automatically after a successful rootfs build. +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" +``` + +Don't invoke the hidden finalizer yourself. The playbook writes a private +handoff and runs finalization locally. `inventory.yml`, +`pb-pxe-controller.vars.yml`, `values-basic-example.yaml`, and +`gpu-access-resolution.json` appear only after success. + +A fresh PXE rootfs can create a missing `render` group and align it with a +unanimous live controller GPU GID after collision checks. A retained rootfs is +never silently changed. It must already contain one valid `render` group and, +when the controller has a resolved GPU GID, the rootfs GID must match. Rebuild +the rootfs or migrate the retained rootfs separately if it doesn't match. +Offline checks don't replace post-boot verification of GPU device ownership, +mode, supplemental groups, and workload access. + +#### Discovery failures and migration + +| Error | Action | +| --- | --- | +| Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | +| `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | +| Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | +| GPU host has no valid `render` group | Install the correct GPU userspace or create one valid system `render` group, then regenerate. | +| GPU render GIDs disagree | Plan and perform a reviewed group migration so every GPU host uses one free GID, then regenerate. | +| CPU host retains GPU access contract, or canonical state/rule conflicts | Inspect `/var/lib/auplc/gpu-access.json` and `/etc/udev/rules.d/70-auplc-gpu-access.rules`. Remove stale project-owned files from a truly CPU-only host, or complete the GPU migration. Never overwrite unknown content. | +| Retained PXE rootfs GID differs from the unanimous live GID | Rebuild the rootfs, or migrate that retained rootfs separately before rerunning the playbook. | + +Old unshipped specs aren't compatible. Remove the former manual GPU policy +fields, regenerate the schema, copy the ordinary node and PXE network values +into it, and set only `pxe.diskless_agents_have_amd_gpus` on PXE deployments. + +## Deployment branch boundary + +This branch and these instructions do not modify or roll out any live +deployment. SHC, FET, and other deployment branches or environments must +backport the automatic discovery and generated-artifact changes before their +own reviewed rollout. From 5fa4856ddc3b19d1240ee4ed05ff87a0e60b8b4b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:54:16 +0800 Subject: [PATCH 058/180] docs(deploy): align automatic GPU workflow references --- deploy/ansible/README.md | 34 +- skills/deploy-aup-learning-cloud/SKILL.md | 304 +++--------- skills/deploy-aup-learning-cloud/reference.md | 458 ++---------------- .../scripts/README.md | 91 +--- 4 files changed, 140 insertions(+), 747 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 3608aec7..108dad42 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -24,32 +24,14 @@ SOFTWARE. K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible/tree/master). -For full instructions, see [Multi-Node Cluster Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). - -## Quick Reference - -```bash -# Configure inventory -vim inventory.yml - -# Base setup -sudo ansible-playbook playbooks/pb-base.yml - -# Deploy K3s cluster -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Install ROCm GPU drivers -sudo ansible-playbook playbooks/pb-rocm.yml - -# Add new nodes (update inventory.yml first) -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Reset cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml - -# Reset single node -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node_name> -``` +For the generator, canonical inventory, validator arguments, and topology-specific +playbook commands, see the authoritative [deployment guide](../README.md). + +Don't write GPU policy into the inventory by hand. SSH generation discovers GPU +hosts and their shared `render` group ID. PXE generation uses only +`pxe.diskless_agents_have_amd_gpus`; when enabled, the controller playbook uses +private bootstrap inputs and publishes canonical files automatically after a +successful rootfs build. ## Prerequisites diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 6946dca6..2fc9725a 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -1,253 +1,107 @@ --- name: deploy-aup-learning-cloud description: >- - Group: Plan & deploy AUP Learning Cloud. Deploys AUP Learning Cloud (a - multi-node JupyterHub-on-k3s platform for AMD - GPUs) onto physical hardware end to end. Use when the user wants to install, - deploy, set up, or stand up AUP Learning Cloud, AUPLC, or "the learning - cloud" on a cluster; mentions a multi-AIPC or 3-node mini-cluster, PXE / - netboot / diskless agents, the Ansible inventory.yml, pb-pxe-controller, - pb-k3s-site, the ROCm GPU device plugin/labeller, an NFS provisioner, or a - JupyterHub values.yaml / Helm chart for this project. Covers both the - PXE-diskless topology and the SSH-preinstalled multi-node topology. Do not - use for the single-node "./auplc-installer install" flow, for building - notebook images, or for non-AUPLC JupyterHub or k3s installs. + Group: Plan and deploy AUP Learning Cloud. Use when the user wants to install + the multi-node JupyterHub-on-k3s platform on physical hardware through either + PXE-diskless or SSH-preinstalled nodes. Do not use for the single-node + ./auplc-installer flow, notebook image builds, or unrelated JupyterHub and + k3s installations. --- # Deploy AUP Learning Cloud -Stand up AUP Learning Cloud on a multi-node k3s cluster: build the cluster with -Ansible, expose AMD GPUs, provide shared storage, and deploy the JupyterHub -chart with Helm so users can log in and spawn GPU notebooks. +Stand up a multi-node AUP Learning Cloud cluster with Ansible, AMD GPU access, +shared storage, and the JupyterHub Helm chart. -This skill is written for any coding agent. Run the commands and edit the files -as described; the full, copy-runnable command sequence and the troubleshooting -table live in **[reference.md](reference.md)**. +Use [deploy/README.md](../../deploy/README.md) as the source of truth for the +generator schema, commands, generated files, validation, and troubleshooting. +This skill defines the interview and safety gates around that procedure. ## Prerequisites -- A checkout of `aup-learning-cloud` on the operator/service machine. -- The service machine runs Ubuntu 24.04 with a reserved/static IP and internet - access. -- `ansible` on the operator machine; `kubectl` and `helm` for the cluster - (reference.md has the Helm install command). -- For GPU scheduling: AMD GPU nodes with a working in-kernel NIC driver. -- The user supplies the physical hardware. **No site values (IPs, subnet, SSH - keys, tokens) ship in the repo** — this skill generates them. +- A checkout of `aup-learning-cloud` on the operator machine. +- Ubuntu 24.04, a reserved controller IP, internet access, and Ansible. +- Physical node, network, storage, and authentication details from the user. +- Passwordless root SSH to every managed host in the SSH topology. -## Helper script paths +Site values and secrets don't ship in the repository. Generate them locally +and never put tokens, private keys, or credentials in tracked files. -Resolve the deploy helpers before running the commands below. From any directory -in an AUP Learning Cloud checkout: +## Phase 1: Interview -```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -``` +Ask for an explicit topology choice before collecting other details or touching +machines. Never infer the choice from the hardware. -When this skill is installed as a plugin rather than used from a checkout, set -`DEPLOY_SKILL_DIR` to the absolute directory containing the loaded `SKILL.md`, -then derive the helpers from that directory: +| Choice | Use when | +| --- | --- | +| **PXE Diskless Netboot** (`pxe-diskless`) | A controller netboots diskless agents. | +| **Multi Node SSH Installation** (`ssh-preinstalled`) | Every node already runs Ubuntu and accepts root SSH. | -```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" -``` +Then collect and confirm: -## Phase 1 — Interview +1. Courses and notebook resources. +2. Controller hostname, static IP, subnet, gateway, and DNS. +3. For SSH, every managed hostname and IP. Don't ask for a GPU host list or a + shared GPU group ID. Generation discovers both over SSH. +4. For PXE, the controller NIC, web port, rootfs SSH public key, and whether + diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU + policy input because agent hardware can't be inferred from the controller. +5. Shared storage location and the Hub access method. -Work through this in order. **The deployment-method choice (1a) is a hard gate: -ask it first and get an explicit answer before collecting anything else or -touching the machines.** +Confirm detected GPU product labels before mapping them to accelerator keys in +the runtime values. -### Phase 1a — Choose the deployment method (ask first, always) +## Phase 2: Generate -Ask the user to pick one. **Never assume or auto-select** — even when the -machines "look like" one case, present both options and let the user decide (you -may recommend, but you still need an explicit choice before continuing): +Create a fresh schema and fill only its current fields. Run the generator rather +than writing inventory or GPU policy by hand. -| Choose | When | -| --- | --- | -| **PXE Diskless Netboot** (`topology: pxe-diskless`) — one service machine netboots diskless agents | Agents have no OS installed; you want zero per-machine install; small teaching lab. This is the [3-node mini-cluster guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html). | -| **Multi Node SSH Installation** (`topology: ssh-preinstalled`) — every node already runs Ubuntu | Each node has an OS and is reachable over SSH; closer to a long-running lab. This is the [multi-node guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). | - -The value in parentheses is the `topology` field for `gen_configs.py` (Phase 3) -and selects the matching section in [reference.md](reference.md). - -### Phase 1b — Collect the rest (some items branch on the choice above) - -Collect, and confirm back to the user, before touching anything: - -1. **Courses** wanted — drives the `values.yaml` course keys + team mappings - (full catalog setup lives in `configure-aup-learning-cloud-courses`). -2. **Node count** and which node is the controller/server, plus its static IP. - - *SSH path only:* also the hostname + IP of every agent node, and confirm - passwordless root SSH already reaches each one. -3. **GPU — do not ask the user to name the model.** Let the tooling find it: the - detectors report the GPUs (`$DEPLOY_SCRIPTS/detect_hardware.sh` in Phase 2) - and the real ROCm `amd.com/gpu.product-name` label - (`$DEPLOY_SCRIPTS/detect_cluster.sh` in Phase 5). Then - **confirm the detected GPU → accelerator-key mapping with the user** before it - goes into the values file. -4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers - (also auto-detected in Phase 2 and cross-checked), plus at least one SSH - public key for the rootfs and the apache web port. - -Login mode (`custom.authMode`) is unchanged — it stays at its `auto-login` -default; switch it later with `configure-aup-learning-cloud-auth` if needed. The -detailed steps for both paths are in [reference.md](reference.md). - -## Phase 2 — Discover - -On the service machine, run the bundled detector and cross-check its JSON -against the Phase 1 answers: - -```bash -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] -``` - -It reports the default-route NIC, the service-machine IP + subnet CIDR, the -gateway, DNS servers, and each AMD GPU (`lspci`, vendor `1002`) with the bound -`kernel_driver`. If a GPU's `kernel_driver` is empty, note its module for -`pxe_initramfs_modules` (PXE path only). Empty fields come back in `warnings` -so you know exactly what to ask the operator for. The detected GPUs are the -source of truth for the accelerator mapping — Phase 1 does not ask the user to -name them, so surface the detected list and confirm it with the user. - -## Phase 3 — Generate config - -Drive `$DEPLOY_SCRIPTS/gen_configs.py` rather than hand-writing YAML — it keeps the -three artifacts consistent, mints the k3s token locally with a CSPRNG (never -printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. - -```bash -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # fill from Phase 1 + 2 -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -``` - -It writes, into `--out-dir`: - -1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for - PXE; listed for SSH) plus the `pxe_controller` group for PXE. -2. `pb-pxe-controller.vars.yml` — PXE path only: extra vars passed to - `deploy/ansible/playbooks/pb-pxe-controller.yml` with `-e @<absolute-path>` (`pxe_network_interface`, - `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, - `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, - `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). -3. `values-basic-example.yaml` — `custom.accelerators.*.nodeSelector` (matched - to real GPU labels in Phase 5), `custom.resources.images`, the storage class - (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). - -Review the artifacts, install the inventory and runtime overlay into the -checkout, and keep the PXE vars in the generated directory. -**Never commit `inventory.yml` — it holds the token.** Field-by-field guidance -is in [reference.md](reference.md). - -Map the generated artifacts into the checkout before Phase 5 validation: - -```bash -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" - -# PXE only: keep this generated secret in place and use its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" -``` - -The generated `gpu.acceleratorKeys` activates the selected accelerators only -for the generic GPU resource. Wire selected accelerators into course resources -separately with `configure-aup-learning-cloud-courses`. - -## Phase 4 — Execute (with confirmation gates) - -Run the install in order. **Pause for explicit user confirmation before each -risky/irreversible step** (see Safety). The PXE path is, in brief: - -1. Install host packages on the service machine. -2. Run `pb-pxe-controller.yml -e @"$PXE_VARS"` to build the PXE/NFS rootfs, - then verify the - controller (dnsmasq, NFS, apache2, TFTP boot files). -3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. -4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. -5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. - -Run the PXE controller step with the generated vars file: - -```bash -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against -the inventory instead. Full commands for both paths are in [reference.md](reference.md). - -## Phase 5 — GPU, storage, and chart - -1. Install the AMD GPU device plugin + ROCm labeller, then read the **real** - cluster state: - - ```bash -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # nodes[], gpu_product_names[], storage_classes[] - ``` - - Confirm the detected GPU → accelerator-key mapping with the user, then patch - `custom.accelerators.*.nodeSelector` so each `amd.com/gpu.product-name` - matches a value in `gpu_product_names`. Gate the install on a clean - pre-flight (exits non-zero on any mismatch): - - ```bash -# Set this to the topology selected in Phase 1a. -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run -``` - -For the PXE path, the validator and Ansible receive the same generated vars -file. Omit `--pxe-vars "$PXE_VARS"` for the SSH path. - -2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` - (storage class `nfs-client`). -3. Deploy the chart: - -```bash -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml -``` - -## Phase 6 — Validate end to end - -```bash -kubectl get nodes -o wide # server + agents Ready -kubectl get pods -A # nothing CrashLoopBackOff/Pending/ImagePullBackOff -kubectl get storageclass # nfs-client present -``` - -Then open the Hub (NodePort example: `http://<SERVICE_IP>:30890`), log in, -spawn a CPU notebook, confirm file persistence across a restart, then spawn a -GPU notebook and confirm its pod lands on a GPU node -(`kubectl get pods -n jupyterhub -o wide`). +For SSH, generation performs read-only discovery on every managed host and +publishes canonical artifacts only after GPU evidence and group IDs agree. -## Safety +For PXE with GPU agents, initial generation creates private bootstrap inventory +and vars. Run the PXE controller playbook with those private files. A successful +rootfs build finalizes generation automatically and publishes the canonical +inventory, PXE vars, runtime overlay, and GPU resolution report. + +Follow the exact generation, installation, and playbook commands in +[deploy/README.md](../../deploy/README.md). Don't invent a separate completion +step. + +## Phase 3: Validate and execute + +Install the canonical generated inventory and runtime overlay into the checkout, +then run the validator with the arguments shown in the deployment guide: -These steps are destructive or hard to reverse — **stop and get explicit user -confirmation before each one**, and never run them silently: +- `--repo` +- `--topology` +- `--inventory` +- `--gpu-resolution` +- both `--values` files +- `--pxe-vars` for PXE only + +Stop on validation failure. After a clean result, follow the topology's Ansible, +storage, device plugin, and Helm sequence in +[deploy/README.md](../../deploy/README.md). + +## Phase 4: Verify + +Check that all expected nodes are Ready, the GPU labels and allocatable resources +match the generated policy, the storage class is available, and JupyterHub pods +are healthy. Open the Hub, start a CPU notebook, verify persistence, then start a +GPU notebook and confirm it schedules on a GPU node. + +## Safety -- Building/rebuilding the PXE rootfs (`pxe_rootfs_force_rebuild: true`). -- Editing `/etc/exports` and restarting `nfs-kernel-server`. -- `kubectl delete node <name>` (debugging only). -- `helm uninstall` or a cluster reset (`pb-k3s-reset.yml`). -- Changing firmware boot order / disabling Secure Boot on agents. +Pause for explicit user confirmation before rebuilding a PXE rootfs, changing +NFS exports, changing firmware boot settings, resetting a cluster, deleting a +node, or uninstalling a Helm release. -Never commit or push. Never write the k3s token, OAuth secrets, or SSH private -keys into tracked files. Preserve the four AUP Learning Cloud attribution -layers (see the project `AGENTS.md`) if any chart/Hub source is touched. +Never commit or push deployment secrets. Preserve the four AUP Learning Cloud +attribution layers described in the project `AGENTS.md` if Hub or chart sources +are changed. ## Reference -Full step-by-step commands for both topologies, the GPU-label-to-accelerator -mapping, the `values.yaml` field guide, and the troubleshooting table: -[reference.md](reference.md). +- [Deployment commands and troubleshooting](../../deploy/README.md) +- [Skill-specific summary](reference.md) diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index e67faf02..4aafa0fa 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -1,442 +1,40 @@ -# Deploy AUP Learning Cloud — Reference +# Deploy AUP Learning Cloud Reference -Full, copy-runnable commands for both deployment topologies, the GPU label -mapping, the `values.yaml` field guide, and the troubleshooting table. The -workflow and confirmation gates are in [SKILL.md](SKILL.md). +The authoritative procedure, command lines, generated file list, and failure +guidance live in [deploy/README.md](../../deploy/README.md). Don't copy those +commands into this reference. -## Contents +## Topology contract -- [Source guides](#source-guides) -- [PXE-diskless topology (3-node mini-cluster)](#pxe-diskless-topology-3-node-mini-cluster) -- [SSH-preinstalled topology (standard multi-node)](#ssh-preinstalled-topology-standard-multi-node) -- [GPU label to accelerator key](#gpu-label-to-accelerator-key) -- [values.yaml field guide](#valuesyaml-field-guide) -- [Troubleshooting](#troubleshooting) - -## Source guides - -- 3-node mini-cluster (PXE diskless): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html> -- Standard multi-node (SSH): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> - -Treat the live docs as the source of truth for version pins; this file -condenses the opinionated path. - -The helper commands are resolved through `DEPLOY_SCRIPTS` as defined in -[SKILL.md](SKILL.md#helper-script-paths), not through a checkout-root -`scripts/` directory. - -The two topology sections below are the two branches of the Phase 1a gate in -[SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → -[PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node -SSH Installation** (`topology: ssh-preinstalled`) → -[SSH-preinstalled topology](#ssh-preinstalled-topology-standard-multi-node). - -## PXE-diskless topology (3-node mini-cluster) - -One service machine (AIPC 1) runs the PXE controller, the single-node k3s -server, NFS, and the apache k3s-credential endpoint. The other machines are -diskless agents that netboot and auto-join. Only AIPC 1 is Ansible-managed. - -### Step 1 — Prepare the service machine - -```bash -sudo apt update -sudo apt install -y git ansible curl ca-certificates jq \ - dnsmasq pxelinux syslinux-common apache2 \ - nfs-kernel-server debootstrap \ - grub-efi-amd64-signed shim-signed - -ip -br addr # record the NIC and IP -ip route # record the gateway -``` - -Give the local `root` a passwordless SSH login (or add `ansible_connection: -local` to the host vars to skip SSH entirely): - -```bash -sudo install -d -m 0700 /root/.ssh -sudo tee -a /root/.ssh/authorized_keys < ~/.ssh/id_ed25519.pub >/dev/null -sudo chmod 0600 /root/.ssh/authorized_keys -ssh root@<SERVICE_IP> true && echo root-ssh-ok -``` - -### Step 2 — Configure the inventory - -Edit `deploy/ansible/inventory.yml`. AIPC 1 is the only host; the `agent` group -stays empty (netboot agents are not Ansible-managed). Generate the token with -`openssl rand -base64 64` and keep it out of chat/VCS. - -```yaml -k3s_cluster: - children: - server: - hosts: - aipc1: - ansible_host: <SERVICE_IP> - agent: - hosts: {} # diskless netboot agents auto-join; do NOT list them here - vars: - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "<paste-a-strong-random-token>" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" - -pxe_controller: - hosts: - aipc1: - ansible_host: <SERVICE_IP> - vars: - ansible_port: 22 - ansible_user: root -``` - -### Step 3 — Prepare the generated PXE controller vars - -The network, controller, server-IP, and SSH-key values are empty by default and -the role asserts on them. `$DEPLOY_SCRIPTS/gen_configs.py` writes these values -to `generated/pb-pxe-controller.vars.yml`. Keep that file at mode `0600`; it can -contain `pxe_rootfs_password`. Resolve its absolute path for the Ansible and -validator commands instead of copying or merging it into the playbook: - -```bash -PXE_VARS="$(realpath ./generated/pb-pxe-controller.vars.yml)" -chmod 0600 "$PXE_VARS" -test "$(stat -c '%a' "$PXE_VARS")" = 600 -``` - -Review the generated values before the first run: - -```yaml -pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) -pxe_network_interface: "enp1s0" # service-machine NIC (Step 1) -pxe_subnet: "192.168.1.0/24" # node subnet, CIDR -pxe_gateway: "192.168.1.1" # default gateway (informational) -pxe_dns_servers: "8.8.8.8,8.8.4.4" -pxe_controller_ip: "192.168.1.10" # this service machine's IP -pxe_k3s_server_ips: - - "192.168.1.10" -pxe_k3s_version: "v1.32.3+k3s1" # MUST match inventory k3s_version -pxe_web_port: 8080 # apache port for the k3s token/kubeconfig (not 80) -pxe_rootfs_password: "" # optional; empty disables password login (use ansible-vault if set) -pxe_rootfs_authorized_keys: - - "ssh-ed25519 AAAA... you@host" # at least one key required -``` - -Set `pxe_rootfs_force_rebuild: false` after the first stable build so you do -not rebuild the rootfs under running agents. The playbook also exposes -`pxe_apt_mirror`, `pxe_rootfs_packages`, and `pxe_initramfs_modules` (add your -NIC module here if it lacks an in-kernel driver) — leave these at their defaults -unless discovery flagged a need. - -### Step 4 — Run the PXE controller playbook - -```bash -cd ~/aup-learning-cloud -REPO_ROOT="$(pwd)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -PXE_VARS="$(realpath "$REPO_ROOT/generated/pb-pxe-controller.vars.yml")" -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" \ - --topology pxe-diskless --pxe-vars "$PXE_VARS" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -### Step 5 — Verify the controller - -```bash -systemctl is-active dnsmasq nfs-kernel-server apache2 -showmount -e localhost -ls -l /srv/tftp/pxelinux.0 /srv/tftp/grubnetx64.efi /srv/tftp/vmlinuz /srv/tftp/initrd.img -curl -I http://127.0.0.1:8080/k3s/ # 403 expected (dir exists, empty) -``` - -The `/k3s/` endpoint is served on port 8080 (k3s owns 80/443 for ingress). - -### Step 6 — Install the single-node k3s server - -Run **without** `sudo` (key-based root SSH already connects as root): - -```bash -cd ~/aup-learning-cloud/deploy/ansible -ansible-playbook -i inventory.yml playbooks/pb-base.yml -ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml -export KUBECONFIG=~/.kube/config # add to ~/.bashrc to persist -kubectl get nodes -o wide -``` - -### Step 7 — Publish k3s credentials for agents - -```bash -sudo install -d -m 0755 /var/www/html/k3s -sudo install -m 0644 /var/lib/rancher/k3s/server/token /var/www/html/k3s/token -sudo sed "s#https://127.0.0.1:6443#https://<SERVICE_IP>:6443#g" \ - /etc/rancher/k3s/k3s.yaml | sudo tee /var/www/html/k3s/kubeconfig >/dev/null -sudo chmod 0644 /var/www/html/k3s/token /var/www/html/k3s/kubeconfig -sudo systemctl reload apache2 - -curl -fsS http://127.0.0.1:8080/k3s/token >/dev/null && echo token-ok -curl -fsS http://127.0.0.1:8080/k3s/kubeconfig >/dev/null && echo kubeconfig-ok -``` - -### Step 8 — Netboot the agents - -On each agent: disable Secure Boot, enable network boot, and put PXE before the -local disk in the firmware boot order. Boot, then watch them register: - -```bash -watch kubectl get nodes -o wide -``` - -Agents appear as `agent-<mac>` nodes and become `Ready`. - -### Step 9 — Validate agent persistence - -Reboot one agent; confirm it rejoins with the same identity. On the agent: - -```bash -mount | grep /var/lib/rancher/k3s -test -f /var/lib/rancher/k3s/node-password && echo node-password-ok -systemctl status mount-local-disk k3s-agent --no-pager -``` - -`kubectl delete node <name>` clears a stale node object — **debugging only**, -confirm with the user first. - -Continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -## SSH-preinstalled topology (standard multi-node) - -Every node already runs Ubuntu 24.04 and is reachable over passwordless SSH. - -### Prepare SSH and inventory - -Helper scripts in `deploy/scripts/` enable root SSH and distribute kubeconfig: - -```bash -./deploy/scripts/edit_sshd.sh -./deploy/scripts/setup_ssh_root_access.sh -./deploy/scripts/deploy-kubeconfig.sh -``` - -Edit `deploy/ansible/inventory.yml` — list every node under `server`/`agent`: - -```yaml -k3s_cluster: - children: - server: - hosts: - <SERVER-HOSTNAME>: - agent: - hosts: - <AGENT-HOSTNAME-1>: - <AGENT-HOSTNAME-2>: - vars: - ansible_port: 22 - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "<strong-random-token>" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" -``` - -### Build the cluster - -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-base.yml # base OS / packages -sudo ansible-playbook playbooks/pb-k3s-site.yml # deploy k3s -sudo ansible-playbook playbooks/pb-rocm.yml # ROCm on GPU nodes -``` - -Related: `pb-k3s-upgrade.yml` (upgrade), `pb-k3s-reset.yml` (reset — RISKY). -Then install `kubectl`/`helm` on the operator machine (see Helm command below) -and continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -### Install Helm - -```bash -wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz -cd /tmp && tar -zxvf helm.tar.gz -sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm -``` - -## Step 10 — AMD GPU device plugin and labeller - -```bash -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml - -kubectl get pods -A | grep -i amd -kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu -``` - -Use the labels that actually appear. Common keys: -`amd.com/gpu.product-name`, `amd.com/gpu.family`, `amd.com/gpu.device-id`. - -## Step 11 — Shared NFS storage for notebook PVCs - -This is separate from the PXE rootfs export. Append the export directly to -`/etc/exports` (on Ubuntu 24.04 `/etc/exports.d/*.conf` is ignored): - -```bash -sudo mkdir -p <NFS_EXPORT> -sudo chown -R nobody:nogroup <NFS_EXPORT> -sudo chmod 0777 <NFS_EXPORT> -echo "<NFS_EXPORT> <CLUSTER_SUBNET>(rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports -sudo exportfs -ra -sudo systemctl restart nfs-kernel-server -showmount -e localhost -``` - -Install the provisioner (storage class `nfs-client`): - -```bash -cd ~/aup-learning-cloud -cp deploy/k8s/nfs-provisioner/values.yaml deploy/k8s/nfs-provisioner/values.local.yaml -# edit values.local.yaml: nfs.server, nfs.path, storageClass.name = nfs-client -helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ -helm repo update -helm upgrade --install nfs-subdir-external-provisioner \ - nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ - --namespace nfs-provisioner --create-namespace \ - -f deploy/k8s/nfs-provisioner/values.local.yaml -kubectl get storageclass -``` - -## Step 12 — Configure JupyterHub values - -The generated `runtime/values-basic-example.yaml` is the canonical deployment -overlay. Review and keep it when Phase 3 generated one. Only when no generated -overlay exists, start a manual overlay from the example: - -```bash -cd ~/aup-learning-cloud/runtime -if [ ! -e values-basic-example.yaml ]; then - cp values-multi-nodes.yaml.example values-basic-example.yaml -fi -``` - -Minimum edits (see the [field guide](#valuesyaml-field-guide)): - -```yaml -custom: - authMode: "auto-login" # single-machine default; avoid "dummy" (login 404s) - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: "<GPU_PRODUCT_LABEL>" # from Step 10 - quotaRate: 3 - resources: - images: - cpu: "<CPU_NOTEBOOK_IMAGE>" - gpu: "<GPU_NOTEBOOK_IMAGE>" -hub: - db: - pvc: - storageClassName: nfs-client -singleuser: - storage: - dynamic: - storageClass: nfs-client -proxy: - service: - type: NodePort - nodePorts: - http: 30890 -``` - -## Step 13 — Deploy AUP Learning Cloud - -```bash -cd ~/aup-learning-cloud -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml - -kubectl get pods -n jupyterhub -o wide -kubectl get svc -n jupyterhub -``` - -For later config changes, re-run the same `helm upgrade --install`. - -## Step 14 — End-to-end validation - -```bash -kubectl get nodes -o wide -kubectl get pods -A -kubectl get storageclass -kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu -``` - -Then browse to `http://<SERVICE_IP>:30890` (or your ingress host), log in, -spawn a CPU notebook, create a file, restart and confirm it persists, then -spawn a GPU notebook and confirm its pod lands on a GPU node. - -## GPU label to accelerator key - -The chart's accelerator catalog (`runtime/values.yaml`) is keyed by accelerator -names; map the ROCm labeller's `amd.com/gpu.product-name` to the right key. The -GPU is auto-detected (Phase 2 and Phase 5), not named by the user in the -interview — use this table to confirm the detected product-name → key mapping -with the user. Verify against the live values file — product names can normalize -differently per fleet. - -| `amd.com/gpu.product-name` (example) | Accelerator key | +| Topology | Generator behavior | | --- | --- | -| `AMD_Radeon_780M_Graphics` | `phx` | -| `AMD_Radeon_890M_Graphics` | `strix` | -| `AMD_Radeon_8060S_Graphics` | `strix-halo` | -| `AMD_Radeon_RX_9070_XT` | `9070xt` | -| `AMD_Radeon_AI_PRO_R9700` | `r9700` | -| `AMD_Radeon_RX_9600_GRE` | `9600gre` | - -If your labeller reports a different product name, update the matching -`custom.accelerators.*.nodeSelector` entry to that exact string. +| `ssh-preinstalled` | Connects to every managed host, discovers GPU hosts and their shared `render` group ID, and publishes canonical files only when discovery is consistent. | +| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input. GPU-enabled first generation emits private bootstrap files; the PXE controller playbook finalizes canonical files after a successful rootfs build. | -## values.yaml field guide +Don't hand-author generated GPU policy. Old unshipped specs should be recreated +from the current `--print-schema` output. -Sections to review in the generated `values-basic-example.yaml`, or in the -manual `values-multi-nodes.yaml.example` copy when generation was not used: - -| Field | Purpose | -| --- | --- | -| `custom.authMode` | `auto-login` for the single-machine example; OAuth modes for real auth | -| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | GitHub OAuth (when not auto-login) | -| `custom.adminUser` | Hub admin | -| `custom.accelerators.*.nodeSelector` | Must match real `amd.com/gpu.*` labels | -| `custom.resources.images` | CPU/GPU/course notebook images | -| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Per-team resources and quotas | -| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | `nfs-client` for multi-node | -| `proxy.service`, `ingress` | NodePort (e.g. 30890) or ingress host | +## Canonical validation inputs -## Troubleshooting +Use the validator command from [deploy/README.md](../../deploy/README.md). It +passes: -| Symptom | Likely cause | First checks | -| --- | --- | --- | -| Playbook fails immediately on an assert | A required PXE var is empty | Re-check `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, and at least one SSH key | -| Agent never shows the PXE menu | Firmware boot order, network boot disabled, or Proxy-DHCP not reaching the client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | -| Agent gets an IP but cannot load boot files | TFTP blocked, missing files, or Secure Boot still on | `/srv/tftp`, firewall, Secure Boot disabled, `dnsmasq` logs | -| Agent has no network during netboot | NIC has no in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | -| Agent kernel boots but cannot mount rootfs | NFS export, subnet ACL, or wrong `pxe_controller_ip` | `showmount -e <SERVICE_IP>`, `/etc/exports`, rootfs kernel args | -| Agent waits for the k3s token | Token not published or apache ACL blocks the subnet | `curl http://<SERVICE_IP>:8080/k3s/token`, apache config | -| Agent joins once but fails after reboot | Missing local k3s persistence or lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | -| Agent fails to join with a version error | Agent rootfs k3s newer than the server | Align `pxe_k3s_version` with `k3s_version`, rebuild rootfs | -| Agent node does not join (SSH path) | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent`, `/etc/hosts` | -| GPU notebook stays Pending | Chart `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub`, node labels | -| PVC stays Pending | StorageClass name mismatch or NFS provisioner cannot mount | `kubectl get storageclass`, provisioner logs, NFS export | -| `kubectl` permission denied on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | +- repository root with `--repo` +- selected topology with `--topology` +- installed inventory with `--inventory` +- generated GPU resolution report with `--gpu-resolution` +- base and generated overlays as two `--values` arguments +- canonical PXE vars with `--pxe-vars` for PXE only -For a complete reset (RISKY — confirm with the user): +Generation and validation must finish before Ansible or Helm changes are made. -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node> # single node -``` +## Operator gates -## Out of scope +Keep the topology choice explicit. Confirm network, node, storage, course, and +access details with the user. For PXE, also confirm the GPU-agent boolean and a +rootfs SSH public key. For SSH, verify passwordless root access to every managed +host. -Zot registry mirror, Cloudflare Tunnel ingress, monitoring/Grafana, HA k3s, -external databases, and NPU setup. Add them only after the minimal deployment -boots agents, schedules GPU notebooks, and persists notebook storage. +Require confirmation before rootfs rebuilds, NFS export changes, firmware boot +changes, cluster resets, node deletion, or Helm uninstall. Keep generated +secrets out of version control. diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 4e212dcc..65bd71c1 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -1,77 +1,36 @@ # Helper scripts -Deterministic helpers the deploy skill runs instead of generating commands ad -hoc. They are dependency-light (`bash` + `python3`, plus the obvious system -tools) and agent-agnostic, and follow the script conventions in -[../../../CONTRIBUTING.md](../../../CONTRIBUTING.md). Each emits JSON or a clear -report and uses exit codes the agent can branch on. +These dependency-light helpers support the multi-node deployment skill. See +[deploy/README.md](../../../deploy/README.md) for the authoritative command +sequence and argument paths. -| Script | Run when | What it does | -| --- | --- | --- | -| `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | -| `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | -| `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | -| `validate.py` | Before each `ansible-playbook` / `helm` run | For `pxe-diskless`, checks required PXE vars and `k3s_version == pxe_k3s_version`; for both topologies, checks GPU labels only for active resource `acceleratorKeys` (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | +| Script | Purpose | +| --- | --- | +| `detect_hardware.sh` | Reports controller network details and local AMD PCI devices as JSON. | +| `detect_cluster.sh` | Reports Kubernetes nodes, AMD GPU labels, storage classes, and GPU DaemonSet state as JSON. | +| `gen_configs.py` | Prints the current spec schema, discovers SSH GPU state, and generates topology-specific deployment artifacts. PXE GPU bootstrap files remain private until the controller playbook finalizes them automatically. | +| `validate.py` | Checks the selected topology against canonical inventory, GPU resolution, values overlays, and PXE vars when applicable. | -## Quick reference +## Generator contract -From any directory in a checkout, resolve helpers with: +The SSH topology discovers GPU hosts and their shared `render` group ID. Users +don't provide either value. The PXE topology has one GPU policy input: +`pxe.diskless_agents_have_amd_gpus`. -```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -``` +Generate specs from fresh `--print-schema` output. Don't hand-edit generated GPU +policy or add a separate PXE completion step. -For an installed plugin, set `DEPLOY_SKILL_DIR` to the absolute directory -containing the loaded `SKILL.md`, then use: +## Validator contract -```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" -``` - -```bash -# Phase 2 — discover the host -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] - -# Phase 3 — generate config from a spec -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # then edit spec.json -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" -# PXE only: keep the generated secret in place and resolve its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" - -# Phase 5 — after k3s + device plugin are up -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] - -# Before running playbooks / helm (set to the selected topology) -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run -``` - -Omit `--pxe-vars "$PXE_VARS"` for `ssh-preinstalled`. For `pxe-diskless`, the -validator and Ansible must receive the same generated file. - -Generated `gpu.acceleratorKeys` wires the selected accelerators to the generic -GPU resource. Use `configure-aup-learning-cloud-courses` to wire course -resources separately. +Use the exact validator command in +[deploy/README.md](../../../deploy/README.md). Its canonical inputs are +`--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` +arguments, and `--pxe-vars` for PXE only. ## Conventions -- **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON - object; partial detection is reported via empty fields + a `warnings` array - rather than failing, so the agent can decide what to ask the operator. -- **Exit codes mean something.** `0` success (warnings allowed), `1` a real - validation failure, `2` a usage / missing-tooling error. -- **Secrets never touch stdout or VCS.** `gen_configs.py` mints the k3s token - with a CSPRNG, writes it only into `inventory.yml`, and `chmod 600`s it. -- **No third-party Python.** `gen_configs.py` / `validate.py` use the stdlib - only (no PyYAML), so they run on a bare operator machine. YAML is emitted - from templates and parsed with targeted scanning. +- Detection data goes to stdout as JSON. Diagnostics go to stderr. +- Exit code `0` means success, `1` means validation failed, and `2` means usage + or required tooling is wrong. +- Generated secrets stay off stdout and out of version control. +- Python helpers use the standard library only. From c781cb5f5a27560b5552cbe7e8ea0877674bfd02 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:20:28 +0800 Subject: [PATCH 059/180] fix(tests): isolate GPU role skill dependencies --- tests/skills/test_gpu_access_role.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index 12f76505..b195fef5 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -2,12 +2,25 @@ """Canonical artifact tests for the multi-node GPU access role.""" +import sys +import types from pathlib import Path import pytest -from ansible.errors import AnsibleFilterError -from jinja2 import Environment +try: + from ansible.errors import AnsibleFilterError +except ModuleNotFoundError: + ansible_module = types.ModuleType("ansible") + errors_module = types.ModuleType("ansible.errors") + + class AnsibleFilterError(Exception): + pass + + errors_module.AnsibleFilterError = AnsibleFilterError + ansible_module.errors = errors_module + sys.modules["ansible"] = ansible_module + sys.modules["ansible.errors"] = errors_module from deploy.ansible.filter_plugins.auplc_json import ( DuplicateJsonKeyError, _reject_duplicate_keys, @@ -25,14 +38,6 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_jinja_integer_test_rejects_boolean_and_float_state_values() -> None: - template = Environment().from_string("{% if value is integer %}integer{% else %}invalid{% endif %}") - - assert template.render(value=993) == "integer" - assert template.render(value=True) == "invalid" - assert template.render(value=993.0) == "invalid" - - def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { "renderGid": 993, From db73537e224dd0949551d41073ab66d19630996b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 060/180] refactor(hub): remove GPU group injection --- runtime/hub/core/config.py | 31 -------------- runtime/hub/core/spawner/kubernetes.py | 26 ------------ runtime/hub/tests/test_spawner_gpu_access.py | 43 ++++++++------------ 3 files changed, 17 insertions(+), 83 deletions(-) diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 28e24ded..3925bbf0 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -45,8 +45,6 @@ import yaml from pydantic import BaseModel, Field, field_validator -MAX_RENDER_GID = (2**32) - 2 - # ============================================================================= # YAML Configuration Models # ============================================================================= @@ -87,25 +85,6 @@ class QuotaSettings(BaseModel): model_config = {"extra": "allow"} -class GpuAccessSettings(BaseModel): - """Host group access settings for GPU-enabled user pods.""" - - renderGid: int | None = None - - @field_validator("renderGid", mode="before") - @classmethod - def validate_render_gid(cls, value: Any) -> int | None: - """Require a native positive integer GID when GPU access is configured.""" - - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_RENDER_GID: - raise ValueError(f"custom.gpuAccess.renderGid must be an integer between 1 and {MAX_RENDER_GID}") - return value - - model_config = {"extra": "allow"} - - class AcceleratorOverride(BaseModel): """Per-accelerator overrides for a resource (image and/or env).""" @@ -232,7 +211,6 @@ class ParsedConfig(BaseModel): accelerators: dict[str, AcceleratorConfig] = Field(default_factory=dict) teams: TeamsConfig = Field(default_factory=TeamsConfig) quota: QuotaSettings = Field(default_factory=QuotaSettings) - gpuAccess: GpuAccessSettings = Field(default_factory=GpuAccessSettings) gitClone: GitCloneSettings = Field(default_factory=GitCloneSettings) hub: HubNetworkSettings = Field(default_factory=HubNetworkSettings) notebook: NotebookNetworkSettings = Field(default_factory=NotebookNetworkSettings) @@ -248,7 +226,6 @@ def from_dicts( accelerators: dict | None = None, teams: dict | None = None, quota: dict | None = None, - gpu_access: dict | None = None, git_clone: dict | None = None, hub: dict | None = None, notebook: dict | None = None, @@ -266,8 +243,6 @@ def from_dicts( raw_config["teams"] = teams if quota: raw_config["quota"] = quota - if gpu_access is not None: - raw_config["gpuAccess"] = gpu_access if git_clone: raw_config["gitClone"] = git_clone if hub: @@ -359,7 +334,6 @@ def init(cls, config_path: str | Path) -> HubConfig: accelerators=raw_config.get("accelerators"), teams=raw_config.get("teams"), quota=raw_config.get("quota"), - gpu_access=raw_config.get("gpuAccess"), git_clone=raw_config.get("gitClone"), hub=raw_config.get("hub"), notebook=raw_config.get("notebook"), @@ -437,11 +411,6 @@ def quota(self) -> QuotaSettings: """Get quota configuration.""" return self._config.quota - @property - def gpu_access(self) -> GpuAccessSettings: - """Get GPU pod access configuration.""" - return self._config.gpuAccess - @property def git_clone(self) -> GitCloneSettings: """Get git clone configuration.""" diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index a99b7fff..dfe2e547 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -40,7 +40,6 @@ from kubespawner import KubeSpawner from tornado import web -from core.config import MAX_RENDER_GID from core.metrics import ( pod_failure_total, repo_clone_failed_total, @@ -94,7 +93,6 @@ class RemoteLabKubeSpawner(KubeSpawner): auth_mode: str = "auto-login" single_node_mode: bool = False quota_enabled: bool | None = False - render_gid: int | None = None # Resource configuration (set from config) resource_images: dict[str, str] = {} @@ -156,7 +154,6 @@ def configure_from_config(cls, config: HubConfig) -> None: cls.default_quota = config.quota.defaultQuota cls.minimum_quota_to_start = config.quota.minimumToStart cls.quota_enabled = config.quota.enabled - cls.render_gid = config.gpu_access.renderGid # Extract git clone settings (single source of truth: GitCloneSettings) git_config = config.git_clone @@ -810,28 +807,6 @@ def _reset_per_spawn_state(self) -> None: self._has_git_init_container = False - def _add_gpu_render_gid(self) -> None: - """Add the configured host render group to a GPU resource's pod.""" - if self.render_gid is None: - raise RuntimeError( - "GPU resource requires custom.gpuAccess.renderGid. " - "Set it to the numeric GID of the host render group before spawning GPU resources." - ) - if ( - isinstance(self.render_gid, bool) - or not isinstance(self.render_gid, int) - or not 1 <= self.render_gid <= MAX_RENDER_GID - ): - raise RuntimeError( - "GPU resource requires a valid custom.gpuAccess.renderGid. " - f"Set it to an integer between 1 and {MAX_RENDER_GID} before spawning GPU resources." - ) - - supplemental_gids = list(self.supplemental_gids or []) - if self.render_gid not in supplemental_gids: - supplemental_gids.append(self.render_gid) - self.supplemental_gids = supplemental_gids - def _configure_spawner(self, resource_type: str, gpu_selection: str | None = None) -> None: """Configure the spawner based on the resource type and GPU selection.""" @@ -896,7 +871,6 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non if "amd.com/gpu" in requirements: self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} - self._add_gpu_render_gid() elif "amd.com/npu" in requirements: self.log.debug("NPU DEVICE PLUGIN are removed, amd.com/npu is no more needed") diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py index edacf4c2..4ea6f476 100644 --- a/runtime/hub/tests/test_spawner_gpu_access.py +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -8,7 +8,6 @@ from unittest.mock import patch import pytest -from pydantic import ValidationError ROOT = Path(__file__).resolve().parents[1] CORE = ROOT / "core" @@ -86,7 +85,6 @@ def load_spawner_module(): return load_module("gpu_access_test_spawner", CORE / "spawner" / "kubernetes.py") -config = load_module("core.config", CORE / "config.py") kubernetes = load_spawner_module() RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner @@ -110,10 +108,9 @@ def get_resource_metadata(self, _resource_type): return ResourceMetadata() -def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = None): +def make_spawner(supplemental_gids: list[int] | None = None): spawner = object.__new__(RemoteLabKubeSpawner) spawner._hub_config = HubConfig() - spawner.render_gid = render_gid spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image"} spawner.resource_requirements = { "cpu": {"cpu": "1", "memory": "1Gi"}, @@ -139,42 +136,36 @@ def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = N return spawner -def test_gpu_render_gid_is_injected_only_for_gpu_pods(): - spawner = make_spawner(render_gid=993) +def test_gpu_pod_requests_accelerator_without_changing_generic_supplemental_groups(): + spawner = make_spawner(supplemental_gids=[1234]) spawner._configure_spawner("gpu", "gpu-a") gpu_manifest = spawner.get_pod_manifest() + + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} + spawner._configure_spawner("cpu") cpu_manifest = spawner.get_pod_manifest() - assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [993]} - assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100} + assert spawner.extra_resource_guarantees == {} + assert spawner.extra_resource_limits == {} + assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} -def test_gpu_render_gid_preserves_existing_supplemental_groups(): - spawner = make_spawner(render_gid=993, supplemental_gids=[1234]) +def test_gpu_pod_without_generic_supplemental_groups_uses_storage_fs_group_only(): + spawner = make_spawner() spawner._configure_spawner("gpu", "gpu-a") - assert spawner.supplemental_gids == [1234, 993] - - -def test_gpu_spawn_requires_a_host_render_gid(): - spawner = make_spawner(render_gid=None) - - with pytest.raises(RuntimeError, match=r"custom\.gpuAccess\.renderGid"): - spawner._configure_spawner("gpu", "gpu-a") - - -def test_gpu_access_config_validates_render_gid(): - assert config.GpuAccessSettings(renderGid=993).renderGid == 993 - assert config.ParsedConfig.from_dicts(gpu_access={"renderGid": 993}).gpuAccess.renderGid == 993 - with pytest.raises(ValidationError, match="renderGid"): - config.GpuAccessSettings(renderGid=True) + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert spawner.get_pod_manifest()["spec"]["securityContext"] == {"fsGroup": 100} def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): - spawner = make_spawner(render_gid=993) + spawner = make_spawner() spawner._resolve_user_resources = lambda: ["cpu"] spawner._configure_spawner = lambda *_args: pytest.fail("unauthorized resource configured the spawner") From 3f2c32238ed1c2a4ea411eabb24825506be5895a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 061/180] refactor(chart): remove GPU GID settings --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 14 -------------- runtime/chart/values.yaml | 5 ----- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index f53227d3..ef7efffc 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"gpuAccess":{"type":"object","additionalProperties":false,"properties":{"renderGid":{"type":["integer","null"],"minimum":1,"maximum":4294967294}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 3eab8688..22ff5fec 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3195,20 +3195,6 @@ properties: Enable auto-admin creation on first install. Credentials will be stored in `jupyterhub-admin-credentials` secret. - gpuAccess: - type: object - additionalProperties: false - description: | - Host group access settings for GPU-enabled user pods. - properties: - renderGid: - type: [integer, "null"] - minimum: 1 - maximum: 4294967294 - description: | - Numeric GID of the host render group. GPU resources receive this - as a supplemental group; CPU resources do not. - notifications: type: object additionalProperties: false diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index e888f22f..a548691f 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -50,11 +50,6 @@ custom: # Define these in runtime/values.yaml, not here accelerators: {} - # Host render-group access for GPU user pods. The installer overlay sets this - # to the detected host render GID when GPU access is provisioned. - gpuAccess: - renderGid: null - # Resource images, requirements, and metadata # Define these in runtime/values.yaml, not here resources: From 075f29d3742ee8d839ac46a72a1884ffa6bb4998 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 062/180] fix(runtime): keep storage group only --- runtime/values-multi-nodes.yaml.example | 12 +++++------- runtime/values.yaml | 7 ++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 63c48382..4af6d259 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -22,7 +22,9 @@ # cp values-multi-nodes.yaml.example values-multi-nodes.yaml # # Prerequisites: -# - Install the AMD GPU device plugin and ROCm node labeller on GPU nodes. +# - The infrastructure owner must deploy and maintain the AMD GPU device plugin +# and ROCm node labeller outside AUPLC. Before Helm, run the readiness and +# capacity checks in deploy/README.md. # - Install an RWX-capable StorageClass for user homes; this example uses the # NFS provisioner from deploy/k8s/nfs-provisioner with class nfs-client. # - Create registry pull secrets only if you use private images. @@ -139,11 +141,6 @@ custom: defaultPersistence: true allowPersistenceChoice: false - # Generated deployment overlays resolve this from corroborated host evidence. - # Keep null in the base example; do not choose a fleet GID manually here. - gpuAccess: - renderGid: null - # -------------------------------------------------------------------------- # Accelerator Configuration # -------------------------------------------------------------------------- @@ -581,7 +578,8 @@ monitoring: enabled: false singleuser: - # Must match the storage ownership group used by the shared volume. + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. fsGid: 100 storage: dynamic: diff --git a/runtime/values.yaml b/runtime/values.yaml index 7bce7308..3dd59180 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -61,10 +61,6 @@ custom: adminUser: enabled: false - # The installer overlay supplies the host render GID for GPU user pods. - gpuAccess: - renderGid: null - # ============================================================================ # Notifications # ============================================================================ @@ -672,7 +668,8 @@ monitoring: enabled: false singleuser: - # Preserve storage volume ownership without replacing KubeSpawner's security context. + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. fsGid: 100 storage: From ee6c31ce3f85826339f1df5a6fe5a8bab117f257 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 063/180] refactor(installer): simplify GPU host policy --- auplc_installer/gpu_access.py | 213 ++------ tests/installer/test_gpu_access.py | 523 +++++--------------- tests/scripts/test_gpu_image_permissions.py | 6 +- 3 files changed, 160 insertions(+), 582 deletions(-) diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py index 7c46991a..c3a946a7 100644 --- a/auplc_installer/gpu_access.py +++ b/auplc_installer/gpu_access.py @@ -1,24 +1,14 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Single-node AMD GPU device-access source of truth. - -The host's existing ``render`` group is authoritative. Its numeric GID is -persisted here so installer reruns and runtime-only commands cannot silently -select a different permission model. -""" +"""Single-node AMD GPU device-access reconciler.""" from __future__ import annotations -import json -from dataclasses import dataclass from pathlib import Path from typing import Protocol from auplc_installer.util import InstallerError, run, run_capture -GPU_ACCESS_STATE_VERSION = 1 -MAX_RENDER_GID = (2**32) - 2 -GPU_ACCESS_STATE_PATH = Path("/var/lib/auplc/gpu-access.json") GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") @@ -46,8 +36,9 @@ UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." CANONICAL_UDEV_RULES = ( f"{UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) _FSYNC_PATH_SCRIPT = ( "import os\n" @@ -59,44 +50,40 @@ " os.close(fd)\n" ) _VERIFY_DEVICE_ACCESS_SCRIPT = ( - "import os, pathlib, stat, sys\n" - "gid = int(sys.argv[1])\n" - "paths = [pathlib.Path('/dev/kfd')]\n" - "for node in pathlib.Path('/sys/class/drm').glob('renderD*'):\n" + "import grp, pathlib, stat\n" + "drm = pathlib.Path('/sys/class/drm')\n" + "devices = [(pathlib.Path('/dev/kfd'), 'render', 0o666)]\n" + "render_nodes = []\n" + "for node in drm.glob('renderD*'):\n" + " driver = node / 'device' / 'driver'\n" + " if driver.exists() and driver.resolve().name == 'amdgpu':\n" + " render_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" + "if not render_nodes: raise SystemExit('no AMD renderD device found')\n" + "devices.extend((path, 'render', 0o666) for path in render_nodes)\n" + "card_nodes = []\n" + "for node in drm.glob('card*'):\n" " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu': paths.append(pathlib.Path('/dev/dri') / node.name)\n" - "if len(paths) == 1: raise SystemExit('no AMD renderD device found')\n" - "for path in paths:\n" + " if driver.exists() and driver.resolve().name == 'amdgpu':\n" + " card_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" + "if not card_nodes: raise SystemExit('no AMD card device found')\n" + "devices.extend((path, 'video', 0o666) for path in card_nodes)\n" + "for path, expected_group, expected_mode in devices:\n" " data = path.lstat()\n" - " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or data.st_gid != gid or stat.S_IMODE(data.st_mode) != 0o660: raise SystemExit(f'bad GPU device access: {path}')\n" + " try:\n" + " group_name = grp.getgrgid(data.st_gid).gr_name\n" + " except KeyError:\n" + " raise SystemExit(f'unknown GPU device group: {path}')\n" + " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or group_name != expected_group or stat.S_IMODE(data.st_mode) != expected_mode:\n" + " raise SystemExit(f'bad GPU device access: {path}')\n" ) -@dataclass(frozen=True) -class GpuAccessState: - """Versioned, immutable record of the host render-group GID.""" - - render_gid: int - version: int = GPU_ACCESS_STATE_VERSION - - def __post_init__(self) -> None: - if self.version != GPU_ACCESS_STATE_VERSION: - raise InstallerError(f"Unsupported GPU access state version: {self.version!r}") - _validate_render_gid(self.render_gid) - - class GpuAccessHost(Protocol): """Privileged host-operation seam for GPU access provisioning.""" - def get_group_entry(self, group_name: str) -> str: - """Return the NSS group record for ``group_name``.""" - def read_text(self, path: Path) -> str | None: """Return a privileged file's text, or ``None`` when it is absent.""" - def write_state_atomically(self, path: Path, text: str) -> None: - """Atomically replace a state file with same-directory persistence.""" - def write_udev_rule(self, path: Path, text: str) -> None: """Write a managed udev rule after reconciliation has authorized it.""" @@ -112,8 +99,8 @@ def settle_udev(self) -> None: def remove_udev_rule(self, path: Path) -> None: """Remove an explicitly recognized legacy udev rule.""" - def verify_device_access(self, render_gid: int) -> None: - """Verify the relevant GPU device inodes use the requested access contract.""" + def verify_device_access(self) -> None: + """Verify the relevant GPU device inodes use the host access contract.""" def is_symlink(self, path: Path) -> bool: """Return whether ``path`` is a symlink without following it.""" @@ -131,12 +118,6 @@ def is_directory(self, path: Path) -> bool: class SystemGpuAccessHost: """Production host adapter using the installer's sudo-aware command helpers.""" - def get_group_entry(self, group_name: str) -> str: - result = run_capture(["getent", "group", group_name], check=False) - if result.returncode != 0: - return "" - return result.stdout or "" - def read_text(self, path: Path) -> str | None: exists = run(["test", "-e", str(path)], sudo=True, check=False) if exists.returncode != 0: @@ -144,10 +125,6 @@ def read_text(self, path: Path) -> str | None: result = run_capture(["cat", str(path)], sudo=True) return result.stdout or "" - def write_state_atomically(self, path: Path, text: str) -> None: - """Durably replace state with a same-directory temporary file.""" - self._write_text_atomically(path, text) - def write_udev_rule(self, path: Path, text: str) -> None: self._write_text_atomically(path, text) @@ -161,7 +138,7 @@ def _write_text_atomically(self, path: Path, text: str) -> None: ) temporary_path = (temporary_result.stdout or "").strip() if not temporary_path: - raise InstallerError(f"Could not create temporary GPU access state beside {path}") + raise InstallerError(f"Could not create temporary GPU access rule beside {path}") try: run(["tee", temporary_path], sudo=True, input_text=text) @@ -188,8 +165,8 @@ def settle_udev(self) -> None: def remove_udev_rule(self, path: Path) -> None: run(["rm", "-f", str(path)], sudo=True) - def verify_device_access(self, render_gid: int) -> None: - run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT, str(render_gid)], sudo=True) + def verify_device_access(self) -> None: + run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT], sudo=True) def is_symlink(self, path: Path) -> bool: return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 @@ -204,117 +181,26 @@ def is_directory(self, path: Path) -> bool: return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 -def serialize_gpu_access_state(state: GpuAccessState) -> str: - """Return the canonical on-disk JSON representation for ``state``.""" - return ( - json.dumps( - {"renderGid": state.render_gid, "version": state.version}, - separators=(",", ":"), - sort_keys=True, - ) - + "\n" - ) - - -def parse_gpu_access_state(text: str) -> GpuAccessState: - """Parse strict versioned GPU access state, failing closed on bad input.""" - try: - payload = json.loads(text) - except (TypeError, json.JSONDecodeError) as exc: - raise InstallerError("Malformed GPU access state") from exc - - if not isinstance(payload, dict) or set(payload) != {"renderGid", "version"}: - raise InstallerError("Malformed GPU access state") - - version = payload["version"] - render_gid = payload["renderGid"] - if type(version) is not int or version != GPU_ACCESS_STATE_VERSION: - raise InstallerError("Unsupported GPU access state version") - _validate_render_gid(render_gid) - return GpuAccessState(render_gid=render_gid, version=version) - - -def resolve_render_gid(getent_output: str) -> int: - """Parse the numeric GID from one ``getent group render`` record.""" - if not isinstance(getent_output, str): - raise InstallerError("Could not resolve the host render group") - - lines = getent_output.splitlines() - if len(lines) != 1: - raise InstallerError("Could not resolve the host render group") - - fields = lines[0].split(":") - if len(fields) != 4 or fields[0] != "render": - raise InstallerError("Could not resolve the host render group") - - raw_gid = fields[2] - if not raw_gid.isascii() or not raw_gid.isdecimal(): - raise InstallerError("Could not resolve the host render group") - - render_gid = int(raw_gid) - _validate_render_gid(render_gid) - return render_gid - - def render_udev_rules() -> str: - """Return the canonical, least-privilege AMD GPU udev rules.""" + """Return the canonical AMD GPU host-device udev rules.""" return CANONICAL_UDEV_RULES -def provision_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: - """Create or reuse immutable state and reconcile the managed udev rule. - - When state is absent, adopt the current host ``render`` GID only after the - udev rule has been applied and verified. Existing state must match the host - group before any mutation occurs. - """ - return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) - - -def load_existing_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: - """Reconcile runtime GPU access, adopting missing state for pre-change installs. - - Runtime, upgrade, and reinstall paths reuse persisted state when present. - For an installation created before GPU access state existed, this performs a - one-time host ``render`` GID adoption after udev verification. A persisted - GID that differs from the current host group remains a hard failure. - """ - return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) - - -def _reconcile_gpu_access(host: GpuAccessHost) -> GpuAccessState: - _validate_parent_chain(host, GPU_ACCESS_STATE_PATH.parent) - _validate_parent_chain(host, GPU_ACCESS_RULES_PATH.parent) - state_text = _read_regular_text(host, GPU_ACCESS_STATE_PATH) - host_gid = resolve_render_gid(host.get_group_entry("render")) - - if state_text is None: - state = GpuAccessState(render_gid=host_gid) - persist_state = True - else: - state = parse_gpu_access_state(state_text) - if state.render_gid != host_gid: - raise InstallerError( - f"Persisted render GID does not match the current host render group ({state.render_gid} != {host_gid})" - ) - persist_state = False - - legacy_paths = _legacy_rules_to_remove(host) - existing_rule = _read_regular_text(host, GPU_ACCESS_RULES_PATH) - rewrite_rule = _should_rewrite_udev_rule(existing_rule) +def provision_gpu_access(host: GpuAccessHost | None = None) -> None: + """Reconcile and verify the canonical AMD GPU host-device policy.""" + active_host = host if host is not None else SystemGpuAccessHost() + _validate_parent_chain(active_host, GPU_ACCESS_RULES_PATH.parent) + legacy_paths = _legacy_rules_to_remove(active_host) + existing_rule = _read_regular_text(active_host, GPU_ACCESS_RULES_PATH) for path in legacy_paths: - host.remove_udev_rule(path) - if rewrite_rule: - host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) - host.reload_udev_rules() - host.trigger_udev() - host.settle_udev() - host.verify_device_access(state.render_gid) - if persist_state: - host.write_state_atomically(GPU_ACCESS_STATE_PATH, serialize_gpu_access_state(state)) - - return state + active_host.remove_udev_rule(path) + if _should_rewrite_udev_rule(existing_rule): + active_host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) + active_host.reload_udev_rules() + active_host.trigger_udev() + active_host.settle_udev() + active_host.verify_device_access() def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: @@ -359,9 +245,4 @@ def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: return False if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") - return True - - -def _validate_render_gid(render_gid: object) -> None: - if type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID: - raise InstallerError(f"Invalid render group GID: {render_gid!r}") + raise InstallerError(f"Refusing to overwrite unrecognized managed GPU udev rule: {GPU_ACCESS_RULES_PATH}") diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index e742c219..230da4a1 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -1,6 +1,6 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Tests for the single-node AMD GPU access source of truth.""" +"""Tests for the single-node AMD GPU host device-access reconciler.""" from __future__ import annotations @@ -12,7 +12,6 @@ from auplc_installer import gpu_access from auplc_installer.gpu_access import ( GPU_ACCESS_RULES_PATH, - GPU_ACCESS_STATE_PATH, LEGACY_AMDGPU_PXE_RULES, LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_RULES_PATH, @@ -20,15 +19,9 @@ LEGACY_KFD_RULES_PATH, LEGACY_ROCM_DEVICES_RULES, LEGACY_ROCM_DEVICES_RULES_PATH, - MAX_RENDER_GID, - GpuAccessState, SystemGpuAccessHost, - load_existing_gpu_access, - parse_gpu_access_state, provision_gpu_access, render_udev_rules, - resolve_render_gid, - serialize_gpu_access_state, ) from auplc_installer.util import InstallerError @@ -36,34 +29,17 @@ class FakeGpuAccessHost: """In-memory adapter for the installer host-operation seam.""" - def __init__(self, *, getent_output: str, files: dict[Path, str] | None = None) -> None: - self.getent_output = getent_output + def __init__(self, *, files: dict[Path, str] | None = None) -> None: self.files = dict(files or {}) self.calls: list[str] = [] self.symlinks: set[Path] = set() self.nonregular_files: set[Path] = set() - self.directories = { - Path("/"), - Path("/etc"), - Path("/etc/udev"), - Path("/etc/udev/rules.d"), - Path("/var"), - Path("/var/lib"), - Path("/var/lib/auplc"), - } - - def get_group_entry(self, group_name: str) -> str: - self.calls.append(f"get-group:{group_name}") - return self.getent_output + self.directories = {Path("/"), Path("/etc"), Path("/etc/udev"), Path("/etc/udev/rules.d")} def read_text(self, path: Path) -> str | None: self.calls.append(f"read:{path}") return self.files.get(path) - def write_state_atomically(self, path: Path, text: str) -> None: - self.calls.append(f"write-state:{path}") - self.files[path] = text - def write_udev_rule(self, path: Path, text: str) -> None: self.calls.append(f"write-rule:{path}") self.files[path] = text @@ -81,8 +57,8 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: self.calls.append("settle-udev") - def verify_device_access(self, render_gid: int) -> None: - self.calls.append(f"verify-devices:{render_gid}") + def verify_device_access(self) -> None: + self.calls.append("verify-devices") def is_symlink(self, path: Path) -> bool: return path in self.symlinks @@ -97,72 +73,34 @@ def is_directory(self, path: Path) -> bool: return path in self.directories -def test_gpu_access_state_round_trips_as_versioned_json() -> None: - state = GpuAccessState(render_gid=993) - - serialized = serialize_gpu_access_state(state) - - assert serialized == '{"renderGid":993,"version":1}\n' - assert parse_gpu_access_state(serialized) == state - - -@pytest.mark.parametrize( - "state_text", - [ - "not json", - '{"renderGid":993,"version":2}', - '{"renderGid":0,"version":1}', - f'{{"renderGid":{MAX_RENDER_GID + 1},"version":1}}', - '{"renderGid":true,"version":1}', - '{"renderGid":993,"unexpected":true,"version":1}', - ], -) -def test_parse_gpu_access_state_rejects_malformed_or_unsupported_state(state_text: str) -> None: - with pytest.raises(RuntimeError): - parse_gpu_access_state(state_text) - - -def test_resolve_render_gid_reads_the_numeric_getent_field() -> None: - assert resolve_render_gid("render:x:993:student\n") == 993 - - -@pytest.mark.parametrize( - "getent_output", - [ - "", - "video:x:44:student\n", - "render:x:0:student\n", - "render:x:not-a-number:student\n", - f"render:x:{MAX_RENDER_GID + 1}:student\n", - "render:x:993:student\nrender:x:994:student\n", - ], -) -def test_resolve_render_gid_rejects_missing_or_invalid_group_records(getent_output: str) -> None: - with pytest.raises(RuntimeError): - resolve_render_gid(getent_output) - - -def test_render_udev_rules_is_the_canonical_least_privilege_policy() -> None: +def test_render_udev_rules_is_the_canonical_host_device_policy() -> None: rules = render_udev_rules() assert rules == ( "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) - assert "card" not in rules - assert "0666" not in rules assert "chmod" not in rules -def test_device_verification_uses_lstat_and_requires_character_devices() -> None: - assert "path.lstat()" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT - assert "stat.S_ISCHR(data.st_mode)" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT +def test_device_verification_checks_kfd_and_amd_render_and_card_nodes_without_a_render_gid() -> None: + script = gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + + assert "path.lstat()" in script + assert "stat.S_ISCHR(data.st_mode)" in script + assert "glob('renderD*')" in script + assert "glob('card*')" in script + assert "'render', 0o666" in script + assert "'video', 0o666" in script + assert "render_gid" not in script + assert "sys.argv[1]" not in script -@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d"), Path("/var/lib/auplc")]) +@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d")]) def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.symlinks.add(unsafe_parent) with pytest.raises(InstallerError, match="symlinked GPU access directory"): @@ -172,7 +110,7 @@ def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_ def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.nonregular_files.add(GPU_ACCESS_RULES_PATH) with pytest.raises(InstallerError, match="non-regular GPU access file"): @@ -182,224 +120,96 @@ def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls -def test_provision_adopts_host_render_gid_and_installs_canonical_rule() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-4:] == [ - "trigger-udev", - "settle-udev", - "verify-devices:993", - f"write-state:{GPU_ACCESS_STATE_PATH}", - ] - - -def test_provision_migrates_exact_legacy_rules_then_verifies_before_persisting_state() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_KFD_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n'), - LEGACY_AMDGPU_RULES_PATH: ( - "# ROCm device permissions\n" - "# Grant render group access to AMD GPU devices\n" - "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" - 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ), - }, - ) +def test_provision_reconciles_the_canonical_rule_without_group_lookup_or_state() -> None: + host = FakeGpuAccessHost() - state = provision_gpu_access(host) + result = provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) - assert LEGACY_KFD_RULES == ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n') - assert LEGACY_AMDGPU_RULES == ( - "# ROCm device permissions\n" - "# Grant render group access to AMD GPU devices\n" - "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" - 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - assert LEGACY_KFD_RULES_PATH not in host.files - assert LEGACY_AMDGPU_RULES_PATH not in host.files - assert host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") < host.calls.index( - f"write-rule:{GPU_ACCESS_RULES_PATH}" - ) - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_provision_migrates_exact_legacy_rocm_devices_rule() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_ROCM_DEVICES_RULES_PATH: ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ), - }, - ) - - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert LEGACY_ROCM_DEVICES_RULES == ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - assert LEGACY_ROCM_DEVICES_RULES_PATH not in host.files - - -def test_provision_migrates_exact_legacy_pxe_rule_at_amdgpu_path() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_AMDGPU_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n'), - }, - ) + assert result is None + assert host.files == {GPU_ACCESS_RULES_PATH: render_udev_rules()} + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] + assert not any("group" in call or "state" in call for call in host.calls) - state = provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) - assert LEGACY_AMDGPU_PXE_RULES == ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n') - assert LEGACY_AMDGPU_RULES_PATH not in host.files - - -def test_near_legacy_pxe_rule_fails_closed_without_removal() -> None: - near_variant = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' - host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_AMDGPU_RULES_PATH: near_variant}) +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_KFD_RULES_PATH, LEGACY_KFD_RULES), + (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_RULES), + (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_PXE_RULES), + (LEGACY_ROCM_DEVICES_RULES_PATH, LEGACY_ROCM_DEVICES_RULES), + ], +) +def test_provision_removes_only_exact_legacy_rules_before_verifying(path: Path, content: str) -> None: + host = FakeGpuAccessHost(files={path: content}) - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) + provision_gpu_access(host) - assert host.files[LEGACY_AMDGPU_RULES_PATH] == near_variant + assert path not in host.files + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls.index(f"remove-rule:{path}") < host.calls.index(f"write-rule:{GPU_ACCESS_RULES_PATH}") + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] -def test_modified_legacy_rocm_devices_rule_fails_closed_without_removal() -> None: - modified = ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_ROCM_DEVICES_RULES_PATH: modified}) +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n'), + ( + LEGACY_ROCM_DEVICES_RULES_PATH, + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n', + ), + ], +) +def test_near_legacy_rule_fails_closed_without_removal(path: Path, content: str) -> None: + host = FakeGpuAccessHost(files={path: content}) with pytest.raises(InstallerError, match="unexpected legacy"): provision_gpu_access(host) - assert host.files[LEGACY_ROCM_DEVICES_RULES_PATH] == modified + assert host.files[path] == content -def test_provision_reapplies_and_verifies_matching_immutable_state() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', - GPU_ACCESS_RULES_PATH: render_udev_rules(), - }, - ) +def test_matching_managed_rule_is_reapplied_and_verified_without_rewriting() -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: render_udev_rules()}) - state = provision_gpu_access(host) + provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) assert not any(call.startswith("write-") for call in host.calls) - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices:993"] + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] -def test_provision_fails_before_mutation_when_persisted_gid_differs_from_host() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:994:student\n", - files={GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n'}, - ) +@pytest.mark.parametrize( + "unexpected_rule", + [ + f"{gpu_access.UDEV_MANAGED_MARKER}\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n', + f'{gpu_access.UDEV_MANAGED_MARKER}\nKERNEL=="kfd", MODE="0666"\n', + ], +) +def test_noncanonical_managed_rule_fails_closed_before_mutation(unexpected_rule: str) -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: unexpected_rule}) - with pytest.raises(RuntimeError, match="does not match"): + with pytest.raises(InstallerError, match="unrecognized managed"): provision_gpu_access(host) - assert not any(call.startswith("write-") for call in host.calls) + assert host.files[GPU_ACCESS_RULES_PATH] == unexpected_rule + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) assert "reload-udev" not in host.calls - assert "trigger-udev" not in host.calls -def test_provision_fails_before_writing_state_when_rule_is_unmanaged() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}, - ) +def test_unmanaged_rule_fails_before_any_mutation() -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}) - with pytest.raises(RuntimeError, match="unmanaged"): + with pytest.raises(InstallerError, match="unmanaged"): provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - assert not any(call.startswith("write-") for call in host.calls) - - -def test_load_existing_gpu_access_adopts_missing_state_after_verification() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - state = load_existing_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_managed_rule_is_reconciled_and_reloaded_when_content_changes() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', - GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", - }, - ) - - state = load_existing_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-5:] == [ - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - "trigger-udev", - "settle-udev", - "verify-devices:993", - ] - - -def test_system_adapter_persists_state_with_a_same_directory_temporary_file(monkeypatch) -> None: - commands: list[list[str]] = [] - capture_commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - return SimpleNamespace(returncode=0) - - def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: - capture_commands.append(command) - return SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n") - - monkeypatch.setattr(gpu_access, "run", fake_run) - monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) - - SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") - - assert capture_commands == [["mktemp", "/var/lib/auplc/.gpu-access.json.XXXXXX"]] - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/var/lib/auplc"], - ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], - ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], - ["mv", "-f", "/var/lib/auplc/.gpu-access.json.temporary", "/var/lib/auplc/gpu-access.json"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc"], - ] + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) + assert "reload-udev" not in host.calls def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: @@ -437,14 +247,19 @@ def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: ] -def test_system_adapter_removes_temporary_file_when_durable_write_fails(monkeypatch) -> None: +def test_system_adapter_removes_temporary_rule_when_durable_write_fails(monkeypatch) -> None: commands: list[list[str]] = [] def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: commands.append(command) if command[:2] == ["test", "-L"]: return SimpleNamespace(returncode=1) - if command == ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"]: + if command == [ + "python3", + "-c", + gpu_access._FSYNC_PATH_SCRIPT, + "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", + ]: raise InstallerError("fsync failed") return SimpleNamespace(returncode=0) @@ -452,56 +267,24 @@ def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: monkeypatch.setattr( gpu_access, "run_capture", - lambda command, **kwargs: SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n"), + lambda command, **kwargs: SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n"), ) with pytest.raises(InstallerError, match="fsync failed"): - SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/var/lib/auplc"], - ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], - ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], - ["rm", "-f", "/var/lib/auplc/.gpu-access.json.temporary"], + ["mkdir", "-p", "/etc/udev/rules.d"], + ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["rm", "-f", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], ] -@pytest.mark.parametrize( - ("failing_method", "expected_calls"), - [ - ( - "write_udev_rule", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - ], - ), - ( - "reload_udev_rules", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - ], - ), - ( - "trigger_udev", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - "trigger-udev", - ], - ), - ], -) -def test_first_install_does_not_persist_state_until_udev_reconciliation_succeeds( - monkeypatch, - failing_method: str, - expected_calls: list[str], -) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") +@pytest.mark.parametrize("failing_method", ["write_udev_rule", "reload_udev_rules", "trigger_udev", "settle_udev"]) +def test_reconciliation_stops_when_udev_mutation_fails(monkeypatch, failing_method: str) -> None: + host = FakeGpuAccessHost() original_method = getattr(host, failing_method) def fail_after_recording(*args: object) -> None: @@ -513,78 +296,14 @@ def fail_after_recording(*args: object) -> None: with pytest.raises(InstallerError, match=f"{failing_method} failed"): provision_gpu_access(host) - assert host.calls[-len(expected_calls) :] == expected_calls - assert GPU_ACCESS_STATE_PATH not in host.files - - -def test_failed_udev_reconciliation_never_rewrites_existing_state(monkeypatch) -> None: - original_state = '{"renderGid":993,"version":1}\n' - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: original_state, - GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", - }, - ) - - def fail_reload() -> None: - host.calls.append("reload-udev") - raise InstallerError("reload failed") - - monkeypatch.setattr(host, "reload_udev_rules", fail_reload) - - with pytest.raises(InstallerError, match="reload failed"): - provision_gpu_access(host) - - assert host.files[GPU_ACCESS_STATE_PATH] == original_state - assert not any(call.startswith("write-state:") for call in host.calls) - assert "trigger-udev" not in host.calls - + assert "verify-devices" not in host.calls -def test_failed_reload_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - def fail_reload() -> None: - host.calls.append("reload-udev") - raise InstallerError("reload failed") - - monkeypatch.setattr(host, "reload_udev_rules", fail_reload) - with pytest.raises(InstallerError, match="reload failed"): - provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - - monkeypatch.setattr(host, "reload_udev_rules", FakeGpuAccessHost.reload_udev_rules.__get__(host)) - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.calls[-2:] == ["verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_failed_settle_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - def fail_settle() -> None: - host.calls.append("settle-udev") - raise InstallerError("settle failed") - - monkeypatch.setattr(host, "settle_udev", fail_settle) - with pytest.raises(InstallerError, match="settle failed"): - provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - assert "verify-devices:993" not in host.calls +def test_failed_inode_verification_leaves_the_reconciled_rule_in_place(monkeypatch) -> None: + host = FakeGpuAccessHost() - monkeypatch.setattr(host, "settle_udev", FakeGpuAccessHost.settle_udev.__get__(host)) - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_failed_inode_verification_does_not_adopt_state(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - def fail_verification(render_gid: int) -> None: - host.calls.append(f"verify-devices:{render_gid}") + def fail_verification() -> None: + host.calls.append("verify-devices") raise InstallerError("device ownership mismatch") monkeypatch.setattr(host, "verify_device_access", fail_verification) @@ -592,36 +311,16 @@ def fail_verification(render_gid: int) -> None: with pytest.raises(InstallerError, match="ownership mismatch"): provision_gpu_access(host) - assert host.calls[-1] == "verify-devices:993" - assert GPU_ACCESS_STATE_PATH not in host.files + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-1] == "verify-devices" @pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.symlinks.add(path) with pytest.raises(InstallerError, match="symlinked"): provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_KFD_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\n'), - (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", GROUP="render", MODE="0660"\n'), - ], -) -def test_one_line_legacy_variants_fail_closed_without_removal(path: Path, content: str) -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={path: content}, - ) - - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) - - assert host.files[path] == content - assert GPU_ACCESS_STATE_PATH not in host.files + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py index ce5f094a..b07047cc 100644 --- a/tests/scripts/test_gpu_image_permissions.py +++ b/tests/scripts/test_gpu_image_permissions.py @@ -13,12 +13,10 @@ def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: dockerfile = DOCKERFILE.read_text(encoding="utf-8") forbidden_patterns = ( - r"groupmod\s+-g\s+992\s+render", - r"groupadd\s+-g\s+992\s+render", - r"usermod\s+-aG\s+video,render\s+\$\{NB_USER\}", - r"\brender\b", + r"\b(?:groupadd|groupmod|usermod)\b.*\b(?:video|render)\b", r"/etc/udev", r"chmod\s+666\b", + r"chmod\b.*(?:/dev/|kfd|render|card)", ) for pattern in forbidden_patterns: assert re.search(pattern, dockerfile) is None, pattern From d5e373c59b09c5c018dadf2c94592a3345922b92 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 064/180] refactor(installer): remove GPU GID workflow --- auplc_installer/cli.py | 30 ++++---- tests/installer/test_cli_gpu_access.py | 97 ++++++++++---------------- tests/installer/test_cli_helpers.py | 1 - 3 files changed, 50 insertions(+), 78 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f2e9b538..609930a9 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -13,7 +13,7 @@ import contextlib import sys import time -from collections.abc import Callable, Sequence +from collections.abc import Sequence from pathlib import Path from typing import NoReturn @@ -23,7 +23,7 @@ detect_and_configure_gpu, refine_gpu_config_from_node_labels, ) -from auplc_installer.gpu_access import GpuAccessState, load_existing_gpu_access, provision_gpu_access +from auplc_installer.gpu_access import provision_gpu_access from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware from auplc_installer.helm import ( deploy_runtime, @@ -326,12 +326,12 @@ def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") -def _render_gid_for_local_hardware(reconcile_gpu_access: Callable[[], GpuAccessState]) -> int | None: +def _provision_gpu_access_for_local_hardware() -> None: match classify_gpu_hardware(): case GpuHardware.GPU: - return reconcile_gpu_access().render_gid + provision_gpu_access() case GpuHardware.CPU: - return None + return case GpuHardware.UNKNOWN: raise InstallerError("Could not determine local AMD GPU hardware; refusing to modify installer state") case unreachable: @@ -355,7 +355,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) with stage("Provisioning GPU device access", idx=2, total=total): - render_gid = _render_gid_for_local_hardware(provision_gpu_access) + _provision_gpu_access_for_local_hardware() paths = state.runtime_paths() with stage("Generating values overlay (initial)", idx=3, total=total): @@ -368,7 +368,6 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) @@ -434,7 +433,6 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) @@ -606,7 +604,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -616,14 +614,13 @@ def cmd_dev_deploy(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths, dev=True) def cmd_dev_upgrade(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -634,14 +631,13 @@ def cmd_dev_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths, dev=True) def cmd_dev_reinstall(state: InstallerState) -> None: - _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -652,7 +648,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -662,14 +658,13 @@ def cmd_rt_install(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths) def cmd_rt_upgrade(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -680,7 +675,6 @@ def cmd_rt_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths) @@ -711,7 +705,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: - _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index ede70311..e4cdaaf9 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -11,20 +11,16 @@ import pytest from auplc_installer import cli -from auplc_installer.gpu_access import GpuAccessState from auplc_installer.gpu_hardware import GpuHardware from auplc_installer.helm import RuntimePaths from auplc_installer.state import InstallerState -@pytest.mark.parametrize( - ("hardware", "expected_render_gid", "expected_provision_count"), - [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], -) -def test_full_install_gates_gpu_access_without_skipping_later_gpu_flow( - monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_provision_count: int +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_full_install_gates_gpu_access_without_passing_it_to_the_overlay( + monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: - events: list[object] = [] + events: list[str] = [] stages: list[tuple[str, int, int]] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) @@ -35,16 +31,15 @@ def fake_stage(label: str, *, idx: int, total: int): yield def fake_overlay(*args: object, **kwargs: object) -> Path: - events.append(("overlay", kwargs["render_gid"])) + assert "render_gid" not in kwargs + events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda: events.append("provision") or GpuAccessState(render_gid=993) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) @@ -60,10 +55,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: assert events.count("provision") == expected_provision_count if expected_provision_count: assert events.index("provision") < events.index("device-plugin") - assert [event for event in events if isinstance(event, tuple)] == [ - ("overlay", expected_render_gid), - ("overlay", expected_render_gid), - ] + assert events.count("overlay") == 2 assert stages == [ ("Detecting GPU", 1, 9), ("Provisioning GPU device access", 2, 9), @@ -77,29 +69,22 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: ] -@pytest.mark.parametrize( - ("hardware", "expected_render_gid", "expected_load_count"), - [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], -) -def test_runtime_upgrade_gates_existing_gpu_access_without_provisioning( - monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_load_count: int +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_runtime_upgrade_gates_host_access_without_provisioning_helm_values( + monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: - events: list[object] = [] + events: list[str] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) def fake_overlay(*args: object, **kwargs: object) -> Path: - events.append(("overlay", kwargs["render_gid"])) + assert "render_gid" not in kwargs + events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr( - cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) - ) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) @@ -108,20 +93,20 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: cli.cmd_rt_upgrade(state) - assert events.count("load") == expected_load_count - assert events[-5:] == ["detect", "refine", "preserve-courses", ("overlay", expected_render_gid), "upgrade-runtime"] + assert events.count("provision") == expected_provision_count + assert events[-5:] == ["detect", "refine", "preserve-courses", "overlay", "upgrade-runtime"] @pytest.mark.parametrize( ("command", "expected_events"), [ - (cli.cmd_dev_deploy, ("detect", "refine", "overlay:None", "deploy-runtime")), - (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), - (cli.cmd_rt_install, ("detect", "refine", "overlay:None", "deploy-runtime")), - (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + (cli.cmd_dev_deploy, ("detect", "refine", "overlay", "deploy-runtime")), + (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), + (cli.cmd_rt_install, ("detect", "refine", "overlay", "deploy-runtime")), + (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), ], ) -def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( +def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] ) -> None: events: list[str] = [] @@ -130,14 +115,16 @@ def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) - monkeypatch.setattr(cli, "load_existing_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not load"))) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + ) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) monkeypatch.setattr( cli, "generate_values_overlay", - lambda *args, **kwargs: events.append(f"overlay:{kwargs['render_gid']}") or paths.overlay_path, + lambda *args, **kwargs: events.append("overlay") or paths.overlay_path, ) monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) @@ -149,16 +136,12 @@ def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( @pytest.mark.parametrize( ("reinstall", "delegate_name"), - [ - (cli.cmd_dev_reinstall, "cmd_dev_deploy"), - (cli.cmd_rt_reinstall, "cmd_rt_install"), - ], + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) @pytest.mark.parametrize( - ("hardware", "expected_access_events"), - [(GpuHardware.GPU, ["load"]), (GpuHardware.CPU, [])], + ("hardware", "expected_access_events"), [(GpuHardware.GPU, ["provision"]), (GpuHardware.CPU, [])] ) -def test_reinstall_gates_existing_gpu_access_before_removing_runtime( +def test_reinstall_gates_host_access_before_removing_runtime( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str, @@ -169,9 +152,7 @@ def test_reinstall_gates_existing_gpu_access_before_removing_runtime( state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr( - cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -188,9 +169,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr( - cli, - "provision_gpu_access", - lambda: (_ for _ in ()).throw(AssertionError("must not provision")), + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) ) with pytest.raises(RuntimeError, match="hardware"): @@ -201,10 +180,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp @pytest.mark.parametrize( ("reinstall", "delegate_name"), - [ - (cli.cmd_dev_reinstall, "cmd_dev_deploy"), - (cli.cmd_rt_reinstall, "cmd_rt_install"), - ], + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str @@ -214,9 +190,7 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr( - cli, - "load_existing_gpu_access", - lambda: (_ for _ in ()).throw(AssertionError("must not load")), + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -225,3 +199,8 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( reinstall(state) assert events == [] + + +def test_cli_exposes_no_render_gid_reconciliation_api() -> None: + assert not hasattr(cli, "_render_gid_for_local_hardware") + assert not hasattr(cli, "load_existing_gpu_access") diff --git a/tests/installer/test_cli_helpers.py b/tests/installer/test_cli_helpers.py index 2cc863e8..bc33cffa 100644 --- a/tests/installer/test_cli_helpers.py +++ b/tests/installer/test_cli_helpers.py @@ -41,7 +41,6 @@ def _write_overlay(path: Path, courses: CourseSelection) -> None: image_tag="v1.0", courses=courses, offline_mode=False, - render_gid=993, overlay_path=path, ) From 3bc557fbee23aa8979ceaeba5545da15a0397181 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 065/180] refactor(installer): remove GPU GID overlays --- auplc_installer/overlay.py | 8 -------- tests/installer/test_overlay.py | 14 +++++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 202b5c6c..5815f391 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -47,7 +47,6 @@ def emit_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, - render_gid: int | None, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" buf = StringIO() @@ -67,11 +66,6 @@ def emit_overlay( buf.write(f"# Env selection : {courses.description()}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") - buf.write(" gpuAccess:\n") - if render_gid is None: - buf.write(" renderGid: null\n") - else: - buf.write(f" renderGid: {render_gid}\n") # --- accelerators --- any_accel_emitted = False @@ -170,7 +164,6 @@ def generate_values_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, - render_gid: int | None, overlay_path: Path, ) -> Path: """Render the overlay and write it to ``overlay_path``. Returns the path.""" @@ -182,7 +175,6 @@ def generate_values_overlay( image_tag=image_tag, courses=courses, offline_mode=offline_mode, - render_gid=render_gid, ) overlay_path.write_text(text, encoding="utf-8") return overlay_path diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index c93e4fc7..677eb115 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -46,7 +46,6 @@ def _render( courses: CourseSelection, offline_mode: bool = False, image_tag: str = "v1.0", - render_gid: int | None = 993, ) -> tuple[str, dict]: text = emit_overlay( cfg, @@ -54,7 +53,6 @@ def _render( image_tag=image_tag, courses=courses, offline_mode=offline_mode, - render_gid=render_gid, ) return text, yaml.safe_load(text) @@ -96,7 +94,6 @@ def _write_and_read_back(courses: CourseSelection) -> CourseSelection | None: image_tag="v1.0", courses=courses, offline_mode=False, - render_gid=993, overlay_path=path, ) return try_load_courses_from_overlay(path) @@ -110,30 +107,29 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] -def test_overlay_emits_explicit_gpu_access_gid_without_global_pod_groups() -> None: +def test_overlay_never_emits_gpu_access_contract() -> None: text = emit_overlay( _strix_halo_cfg(), image_registry="ghcr.io/amdresearch", image_tag="v1.0", courses=CourseSelection.default(), offline_mode=False, - render_gid=993, ) parsed = yaml.safe_load(text) - assert parsed["custom"]["gpuAccess"]["renderGid"] == 993 + assert "gpuAccess" not in parsed["custom"] + assert "renderGid" not in text assert "supplementalGroups" not in text -def test_overlay_emits_null_render_gid_without_removing_gpu_resources() -> None: +def test_overlay_keeps_gpu_resources_without_gpu_access_contract() -> None: _, parsed = _render( _strix_halo_cfg(), courses=CourseSelection.default(), - render_gid=None, ) custom = parsed["custom"] - assert custom["gpuAccess"]["renderGid"] is None + assert "gpuAccess" not in custom assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) assert "teams" not in custom From e5252db7b99565cd94f95652c474bfb609d827f0 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 066/180] refactor(ansible): simplify GPU access role --- deploy/ansible/filter_plugins/auplc_json.py | 41 -- .../roles/gpu_access/defaults/main.yml | 4 - .../ansible/roles/gpu_access/tasks/apply.yml | 152 +++--- .../roles/gpu_access/tasks/preflight.yml | 120 +---- .../roles/gpu_access/tasks/validate.yml | 9 - .../templates/70-auplc-gpu-access.rules.j2 | 5 +- .../gpu_access/templates/gpu-access.json.j2 | 1 - tests/skills/test_gpu_access_role.py | 503 ++++-------------- 8 files changed, 220 insertions(+), 615 deletions(-) delete mode 100644 deploy/ansible/filter_plugins/auplc_json.py delete mode 100644 deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 diff --git a/deploy/ansible/filter_plugins/auplc_json.py b/deploy/ansible/filter_plugins/auplc_json.py deleted file mode 100644 index 080de6af..00000000 --- a/deploy/ansible/filter_plugins/auplc_json.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - -"""Strict JSON filters used by AUP Learning Cloud Ansible roles.""" - -import json -from collections.abc import Callable -from dataclasses import dataclass -from typing import TypeAlias - -from ansible.errors import AnsibleFilterError - -JSONValue: TypeAlias = None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] - - -@dataclass(frozen=True, slots=True) -class DuplicateJsonKeyError(ValueError): - key: str - - def __str__(self) -> str: - return f"Duplicate JSON object key: {self.key!r}" - - -def _reject_duplicate_keys(pairs: list[tuple[str, JSONValue]]) -> dict[str, JSONValue]: - result: dict[str, JSONValue] = {} - for key, value in pairs: - if key in result: - raise DuplicateJsonKeyError(key) - result[key] = value - return result - - -def auplc_from_json_strict(value: str) -> JSONValue: - try: - return json.loads(value, object_pairs_hook=_reject_duplicate_keys) - except (TypeError, DuplicateJsonKeyError, json.JSONDecodeError): - raise AnsibleFilterError("Invalid JSON value") from None - - -class FilterModule: - def filters(self) -> dict[str, Callable[[str], JSONValue]]: - return {"auplc_from_json_strict": auplc_from_json_strict} diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 74e40ed3..8f5ff6c7 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -1,10 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -# Set this explicitly in cluster inventory or PXE extra vars. The role never -# assumes a site-specific GID. -auplc_render_gid: null -auplc_normalize_render_gid: false auplc_gpu_access_enabled: false # Set for a PXE rootfs. Leave empty to configure the live host. auplc_rootfs_path: "" diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml index 1def00de..f830b935 100644 --- a/deploy/ansible/roles/gpu_access/tasks/apply.yml +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -1,6 +1,39 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- +- name: Inspect canonical GPU access rule before apply + ansible.builtin.stat: + path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + follow: false + register: _auplc_apply_destination_rule + +- name: Reject unsafe canonical GPU access rule before apply + ansible.builtin.assert: + that: + - not _auplc_apply_destination_rule.stat.exists or + (_auplc_apply_destination_rule.stat.isreg and not _auplc_apply_destination_rule.stat.islnk) + fail_msg: Unsafe canonical GPU access destination. + +- name: Read canonical GPU access rule before apply + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + register: _auplc_apply_existing_rule + when: _auplc_apply_destination_rule.stat.exists + +- name: Define canonical GPU access rule contents for apply + ansible.builtin.set_fact: + _auplc_apply_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" + +- name: Recheck canonical GPU access rule before apply + ansible.builtin.assert: + that: (_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule + fail_msg: Unmanaged canonical GPU access rule. + when: _auplc_apply_destination_rule.stat.exists + - name: Inspect recognized project-owned legacy GPU rules for apply ansible.builtin.stat: path: "{{ item.path }}" @@ -37,38 +70,6 @@ loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) -- name: Normalize live render GID - ansible.builtin.command: - argv: [groupmod, -g, "{{ auplc_render_gid | string }}", render] - when: - - _auplc_target_root | length == 0 - - (_auplc_current_render_gid | int) != (auplc_render_gid | int) - - auplc_normalize_render_gid | bool - -- name: Normalize rootfs render GID - ansible.builtin.command: - argv: [chroot, "{{ _auplc_target_root }}", groupmod, -g, "{{ auplc_render_gid | string }}", render] - when: - - _auplc_target_root | length > 0 - - (_auplc_current_render_gid | int) != (auplc_render_gid | int) - - auplc_normalize_render_gid | bool - -- name: Verify target render GID - ansible.builtin.command: - argv: >- - {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} - register: _auplc_verified_render_group - changed_when: false - -- name: Require verified render GID - ansible.builtin.assert: - that: - - _auplc_verified_render_group.stdout_lines | length == 1 - - _auplc_verified_render_group.stdout.split(':')[0] == 'render' - - (_auplc_verified_render_group.stdout.split(':')[2] | int) == (auplc_render_gid | int) - fail_msg: Target render group did not resolve to auplc_render_gid. - - name: Create target udev rules directory ansible.builtin.file: path: "{{ _auplc_target_root }}/etc/udev/rules.d" @@ -116,9 +117,9 @@ - _auplc_kfd.stat.exists - _auplc_kfd.stat.ischr - _auplc_kfd.stat.uid == 0 - - _auplc_kfd.stat.gid == (auplc_render_gid | int) - - _auplc_kfd.stat.mode == '0660' - fail_msg: /dev/kfd is not root:render with mode 0660 after reconciliation. + - _auplc_kfd.stat.gr_name == 'render' + - _auplc_kfd.stat.mode == '0666' + fail_msg: /dev/kfd is not root:render with mode 0666 after reconciliation. when: _auplc_target_root | length == 0 - name: Find live DRM render nodes @@ -130,11 +131,20 @@ register: _auplc_render_nodes when: _auplc_target_root | length == 0 -- name: Resolve live DRM render node driver symlinks +- name: Find live DRM card nodes + ansible.builtin.find: + paths: /dev/dri + patterns: card* + file_type: any + recurse: false + register: _auplc_card_nodes + when: _auplc_target_root | length == 0 + +- name: Resolve live DRM node driver symlinks ansible.builtin.command: argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] - loop: "{{ _auplc_render_nodes.files }}" - register: _auplc_render_node_drivers + loop: "{{ (_auplc_render_nodes.files | default([])) + (_auplc_card_nodes.files | default([])) }}" + register: _auplc_drm_node_drivers changed_when: false failed_when: false when: _auplc_target_root | length == 0 @@ -143,48 +153,68 @@ ansible.builtin.set_fact: _auplc_amd_render_nodes: >- {{ (_auplc_amd_render_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' else []) }} - loop: "{{ _auplc_render_node_drivers.results }}" + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and + (item.item.path | basename) is match('^renderD') else []) }} + loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" + when: _auplc_target_root | length == 0 + +- name: Select AMD live DRM card nodes + ansible.builtin.set_fact: + _auplc_amd_card_nodes: >- + {{ (_auplc_amd_card_nodes | default([])) + + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and + (item.item.path | basename) is match('^card') else []) }} + loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" when: _auplc_target_root | length == 0 - name: Require AMD live DRM render nodes ansible.builtin.assert: - that: _auplc_amd_render_nodes | length > 0 + that: (_auplc_amd_render_nodes | default([])) | length > 0 fail_msg: No AMD renderD node was available for GPU access verification. when: _auplc_target_root | length == 0 +- name: Require AMD live DRM card nodes + ansible.builtin.assert: + that: (_auplc_amd_card_nodes | default([])) | length > 0 + fail_msg: No AMD card node was available for GPU access verification. + when: _auplc_target_root | length == 0 + - name: Inspect AMD live DRM render nodes ansible.builtin.stat: path: "{{ item }}" follow: false - loop: "{{ _auplc_amd_render_nodes }}" + loop: "{{ _auplc_amd_render_nodes | default([]) }}" register: _auplc_amd_render_node_stats when: _auplc_target_root | length == 0 +- name: Inspect AMD live DRM card nodes + ansible.builtin.stat: + path: "{{ item }}" + follow: false + loop: "{{ _auplc_amd_card_nodes | default([]) }}" + register: _auplc_amd_card_node_stats + when: _auplc_target_root | length == 0 + - name: Verify AMD render node ownership and mode ansible.builtin.assert: that: - item.stat.exists - item.stat.ischr - item.stat.uid == 0 - - item.stat.gid == (auplc_render_gid | int) - - item.stat.mode == '0660' - fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0660." - loop: "{{ _auplc_amd_render_node_stats.results }}" + - item.stat.gr_name == 'render' + - item.stat.mode == '0666' + fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0666." + loop: "{{ _auplc_amd_render_node_stats.results | default([]) }}" when: _auplc_target_root | length == 0 -- name: Create target GPU access state directory - ansible.builtin.file: - path: "{{ _auplc_target_root }}/var/lib/auplc" - state: directory - owner: root - group: root - mode: "0755" - -- name: Persist target GPU access state - ansible.builtin.template: - src: gpu-access.json.j2 - dest: "{{ _auplc_target_root }}/var/lib/auplc/gpu-access.json" - owner: root - group: root - mode: "0644" +- name: Verify AMD card node ownership and mode + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.ischr + - item.stat.uid == 0 + - item.stat.gr_name == 'video' + - item.stat.mode == '0666' + fail_msg: "AMD card node {{ item.item }} is not root:video with mode 0666." + loop: "{{ _auplc_amd_card_node_stats.results | default([]) }}" + when: _auplc_target_root | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml index fcb4808b..23d448e4 100644 --- a/deploy/ansible/roles/gpu_access/tasks/preflight.yml +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -27,9 +27,6 @@ - /etc - /etc/udev - /etc/udev/rules.d - - /var - - /var/lib - - /var/lib/auplc register: _auplc_destination_parent_stats - name: Reject unsafe canonical GPU access destination parents @@ -39,52 +36,38 @@ fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" loop: "{{ _auplc_destination_parent_stats.results }}" -- name: Inspect canonical GPU access destinations +- name: Inspect canonical GPU access destination ansible.builtin.stat: - path: "{{ _auplc_target_root }}{{ item }}" + path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" follow: false - loop: - - /etc/udev/rules.d/70-auplc-gpu-access.rules - - /var/lib/auplc/gpu-access.json - register: _auplc_destination_stats + register: _auplc_destination_rule -- name: Reject unsafe canonical GPU access destinations +- name: Reject unsafe canonical GPU access destination ansible.builtin.assert: that: - - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) - fail_msg: "Unsafe canonical GPU access destination: {{ item.item }}" - loop: "{{ _auplc_destination_stats.results }}" + - not _auplc_destination_rule.stat.exists or + (_auplc_destination_rule.stat.isreg and not _auplc_destination_rule.stat.islnk) + fail_msg: Unsafe canonical GPU access destination. -- name: Read existing canonical GPU access destinations +- name: Read existing canonical GPU access rule ansible.builtin.slurp: - src: "{{ _auplc_target_root }}{{ item.item }}" - loop: "{{ _auplc_destination_stats.results }}" - when: item.stat.exists - register: _auplc_existing_destinations + src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + register: _auplc_existing_rule + when: _auplc_destination_rule.stat.exists -- name: Define canonical GPU access rule content +- name: Define canonical GPU access rule contents ansible.builtin.set_fact: _auplc_canonical_rule: | # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - name: Reject unmanaged canonical GPU access rule ansible.builtin.assert: - that: (item.content | b64decode) == _auplc_canonical_rule - fail_msg: "Unmanaged canonical GPU access rule: {{ item.item.item }}" - loop: "{{ _auplc_existing_destinations.results }}" - when: - - not item.skipped | default(false) - - item.item.item.endswith('70-auplc-gpu-access.rules') - -- name: Parse existing canonical GPU access state - ansible.builtin.set_fact: - _auplc_existing_state: "{{ item.content | b64decode | auplc_from_json_strict }}" - loop: "{{ _auplc_existing_destinations.results }}" - when: - - not item.skipped | default(false) - - item.item.item.endswith('gpu-access.json') + that: (_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule + fail_msg: Unmanaged canonical GPU access rule. + when: _auplc_destination_rule.stat.exists - name: Define recognized project-owned legacy GPU rules ansible.builtin.set_fact: @@ -137,70 +120,3 @@ fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) - -- name: Read target render group - ansible.builtin.command: - argv: >- - {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} - register: _auplc_render_group - changed_when: false - failed_when: false - -- name: Require target render group - ansible.builtin.assert: - that: - - _auplc_render_group.rc == 0 - - _auplc_render_group.stdout_lines | length == 1 - - _auplc_render_group.stdout.split(':') | length == 4 - - _auplc_render_group.stdout.split(':')[0] == 'render' - - _auplc_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _auplc_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Target has no valid render group; this role never creates groups. - -- name: Record target render GID - ansible.builtin.set_fact: - _auplc_current_render_gid: "{{ _auplc_render_group.stdout.split(':')[2] | int }}" - -- name: Reject invalid canonical GPU access state except Interrupted normalization retry - ansible.builtin.assert: - that: - - _auplc_existing_state is mapping - - _auplc_existing_state.keys() | list | sort == ['renderGid', 'version'] - - _auplc_existing_state.version is integer - - _auplc_existing_state.version == 1 - - _auplc_existing_state.renderGid is integer - - _auplc_existing_state.renderGid >= 1 - - _auplc_existing_state.renderGid <= 4294967294 - - >- - _auplc_existing_state.renderGid == auplc_render_gid or - ((auplc_normalize_render_gid | bool) and - (_auplc_existing_state.renderGid == _auplc_current_render_gid or - _auplc_current_render_gid == auplc_render_gid)) - fail_msg: Invalid canonical GPU access state. - when: _auplc_existing_state is defined - -- name: List target groups for desired GID collision - ansible.builtin.command: - argv: >- - {{ ['getent', 'group'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group'] }} - register: _auplc_all_groups - changed_when: false - failed_when: false - -- name: Reject desired GID collision - ansible.builtin.assert: - that: - - _auplc_all_groups.rc == 0 - - >- - _auplc_all_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (auplc_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: auplc_render_gid is already assigned to another target group. - -- name: Reject render GID mismatch without normalization - ansible.builtin.assert: - that: - - _auplc_current_render_gid == auplc_render_gid or (auplc_normalize_render_gid | bool) - fail_msg: Target render GID differs from auplc_render_gid and normalization is disabled. diff --git a/deploy/ansible/roles/gpu_access/tasks/validate.yml b/deploy/ansible/roles/gpu_access/tasks/validate.yml index 3e32ee7a..dbfa855e 100644 --- a/deploy/ansible/roles/gpu_access/tasks/validate.yml +++ b/deploy/ansible/roles/gpu_access/tasks/validate.yml @@ -1,15 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -- name: Validate desired render GID - ansible.builtin.assert: - that: - - auplc_render_gid is not none - - auplc_render_gid is integer - - auplc_render_gid >= 1 - - auplc_render_gid <= 4294967294 - fail_msg: auplc_render_gid must be an explicit integer between 1 and 4294967294. - - name: Validate GPU access rootfs path syntax ansible.builtin.assert: that: diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 index c75ec1e2..f57d023a 100644 --- a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 +++ b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 @@ -1,3 +1,4 @@ # Managed by auplc-installer: AMD GPU device access. -KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" -SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" +KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" +SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" +SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" diff --git a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 deleted file mode 100644 index 89b9110a..00000000 --- a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 +++ /dev/null @@ -1 +0,0 @@ -{"renderGid":{{ auplc_render_gid | int }},"version":1} diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index b195fef5..ccece9ce 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,32 +1,9 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Canonical artifact tests for the multi-node GPU access role.""" +"""Contract tests for the Ansible GPU device-mode role.""" -import sys -import types from pathlib import Path -import pytest - -try: - from ansible.errors import AnsibleFilterError -except ModuleNotFoundError: - ansible_module = types.ModuleType("ansible") - errors_module = types.ModuleType("ansible.errors") - - class AnsibleFilterError(Exception): - pass - - errors_module.AnsibleFilterError = AnsibleFilterError - ansible_module.errors = errors_module - sys.modules["ansible"] = ansible_module - sys.modules["ansible.errors"] = errors_module -from deploy.ansible.filter_plugins.auplc_json import ( - DuplicateJsonKeyError, - _reject_duplicate_keys, - auplc_from_json_strict, -) - ROOT = Path(__file__).resolve().parents[2] ANSIBLE = ROOT / "deploy" / "ansible" GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" @@ -38,423 +15,159 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: - assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { - "renderGid": 993, - "version": 1, - } - - -def test_duplicate_json_key_error_preserves_typed_key() -> None: - with pytest.raises(DuplicateJsonKeyError) as error: - _reject_duplicate_keys([("version", 1), ("version", 2)]) - - assert error.value.key == "version" - assert str(error.value) == "Duplicate JSON object key: 'version'" - - -@pytest.mark.parametrize( - "value", - [ - '{"renderGid":1,"renderGid":993,"version":1}', - '{"renderGid":1,"render\\u0047id":993,"version":1}', - '{"renderGid":1,"version":1,"version":2}', - '{"outer":{"version":1,"version":2}}', - ], -) -def test_strict_json_filter_rejects_semantic_duplicate_keys(value: str) -> None: - with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): - auplc_from_json_strict(value) - - -@pytest.mark.parametrize("value", ["{", '{"renderGid":1,}']) -def test_strict_json_filter_rejects_malformed_json(value: str) -> None: - with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): - auplc_from_json_strict(value) - - -def test_gpu_access_role_renders_the_unified_render_gid_contract() -> None: +def test_gpu_access_role_uses_shc_proven_device_mode_contract() -> None: defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") - state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") - assert "auplc_render_gid: null" in defaults - assert "auplc_normalize_render_gid: false" in defaults + assert "auplc_gpu_access_enabled: false" in defaults assert 'auplc_rootfs_path: ""' in defaults - assert "getent" in tasks - assert "groupmod" in tasks - assert "auplc_normalize_render_gid" in tasks - assert "_auplc_all_groups" in tasks - assert "reject('match', '^render:')" in tasks - assert "_auplc_render_group.stdout.split(':')[2] | int <= 4294967294" in tasks - assert "notify:" not in tasks - assert "Reload live udev rules on every apply" in tasks - assert "Trigger live udev rules on every apply" in tasks - assert "ansible.builtin.group:" not in tasks + assert "auplc_render_gid" not in defaults + assert "normalize" not in defaults + assert "gpu-access.json" not in preflight + assert "groupmod" not in apply + assert "GID collision" not in preflight assert rules == ( "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) - assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' - - -def test_gpu_access_role_is_wired_for_live_hosts_and_pxe_rootfs_without_legacy_udev_paths() -> None: - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - rocm_tasks = read(ANSIBLE / "roles" / "rocm" / "tasks" / "main.yml") - pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) - pxe_chroot = read(ANSIBLE / "roles" / "pxe_controller" / "templates" / "chroot-setup.sh.j2") - - assert "name: gpu_access" in rocm_playbook - assert "name: gpu_access" in udev_playbook - assert "udev-rocm" not in udev_playbook - assert "render:993" not in rocm_tasks - assert "70-amdgpu.rules" not in rocm_tasks - assert "include_tasks: gpu_access.yml" in pxe_tasks - assert "name: gpu_access" in pxe_gpu_tasks - assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in pxe_gpu_tasks - assert "0666" not in pxe_chroot - assert not (ANSIBLE / "roles" / "udev" / "main.yml").exists() -def test_gpu_access_live_host_playbooks_abort_all_hosts_on_preflight_failure() -> None: - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - - assert "any_errors_fatal: true" in rocm_playbook - assert "any_errors_fatal: true" in udev_playbook - - -def test_gpu_access_role_migrates_only_recognized_legacy_rules_and_reconciles_live_devices() -> None: - tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert "70-kfd.rules" in tasks - assert "70-amdgpu.rules" in tasks - assert "contents:" in tasks - assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in tasks - assert "70-rocm-devices.rules" in tasks - assert 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"' in tasks - assert "islnk" in tasks - assert "ansible.builtin.slurp" in tasks - assert "Define recognized project-owned legacy GPU rules" in tasks - assert "Unexpected legacy GPU rule content" in tasks - assert "udevadm" in tasks - assert "Verify /dev/kfd ownership and mode" in tasks - assert "Verify AMD render node ownership and mode" in tasks - assert "Settle live udev events before inode verification" in tasks - assert ( - tasks.index("Trigger live udev rules on every apply") - < tasks.index("Settle live udev events before inode verification") - < tasks.index("Inspect /dev/kfd after live reconciliation") - ) - assert tasks.index("Verify AMD render node ownership and mode") < tasks.index("Persist target GPU access state") - assert "/sys/class/drm" in tasks - assert "readlink" in tasks - assert "basename" in tasks - assert "DRIVER=amdgpu" not in tasks - assert "notify:" not in tasks - - -def test_gpu_access_role_validates_legacy_rules_before_render_gid_normalization() -> None: +def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() -> None: + validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - tasks = preflight + apply + assert "realpath" in validation + assert "auplc_rootfs_path != '/'" in validation + assert "_auplc_canonical_allowed_root" in validation + assert "Inspect GPU access rootfs target" in preflight + assert "follow: false" in preflight + assert "Reject unsafe canonical GPU access destination parents" in preflight + assert "Reject unsafe canonical GPU access destination" in preflight assert "Define recognized project-owned legacy GPU rules" in preflight - assert "Inspect recognized project-owned legacy GPU rules" in preflight - assert "Reject legacy GPU rule symlinks and non-regular files" in preflight - assert "Read recognized project-owned legacy GPU rules" in preflight assert "Reject unexpected legacy GPU rule content" in preflight - assert "follow: false" in preflight - assert "contents:" in preflight - assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in preflight - assert "not item.skipped | default(false)" in preflight - assert "(item.content | b64decode) in item.item.item.contents" in preflight - assert preflight.index("Inspect recognized project-owned legacy GPU rules") < preflight.index( - "Reject legacy GPU rule symlinks and non-regular files" - ) - assert preflight.index("Reject legacy GPU rule symlinks and non-regular files") < preflight.index( - "Read recognized project-owned legacy GPU rules" - ) - assert preflight.index("Read recognized project-owned legacy GPU rules") < preflight.index( - "Reject unexpected legacy GPU rule content" - ) - assert tasks.index("Reject unexpected legacy GPU rule content") < tasks.index("Normalize live render GID") + assert "70-kfd.rules" in preflight + assert "70-amdgpu.rules" in preflight + assert "70-rocm-devices.rules" in preflight assert "Remove recognized project-owned legacy GPU rules" in apply - assert "Inspect recognized project-owned legacy GPU rules for apply" in apply - assert "Reject legacy GPU rule symlinks and non-regular files before apply" in apply - assert "Read recognized project-owned legacy GPU rules for apply" in apply - assert "Reject unexpected legacy GPU rule content before apply" in apply - assert "register: _auplc_apply_legacy_gpu_rule_stats" in apply - assert "register: _auplc_apply_legacy_gpu_rule_contents" in apply - assert "_auplc_apply_legacy_gpu_rule_contents.results" in apply - assert "_auplc_legacy_gpu_rule_contents.results" not in apply - assert apply.index("Reject legacy GPU rule symlinks and non-regular files before apply") < apply.index( - "Read recognized project-owned legacy GPU rules for apply" - ) - assert apply.index("Read recognized project-owned legacy GPU rules for apply") < apply.index( - "Reject unexpected legacy GPU rule content before apply" - ) assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( "Remove recognized project-owned legacy GPU rules" ) - assert apply.index("Remove recognized project-owned legacy GPU rules") < apply.index("Normalize live render GID") - -def test_pxe_rootfs_lifecycle_uses_an_independent_trusted_parent() -> None: - defaults = read(ANSIBLE / "roles" / "pxe_controller" / "defaults" / "main.yml") - tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - gpu_tasks = read(PXE_GPU_ACCESS_TASKS) - assert 'pxe_nfs_allowed_root: "/srv/nfs"' in defaults - assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in gpu_tasks - assert "Constrain canonical PXE rootfs before lifecycle changes" in tasks - assert tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < tasks.index( - "Admit retained PXE GPU rootfs read-only before lifecycle changes" - ) - - -def test_pxe_rootfs_is_canonicalized_before_gpu_admission() -> None: - tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") +def test_gpu_access_role_reconciles_and_verifies_live_devices_only() -> None: + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "Canonicalize PXE rootfs before lifecycle changes" in tasks - assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( - "Stop NFS before rootfs rebuild" - ) - assert "_pxe_canonical_nfs_root" in tasks - assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( - "Admit retained PXE GPU rootfs read-only before lifecycle changes" + assert "Reload live udev rules on every apply" in apply + assert "Trigger live udev rules on every apply" in apply + assert "Settle live udev events before inode verification" in apply + assert "Inspect /dev/kfd after live reconciliation" in apply + assert "Verify /dev/kfd ownership and mode" in apply + assert "Find live DRM render nodes" in apply + assert "Verify AMD render node ownership and mode" in apply + assert "Find live DRM card nodes" in apply + assert "Verify AMD card node ownership and mode" in apply + assert "/sys/class/drm" in apply + assert "readlink" in apply + assert "basename" in apply + assert "_auplc_kfd.stat.mode == '0666'" in apply + assert "item.stat.mode == '0666'" in apply + assert "item.stat.mode == '0666'" in apply + assert "item.stat.gr_name == 'render'" in apply + assert "item.stat.gr_name == 'video'" in apply + assert ( + apply.index("Trigger live udev rules on every apply") + < apply.index("Settle live udev events before inode verification") + < apply.index("Inspect /dev/kfd after live reconciliation") ) + assert "when: _auplc_target_root | length == 0" in apply + assert "gpu-access.json" not in apply -def test_gpu_access_preflight_refuses_unmanaged_canonical_destinations() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Inspect canonical GPU access destinations" in preflight - assert "70-auplc-gpu-access.rules" in preflight - assert "gpu-access.json" in preflight - assert "follow: false" in preflight - assert "Reject unmanaged canonical GPU access rule" in preflight - assert "Reject invalid canonical GPU access state" in preflight - assert "auplc_from_json_strict" in preflight - assert "| from_json" not in preflight - assert "renderGid" in preflight - assert "version" in preflight - assert 'src: "{{ _auplc_target_root }}{{ item.item }}"' in preflight - assert "Interrupted normalization retry" in preflight - assert "_auplc_current_render_gid == auplc_render_gid" in preflight - assert "_auplc_existing_state.version is integer" in preflight - assert "_auplc_existing_state.renderGid is integer" in preflight - assert "_auplc_existing_state.renderGid | int" not in preflight - - -def test_gpu_access_roles_use_strict_json_for_canonical_state_readers() -> None: +def test_gpu_access_role_rejects_noncanonical_managed_rule_content() -> None: preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") pxe_tasks = read(PXE_GPU_ACCESS_TASKS) - assert "Parse existing canonical GPU access state" in preflight - assert "auplc_from_json_strict" in preflight - assert "auplc_from_json_strict" in pxe_tasks - assert "| from_json" not in preflight - assert "| from_json" not in pxe_tasks - - -def test_canonical_gpu_access_state_contract_is_exact_json() -> None: - state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") - - assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + assert "_auplc_previous_canonical_rule" not in preflight + assert "(_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule" in preflight + assert "Reject unmanaged canonical GPU access rule" in preflight + assert "_auplc_apply_previous_canonical_rule" not in apply + assert "Recheck canonical GPU access rule before apply" in apply + assert "(_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule" in apply + assert "Unmanaged canonical GPU access rule." in apply + assert "_pxe_retained_previous_canonical_rule" not in pxe_tasks + assert "(_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule" in pxe_tasks + assert "non-canonical GPU access rule" in pxe_tasks -def test_gpu_access_role_splits_safe_preflight_and_rootfs_apply() -> None: - defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") - pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) +def test_pxe_gpu_access_installs_rules_without_gid_or_state_contract() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) - assert "auplc_gpu_access_enabled: false" in defaults - assert "auplc_rootfs_allowed_root" in defaults - assert "realpath" in validation - assert "auplc_rootfs_path != '/'" in validation - assert "islnk" in preflight - assert "include_tasks: gpu_access.yml" in pxe_tasks - assert "tasks_from: validate" not in pxe_tasks - assert "Constrain canonical PXE rootfs before lifecycle changes" in pxe_tasks - assert pxe_tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < pxe_tasks.index( + assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main + assert "Re-preflight PXE GPU rootfs before TFTP" in main + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( "Stop NFS before rootfs rebuild" ) - assert "tasks_from: preflight" in pxe_gpu_tasks - assert "tasks_from: apply" in pxe_gpu_tasks - assert "auplc_normalize_render_gid:" in pxe_gpu_tasks - assert "pxe_rootfs_force_rebuild | bool" in pxe_tasks - assert "pxe_gpu_access_normalize_render_gid | bool" not in pxe_tasks - assert "pxe_gpu_access_normalize_render_gid" not in read(PXE_CONTROLLER_ROLE / "defaults" / "main.yml") + assert "Inspect retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU rule" in tasks + assert "tasks_from: preflight" in tasks + assert "tasks_from: apply" in tasks + assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks + assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks + assert "auplc_render_gid" not in tasks + assert "render_gid" not in tasks + assert "groupadd" not in tasks + assert "groupmod" not in tasks + assert "collision" not in tasks.lower() + assert "gpu-access.json" not in tasks + assert "/dev/kfd" not in tasks + assert "/dev/dri" not in tasks -def test_live_playbooks_preflight_gpu_hosts_before_mutating_roles() -> None: +def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") - assert "pre_tasks:" in rocm_playbook - assert "Assert explicit GPU access enablement" in rocm_playbook - assert "auplc_gpu_access_enabled is defined" in rocm_playbook - assert "auplc_gpu_access_enabled is boolean" in rocm_playbook - assert "default(false)" not in rocm_playbook - assert "tasks_from: preflight" in rocm_playbook + assert "any_errors_fatal: true" in rocm_playbook + assert "any_errors_fatal: true" in udev_playbook assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") - assert "- role: rocm" in rocm_playbook - assert rocm_playbook.count("auplc_gpu_access_enabled") >= 3 assert "tasks_from: apply" in rocm_playbook - assert "auplc_gpu_access_enabled" in rocm_playbook - assert "pre_tasks:" in udev_playbook - assert "Assert explicit GPU access enablement" in udev_playbook - assert "auplc_gpu_access_enabled is defined" in udev_playbook - assert "auplc_gpu_access_enabled is boolean" in udev_playbook - assert "default(false)" not in udev_playbook assert "tasks_from: preflight" in udev_playbook assert "tasks_from: apply" in udev_playbook + assert "render_gid" not in pxe_playbook -def test_gpu_access_discovery_playbook_is_read_only_and_serializes_live_host_evidence() -> None: - playbook = read(ANSIBLE / "playbooks" / "pb-gpu-access-discovery.yml") - - assert "hosts: k3s_cluster" in playbook - assert "gather_facts: false" in playbook - assert "ignore_unreachable: true" in playbook - assert "ansible.builtin.command:" in playbook - assert "ansible.builtin.stat:" in playbook - assert "ansible.builtin.slurp:" in playbook - assert "ansible.builtin.shell:" not in playbook - assert "changed_when: false" in playbook - assert "lspci" in playbook - assert '"1002::0300"' in playbook - assert '"1002::0302"' in playbook - assert '"1002::0380"' in playbook - assert "getent" in playbook - assert "/sys/bus/pci/devices" in playbook - assert "gpu_access_discovery_output_path" in playbook - assert "delegate_to: localhost" in playbook - assert "ansible.builtin.copy:" in playbook - assert "to_json" in playbook - assert "stat_success" in playbook - assert "content_success" in playbook - assert "legacy_rules" in playbook - assert "/etc/udev/rules.d/70-kfd.rules" in playbook - assert "/etc/udev/rules.d/70-amdgpu.rules" in playbook - assert "/etc/udev/rules.d/70-rocm-devices.rules" in playbook - file_probes = playbook[ - playbook.index("Inspect persisted GPU access state") : playbook.index( - "Record machine-readable GPU access discovery evidence" - ) - ] - assert file_probes.count("ignore_errors: true") == 10 - assert "failed_when: false" not in file_probes - assert 'mode: "0600"' in playbook - assert "hosts: pxe_controller" not in playbook +def test_pxe_controller_playbook_has_no_obsolete_finalizer_post_tasks() -> None: + pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + assert "post_tasks:" not in pxe_playbook + assert "pxe_finalizer_" not in pxe_playbook + assert "--finalize-pxe" not in pxe_playbook -def test_pxe_gpu_admission_resolves_fresh_rootfs_and_refuses_retained_migrations() -> None: - assert PXE_GPU_ACCESS_TASKS.exists() - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Record PXE rootfs state before lifecycle changes" in main - assert "_pxe_rootfs_existed_at_start" in main - assert "_pxe_rootfs_rebuilt_this_run" in main - assert "include_tasks: gpu_access.yml" in main - assert main.index("Record PXE rootfs state before lifecycle changes") < main.index("Stop NFS before rootfs rebuild") - assert main.index("include_tasks: gpu_access.yml") < main.index("Find latest kernel in rootfs") - assert "tasks_from: validate" not in main - assert "tasks_from: preflight" not in main - assert "tasks_from: apply" not in main - - assert "_pxe_rootfs_disposition" in tasks - assert "_pxe_unanimous_live_render_gid" in tasks - assert "_pxe_resolved_render_gid" in tasks - assert "fresh" in tasks - assert "retained" in tasks - assert "groupadd" in tasks - assert "--system" in tasks - assert "groupmod" not in tasks - assert "getent" in tasks - assert 'auplc_render_gid: "{{ _pxe_resolved_render_gid }}"' in tasks - assert "auplc_normalize_render_gid: \"{{ _pxe_rootfs_disposition == 'fresh' }}\"" in tasks - assert "Require retained PXE legacy GPU rules absent" in tasks - assert "Require retained PXE render GID matches unanimous live GID" in tasks - assert "tasks_from: preflight" in tasks - assert "tasks_from: apply" in tasks - assert "render:993" not in tasks - assert "lspci" not in tasks - assert "pxe_gpu_access_normalize_render_gid" not in tasks - - -def test_pxe_gpu_admission_preflights_retained_rootfs_before_lifecycle_mutation() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - - retained_admission = "Admit retained PXE GPU rootfs read-only before lifecycle changes" - final_admission = "Re-preflight PXE GPU rootfs before TFTP" - - assert main.count("include_tasks: gpu_access.yml") == 2 - assert main.index("Record PXE rootfs state before lifecycle changes") < main.index(retained_admission) - assert main.index(retained_admission) < main.index("Stop NFS before rootfs rebuild") - assert main.index("Remove chroot setup script") < main.index(final_admission) - assert main.index(final_admission) < main.index("Find latest kernel in rootfs") - - retained_branch = main[main.index(retained_admission) : main.index("Stop NFS before rootfs rebuild")] - final_branch = main[main.index(final_admission) : main.index("Find latest kernel in rootfs")] - - assert "pxe_gpu_access_enabled | bool" in retained_branch - assert "not (_pxe_rootfs_rebuilt_this_run | bool)" in retained_branch - assert "pxe_gpu_access_enabled | bool" in final_branch - assert "pxe_gpu_admission_phase: final" in final_branch - - -def test_pxe_rootfs_disposition_uses_initial_root_path_and_rejects_partial_retained_trees() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - - assert "Require existing PXE rootfs is a directory" in main - assert "Require incomplete PXE rootfs force rebuild" in main - assert '_pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}"' in main - assert "not (_pxe_rootfs_lstat.stat.exists | bool)" in main - assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("Stop NFS before rootfs rebuild") - assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("- name: Build NFS rootfs") - - -def test_pxe_retained_admission_is_read_only_until_post_chroot_repreflight_and_apply() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main - assert "Re-preflight PXE GPU rootfs before TFTP" in main - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Stop NFS before rootfs rebuild" +def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() -> None: + forbidden = ( + "auplc_render_gid", + "auplc_normalize_render_gid", + "gpu-access.json", + "auplc_from_json_strict", + "groupmod", + "render GID collision", ) - assert main.index("Remove chroot setup script") < main.index("Re-preflight PXE GPU rootfs before TFTP") - assert "pxe_gpu_admission_phase: retained-read-only" in main - assert "pxe_gpu_admission_phase: final" in main - assert "Require retained PXE canonical GPU rule" in tasks - assert "Require retained PXE canonical GPU state" in tasks - assert "Apply GPU access after final PXE re-preflight" in tasks - retained_read_only = tasks[: tasks.index("Preflight GPU access after final PXE re-preflight")] - assert "tasks_from: apply" not in retained_read_only - assert "pxe_gpu_admission_phase == 'final'" in tasks - -def test_pxe_retained_admission_checks_canonical_parent_chain_before_lifecycle_or_chroot_mutation() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Inspect retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU access parents" in tasks - for parent in ("/etc", "/etc/udev", "/etc/udev/rules.d", "/var", "/var/lib", "/var/lib/auplc"): - assert parent in tasks - assert "item.stat.exists" in tasks - assert "item.stat.isdir" in tasks - assert "not item.stat.islnk" in tasks - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Execute chroot setup" + ansible_text = "\n".join( + path.read_text(encoding="utf-8") + for path in ANSIBLE.rglob("*") + if path.is_file() and "__pycache__" not in path.parts ) + + for term in forbidden: + assert term not in ansible_text From b1fd9a35bf5597497432b2a727ffe96d00df2332 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 067/180] refactor(ansible): simplify PXE GPU policy --- .../ansible/playbooks/pb-pxe-controller.yml | 38 --- .../roles/pxe_controller/tasks/gpu_access.yml | 221 ++---------------- 2 files changed, 14 insertions(+), 245 deletions(-) diff --git a/deploy/ansible/playbooks/pb-pxe-controller.yml b/deploy/ansible/playbooks/pb-pxe-controller.yml index 250508cc..425b939e 100644 --- a/deploy/ansible/playbooks/pb-pxe-controller.yml +++ b/deploy/ansible/playbooks/pb-pxe-controller.yml @@ -102,41 +102,3 @@ roles: - role: pxe_controller - - post_tasks: - - name: Write private PXE finalizer handoff from resolved rootfs facts - ansible.builtin.copy: - content: >- - {{ { - 'version': 1, - 'generation': pxe_finalizer_generation, - 'spec_sha256': pxe_finalizer_spec_sha256, - 'topology': 'pxe-diskless', - 'pxe_gpu_access_enabled': pxe_gpu_access_enabled | bool, - 'render_gid': _pxe_resolved_render_gid | default(none) - } | to_json }} - dest: "{{ pxe_finalizer_handoff }}" - mode: "0600" - delegate_to: localhost - run_once: true - become: false - no_log: true - when: pxe_finalizer_context is defined - - - name: Finalize generated PXE GPU policy from resolved rootfs facts - ansible.builtin.command: - argv: - - "{{ pxe_finalizer_script }}" - - --finalize-pxe - - --out-dir - - "{{ pxe_finalizer_context | dirname }}" - - --context - - "{{ pxe_finalizer_context }}" - - --handoff - - "{{ pxe_finalizer_handoff }}" - delegate_to: localhost - run_once: true - become: false - no_log: true - changed_when: false - when: pxe_finalizer_context is defined diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml index 06b5d5f8..d0a0bd5b 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -4,171 +4,12 @@ - name: Record PXE GPU admission disposition ansible.builtin.set_fact: _pxe_rootfs_disposition: "{{ 'fresh' if _pxe_rootfs_rebuilt_this_run | bool else 'retained' }}" - _pxe_unanimous_live_render_gid: "{{ auplc_render_gid if auplc_render_gid is defined and auplc_render_gid is not none else none }}" - _pxe_resolved_render_gid: null - name: Assert PXE GPU admission phase ansible.builtin.assert: that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] fail_msg: PXE GPU admission phase is invalid. -- name: Validate optional unanimous live render GID - ansible.builtin.assert: - that: - - _pxe_unanimous_live_render_gid is integer - - _pxe_unanimous_live_render_gid >= 1 - - _pxe_unanimous_live_render_gid <= 4294967294 - fail_msg: auplc_render_gid must be an integer between 1 and 4294967294 when supplied for a PXE GPU rootfs. - when: - - pxe_gpu_access_enabled | bool - - _pxe_unanimous_live_render_gid is not none - -- name: Inspect PXE rootfs render group for GPU admission - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] - register: _pxe_admission_render_group - changed_when: false - failed_when: false - when: pxe_gpu_access_enabled | bool - -- name: Require fresh PXE render group lookup outcome - ansible.builtin.assert: - that: _pxe_admission_render_group.rc in [0, 2] - fail_msg: Unable to determine whether the fresh PXE rootfs has a render group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' - -- name: Require strict existing fresh PXE render group - ansible.builtin.assert: - that: - - _pxe_admission_render_group.stdout_lines | length == 1 - - _pxe_admission_render_group.stdout.split(':') | length == 4 - - _pxe_admission_render_group.stdout.split(':')[0] == 'render' - - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Fresh PXE rootfs render group is malformed. - when: >- - pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' and - _pxe_admission_render_group.rc == 0 - -- name: Require strict retained PXE render group - ansible.builtin.assert: - that: - - _pxe_admission_render_group.rc == 0 - - _pxe_admission_render_group.stdout_lines | length == 1 - - _pxe_admission_render_group.stdout.split(':') | length == 4 - - _pxe_admission_render_group.stdout.split(':')[0] == 'render' - - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Retained PXE rootfs must already have one valid render group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Record existing PXE render GID - ansible.builtin.set_fact: - _pxe_existing_render_gid: "{{ _pxe_admission_render_group.stdout.split(':')[2] | int }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_admission_render_group.rc == 0 - -- name: List fresh PXE rootfs groups before render GID creation - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group] - register: _pxe_fresh_groups - changed_when: false - failed_when: false - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - - _pxe_unanimous_live_render_gid is not none - -- name: Reject fresh PXE render GID collision - ansible.builtin.assert: - that: - - _pxe_fresh_groups.rc == 0 - - >- - _pxe_fresh_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (_pxe_unanimous_live_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: Fresh PXE rootfs render GID is already assigned to another group. - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - - _pxe_unanimous_live_render_gid is not none - -- name: Create missing fresh PXE render group - ansible.builtin.command: - argv: >- - {{ ['chroot', pxe_nfs_root, 'groupadd', '--system', '-g', (_pxe_unanimous_live_render_gid | string), 'render'] - if _pxe_unanimous_live_render_gid is not none - else ['chroot', pxe_nfs_root, 'groupadd', '--system', 'render'] }} - changed_when: true - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Read fresh PXE render group after creation - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] - register: _pxe_created_render_group - changed_when: false - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Resolve newly created fresh PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_created_render_group.stdout.split(':')[2] | int }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Resolve existing fresh PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is not none - -- name: Resolve retained PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - -- name: List retained PXE rootfs groups for render GID collision check - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group] - register: _pxe_retained_groups - changed_when: false - failed_when: false - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Reject retained PXE render GID collision - ansible.builtin.assert: - that: - - _pxe_retained_groups.rc == 0 - - >- - _pxe_retained_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (_pxe_resolved_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: Retained PXE rootfs render GID is already assigned to another group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE render GID matches unanimous live GID - ansible.builtin.assert: - that: _pxe_resolved_render_gid == _pxe_unanimous_live_render_gid - fail_msg: Retained PXE rootfs render GID differs from the supplied unanimous live render GID; rebuild or migrate it separately. - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - _pxe_unanimous_live_render_gid is not none - - name: Inspect retained PXE canonical GPU access parents ansible.builtin.stat: path: "{{ pxe_nfs_root }}{{ item }}" @@ -177,9 +18,6 @@ - /etc - /etc/udev - /etc/udev/rules.d - - /var - - /var/lib - - /var/lib/auplc register: _pxe_retained_canonical_gpu_parent_stats when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' @@ -202,7 +40,6 @@ - /etc/udev/rules.d/70-amdgpu.rules - /etc/udev/rules.d/70-rocm-devices.rules - /etc/udev/rules.d/70-auplc-gpu-access.rules - - /var/lib/auplc/gpu-access.json register: _pxe_retained_gpu_policy_stats when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' @@ -213,60 +50,34 @@ loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Require retained PXE canonical GPU destinations +- name: Require retained PXE canonical GPU rule destination ansible.builtin.assert: that: - - item.stat.exists - - item.stat.isreg - - not item.stat.islnk - fail_msg: "Retained PXE rootfs requires an exact canonical GPU access destination: {{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + - _pxe_retained_gpu_policy_stats.results[3].stat.exists + - _pxe_retained_gpu_policy_stats.results[3].stat.isreg + - not _pxe_retained_gpu_policy_stats.results[3].stat.islnk + fail_msg: Retained PXE rootfs requires an exact canonical GPU access rule. when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Read retained PXE canonical GPU access destinations +- name: Read retained PXE canonical GPU rule ansible.builtin.slurp: - src: "{{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" - register: _pxe_retained_canonical_gpu_destinations + src: "{{ _pxe_retained_gpu_policy_stats.results[3].item }}" + register: _pxe_retained_canonical_gpu_rule when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Define retained PXE canonical GPU rule +- name: Define retained PXE canonical GPU rules ansible.builtin.set_fact: _pxe_retained_canonical_rule: | # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - name: Require retained PXE canonical GPU rule ansible.builtin.assert: - that: (item.content | b64decode) == _pxe_retained_canonical_rule - fail_msg: "Retained PXE rootfs has a non-canonical GPU access rule: {{ item.item.item }}" - loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - item.item.item.endswith('70-auplc-gpu-access.rules') - -- name: Parse retained PXE canonical GPU state - ansible.builtin.set_fact: - _pxe_retained_canonical_state: "{{ item.content | b64decode | auplc_from_json_strict }}" - loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - item.item.item.endswith('gpu-access.json') - -- name: Require retained PXE canonical GPU state - ansible.builtin.assert: - that: - - _pxe_retained_canonical_state is mapping - - _pxe_retained_canonical_state.keys() | list | sort == ['renderGid', 'version'] - - _pxe_retained_canonical_state.version is integer - - _pxe_retained_canonical_state.version == 1 - - _pxe_retained_canonical_state.renderGid is integer - - _pxe_retained_canonical_state.renderGid == _pxe_resolved_render_gid - fail_msg: Retained PXE rootfs has a non-canonical GPU access state. + that: (_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule + fail_msg: Retained PXE rootfs has a non-canonical GPU access rule. when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - name: Preflight GPU access after final PXE re-preflight @@ -276,8 +87,6 @@ vars: auplc_rootfs_path: "{{ pxe_nfs_root }}" auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" - auplc_render_gid: "{{ _pxe_resolved_render_gid }}" - auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" when: - pxe_gpu_access_enabled | bool - pxe_gpu_admission_phase == 'final' @@ -289,8 +98,6 @@ vars: auplc_rootfs_path: "{{ pxe_nfs_root }}" auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" - auplc_render_gid: "{{ _pxe_resolved_render_gid }}" - auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" when: - pxe_gpu_access_enabled | bool - pxe_gpu_admission_phase == 'final' From 117b43b7551a92c6db7751335342b619ab82deec Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 068/180] refactor(deploy): simplify GPU discovery --- .../playbooks/pb-gpu-access-discovery.yml | 249 +----------------- .../scripts/gpu_access_resolution.py | 159 +---------- tests/skills/test_gpu_access_resolution.py | 244 +++-------------- 3 files changed, 54 insertions(+), 598 deletions(-) diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml index cd8c366a..f788e0d9 100644 --- a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -1,4 +1,4 @@ -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- - name: Discover fleet GPU-access evidence hosts: k3s_cluster @@ -14,48 +14,6 @@ sysfs: rc: 255 stdout: "" - render_group: - rc: 255 - stdout: "" - groups: - rc: 255 - stdout: "" - state: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - rule: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - legacy_rules: - kfd: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - amdgpu: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - rocm_devices: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" pre_tasks: - name: Require a safe local discovery evidence output path ansible.builtin.assert: @@ -63,9 +21,7 @@ - gpu_access_discovery_output_path is defined - gpu_access_discovery_output_path is string - gpu_access_discovery_output_path is match('^/') - fail_msg: >- - Set gpu_access_discovery_output_path to an absolute controller-local - path before running discovery. + fail_msg: Set gpu_access_discovery_output_path to an absolute controller-local path before running discovery. delegate_to: localhost run_once: true changed_when: false @@ -113,33 +69,21 @@ tasks: - name: Discover AMD VGA display BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0300" + argv: [lspci, -Dnn, -d, "1002::0300"] register: _auplc_discovery_lspci_vga changed_when: false failed_when: false - name: Discover AMD 3D display BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0302" + argv: [lspci, -Dnn, -d, "1002::0302"] register: _auplc_discovery_lspci_3d changed_when: false failed_when: false - name: Discover AMD display-controller BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0380" + argv: [lspci, -Dnn, -d, "1002::0380"] register: _auplc_discovery_lspci_display changed_when: false failed_when: false @@ -172,120 +116,6 @@ changed_when: false failed_when: false - - name: Read render group record - ansible.builtin.command: - argv: - - getent - - group - - render - register: _auplc_discovery_render_group - changed_when: false - failed_when: false - - - name: Read all group records for render GID collision detection - ansible.builtin.command: - argv: - - getent - - group - register: _auplc_discovery_groups - changed_when: false - failed_when: false - - - name: Inspect persisted GPU access state - ansible.builtin.stat: - path: /var/lib/auplc/gpu-access.json - follow: false - register: _auplc_discovery_state - changed_when: false - ignore_errors: true - - - name: Read persisted GPU access state - ansible.builtin.slurp: - src: /var/lib/auplc/gpu-access.json - register: _auplc_discovery_state_content - when: - - _auplc_discovery_state.stat.exists | default(false) - - _auplc_discovery_state.stat.isreg | default(false) - - not (_auplc_discovery_state.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect canonical GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-auplc-gpu-access.rules - follow: false - register: _auplc_discovery_rule - changed_when: false - ignore_errors: true - - - name: Read canonical GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-auplc-gpu-access.rules - register: _auplc_discovery_rule_content - when: - - _auplc_discovery_rule.stat.exists | default(false) - - _auplc_discovery_rule.stat.isreg | default(false) - - not (_auplc_discovery_rule.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy kfd GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-kfd.rules - follow: false - register: _auplc_discovery_legacy_kfd - changed_when: false - ignore_errors: true - - - name: Read legacy kfd GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-kfd.rules - register: _auplc_discovery_legacy_kfd_content - when: - - _auplc_discovery_legacy_kfd.stat.exists | default(false) - - _auplc_discovery_legacy_kfd.stat.isreg | default(false) - - not (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy amdgpu GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-amdgpu.rules - follow: false - register: _auplc_discovery_legacy_amdgpu - changed_when: false - ignore_errors: true - - - name: Read legacy amdgpu GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-amdgpu.rules - register: _auplc_discovery_legacy_amdgpu_content - when: - - _auplc_discovery_legacy_amdgpu.stat.exists | default(false) - - _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) - - not (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy ROCm devices GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-rocm-devices.rules - follow: false - register: _auplc_discovery_legacy_rocm_devices - changed_when: false - ignore_errors: true - - - name: Read legacy ROCm devices GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-rocm-devices.rules - register: _auplc_discovery_legacy_rocm_devices_content - when: - - _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) - - _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) - - not (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - name: Record machine-readable GPU access discovery evidence ansible.builtin.set_fact: _auplc_gpu_access_discovery_evidence: @@ -297,79 +127,12 @@ sysfs: rc: "{{ _auplc_discovery_sysfs.rc }}" stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" - render_group: - rc: "{{ _auplc_discovery_render_group.rc }}" - stdout: "{{ _auplc_discovery_render_group.stdout | default('') }}" - groups: - rc: "{{ _auplc_discovery_groups.rc }}" - stdout: "{{ _auplc_discovery_groups.stdout | default('') }}" - state: - stat_success: "{{ not (_auplc_discovery_state.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_state.failed | default(false)) and - (not (_auplc_discovery_state.stat.exists | default(false)) or - not (_auplc_discovery_state.stat.isreg | default(false)) or - (_auplc_discovery_state.stat.islnk | default(false)) or - not (_auplc_discovery_state_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_state.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_state.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_state.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_state_content.content | default('') | b64decode }}" - rule: - stat_success: "{{ not (_auplc_discovery_rule.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_rule.failed | default(false)) and - (not (_auplc_discovery_rule.stat.exists | default(false)) or - not (_auplc_discovery_rule.stat.isreg | default(false)) or - (_auplc_discovery_rule.stat.islnk | default(false)) or - not (_auplc_discovery_rule_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_rule.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_rule.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_rule.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_rule_content.content | default('') | b64decode }}" - legacy_rules: - kfd: - stat_success: "{{ not (_auplc_discovery_legacy_kfd.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_kfd.failed | default(false)) and - (not (_auplc_discovery_legacy_kfd.stat.exists | default(false)) or - not (_auplc_discovery_legacy_kfd.stat.isreg | default(false)) or - (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_kfd_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_kfd.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_kfd.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_kfd.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_kfd_content.content | default('') | b64decode }}" - amdgpu: - stat_success: "{{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) and - (not (_auplc_discovery_legacy_amdgpu.stat.exists | default(false)) or - not (_auplc_discovery_legacy_amdgpu.stat.isreg | default(false)) or - (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_amdgpu_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_amdgpu.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_amdgpu.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_amdgpu_content.content | default('') | b64decode }}" - rocm_devices: - stat_success: "{{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) and - (not (_auplc_discovery_legacy_rocm_devices.stat.exists | default(false)) or - not (_auplc_discovery_legacy_rocm_devices.stat.isreg | default(false)) or - (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_rocm_devices_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_rocm_devices.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_rocm_devices_content.content | default('') | b64decode }}" changed_when: false - name: Write machine-readable GPU access discovery evidence locally ansible.builtin.copy: content: | - {"version":2,"hosts":[{% for discovery_host in ansible_play_hosts_all %} + {"version":1,"hosts":[{% for discovery_host in ansible_play_hosts_all %} {{ ( hostvars[discovery_host]._auplc_gpu_access_discovery_evidence | default( diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py index b618b2f8..555c456d 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py @@ -11,13 +11,7 @@ from config_common import DuplicateJsonKeyError, strict_json_loads from gpu_resolution_manifest import ResolutionManifest, build_resolution_manifest -EVIDENCE_VERSION: Final = 2 -MAX_RENDER_GID: Final = 4_294_967_294 -CANONICAL_RULE: Final = ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' -) +EVIDENCE_VERSION: Final = 1 BDF_PATTERN: Final = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") @@ -58,41 +52,18 @@ class CommandEvidence: stdout: str -@dataclass(frozen=True, slots=True) -class FileEvidence: - stat_success: bool - content_success: bool - exists: bool - regular: bool - symlink: bool - content: str - - -@dataclass(frozen=True, slots=True) -class LegacyRuleEvidence: - kfd: FileEvidence - amdgpu: FileEvidence - rocm_devices: FileEvidence - - @dataclass(frozen=True, slots=True) class HostEvidence: target: InventoryTarget reachable: bool lspci: CommandEvidence sysfs: CommandEvidence - render_group: CommandEvidence - groups: CommandEvidence - state: FileEvidence - rule: FileEvidence - legacy_rules: LegacyRuleEvidence @dataclass(frozen=True, slots=True) class HostResolution: target: InventoryTarget status: HostStatus - render_gid: int | None reason: str | None @@ -100,7 +71,6 @@ class HostResolution: class FleetResolution: status: FleetStatus hosts: tuple[HostResolution, ...] - render_gid: int | None reason: str | None @@ -135,26 +105,21 @@ def resolve_fleet(expected_targets: tuple[InventoryTarget, ...], evidence: tuple return _blocked(resolutions, "unknown host evidence") gpu_hosts = tuple(host for host in resolutions if host.status is HostStatus.GPU) if not gpu_hosts: - return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None, None) - gids = {host.render_gid for host in gpu_hosts} - if len(gids) != 1: - return _blocked(resolutions, "GPU render GIDs disagree") - return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, next(iter(gids)), None) + return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None) + return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, None) def resolution_manifest(resolution: FleetResolution) -> ResolutionManifest: """Build the public serialized manifest for a resolved fleet.""" return build_resolution_manifest( - version=1, status=resolution.status.value, - render_gid=resolution.render_gid, hosts={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, ) def _parse_host(raw, field: str) -> HostEvidence: _require_mapping(raw, field) - required = {"host", "reachable", "lspci", "sysfs", "render_group", "groups", "state", "rule", "legacy_rules"} + required = {"host", "reachable", "lspci", "sysfs"} if set(raw) != required or type(raw["host"]) is not str or not raw["host"]: raise EvidenceParseError(field=field) if type(raw["reachable"]) is not bool: @@ -164,11 +129,6 @@ def _parse_host(raw, field: str) -> HostEvidence: reachable=raw["reachable"], lspci=_parse_command(raw["lspci"], f"{field}.lspci"), sysfs=_parse_command(raw["sysfs"], f"{field}.sysfs"), - render_group=_parse_command(raw["render_group"], f"{field}.render_group"), - groups=_parse_command(raw["groups"], f"{field}.groups"), - state=_parse_file(raw["state"], f"{field}.state"), - rule=_parse_file(raw["rule"], f"{field}.rule"), - legacy_rules=_parse_legacy_rules(raw["legacy_rules"], f"{field}.legacy_rules"), ) @@ -179,29 +139,6 @@ def _parse_command(raw, field: str) -> CommandEvidence: return CommandEvidence(rc=raw["rc"], stdout=raw["stdout"]) -def _parse_file(raw, field: str) -> FileEvidence: - _require_mapping(raw, field) - required = {"stat_success", "content_success", "exists", "regular", "symlink", "content"} - if set(raw) != required or any( - type(raw[key]) is not bool for key in ("stat_success", "content_success", "exists", "regular", "symlink") - ): - raise EvidenceParseError(field=field) - if type(raw["content"]) is not str: - raise EvidenceParseError(field=f"{field}.content") - return FileEvidence(**raw) - - -def _parse_legacy_rules(raw, field: str) -> LegacyRuleEvidence: - _require_mapping(raw, field) - if set(raw) != {"kfd", "amdgpu", "rocm_devices"}: - raise EvidenceParseError(field=field) - return LegacyRuleEvidence( - kfd=_parse_file(raw["kfd"], f"{field}.kfd"), - amdgpu=_parse_file(raw["amdgpu"], f"{field}.amdgpu"), - rocm_devices=_parse_file(raw["rocm_devices"], f"{field}.rocm_devices"), - ) - - def _require_mapping(value, field: str) -> None: if type(value) is not dict: raise EvidenceParseError(field=field) @@ -210,20 +147,13 @@ def _require_mapping(value, field: str) -> None: def _resolve_host(evidence: HostEvidence) -> HostResolution: if not evidence.reachable or evidence.lspci.rc != 0 or evidence.sysfs.rc != 0: return _unknown(evidence, "GPU discovery probe failed") - if not _file_probes_succeeded(evidence): - return _unknown(evidence, "GPU access file probe failed") lspci_bdfs = _bdfs(evidence.lspci.stdout) sysfs_bdfs = _bdfs(evidence.sysfs.stdout) if lspci_bdfs is None or sysfs_bdfs is None or lspci_bdfs != sysfs_bdfs: return _unknown(evidence, "AMD GPU BDF probes disagree") if not lspci_bdfs: - if evidence.state.exists or evidence.rule.exists or _legacy_rule_exists(evidence.legacy_rules): - return _unknown(evidence, "CPU host retains GPU access contract") - return HostResolution(evidence.target, HostStatus.CPU, None, None) - render_gid = _render_gid(evidence) - if render_gid is None or not _safe_gpu_files(evidence, render_gid): - return _unknown(evidence, "GPU access contract is unsafe") - return HostResolution(evidence.target, HostStatus.GPU, render_gid, None) + return HostResolution(evidence.target, HostStatus.CPU, None) + return HostResolution(evidence.target, HostStatus.GPU, None) def _bdfs(stdout: str) -> frozenset[str] | None: @@ -233,82 +163,9 @@ def _bdfs(stdout: str) -> frozenset[str] | None: return None -def _render_gid(evidence: HostEvidence) -> int | None: - if evidence.render_group.rc != 0 or evidence.groups.rc != 0: - return None - record = _group_record(evidence.render_group.stdout) - if record is None or record[0] != "render": - return None - gid = record[1] - groups = tuple(_group_record(line) for line in evidence.groups.stdout.splitlines()) - if not groups or any(group is None for group in groups): - return None - if sum(group[0] == "render" and group[1] == gid for group in groups) != 1: - return None - if any(group[0] != "render" and group[1] == gid for group in groups): - return None - return gid - - -def _group_record(record: str) -> tuple[str, int] | None: - fields = record.split(":") - if len(fields) != 4 or not fields[0] or not fields[2].isascii() or not fields[2].isdecimal(): - return None - gid = int(fields[2]) - if 1 <= gid <= MAX_RENDER_GID: - return fields[0], gid - return None - - -def _safe_gpu_files(evidence: HostEvidence, render_gid: int) -> bool: - if not _safe_file(evidence.state) or not _safe_file(evidence.rule): - return False - if evidence.state.exists and _state_gid(evidence.state.content) != render_gid: - return False - return not evidence.rule.exists or evidence.rule.content == CANONICAL_RULE - - -def _safe_file(evidence: FileEvidence) -> bool: - if not evidence.stat_success or not evidence.content_success: - return False - if evidence.exists: - return evidence.regular and not evidence.symlink - return not evidence.regular and not evidence.symlink and not evidence.content - - -def _file_probes_succeeded(evidence: HostEvidence) -> bool: - return all( - file_evidence.stat_success and file_evidence.content_success - for file_evidence in ( - evidence.state, - evidence.rule, - evidence.legacy_rules.kfd, - evidence.legacy_rules.amdgpu, - evidence.legacy_rules.rocm_devices, - ) - ) - - -def _legacy_rule_exists(evidence: LegacyRuleEvidence) -> bool: - return any(file_evidence.exists for file_evidence in (evidence.kfd, evidence.amdgpu, evidence.rocm_devices)) - - -def _state_gid(raw: str) -> int | None: - try: - state = strict_json_loads(raw) - except (DuplicateJsonKeyError, TypeError, json.JSONDecodeError): - return None - if type(state) is not dict or set(state) != {"renderGid", "version"}: - return None - gid = state["renderGid"] - if type(gid) is not int or type(state["version"]) is not int or state["version"] != 1: - return None - return gid if 1 <= gid <= MAX_RENDER_GID else None - - def _unknown(evidence: HostEvidence, reason: str) -> HostResolution: - return HostResolution(evidence.target, HostStatus.UNKNOWN, None, reason) + return HostResolution(evidence.target, HostStatus.UNKNOWN, reason) def _blocked(hosts: tuple[HostResolution, ...], reason: str) -> FleetResolution: - return FleetResolution(FleetStatus.BLOCKED, hosts, None, reason) + return FleetResolution(FleetStatus.BLOCKED, hosts, reason) diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index ea853f00..37003f28 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -6,6 +6,7 @@ import importlib.util import json +import re import sys from pathlib import Path @@ -14,6 +15,7 @@ ROOT = Path(__file__).resolve().parents[2] RESOLUTION = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_access_resolution.py" MANIFEST = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_resolution_manifest.py" +DISCOVERY_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" GPU_BDF = "0000:03:00.0" @@ -45,13 +47,6 @@ def host_evidence( lspci_rc: int = 0, sysfs_rc: int = 0, reachable: bool = True, - render_gid: int = 993, - group_listing: str | None = None, - state: str | None = None, - rule: str | None = None, - state_stat_success: bool = True, - state_content_success: bool = True, - legacy_rules: dict[str, str | None] | None = None, ) -> dict: lspci = "\n".join(lspci_bdfs or []) sysfs = "\n".join(sysfs_bdfs if sysfs_bdfs is not None else lspci_bdfs or []) @@ -60,67 +55,44 @@ def host_evidence( "reachable": reachable, "lspci": {"rc": lspci_rc, "stdout": lspci}, "sysfs": {"rc": sysfs_rc, "stdout": sysfs}, - "render_group": {"rc": 0, "stdout": f"render:x:{render_gid}:\n"}, - "groups": {"rc": 0, "stdout": group_listing or f"render:x:{render_gid}:\n"}, - "state": { - "stat_success": state_stat_success, - "content_success": state_content_success, - "exists": state is not None, - "regular": state is not None, - "symlink": False, - "content": state or "", - }, - "rule": { - "stat_success": True, - "content_success": True, - "exists": rule is not None, - "regular": rule is not None, - "symlink": False, - "content": rule or "", - }, - "legacy_rules": { - key: { - "stat_success": True, - "content_success": True, - "exists": content is not None, - "regular": content is not None, - "symlink": False, - "content": content or "", - } - for key, content in (legacy_rules or {}).items() - } - | { - key: { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - } - for key in ("kfd", "amdgpu", "rocm_devices") - if key not in (legacy_rules or {}) - }, } def evidence_document(*hosts: dict) -> str: - return json.dumps({"version": 2, "hosts": list(hosts)}) + return json.dumps({"version": 1, "hosts": list(hosts)}) def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) +def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> None: + playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") + evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( + " changed_when:", maxsplit=1 + )[0] + fallback_block = playbook.split("_auplc_gpu_access_unknown_evidence:", maxsplit=1)[1].split( + " pre_tasks:", maxsplit=1 + )[0] + evidence_keys = re.findall(r"^ ([a-z_]+):", evidence_block, re.MULTILINE) + fallback_keys = re.findall(r"^ ([a-z_]+):", fallback_block, re.MULTILINE) + + assert evidence_keys == ["host", "reachable", "lspci", "sysfs"] + assert fallback_keys == ["reachable", "lspci", "sysfs"] + assert "combine({'host': discovery_host})" in playbook + assert '{"version":1,"hosts":[' in playbook + assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook + assert "| to_json" in playbook + + def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: module = load_resolution_module() - raw = evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF])) - evidence = module.parse_fleet_evidence(raw) + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) assert evidence[0].target == module.InventoryTarget(name="gpu-1") assert evidence[0].lspci.stdout == GPU_BDF - assert evidence[0].state.exists is False + assert evidence[0].sysfs.stdout == GPU_BDF @pytest.mark.parametrize( @@ -143,38 +115,7 @@ def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: module = load_resolution_module() with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): - module.parse_fleet_evidence('{"version":2,"version":2,"hosts":[]}') - - -def test_resolve_fleet_blocks_duplicate_persisted_state_render_gid() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state='{"renderGid":993,"renderGid":993,"version":1}') - ) - ) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - - -@pytest.mark.parametrize( - "state", - [ - '{"renderGid":993,"version":1,"version":1}', - '{"renderGid":993,"version":1,"r\\u0065nderGid":993}', - '{"renderGid":993,"version":1,"v\\u0065rsion":1}', - ], - ids=["duplicate-version", "escaped-render-gid", "escaped-version"], -) -def test_resolve_fleet_blocks_semantic_duplicate_persisted_state_keys(state: str) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state=state))) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED + module.parse_fleet_evidence('{"version":1,"version":1,"hosts":[]}') def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: @@ -184,7 +125,6 @@ def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), evidence) assert resolution.status is module.FleetStatus.GPU_RESOLVED - assert resolution.render_gid == 993 assert resolution.hosts[0].status is module.HostStatus.GPU @@ -195,7 +135,6 @@ def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), evidence) assert resolution.status is module.FleetStatus.CPU_ONLY - assert resolution.render_gid is None assert resolution.hosts[0].status is module.HostStatus.CPU @@ -235,174 +174,71 @@ def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.render_gid is None + assert resolution.reason == "incomplete host coverage" -def test_resolve_fleet_blocks_render_gid_collisions() -> None: +def test_resolve_fleet_accepts_gpu_hosts_without_a_shared_gid() -> None: module = load_resolution_module() parsed = module.parse_fleet_evidence( evidence_document( - host_evidence( - "gpu-1", - lspci_bdfs=[GPU_BDF], - group_listing="render:x:993:\nother:x:993:\n", - ) + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"]), ) ) - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_blocks_cpu_hosts_with_persisted_gpu_access_contracts() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", state='{"renderGid":993,"version":1}')) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -@pytest.mark.parametrize("legacy_key", ["kfd", "amdgpu", "rocm_devices"]) -def test_resolve_fleet_blocks_cpu_hosts_with_any_legacy_gpu_access_rule(legacy_key: str) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", legacy_rules={legacy_key: 'KERNEL=="kfd", MODE="0666"\n'})) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_keeps_gpu_legacy_rule_admission_for_the_later_exact_migration() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], legacy_rules={"amdgpu": "legacy\n"})) - ) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert [host.status for host in resolution.hosts] == [module.HostStatus.GPU, module.HostStatus.GPU] -def test_resolve_fleet_blocks_file_probe_failures_instead_of_treating_them_as_absence() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", state_stat_success=False, state_content_success=False)) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -@pytest.mark.parametrize( - "contracts", - [ - {"state": '{"renderGid":994,"version":1}'}, - {"rule": 'KERNEL=="kfd", MODE="0666"\n'}, - ], -) -def test_resolve_fleet_blocks_gpu_hosts_with_unsafe_persisted_contracts(contracts: dict) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], **contracts))) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_requires_unanimous_gpu_render_gid() -> None: - module = load_resolution_module() - same_gid = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), - host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=993), - ) - ) - mixed_gid = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), - host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=994), - ) - ) - - resolved = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), same_gid) - blocked = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), mixed_gid) - - assert resolved.status is module.FleetStatus.GPU_RESOLVED - assert resolved.render_gid == 993 - assert blocked.status is module.FleetStatus.BLOCKED - assert blocked.render_gid is None - - -def test_resolution_manifest_preserves_explicit_host_booleans_and_unanimous_gid() -> None: +def test_resolution_manifest_preserves_explicit_host_booleans() -> None: module = load_resolution_module() parsed = module.parse_fleet_evidence( evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), host_evidence("cpu-1"), ) ) - resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed) - manifest = module.resolution_manifest(resolution) + manifest = module.resolution_manifest(module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed)) assert manifest == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"cpu-1": False, "gpu-1": True}, } def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: manifest = load_manifest_module().build_resolution_manifest( - version=1, status="gpu_resolved", - render_gid=993, hosts={"zeta": True, "alpha": False}, ) assert type(manifest) is dict - assert list(manifest) == ["version", "status", "render_gid", "hosts"] + assert list(manifest) == ["version", "status", "hosts"] assert list(manifest["hosts"]) == ["alpha", "zeta"] - assert set(manifest) == {"version", "status", "render_gid", "hosts"} + assert set(manifest) == {"version", "status", "hosts"} def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: module = load_manifest_module() base = module.build_resolution_manifest( - version=1, status="gpu_resolved", - render_gid=993, hosts={"gpu-2": True, "gpu-1": True}, ) manifest = module.build_pxe_resolution_manifest( - version=base["version"], - status=base["status"], - render_gid=base["render_gid"], - hosts=base["hosts"], + base, gpu_access_enabled=True, - pxe_render_gid=994, ) assert base == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"gpu-1": True, "gpu-2": True}, } - assert list(manifest) == ["version", "status", "render_gid", "hosts", "pxe_rootfs"] - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 994} - assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled", "render_gid"} + assert list(manifest) == ["version", "status", "hosts", "pxe_rootfs"] + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} + assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled"} From f217d1fc984c81957f842f29fc2e1f94639adc5c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 069/180] refactor(deploy): remove draft GPU inputs --- .../scripts/config_generation.py | 7 +------ tests/skills/test_config_generation_security.py | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py index 5fdcf84a..c0e881ff 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -8,12 +8,11 @@ import re from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote -from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values +from config_rendering import render_inventory, render_pxe_vars, render_values __all__ = [ "DEFAULT_ACCEL_LABELS", "HEADER_HASH", - "ResolvedGpuPolicy", "SCHEMA", "die", "render_inventory", @@ -200,10 +199,6 @@ def validate_spec(spec: dict) -> str: server_name = _validate_server(require(spec, "server"), "spec.server") _validate_agents(spec, server_name) _validate_rendered_options(spec) - if "render_gid" in spec: - die("spec.render_gid is no longer accepted; GPU policy is discovered automatically") - if "gpu_access" in spec: - die("spec.gpu_access is no longer accepted; GPU policy is discovered automatically") if topo == "pxe-diskless": _validate_pxe(spec) return topo diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py index 6ec39bf8..adb6c282 100644 --- a/tests/skills/test_config_generation_security.py +++ b/tests/skills/test_config_generation_security.py @@ -73,6 +73,15 @@ def test_generator_rejects_an_invalid_k3s_version_before_discovery(capsys: pytes assert "spec.k3s_version" in capsys.readouterr().err +def test_generator_applies_the_normal_unknown_field_policy_to_draft_gpu_fields() -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["render_gid"] = 993 + spec["gpu_access"] = {"hosts": []} + + assert module.validate_spec(spec) == "ssh-preinstalled" + + @pytest.mark.parametrize( "raw", [ From 2d28d6c87b413bce11a43987ec0fdae324d057b5 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 070/180] refactor(deploy): render boolean GPU policy --- .../scripts/config_rendering.py | 24 +-- .../scripts/gen_configs.py | 66 +++----- .../scripts/gpu_artifact_generation.py | 13 +- tests/skills/test_gpu_artifact_generation.py | 158 +++++++----------- 4 files changed, 93 insertions(+), 168 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py index 2ac59a86..b66a0064 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -4,24 +4,14 @@ from __future__ import annotations -from dataclasses import dataclass - from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote from gpu_access_resolution import FleetResolution, HostStatus -@dataclass(frozen=True, slots=True) -class ResolvedGpuPolicy: - host_gpu_enabled: dict[str, bool] - render_gid: int | None - pxe_gpu_enabled: bool - - def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str: topo = spec["topology"] server = spec["server"] k3s_version = spec["k3s_version"] - render_gid = resolution.render_gid host_gpu_enabled = {host.target.name: host.status is HostStatus.GPU for host in resolution.hosts} lines = [ HEADER_HASH, @@ -49,7 +39,6 @@ def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str " ansible_port: 22", " ansible_user: root", f" k3s_version: {yaml_quote(k3s_version)}", - f" auplc_render_gid: {'null' if render_gid is None else render_gid}", f" token: {yaml_quote(token)}", " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", ] @@ -67,7 +56,7 @@ def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str return "\n".join(lines) + "\n" -def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: str | None = None) -> str: +def render_pxe_vars(spec: dict, pxe_gpu_access_enabled: bool) -> str: net = require(spec, "network") pxe = spec.get("pxe", {}) keys = pxe.get("authorized_keys", []) @@ -75,7 +64,6 @@ def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: st die("pxe.authorized_keys must contain at least one SSH public key") server_ip = spec["server"]["ip"] k3s_version = spec["k3s_version"] - render_gid = policy.render_gid lines = [ HEADER_HASH, "# Pass this file to pb-pxe-controller.yml with", @@ -91,26 +79,22 @@ def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: st "pxe_k3s_server_ips:", f" - {yaml_quote(server_ip)}", f"pxe_k3s_version: {yaml_quote(k3s_version)}", - f"auplc_render_gid: {'null' if render_gid is None else render_gid}", - f"pxe_gpu_access_enabled: {'true' if policy.pxe_gpu_enabled else 'false'}", + f"pxe_gpu_access_enabled: {'true' if pxe_gpu_access_enabled else 'false'}", f"pxe_web_port: {int(pxe.get('web_port', 8080))}", f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", "pxe_rootfs_authorized_keys:", ] for key in keys: lines.append(f" - {yaml_quote(key)}") - if finalizer_context is not None: - lines.append(f"pxe_finalizer_context: {yaml_quote(finalizer_context)}") return "\n".join(lines) + "\n" -def render_values(spec: dict, resolution: FleetResolution) -> str: +def render_values(spec: dict) -> str: accel = spec.get("accelerators") or {} storage_class = (spec.get("storage") or {}).get("class", "nfs-client") node_port = (spec.get("proxy") or {}).get("node_port", 30890) auth_mode = spec.get("auth_mode", "auto-login") images = spec.get("images") or {} - render_gid = resolution.render_gid lines = [ "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", "# Helm overlay generated by auplc-skills gen_configs.py.", @@ -119,8 +103,6 @@ def render_values(spec: dict, resolution: FleetResolution) -> str: "# --create-namespace -f runtime/values.yaml -f <this file>", "custom:", f" authMode: {yaml_quote(auth_mode)}", - " gpuAccess:", - f" renderGid: {'null' if render_gid is None else render_gid}", ] if accel: lines.append(" accelerators:") diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index 3a3bdb9f..e9665c10 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -3,8 +3,8 @@ """Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. Given a JSON cluster-spec (see ``--print-schema``), discover the managed hosts' -GPU policy. SSH and PXE without GPU-enabled diskless agents immediately write -mutually consistent canonical deployment artifacts: +GPU policy. Both topologies immediately write mutually consistent canonical +deployment artifacts: 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the @@ -12,26 +12,19 @@ 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to pb-pxe-controller.yml with ``-e @<absolute-path>``. - 3. ``values-basic-example.yaml`` -- Helm overlay: resolved render GID, - storage, proxy, and authentication. + 3. ``values-basic-example.yaml`` -- Helm overlay: storage, proxy, and + authentication. 4. ``gpu-access-resolution.json`` -- Machine-readable resolved host policy. -GPU-enabled PXE instead writes private ``.pxe-bootstrap.inventory.yml``, -``.pxe-bootstrap.vars.yml``, and ``.pxe-finalizer-context.json`` files while -canonical artifacts remain absent. The controller playbook publishes the -canonical artifacts only after it resolves the rootfs GID and succeeds. - Design choices (deliberate): * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this runs on a bare operator machine. YAML is emitted from templates, not a serialiser -- the output is small, fixed-shape, and carries the copyright header. - * The k3s token is generated locally with ``secrets`` (CSPRNG). Immediate - canonical output writes it only into ``inventory.yml``. Pending GPU-enabled - PXE stores it only in private ``.pxe-finalizer-context.json`` until the - controller succeeds and finalization writes ``inventory.yml``. It is never - printed to stdout/stderr. Pass ``--token-file`` to reuse an existing token - instead of minting one. + * The k3s token is generated locally with ``secrets`` (CSPRNG). Canonical + output writes it only into ``inventory.yml``. It is never printed to + stdout/stderr. Pass ``--token-file`` to reuse an existing token instead of + minting one. * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never be newer than the server (k3s refuses that). * Existing files are not overwritten unless ``--force`` is given. @@ -59,12 +52,12 @@ SCHEMA, die, render_inventory, + render_pxe_vars, render_values, validate_spec, validate_yaml_scalar, ) from gpu_artifact_generation import DiscoveryFailure, canonical_paths, discover_gpu_policy, manifest_content -from pxe_finalization import FinalizationError, finalize, publish_disabled_rootfs, stage_pending def gen_token() -> str: @@ -79,22 +72,11 @@ def main(argv=None) -> int: ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") ap.add_argument("--force", action="store_true", help="overwrite existing files") ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") - ap.add_argument("--finalize-pxe", action="store_true", help=argparse.SUPPRESS) - ap.add_argument("--context", help=argparse.SUPPRESS) - ap.add_argument("--handoff", help=argparse.SUPPRESS) args = ap.parse_args(argv) if args.print_schema: print(json.dumps(SCHEMA, indent=2)) return 0 - if args.finalize_pxe: - if args.spec or args.token_file or args.context is None or args.handoff is None: - die("--finalize-pxe requires --out-dir, --context, and --handoff", 2) - try: - finalize(Path(args.out_dir), Path(args.context), Path(args.handoff)) - except FinalizationError as error: - die(str(error)) - return 0 if not args.spec: die("--spec is required (or use --print-schema)", 2) @@ -116,24 +98,20 @@ def main(argv=None) -> int: discovery = discover_gpu_policy(spec, out) except DiscoveryFailure as error: die(str(error)) + inventory, values, manifest = canonical_paths(out) + artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] + pxe_gpu_access_enabled = None if topo == "pxe-diskless": - try: - if spec["pxe"]["diskless_agents_have_amd_gpus"]: - stage_pending(spec, token, discovery.resolution, out, args.force) - print("PXE GPU rootfs is pending finalization after pb-pxe-controller.yml resolves its render GID.") - else: - publish_disabled_rootfs(spec, token, discovery.resolution, out, args.force) - except FinalizationError as error: - die(str(error)) - else: - inventory, values, manifest = canonical_paths(out) - artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] - artifacts += [ - (values, render_values(spec, discovery.resolution), 0o644, False), - (manifest, manifest_content(discovery), 0o644, False), - ] - preflight_destinations([path for path, _, _, _ in artifacts], args.force) - publish_artifacts(artifacts, args.force) + pxe_gpu_access_enabled = spec["pxe"]["diskless_agents_have_amd_gpus"] + artifacts.append( + (out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec, pxe_gpu_access_enabled), 0o600, True) + ) + artifacts += [ + (values, render_values(spec), 0o644, False), + (manifest, manifest_content(discovery, pxe_gpu_access_enabled), 0o644, False), + ] + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) print( "\nNext: review the files, then copy them into your aup-learning-cloud " diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py index 3c47e1d2..e16fac14 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py @@ -25,6 +25,7 @@ resolution_manifest, resolve_fleet, ) +from gpu_resolution_manifest import build_pxe_resolution_manifest DISCOVERY_TIMEOUT_BASE_SECONDS = 30 DISCOVERY_TIMEOUT_PER_TARGET_SECONDS = 15 @@ -213,6 +214,14 @@ def read_regular_file(path: Path) -> str: raise DiscoveryFailure("GPU discovery evidence could not be read") from error -def manifest_content(result: DiscoveryResult) -> str: - document = resolution_manifest(result.resolution) +def manifest_content(result: DiscoveryResult, pxe_gpu_access_enabled: bool | None = None) -> str: + base = resolution_manifest(result.resolution) + document = ( + base + if pxe_gpu_access_enabled is None + else build_pxe_resolution_manifest( + base, + gpu_access_enabled=pxe_gpu_access_enabled, + ) + ) return json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index 92924077..4e40a624 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -16,42 +16,13 @@ GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" -def evidence_host(name: str, *, gpu: bool = False, gid: int = 993, reachable: bool = True) -> dict: +def evidence_host(name: str, *, gpu: bool = False, reachable: bool = True) -> dict: bdf = "0000:03:00.0" if gpu else "" return { "host": name, "reachable": reachable, "lspci": {"rc": 0, "stdout": bdf}, "sysfs": {"rc": 0, "stdout": bdf}, - "render_group": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, - "groups": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, - "state": { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - }, - "rule": { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - }, - "legacy_rules": { - key: { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - } - for key in ("kfd", "amdgpu", "rocm_devices") - }, } @@ -96,11 +67,26 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document return record +def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def write_json(path: Path, document: dict) -> Path: + path.write_text(json.dumps(document), encoding="utf-8") + return path + + def test_generator_forces_repository_host_key_checking_over_disabled_environment( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible( - tmp_path, monkeypatch, {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]} + tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} ) environment_record = tmp_path / "ansible-environment.json" monkeypatch.setenv("FAKE_ANSIBLE_ENV_RECORD", str(environment_record)) @@ -113,9 +99,8 @@ def test_generator_forces_repository_host_key_checking_over_disabled_environment monkeypatch.setenv("ANSIBLE_SCP_IF_SSH", "True") monkeypatch.setenv("ANSIBLE_SCP_EXTRA_ARGS", "-o UserKnownHostsFile=/dev/null") monkeypatch.setenv("ANSIBLE_SFTP_EXTRA_ARGS", "-o StrictHostKeyChecking=no") - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) - result = run_generator(spec_path, tmp_path / "generated") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") assert result.returncode == 0, result.stderr assert json.loads(environment_record.read_text(encoding="utf-8")) == { @@ -143,9 +128,8 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( ) fake_ansible.chmod(0o755) monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) - result = run_generator(spec_path, tmp_path / "generated") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") assert result.returncode == 1 assert "exit code 2" in result.stderr @@ -154,44 +138,27 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( assert "token=<redacted>" in result.stderr -def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], - capture_output=True, - check=False, - text=True, - timeout=30, - ) - - -def write_json(path: Path, document: dict) -> Path: - path.write_text(json.dumps(document), encoding="utf-8") - return path - - def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: record = write_fake_ansible( tmp_path, monkeypatch, - {"version": 2, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, + {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, ) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: 993" in inventory assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": False, "server": True}, } discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" @@ -207,29 +174,46 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( ] -def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( +def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible( tmp_path, monkeypatch, - {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]}, + {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent", gpu=True)]}, ) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: null" in inventory - assert inventory.count("auplc_gpu_access_enabled: false") == 2 - assert "renderGid: null" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest == { + assert inventory.count("auplc_gpu_access_enabled: true") == 2 + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in values + assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} + + +def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} + ) + out_dir = tmp_path / "generated" + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "cpu_only", - "render_gid": None, "hosts": {"agent": False, "server": False}, } @@ -238,7 +222,6 @@ def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str ) -> None: - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -248,7 +231,7 @@ def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( fake_ansible.chmod(0o755) monkeypatch.setenv("PATH", str(fake_bin)) - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 1 assert not (out_dir / "inventory.yml").exists() @@ -256,22 +239,14 @@ def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( assert not (out_dir / "gpu-access-resolution.json").exists() -@pytest.mark.parametrize( - "document", - [ - {"version": 2, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, - { - "version": 2, - "hosts": [evidence_host("server", gpu=True, gid=993), evidence_host("agent", gpu=True, gid=994)], - }, - ], - ids=["unknown", "gid-disagreement"], -) def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch, document) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 1, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, + ) out_dir = tmp_path / "generated" out_dir.mkdir() inventory = out_dir / "inventory.yml" @@ -281,28 +256,9 @@ def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( values.write_text("previous values\n", encoding="utf-8") manifest.write_text("previous manifest\n", encoding="utf-8") - result = run_generator(spec_path, out_dir, "--force") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir, "--force") assert result.returncode == 1 assert inventory.read_text(encoding="utf-8") == "previous inventory\n" assert values.read_text(encoding="utf-8") == "previous values\n" assert manifest.read_text(encoding="utf-8") == "previous manifest\n" - - -@pytest.mark.parametrize( - "field", - ["render_gid", "gpu_access"], -) -def test_generator_rejects_removed_public_gpu_fields_before_running_discovery( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str -) -> None: - record = write_fake_ansible(tmp_path, monkeypatch, {"version": 2, "hosts": []}) - spec = ssh_spec() - spec[field] = 993 if field == "render_gid" else {"hosts": []} - spec_path = write_json(tmp_path / "spec.json", spec) - - result = run_generator(spec_path, tmp_path / "generated") - - assert result.returncode == 1 - assert f"spec.{field} is no longer accepted" in result.stderr - assert not record.exists() From 709ee3f656ad8151feedb91551eb7452412ca3e6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 071/180] refactor(deploy): simplify GPU manifests --- .../scripts/gpu_resolution_manifest.py | 23 +--- .../scripts/gpu_resolution_parsing.py | 103 ++---------------- .../scripts/gpu_resolution_validation.py | 25 +---- .../scripts/validate.py | 5 +- tests/skills/test_deploy_scripts.py | 83 ++------------ 5 files changed, 35 insertions(+), 204 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py index 60a59fd7..e80d0eaf 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py @@ -5,7 +5,9 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TypedDict +from typing import Final, TypedDict + +MANIFEST_VERSION: Final = 1 class ResolutionManifest(TypedDict): @@ -13,7 +15,6 @@ class ResolutionManifest(TypedDict): version: int status: str - render_gid: int | None hosts: dict[str, bool] @@ -21,7 +22,6 @@ class PxeRootfsManifest(TypedDict): """Serialized GPU policy applied to the PXE root filesystem.""" gpu_access_enabled: bool - render_gid: int | None class PxeResolutionManifest(ResolutionManifest): @@ -32,37 +32,26 @@ class PxeResolutionManifest(ResolutionManifest): def build_resolution_manifest( *, - version: int, status: str, - render_gid: int | None, hosts: Mapping[str, bool], ) -> ResolutionManifest: """Build a deterministic ordinary dictionary for fleet resolution.""" return { - "version": version, + "version": MANIFEST_VERSION, "status": status, - "render_gid": render_gid, "hosts": {name: hosts[name] for name in sorted(hosts)}, } def build_pxe_resolution_manifest( + resolution: ResolutionManifest, *, - version: int, - status: str, - render_gid: int | None, - hosts: Mapping[str, bool], gpu_access_enabled: bool, - pxe_render_gid: int | None, ) -> PxeResolutionManifest: """Build a PXE manifest without mutating a base fleet manifest.""" return { - "version": version, - "status": status, - "render_gid": render_gid, - "hosts": {name: hosts[name] for name in sorted(hosts)}, + **resolution, "pxe_rootfs": { "gpu_access_enabled": gpu_access_enabled, - "render_gid": pxe_render_gid, }, } diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py index 7f14faad..6fe4c6d1 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -5,29 +5,24 @@ from pathlib import Path from config_common import DuplicateJsonKeyError, strict_json_loads - -MAX_RENDER_GID = 4_294_967_294 +from gpu_resolution_manifest import MANIFEST_VERSION @dataclass(frozen=True, slots=True) class GpuInventory: hosts: dict[str, bool] - render_gid: int | None @dataclass(frozen=True, slots=True) class GpuResolution: status: str hosts: dict[str, bool] - render_gid: int | None pxe_rootfs_enabled: bool | None - pxe_rootfs_gid: int | None @dataclass(frozen=True, slots=True) class PxeGpuPolicy: enabled: bool - render_gid: int | None def configured_path(repo: Path, value: str) -> Path: @@ -35,17 +30,6 @@ def configured_path(repo: Path, value: str) -> Path: return path if path.is_absolute() else repo / path -def parse_gpu_gid(value: str) -> int | None | str: - normalized = value.strip() - if normalized in {"null", "~"}: - return None - if normalized.isascii() and normalized.isdecimal(): - gid = int(normalized) - if 1 <= gid <= MAX_RENDER_GID: - return gid - return "invalid" - - def parse_gpu_boolean(value: str) -> bool | None: normalized = value.strip() if normalized == "true": @@ -62,7 +46,6 @@ def yaml_indent(line: str) -> int: def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: host_values: dict[str, list[str]] = {} host_names: list[str] = [] - render_gids: list[str] = [] stack: list[tuple[int, str]] = [] for raw_line in text.splitlines(): @@ -95,8 +78,6 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: and key == "auplc_gpu_access_enabled" ): host_values.setdefault(path[4], []).append(value) - elif path == ("k3s_cluster", "vars") and key == "auplc_render_gid": - render_gids.append(value) stack.append((indent, key)) parse_errors: list[str] = [] @@ -115,64 +96,9 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") continue hosts[host] = enabled - if len(render_gids) != 1: - parse_errors.append("inventory must define exactly one k3s_cluster.vars.auplc_render_gid") - return None, parse_errors - render_gid = parse_gpu_gid(render_gids[0]) - if render_gid == "invalid": - parse_errors.append("inventory has malformed auplc_render_gid") - return None, parse_errors if parse_errors: return None, parse_errors - return GpuInventory(hosts=hosts, render_gid=render_gid), parse_errors - - -def parse_values_gpu_gid(text: str) -> tuple[int | None, bool, list[str]]: - render_gids: list[str] = [] - stack: list[tuple[int, str]] = [] - for raw_line in text.splitlines(): - line = raw_line.split("#", 1)[0].rstrip() - if not line.strip(): - continue - indent = yaml_indent(line) - stripped = line.strip() - while stack and indent <= stack[-1][0]: - stack.pop() - path = tuple(key for _, key in stack) - mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) - if not mapping_match: - continue - key = mapping_match.group(1).strip("\"'") - value = (mapping_match.group(2) or "").strip() - if path == ("custom", "gpuAccess") and key == "renderGid": - render_gids.append(value) - stack.append((indent, key)) - if not render_gids: - return None, False, [] - if len(render_gids) != 1: - return None, True, ["custom.gpuAccess.renderGid is duplicated"] - render_gid = parse_gpu_gid(render_gids[0]) - if render_gid == "invalid": - return None, True, ["custom.gpuAccess.renderGid is malformed"] - return render_gid, True, [] - - -def collect_effective_gpu_gid(repo: Path, values: list[str]) -> tuple[int | None, list[str]]: - effective_gid: int | None = None - found = False - parse_errors: list[str] = [] - for rel in values or ["runtime/values.yaml"]: - path = configured_path(repo, rel) - if not path.exists(): - continue - render_gid, present, file_errors = parse_values_gpu_gid(path.read_text(encoding="utf-8")) - parse_errors.extend(f"{path}: {error}" for error in file_errors) - if present and not file_errors: - effective_gid = render_gid - found = True - if not found: - parse_errors.append("effective values have no custom.gpuAccess.renderGid") - return effective_gid, parse_errors + return GpuInventory(hosts=hosts), parse_errors def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: @@ -184,13 +110,13 @@ def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None return None, [f"GPU resolution manifest is malformed: {exc}"] if type(document) is not dict: return None, ["GPU resolution manifest must be a JSON object"] - expected_keys = {"version", "status", "render_gid", "hosts"} + expected_keys = {"version", "status", "hosts"} if topology == "pxe-diskless": expected_keys.add("pxe_rootfs") if set(document) != expected_keys: return None, ["GPU resolution manifest has an unexpected schema"] - if type(document["version"]) is not int or document["version"] != 1: - return None, ["GPU resolution manifest version must be integer 1"] + if type(document["version"]) is not int or document["version"] != MANIFEST_VERSION: + return None, [f"GPU resolution manifest version must be integer {MANIFEST_VERSION}"] status = document["status"] if type(status) is not str or status not in {"cpu_only", "gpu_resolved"}: return None, ["GPU resolution manifest status must be cpu_only or gpu_resolved"] @@ -200,25 +126,19 @@ def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None type(host) is not str or not host or type(enabled) is not bool for host, enabled in document["hosts"].items() ): return None, ["GPU resolution manifest hosts must map non-empty names to booleans"] - render_gid = document["render_gid"] - if render_gid is not None and (type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID): - return None, ["GPU resolution manifest render_gid must be an integer or null"] if topology == "ssh-preinstalled": - return GpuResolution(status, document["hosts"], render_gid, None, None), [] + return GpuResolution(status, document["hosts"], None), [] rootfs = document["pxe_rootfs"] - if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled", "render_gid"}: + if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled"}: return None, ["GPU resolution manifest pxe_rootfs has an unexpected schema"] rootfs_enabled = rootfs["gpu_access_enabled"] - rootfs_gid = rootfs["render_gid"] if type(rootfs_enabled) is not bool: return None, ["GPU resolution manifest pxe_rootfs.gpu_access_enabled must be boolean"] - if rootfs_gid is not None and (type(rootfs_gid) is not int or not 1 <= rootfs_gid <= MAX_RENDER_GID): - return None, ["GPU resolution manifest pxe_rootfs.render_gid must be an integer or null"] - return GpuResolution(status, document["hosts"], render_gid, rootfs_enabled, rootfs_gid), [] + return GpuResolution(status, document["hosts"], rootfs_enabled), [] def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: - values: dict[str, list[str]] = {"auplc_render_gid": [], "pxe_gpu_access_enabled": []} + values: dict[str, list[str]] = {"pxe_gpu_access_enabled": []} for raw_line in text.splitlines(): line = raw_line.split("#", 1)[0].rstrip() if not line.strip() or yaml_indent(line) != 0: @@ -235,12 +155,9 @@ def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: parse_errors.append(f"PXE vars must define exactly one {key}") if parse_errors: return None, parse_errors - render_gid = parse_gpu_gid(values["auplc_render_gid"][0]) enabled = parse_gpu_boolean(values["pxe_gpu_access_enabled"][0]) - if render_gid == "invalid": - parse_errors.append("PXE vars have malformed auplc_render_gid") if enabled is None: parse_errors.append("PXE vars have malformed pxe_gpu_access_enabled") if parse_errors: return None, parse_errors - return PxeGpuPolicy(enabled=enabled, render_gid=render_gid), [] + return PxeGpuPolicy(enabled=enabled), [] diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index 1fd8c041..f33c5cc8 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -4,7 +4,6 @@ from pathlib import Path from gpu_resolution_parsing import ( - collect_effective_gpu_gid, configured_path, parse_gpu_inventory, parse_gpu_resolution, @@ -17,7 +16,6 @@ class GpuArtifactValidationRequest: repo: Path inventory_path: str resolution_path: str - values: list[str] topology: str pxe_vars_path: Path has_prior_errors: bool @@ -85,8 +83,7 @@ def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactVal return GpuArtifactValidationResult([f"GPU resolution manifest not found: {resolution_file}"], []) inventory, inventory_errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) resolution, resolution_errors = parse_gpu_resolution(resolution_file.read_text(encoding="utf-8"), request.topology) - helm_gid, helm_errors = collect_effective_gpu_gid(request.repo, request.values) - errors.extend([*inventory_errors, *resolution_errors, *helm_errors]) + errors.extend([*inventory_errors, *resolution_errors]) if inventory is None or resolution is None or errors: return GpuArtifactValidationResult(errors, []) if set(inventory.hosts) != set(resolution.hosts): @@ -104,22 +101,10 @@ def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactVal return GpuArtifactValidationResult(errors, []) if pxe_policy.enabled != resolution.pxe_rootfs_enabled: errors.append("PXE pxe_gpu_access_enabled disagrees with GPU resolution manifest pxe_rootfs") - if resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is None: - errors.append("GPU-enabled PXE rootfs requires a numeric render GID") - if not resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is not None: - errors.append("GPU-disabled PXE rootfs requires a null render GID") - if resolution.pxe_rootfs_enabled and pxe_policy.render_gid != resolution.pxe_rootfs_gid: - errors.append("PXE auplc_render_gid disagrees with GPU resolution manifest pxe_rootfs render_gid") - gids = [inventory.render_gid, helm_gid, resolution.render_gid] - if pxe_policy is not None: - gids.append(pxe_policy.render_gid) - if len(set(gids)) != 1: - errors.append("inventory, Helm, PXE, and GPU resolution render GIDs disagree") - enabled_scope = any(resolution.hosts.values()) or resolution.pxe_rootfs_enabled is True if resolution.status == "cpu_only": - if enabled_scope or resolution.render_gid is not None or any(gid is not None for gid in gids): - errors.append("cpu_only GPU resolution requires all host/rootfs booleans false and all render GIDs null") - elif not enabled_scope or resolution.render_gid is None: - errors.append("gpu_resolved GPU resolution requires an enabled scope and a numeric render GID") + if any(resolution.hosts.values()): + errors.append("cpu_only GPU resolution requires all host booleans false") + elif not any(resolution.hosts.values()): + errors.append("gpu_resolved GPU resolution requires an enabled host") passed = [] if request.has_prior_errors or errors else ["GPU access artifacts agree"] return GpuArtifactValidationResult(errors, passed) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index ccb5bb6b..8e3982c1 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,8 +12,8 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * generated inventory, GPU-resolution manifest, Helm render GID, and PXE - rootfs policy agree when generated artifacts are supplied; + * generated inventory, GPU-resolution manifest, and PXE rootfs policy agree + when generated artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -254,7 +254,6 @@ def main(argv=None) -> int: repo=repo, inventory_path=args.inventory, resolution_path=args.gpu_resolution, - values=args.values, topology=args.topology, pxe_vars_path=pxe_vars_path(repo, args.pxe_vars), has_prior_errors=bool(errors), diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 0469f4fc..c57f916d 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -80,20 +80,12 @@ def fake_ansible_playbook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> No output = next(value.split('=', 1)[1] for value in arguments if value.startswith('gpu_access_discovery_output_path=')) hosts = [line.strip()[:-1] for line in inventory.read_text(encoding='utf-8').splitlines() if line.startswith(' ') and line.rstrip().endswith(':')] evidence = { - 'version': 2, + 'version': 1, 'hosts': [{ 'host': host, 'reachable': True, 'lspci': {'rc': 0, 'stdout': ''}, 'sysfs': {'rc': 0, 'stdout': ''}, - 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'legacy_rules': { - key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} - for key in ('kfd', 'amdgpu', 'rocm_devices') - }, } for host in hosts], } Path(output).write_text(json.dumps(evidence), encoding='utf-8') @@ -123,15 +115,11 @@ def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, ) values = write_file( repo / "generated/values-basic-example.yaml", """custom: - gpuAccess: - renderGid: 993 resources: metadata: {} """, @@ -142,7 +130,6 @@ def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": False, "server": True}, } ), @@ -921,15 +908,11 @@ def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: null """, ) values = write_file( repo / "generated/values-basic-example.yaml", """custom: - gpuAccess: - renderGid: null resources: metadata: {} """, @@ -940,7 +923,6 @@ def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> { "version": 1, "status": "cpu_only", - "render_gid": None, "hosts": {"agent": False, "server": False}, } ), @@ -991,15 +973,15 @@ def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> [ ("not JSON", "GPU resolution manifest is malformed"), ( - '{"version":1,"status":"pending","render_gid":993,"hosts":{"agent":false,"server":true}}', + '{"version":1,"status":"pending","hosts":{"agent":false,"server":true}}', "GPU resolution manifest status must be cpu_only or gpu_resolved", ), ( - '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"server":true,"server":false}}', + '{"version":1,"status":"gpu_resolved","hosts":{"server":true,"server":false}}', "duplicate JSON key 'server'", ), ( - '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"ser\\u0076er":true,"server":false}}', + '{"version":1,"status":"gpu_resolved","hosts":{"ser\\u0076er":true,"server":false}}', "duplicate JSON key 'server'", ), ], @@ -1044,8 +1026,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' must define exactly one auplc_gpu_access_enabled", ), @@ -1062,8 +1042,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' has malformed auplc_gpu_access_enabled", ), @@ -1081,8 +1059,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' must define exactly one auplc_gpu_access_enabled", ), @@ -1136,24 +1112,14 @@ def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: P assert "GPU resolution manifest not found" in result.stdout -def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path) -> None: +def test_validator_rejects_mismatched_host_boolean(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, resolution = write_resolved_gpu_artifacts(repo) - values.write_text( - """custom: - gpuAccess: - renderGid: 994 - resources: - metadata: {} -""", - encoding="utf-8", - ) resolution.write_text( json.dumps( { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": True, "server": True}, } ), @@ -1176,10 +1142,9 @@ def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path assert result.returncode == 1 assert "inventory host 'agent' GPU access boolean disagrees" in result.stdout - assert "render GIDs disagree" in result.stdout -def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) -> None: +def test_validator_rejects_pxe_rootfs_boolean_mismatch(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory = write_file( repo / "generated/inventory.yml", @@ -1192,20 +1157,17 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - auplc_gpu_access_enabled: false agent: hosts: {} - vars: - auplc_render_gid: 993 """, ) - values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n gpuAccess:\n renderGid: 993\n") + values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n resources:\n metadata: {}\n") resolution = write_file( repo / "generated/gpu-access-resolution.json", json.dumps( { "version": 1, - "status": "gpu_resolved", - "render_gid": 993, + "status": "cpu_only", "hosts": {"server": False}, - "pxe_rootfs": {"gpu_access_enabled": True, "render_gid": 993}, + "pxe_rootfs": {"gpu_access_enabled": True}, } ), ) @@ -1218,7 +1180,6 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - pxe_k3s_server_ips: [192.168.1.10] pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] pxe_k3s_version: v1.32.3+k3s1 -auplc_render_gid: 994 pxe_gpu_access_enabled: false """, ) @@ -1241,7 +1202,6 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - assert result.returncode == 1 assert "pxe_gpu_access_enabled disagrees" in result.stdout - assert "PXE auplc_render_gid disagrees" in result.stdout def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: @@ -1611,20 +1571,6 @@ def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: assert callable(artifacts.publish_artifacts) -def test_generator_rejects_legacy_public_gpu_policy_fields_before_discovery(tmp_path: Path) -> None: - spec = generator_spec() - spec["render_gid"] = 1055 - spec["gpu_access"] = {"hosts": [], "pxe_rootfs_enabled": False} - spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) - out_dir = tmp_path / "generated" - - result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "spec.render_gid is no longer accepted" in result.stderr - assert not out_dir.exists() - - def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -1640,13 +1586,8 @@ def host(name, bdf): return { 'host': name, 'reachable': True, 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, - 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'legacy_rules': {key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} for key in ('kfd', 'amdgpu', 'rocm_devices')}, } -pathlib.Path(output).write_text(json.dumps({'version': 2, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') +pathlib.Path(output).write_text(json.dumps({'version': 1, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') """, encoding="utf-8", ) @@ -1665,9 +1606,9 @@ def host(name, bdf): assert result.returncode == 0, result.stdout + result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: 993" in inventory assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert manifest["hosts"] == {"agent": False, "server": True} From 131f2f1d5f9de28fbc929c74435ff6f72d1b6578 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 072/180] refactor(deploy): remove PXE finalizer --- .../scripts/pxe_finalization.py | 173 ------- .../scripts/pxe_finalization_support.py | 257 ---------- tests/skills/test_pxe_finalization.py | 457 +++--------------- 3 files changed, 63 insertions(+), 824 deletions(-) delete mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py delete mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py deleted file mode 100644 index cb9c7c9f..00000000 --- a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Orchestrate transactional PXE configuration finalization.""" - -from __future__ import annotations - -import json -import os -import secrets -from pathlib import Path - -import pxe_finalization_support as _support -from artifact_store import preflight_destinations, publish_artifacts -from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values -from gpu_access_resolution import ( - FleetResolution, - FleetStatus, - HostStatus, - resolution_manifest, -) -from gpu_access_resolution import HostResolution as HostResolution -from gpu_resolution_manifest import build_pxe_resolution_manifest -from pxe_finalization_support import MAX_RENDER_GID as MAX_RENDER_GID -from pxe_finalization_support import ( - VERSION, - Artifact, - JsonDocument, -) -from pxe_finalization_support import FinalizationError as FinalizationError -from pxe_finalization_support import PxePaths as PxePaths -from pxe_finalization_support import paths as paths - -_artifact_attestations = _support.artifact_attestations -_completion = _support.completion -_controller_resolution = _support.controller_resolution -_exclusive_lock = _support.exclusive_lock -_final_resolution = _support.final_resolution -_generation_paths = _support.generation_paths -_read_artifact_attestation = _support.read_artifact_attestation -_read_document = _support.read_document -_spec_sha256 = _support.spec_sha256 -_target = _support.target -_valid_gid = _support.valid_gid -_validate = _support.validate -_verify_canonical_artifacts = _support.verify_canonical_artifacts - - -def stage_pending(spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool) -> PxePaths: - pending = paths(out_dir) - if controller.status is FleetStatus.BLOCKED: - raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - context: JsonDocument = { - "version": VERSION, - "generation": secrets.token_urlsafe(32), - "spec_sha256": _spec_sha256(spec), - "topology": "pxe-diskless", - "spec": spec, - "token": token, - "controller": resolution_manifest(controller), - } - bootstrap = render_pxe_vars(spec, _controller_policy(controller, True), str(pending.context)) - bootstrap += "\n".join( - [ - f"pxe_finalizer_handoff: {_yaml_quote(str(pending.handoff))}", - f"pxe_finalizer_generation: {_yaml_quote(context['generation'])}", - f"pxe_finalizer_spec_sha256: {_yaml_quote(context['spec_sha256'])}", - f"pxe_finalizer_script: {_yaml_quote(str(Path(__file__).with_name('gen_configs.py').resolve()))}", - "", - ] - ) - artifacts: list[Artifact] = [ - (pending.bootstrap_inventory, _render_bootstrap_inventory(spec), 0o600, True), - (pending.bootstrap_vars, bootstrap, 0o600, True), - (pending.context, json.dumps(context, sort_keys=True) + "\n", 0o600, True), - ] - if not force: - preflight_destinations(_generation_paths(pending), False) - publish_artifacts(artifacts, force, _generation_paths(pending)) - return pending - - -def publish_disabled_rootfs( - spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool -) -> None: - pending = paths(out_dir) - if controller.status is FleetStatus.BLOCKED: - raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - policy = _controller_policy(controller, False) - artifacts: list[Artifact] = [ - (pending.inventory, render_inventory(spec, token, controller), 0o600, True), - (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), - (pending.values, render_values(spec, controller), 0o644, False), - (pending.manifest, _manifest(controller, False, None), 0o644, False), - ] - if not force: - preflight_destinations(_generation_paths(pending), False) - publish_artifacts(artifacts, force, _generation_paths(pending)) - - -def finalize(out_dir: Path, context_path: Path, handoff_path: Path) -> None: - pending = paths(out_dir) - if context_path.resolve() != pending.context or handoff_path.resolve() != pending.handoff: - raise FinalizationError("PXE finalizer context and handoff paths must be the generated private paths") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - context = _read_document(pending.context, "PXE finalizer context") - handoff = _read_document(pending.handoff, "PXE finalizer handoff") - spec, controller, rootfs_gid = _validate(context, handoff) - resolution = _final_resolution(controller, rootfs_gid) - policy = _controller_policy(resolution, True) - artifacts: list[Artifact] = [ - (pending.inventory, render_inventory(spec, context["token"], resolution), 0o600, True), - (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), - (pending.values, render_values(spec, resolution), 0o644, False), - (pending.manifest, _manifest(resolution, True, rootfs_gid), 0o644, False), - ] - completion = _completion(context, handoff, _artifact_attestations(artifacts)) - if os.path.lexists(pending.completion): - if _read_document(pending.completion, "PXE finalizer completion") != completion: - raise FinalizationError("PXE finalizer completion does not match the supplied handoff") - _verify_canonical_artifacts(pending, completion["artifacts"]) - return - published: list[Artifact] = [ - *artifacts, - (pending.completion, json.dumps(completion, sort_keys=True) + "\n", 0o600, True), - ] - preflight_destinations([path for path, _, _, _ in published], False) - publish_artifacts(published, False) - - -def _controller_policy(resolution: FleetResolution, rootfs_enabled: bool) -> ResolvedGpuPolicy: - return ResolvedGpuPolicy( - host_gpu_enabled={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, - render_gid=resolution.render_gid, - pxe_gpu_enabled=rootfs_enabled, - ) - - -def _manifest(resolution: FleetResolution, rootfs_enabled: bool, rootfs_gid: int | None) -> str: - base = resolution_manifest(resolution) - document = build_pxe_resolution_manifest( - version=base["version"], - status=base["status"], - render_gid=base["render_gid"], - hosts=base["hosts"], - gpu_access_enabled=rootfs_enabled, - pxe_render_gid=rootfs_gid, - ) - return json.dumps(document, indent=2, sort_keys=True) + "\n" - - -def _render_bootstrap_inventory(spec: JsonDocument) -> str: - server = spec["server"] - return "\n".join( - [ - "pxe_controller:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " vars:", - " ansible_port: 22", - " ansible_user: root", - "", - ] - ) - - -def _yaml_quote(value: str) -> str: - return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py deleted file mode 100644 index e94923f7..00000000 --- a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Typed security and verification support for PXE finalization.""" - -from __future__ import annotations - -import fcntl -import hashlib -import json -import os -import stat -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import Final, TypeAlias, TypedDict - -from config_common import DuplicateJsonKeyError, strict_json_loads -from gpu_access_resolution import FleetResolution, FleetStatus, HostResolution, HostStatus, InventoryTarget - -VERSION: Final = 1 -MAX_RENDER_GID: Final = 4_294_967_294 - -JsonScalar: TypeAlias = str | int | float | bool | None -JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -JsonDocument: TypeAlias = dict[str, JsonValue] -Artifact: TypeAlias = tuple[Path, str, int, bool] - - -class ArtifactAttestation(TypedDict): - sha256: str - mode: int - owner_uid: int - - -ArtifactAttestations: TypeAlias = dict[str, ArtifactAttestation] - - -@dataclass(frozen=True, slots=True) -class FinalizationError(Exception): - reason: str - - def __str__(self) -> str: - return self.reason - - -@dataclass(frozen=True, slots=True) -class PxePaths: - bootstrap_inventory: Path - bootstrap_vars: Path - context: Path - handoff: Path - completion: Path - lock: Path - inventory: Path - pxe_vars: Path - values: Path - manifest: Path - - -def paths(out_dir: Path) -> PxePaths: - root = out_dir.resolve() - return PxePaths( - bootstrap_inventory=root / ".pxe-bootstrap.inventory.yml", - bootstrap_vars=root / ".pxe-bootstrap.vars.yml", - context=root / ".pxe-finalizer-context.json", - handoff=root / ".pxe-finalizer-handoff.json", - completion=root / ".pxe-finalizer-completion.json", - lock=root / ".pxe-finalizer.lock", - inventory=root / "inventory.yml", - pxe_vars=root / "pb-pxe-controller.vars.yml", - values=root / "values-basic-example.yaml", - manifest=root / "gpu-access-resolution.json", - ) - - -def spec_sha256(spec: JsonDocument) -> str: - encoded = json.dumps(spec, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).hexdigest() - - -def valid_gid(value: JsonValue) -> bool: - return type(value) is int and 1 <= value <= MAX_RENDER_GID - - -def read_document(path: Path, label: str) -> JsonDocument: - try: - descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) - with os.fdopen(descriptor, encoding="utf-8") as source: - mode = os.fstat(source.fileno()).st_mode - if not stat.S_ISREG(mode): - raise FinalizationError(f"{label} must be a regular file") - document = strict_json_loads(source.read()) - except FinalizationError: - raise - except (DuplicateJsonKeyError, FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: - raise FinalizationError(f"{label} cannot be read") from error - if type(document) is not dict: - raise FinalizationError(f"{label} must be a JSON object") - return document - - -def validate(context: JsonDocument, handoff: JsonDocument) -> tuple[JsonDocument, FleetResolution, int]: - required_context = {"version", "generation", "spec_sha256", "topology", "spec", "token", "controller"} - required_handoff = {"version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid"} - if set(context) != required_context or set(handoff) != required_handoff: - raise FinalizationError("PXE finalizer context or handoff has an unexpected schema") - if type(context["version"]) is not int or type(handoff["version"]) is not int: - raise FinalizationError("PXE finalizer context or handoff version is invalid") - if context["version"] != VERSION or handoff["version"] != VERSION: - raise FinalizationError("PXE finalizer context or handoff version is unsupported") - if context["topology"] != "pxe-diskless" or handoff["topology"] != "pxe-diskless": - raise FinalizationError("PXE finalizer topology is invalid") - generation = context["generation"] - if type(generation) is not str or not generation or handoff["generation"] != generation: - raise FinalizationError("PXE finalizer generation does not match") - spec = context["spec"] - if type(spec) is not dict or spec_sha256(spec) != context["spec_sha256"]: - raise FinalizationError("PXE finalizer context spec does not match its digest") - if handoff["spec_sha256"] != context["spec_sha256"] or spec.get("topology") != "pxe-diskless": - raise FinalizationError("PXE finalizer handoff does not match its pending spec") - if "render_gid" in spec or "gpu_access" in spec: - raise FinalizationError("PXE finalizer context contains removed public GPU policy fields") - if type(context["token"]) is not str or not context["token"]: - raise FinalizationError("PXE finalizer context token is invalid") - pxe = spec.get("pxe") - if type(pxe) is not dict or pxe.get("diskless_agents_have_amd_gpus") is not True: - raise FinalizationError("PXE finalizer context is not for GPU-enabled diskless agents") - rootfs_gid = handoff["render_gid"] - if handoff["pxe_gpu_access_enabled"] is not True or not valid_gid(rootfs_gid): - raise FinalizationError("PXE finalizer handoff has no valid resolved rootfs GID") - controller = controller_resolution(spec, context["controller"]) - if controller.render_gid is not None and controller.render_gid != rootfs_gid: - raise FinalizationError("PXE rootfs render GID disagrees with the GPU-enabled controller") - return spec, controller, rootfs_gid - - -def controller_resolution(spec: JsonDocument, raw: JsonValue) -> FleetResolution: - if type(raw) is not dict or set(raw) != {"version", "status", "render_gid", "hosts"}: - raise FinalizationError("PXE finalizer context controller evidence is invalid") - server = spec.get("server") - name = server.get("name") if type(server) is dict else None - hosts = raw["hosts"] - if type(name) is not str or type(hosts) is not dict or set(hosts) != {name} or type(hosts[name]) is not bool: - raise FinalizationError("PXE finalizer context controller host is invalid") - enabled = hosts[name] - gid = raw["render_gid"] - if enabled and not valid_gid(gid): - raise FinalizationError("PXE finalizer context controller GID is invalid") - if not enabled and gid is not None: - raise FinalizationError("CPU-only PXE controller must not publish a render GID") - status = HostStatus.GPU if enabled else HostStatus.CPU - fleet_status = FleetStatus.GPU_RESOLVED if enabled else FleetStatus.CPU_ONLY - if type(raw["version"]) is not int or raw["version"] != VERSION or raw["status"] != fleet_status.value: - raise FinalizationError("PXE finalizer context controller status is invalid") - host = HostResolution(target=target(name), status=status, render_gid=gid, reason=None) - return FleetResolution(fleet_status, (host,), gid, None) - - -def target(name: str) -> InventoryTarget: - return InventoryTarget(name=name) - - -def final_resolution(controller: FleetResolution, rootfs_gid: int) -> FleetResolution: - return FleetResolution(FleetStatus.GPU_RESOLVED, controller.hosts, rootfs_gid, None) - - -def generation_paths(pending: PxePaths) -> tuple[Path, ...]: - return ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - pending.inventory, - pending.pxe_vars, - pending.values, - pending.manifest, - ) - - -def artifact_attestations(artifacts: list[Artifact]) -> ArtifactAttestations: - return { - path.name: {"sha256": hashlib.sha256(content.encode()).hexdigest(), "mode": mode, "owner_uid": os.geteuid()} - for path, content, mode, _ in artifacts - } - - -def verify_canonical_artifacts(pending: PxePaths, expected: JsonValue) -> None: - canonical = (pending.inventory, pending.pxe_vars, pending.values, pending.manifest) - if type(expected) is not dict or set(expected) != {path.name for path in canonical}: - raise FinalizationError("PXE finalizer completion artifacts are invalid") - for path in canonical: - attestation = expected[path.name] - if ( - type(attestation) is not dict - or set(attestation) != {"sha256", "mode", "owner_uid"} - or type(attestation["sha256"]) is not str - or type(attestation["mode"]) is not int - or type(attestation["owner_uid"]) is not int - or read_artifact_attestation(path) != attestation - ): - raise FinalizationError(f"PXE finalizer canonical artifact is missing or corrupted: {path.name}") - - -def read_artifact_attestation(path: Path) -> ArtifactAttestation: - try: - descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) - with os.fdopen(descriptor, "rb") as source: - artifact_stat = os.fstat(source.fileno()) - if not stat.S_ISREG(artifact_stat.st_mode): - raise FinalizationError("PXE finalizer canonical artifact must be a regular file") - digest = hashlib.sha256() - while chunk := source.read(65_536): - digest.update(chunk) - return { - "sha256": digest.hexdigest(), - "mode": stat.S_IMODE(artifact_stat.st_mode), - "owner_uid": artifact_stat.st_uid, - } - except FinalizationError: - raise - except (FileNotFoundError, OSError) as error: - raise FinalizationError("PXE finalizer canonical artifact cannot be read") from error - - -def completion(context: JsonDocument, handoff: JsonDocument, artifacts: ArtifactAttestations) -> JsonDocument: - return { - **{ - key: handoff[key] - for key in ("version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid") - }, - "artifacts": artifacts, - } - - -@contextmanager -def exclusive_lock(path: Path) -> Iterator[None]: - descriptor = -1 - locked = False - try: - descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600) - lock_stat = os.fstat(descriptor) - if not stat.S_ISREG(lock_stat.st_mode): - raise FinalizationError("PXE finalizer lock must be a regular file") - if lock_stat.st_uid != os.geteuid() or stat.S_IMODE(lock_stat.st_mode) != 0o600: - raise FinalizationError("PXE finalizer lock has unsafe owner or mode") - fcntl.flock(descriptor, fcntl.LOCK_EX) - locked = True - yield - except OSError as error: - raise FinalizationError("PXE finalizer lock cannot be opened") from error - finally: - if locked: - fcntl.flock(descriptor, fcntl.LOCK_UN) - if descriptor >= 0: - os.close(descriptor) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index effccb02..554d6049 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -1,18 +1,19 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for immediate PXE GPU policy generation.""" + from __future__ import annotations import json import os import subprocess import sys -from dataclasses import FrozenInstanceError -from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" -PXE_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-pxe-controller.yml" def pxe_spec(gpu_agents: bool) -> dict: @@ -49,16 +50,11 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, contr output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) Path(output).write_text(json.dumps({{ - 'version': 2, + 'version': 1, 'hosts': [{{ 'host': 'controller', 'reachable': True, 'lspci': {{'rc': 0, 'stdout': {bdf!r}}}, 'sysfs': {{'rc': 0, 'stdout': {bdf!r}}}, - 'render_group': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, - 'groups': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, - 'state': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, - 'rule': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, - 'legacy_rules': {{key: {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}} for key in ('kfd', 'amdgpu', 'rocm_devices')}}, }}], }}), encoding='utf-8') """, @@ -78,439 +74,112 @@ def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: ) -def load_finalizer_module(): - scripts = GEN_CONFIGS.parent - sys.path.insert(0, str(scripts)) - try: - spec = spec_from_file_location("test_pxe_finalizer", scripts / "pxe_finalization.py") - assert spec is not None and spec.loader is not None - module = module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - finally: - sys.path.pop(0) - - -def pending_handoff(out_dir: Path, *, render_gid: int = 995) -> tuple[Path, Path]: - context_path = out_dir / ".pxe-finalizer-context.json" - context = json.loads(context_path.read_text(encoding="utf-8")) - handoff_path = out_dir / ".pxe-finalizer-handoff.json" - write_json( - handoff_path, - { - "version": 1, - "generation": context["generation"], - "spec_sha256": context["spec_sha256"], - "topology": "pxe-diskless", - "pxe_gpu_access_enabled": True, - "render_gid": render_gid, - }, - ) - return context_path, handoff_path - - def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: return ( out_dir / "inventory.yml", out_dir / "pb-pxe-controller.vars.yml", out_dir / "values-basic-example.yaml", out_dir / "gpu-access-resolution.json", - out_dir / ".pxe-finalizer-completion.json", ) -def cpu_controller(finalizer): - return finalizer.FleetResolution( - finalizer.FleetStatus.CPU_ONLY, - (finalizer.HostResolution(finalizer._target("controller"), finalizer.HostStatus.CPU, None, None),), - None, - None, - ) - - -def test_pxe_finalizer_preserves_moved_imports_as_immutable_support_types(tmp_path: Path) -> None: - finalizer = load_finalizer_module() - support = sys.modules["pxe_finalization_support"] - - assert finalizer.FinalizationError is support.FinalizationError - assert finalizer.PxePaths is support.PxePaths - assert finalizer.paths is support.paths - assert finalizer.VERSION == support.VERSION == 1 - assert finalizer.MAX_RENDER_GID == support.MAX_RENDER_GID == 4_294_967_294 - assert finalizer._read_document is support.read_document - assert finalizer._generation_paths is support.generation_paths - assert finalizer._artifact_attestations is support.artifact_attestations - assert finalizer._completion is support.completion - assert finalizer._verify_canonical_artifacts is support.verify_canonical_artifacts - assert finalizer._exclusive_lock is support.exclusive_lock - - error = finalizer.FinalizationError("immutable") - pending = finalizer.paths(tmp_path) - with pytest.raises(FrozenInstanceError): - error.reason = "changed" - with pytest.raises(FrozenInstanceError): - pending.context = tmp_path / "changed.json" - - -def test_pxe_gpu_agents_stage_only_private_bootstrap_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 0, result.stderr - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - assert "pxe_controller:" in (out_dir / ".pxe-bootstrap.inventory.yml").read_text(encoding="utf-8") - bootstrap = (out_dir / ".pxe-bootstrap.vars.yml").read_text(encoding="utf-8") - assert "pxe_gpu_access_enabled: true" in bootstrap - assert "pxe_finalizer_context:" in bootstrap - assert (out_dir / ".pxe-finalizer-context.json").stat().st_mode & 0o777 == 0o600 - - -def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(False)) - out_dir = tmp_path / "generated" - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 0, result.stderr - assert "pxe_gpu_access_enabled: false" in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: null" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False, "render_gid": None} - - -def test_pxe_disabled_rootfs_force_replaces_private_generation_state_under_the_generation_lock(tmp_path: Path) -> None: - finalizer = load_finalizer_module() - out_dir = tmp_path / "generated" - pending = finalizer.paths(out_dir) - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("stale\n", encoding="utf-8") - - finalizer.publish_disabled_rootfs(pxe_spec(False), "token", cpu_controller(finalizer), out_dir, True) - - assert all( - not path.exists() - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ) - ) - assert all(path.exists() for path in canonical_artifacts(out_dir)[:-1]) - - -def test_pxe_disabled_rootfs_force_restores_private_and_canonical_generation_when_publication_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - out_dir = tmp_path / "generated" - pending = finalizer.paths(out_dir) - finalizer.publish_disabled_rootfs(pxe_spec(False), "old-token", cpu_controller(finalizer), out_dir, False) - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ): - path.write_text(f"old {path.name}\n", encoding="utf-8") - tracked = ( - *canonical_artifacts(out_dir)[:-1], - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ) - before = {path.name: path.read_bytes() for path in tracked} - original_replace = artifact_store.os.replace - - def fail_values_replace(source, destination): - if Path(destination) == pending.values and ".backup." not in str(source): - raise OSError("injected disabled-rootfs publication failure") - return original_replace(source, destination) - - monkeypatch.setattr(artifact_store.os, "replace", fail_values_replace) - with pytest.raises(SystemExit): - finalizer.publish_disabled_rootfs(pxe_spec(False), "new-token", cpu_controller(finalizer), out_dir, True) - - assert {path.name: path.read_bytes() for path in tracked} == before - - -def test_pxe_finalizer_publishes_resolved_policy_idempotently_without_secret_output( +def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - pending = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - context, handoff = pending_handoff(out_dir) - first = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) - second = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) - assert pending.returncode == 0, pending.stderr - assert first.returncode == 0, first.stderr - assert second.returncode == 0, second.stderr - assert "do-not-print-this-secret" not in first.stdout + first.stderr + second.stdout + second.stderr - assert "auplc_render_gid: 995" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_gpu_access_enabled: false" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "renderGid: 995" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 995} - completion = json.loads((out_dir / ".pxe-finalizer-completion.json").read_text(encoding="utf-8")) - assert completion["artifacts"]["inventory.yml"]["mode"] == 0o600 - assert completion["artifacts"]["inventory.yml"]["owner_uid"] == os.geteuid() + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in values + assert "pxe_gpu_access_enabled: true" in pxe_vars + assert manifest == { + "version": 1, + "status": "cpu_only", + "hosts": {"controller": False}, + "pxe_rootfs": {"gpu_access_enabled": True}, + } + assert not list(out_dir.glob(".pxe-finalizer-*")) + assert "do-not-print-this-secret" not in result.stdout + result.stderr -def test_pxe_finalizer_retry_rejects_canonical_mode_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - (out_dir / "inventory.yml").chmod(0o644) result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(False))), "--out-dir", str(out_dir) ) - assert result.returncode == 1 - - -def test_pxe_pending_generation_rejects_existing_private_or_canonical_state_without_force( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "refusing to overwrite" in result.stderr + assert result.returncode == 0, result.stderr + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert "pxe_gpu_access_enabled: false" in pxe_vars + assert "auplc_render_gid" not in pxe_vars + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} -def test_pxe_forced_pending_generation_hides_prior_public_and_private_generation( +def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + write_fake_ansible(tmp_path, monkeypatch, controller_gpu=True) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - old_generation = json.loads(context.read_text(encoding="utf-8"))["generation"] - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 0, result.stderr - assert json.loads(context.read_text(encoding="utf-8"))["generation"] != old_generation - assert not handoff.exists() - assert all(not path.exists() for path in canonical_artifacts(out_dir)) + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert "auplc_gpu_access_enabled: true" in inventory + assert manifest["status"] == "gpu_resolved" + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} -def test_pxe_forced_pending_generation_restores_prior_generation_if_staging_fails( +def test_pxe_generator_refuses_existing_canonical_artifacts_without_force( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - previous = {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - original_replace = artifact_store.os.replace - - def fail_new_bootstrap(source, destination): - if Path(destination) == out_dir / ".pxe-bootstrap.inventory.yml" and ".backup." not in str(source): - raise OSError("injected staging failure") - return original_replace(source, destination) - - monkeypatch.setattr(artifact_store.os, "replace", fail_new_bootstrap) - controller = finalizer._controller_resolution( - pxe_spec(True), json.loads(context.read_text(encoding="utf-8"))["controller"] - ) - with pytest.raises(SystemExit): - finalizer.stage_pending(pxe_spec(True), "replacement-token", controller, out_dir, True) - - assert {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} == previous + out_dir.mkdir() + existing = out_dir / "values-basic-example.yaml" + existing.write_text("existing\n", encoding="utf-8") - -@pytest.mark.parametrize("document_name", (".pxe-finalizer-context.json", ".pxe-finalizer-handoff.json")) -def test_pxe_finalizer_rejects_duplicate_keys_in_private_documents( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document_name: str -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - document_path = out_dir / document_name - document_path.write_text( - '{"generation":"duplicate",' + document_path.read_text(encoding="utf-8")[1:], encoding="utf-8" - ) - - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) - - assert result.returncode == 1 - assert all(not path.exists() for path in canonical_artifacts(out_dir)) - - -@pytest.mark.parametrize("mutation", ("missing", "tampered")) -def test_pxe_finalizer_retry_rejects_missing_or_tampered_canonical_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - inventory = out_dir / "inventory.yml" - if mutation == "missing": - inventory.unlink() - else: - inventory.write_text("tampered\n", encoding="utf-8") - - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert existing.read_text(encoding="utf-8") == "existing\n" + assert all(not path.exists() for path in canonical_artifacts(out_dir) if path != existing) -def test_pxe_finalizer_rejects_symlink_lock_without_touching_its_target( +def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - target = tmp_path / "lock-target" - target.write_text("unchanged\n", encoding="utf-8") - target.chmod(0o644) - lock = out_dir / ".pxe-finalizer.lock" - lock.unlink() - lock.symlink_to(target) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import sys +from pathlib import Path - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({'version': 1, 'hosts': [{'host': 'controller', 'reachable': False, 'lspci': {'rc': 0, 'stdout': ''}, 'sysfs': {'rc': 0, 'stdout': ''}}]}), encoding='utf-8') +""", + encoding="utf-8", ) - - assert result.returncode == 1 - assert target.read_text(encoding="utf-8") == "unchanged\n" - assert target.stat().st_mode & 0o777 == 0o644 - - -@pytest.mark.parametrize( - ("field", "value"), - [("generation", "stale"), ("topology", "ssh-preinstalled"), ("render_gid", None), ("version", True)], -) -def test_pxe_finalizer_rejects_invalid_handoffs_without_publishing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: str | int | None -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - document = json.loads(handoff.read_text(encoding="utf-8")) - document[field] = value - write_json(handoff, document) - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - - -def test_pxe_finalizer_rolls_back_if_late_canonical_publication_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - original_link = artifact_store.os.link - - def fail_values_link(source, destination): - if Path(destination).name == "values-basic-example.yaml": - raise OSError("injected publication failure") - return original_link(source, destination) - - monkeypatch.setattr(artifact_store.os, "link", fail_values_link) - with pytest.raises(SystemExit): - finalizer.finalize(out_dir, context, handoff) - - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "pb-pxe-controller.vars.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - assert not (out_dir / ".pxe-finalizer-completion.json").exists() - - -def test_pxe_playbook_writes_and_finalizes_private_rootfs_handoff_locally() -> None: - playbook = PXE_PLAYBOOK.read_text(encoding="utf-8") - - assert "pxe_finalizer_handoff" in playbook - assert "pxe_finalizer_context" in playbook - assert "--finalize-pxe" in playbook - assert "delegate_to: localhost" in playbook - assert "run_once: true" in playbook - assert "become: false" in playbook - assert "argv:" in playbook + assert all(not path.exists() for path in canonical_artifacts(out_dir)) From 0d095733722b2020876b47ff4a17c10b74cbc3ae Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 073/180] docs: update installer stages --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index efcc2ae7..f19aa4b8 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,15 @@ A successful install looks like this: ```text This operation needs root privileges. Requesting sudo password... - ✓ [1/8] Detecting GPU (0.2s) - ✓ [2/8] Generating values overlay (initial) (0.0s) - ✓ [3/8] Installing helm + k9s (0.0s) - ✓ [4/8] Installing K3s (single-node) (3.8s) - ✓ [5/8] Pulling custom + external images (25.0s) - ✓ [6/8] Deploying ROCm GPU device plugin + node labeller (0.2s) - ✓ [7/8] Refreshing values overlay from node labels (0.2s) - ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) (9.2s) + ✓ [1/9] Detecting GPU (0.2s) + ✓ [2/9] Provisioning GPU device access (0.1s) + ✓ [3/9] Generating values overlay (initial) (0.0s) + ✓ [4/9] Installing helm + k9s (0.0s) + ✓ [5/9] Installing K3s (single-node) (3.8s) + ✓ [6/9] Pulling custom + external images (25.0s) + ✓ [7/9] Deploying ROCm GPU device plugin + node labeller (0.2s) + ✓ [8/9] Refreshing values overlay from node labels (0.2s) + ✓ [9/9] Deploying JupyterHub runtime (helm install + wait) (9.2s) _ _ _ ____ _ _ ____ _ _ / \ | | | | _ \ | | ___ __ _ _ __ _ __ (_)_ __ __ _ / ___| | ___ _ _ __| | From d9d973daf7f92a2c7654857a9c55c2ce24a4882b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 074/180] docs(deploy): document GPU infrastructure contract --- deploy/README.md | 100 ++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index d2a1a72f..23703560 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,12 +53,16 @@ sudo ./auplc-installer install ### Multi-Node Cluster -Generate the spec, fill in the normal network and node details, then let the -generator discover GPU hosts and their shared `render` GID. The SSH flow asks -for no GPU host list and no GID. A PXE spec asks one extra GPU question: +Generate the spec and fill in the network and node details. The SSH flow needs +only the managed host details. A PXE spec asks one extra GPU question: `pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless agents' hardware is not inferred from the controller. +The AMD device plugin and ROCm node labeller are cluster infrastructure +prerequisites owned outside AUPLC. The infrastructure owner must deploy and +maintain them according to AMD's official guidance. Before Helm, verify that the +existing DaemonSets are ready and that GPU capacity is advertised. + #### SSH-preinstalled ```bash @@ -82,6 +86,10 @@ sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + cd "$REPO_ROOT" helm upgrade --install jupyterhub ./runtime/chart \ --namespace jupyterhub --create-namespace \ @@ -92,34 +100,35 @@ helm upgrade --install jupyterhub ./runtime/chart \ Generation runs read-only Ansible discovery against every managed host. It cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class records under `/sys/bus/pci/devices`; it does not require the devices to be -attached to `amdgpu` before ROCm installation. It checks -the `render` group and existing GPU access files, and publishes only when every -GPU host agrees on one GID. CPU-only fleets publish `null` for the generated -inventory and Helm render GID. GPU policy details in generated files are -internal outputs, not fields to maintain by hand. +attached to `amdgpu` before ROCm installation. The resulting GPU resolution +report records which managed hosts have AMD display hardware. + +The GPU permission contract is fixed across GPU hosts and PXE root filesystems: -Configure notebook storage ownership with `singleuser.fsGid: 100`. Never set -storage `fsGroup` through `extraPodConfig.securityContext`, because that Pod -security-context override can replace the GPU resource's generated -`supplementalGroups`. +- `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render` with mode `0666`. +- AMD `/dev/dri/card*` nodes are `root:video` with mode `0666`. +- Every GPU device node injected into a Pod therefore has mode `0666`. +- Host provisioning owns device-node discretionary access control. +- AUPLC Hub adds no GPU supplemental group to user Pods. +- AMD device-plugin allocation is the visibility boundary: only Pods that + request `amd.com/gpu` receive GPU device nodes. The plugin does not set Unix + ownership or modes on host device nodes. + +`singleuser.fsGid: 100` controls shared notebook storage ownership only. It is +not part of GPU access and must not be treated as a GPU group setting. #### PXE-diskless After setting `topology` to `pxe-diskless`, fill the PXE network fields and set -only `pxe.diskless_agents_have_amd_gpus` for GPU policy. When it is `true`, the -first generation is pending and creates private bootstrap files instead of -canonical deployment files. +`pxe.diskless_agents_have_amd_gpus` explicitly. Generation writes the canonical +inventory, controller vars, runtime overlay, and GPU resolution report directly. +These artifacts express the desired deployment inputs; their existence is not +proof that rootfs provisioning succeeded. Review and install them before running +the controller playbook, whose successful completion provisions the rootfs. ```bash cd "$REPO_ROOT" python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -cd "$REPO_ROOT/deploy/ansible" -sudo ansible-playbook \ - -i "$GENERATED_DIR/.pxe-bootstrap.inventory.yml" \ - playbooks/pb-pxe-controller.yml \ - -e @"$GENERATED_DIR/.pxe-bootstrap.vars.yml" - -# pb-pxe-controller finalizes automatically after a successful rootfs build. install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ @@ -128,40 +137,41 @@ python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskles --values "$REPO_ROOT/runtime/values.yaml" \ --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" -``` -Don't invoke the hidden finalizer yourself. The playbook writes a private -handoff and runs finalization locally. `inventory.yml`, -`pb-pxe-controller.vars.yml`, `values-basic-example.yaml`, and -`gpu-access-resolution.json` appear only after success. +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` -A fresh PXE rootfs can create a missing `render` group and align it with a -unanimous live controller GPU GID after collision checks. A retained rootfs is -never silently changed. It must already contain one valid `render` group and, -when the controller has a resolved GPU GID, the rootfs GID must match. Rebuild -the rootfs or migrate the retained rootfs separately if it doesn't match. -Offline checks don't replace post-boot verification of GPU device ownership, -mode, supplemental groups, and workload access. +A fresh PXE rootfs receives the fixed udev rule during the controller playbook. +A retained rootfs is accepted only when it already contains that exact canonical +rule and no conflicting legacy GPU rule. Rebuild or correct a retained rootfs +separately if that safety check fails. -#### Discovery failures and migration +#### Discovery failures | Error | Action | | --- | --- | | Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | | `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | | Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | -| GPU host has no valid `render` group | Install the correct GPU userspace or create one valid system `render` group, then regenerate. | -| GPU render GIDs disagree | Plan and perform a reviewed group migration so every GPU host uses one free GID, then regenerate. | -| CPU host retains GPU access contract, or canonical state/rule conflicts | Inspect `/var/lib/auplc/gpu-access.json` and `/etc/udev/rules.d/70-auplc-gpu-access.rules`. Remove stale project-owned files from a truly CPU-only host, or complete the GPU migration. Never overwrite unknown content. | -| Retained PXE rootfs GID differs from the unanimous live GID | Rebuild the rootfs, or migrate that retained rootfs separately before rerunning the playbook. | - -Old unshipped specs aren't compatible. Remove the former manual GPU policy -fields, regenerate the schema, copy the ordinary node and PXE network values -into it, and set only `pxe.diskless_agents_have_amd_gpus` on PXE deployments. +| Retained PXE rootfs has a legacy or non-canonical GPU rule | Rebuild the rootfs, or replace the conflicting rule through a separate reviewed maintenance action before rerunning the playbook. | ## Deployment branch boundary This branch and these instructions do not modify or roll out any live deployment. SHC, FET, and other deployment branches or environments must -backport the automatic discovery and generated-artifact changes before their -own reviewed rollout. +backport the host permission and immediate artifact publication changes before +their own reviewed rollout. From e74c40d1d26efcc4b0c4b6196772a623a5a5844c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 075/180] docs(ansible): document GID-free policy --- deploy/ansible/README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 108dad42..d698183b 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -22,16 +22,24 @@ SOFTWARE. # Ansible Playbooks -K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible/tree/master). +K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible). For the generator, canonical inventory, validator arguments, and topology-specific playbook commands, see the authoritative [deployment guide](../README.md). Don't write GPU policy into the inventory by hand. SSH generation discovers GPU -hosts and their shared `render` group ID. PXE generation uses only -`pxe.diskless_agents_have_amd_gpus`; when enabled, the controller playbook uses -private bootstrap inputs and publishes canonical files automatically after a -successful rootfs build. +hosts from managed-host evidence. PXE generation uses only +`pxe.diskless_agents_have_amd_gpus` and writes canonical files before the +controller playbook runs. + +The GPU access role sets AMD device-node policy on GPU hosts and GPU-enabled PXE +root filesystems. `/dev/kfd` and AMD `renderD*` nodes are `root:render 0666`; +AMD `card*` nodes are `root:video 0666`. All injected GPU device nodes therefore +use mode `0666`. Device-plugin allocation is the visibility boundary: only Pods +requesting `amd.com/gpu` receive the nodes. The plugin does not change host inode +permissions, and AUPLC Hub adds no GPU supplemental group to user Pods. Ordinary +container group membership does not participate in GPU permissions; host +provisioning owns device-node discretionary access control. ## Prerequisites @@ -39,3 +47,6 @@ successful rootfs build. - **Python**: 3.12 - **SSH**: Root login with key-based auth to all nodes - **Hosts**: Consistent `/etc/hosts` entries across all nodes +- **GPU integration**: The infrastructure owner must deploy and maintain the AMD + device plugin and ROCm node labeller outside AUPLC. Before Helm, run the + readiness and capacity checks in the [deployment guide](../README.md). From a87aba0c0c49b9285ecbdd160dce4c2c8e4d9c93 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 076/180] docs(k8s): require external GPU device management --- deploy/k8s/README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index be3c471c..ce25b6c2 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -40,17 +40,18 @@ which `runtime/values.yaml` uses as `nodeSelector`s. The installer pins the accelerator `nodeSelector` to the real `amd.com/gpu.product-name` detected on the host, so no manual labelling is needed on single-machine deployments. -If you are deploying manually instead: +For multi-node deployments, the AMD device plugin and ROCm node labeller are +cluster infrastructure prerequisites owned outside AUPLC. The infrastructure +owner must select, deploy, and maintain them according to the +[official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). +AUPLC documentation does not install these privileged components. -```bash -# Deploy AMD GPU device plugin -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml - -# Deploy AMD GPU node labeller (publishes amd.com/gpu.* labels) -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml +Before deploying the AUPLC Helm release, verify the existing infrastructure: -# Verify GPU detection and labels -kubectl describe node <node-name> | grep amd.com/gpu +```bash +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' ``` `runtime/values-multi-nodes.yaml.example` now follows `runtime/values.yaml` and From 8c3ba1f50007fc48953b676b7e4794a4ce132cdb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 077/180] docs(skills): align GPU deployment workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 32 +++++++++++++------ skills/deploy-aup-learning-cloud/reference.md | 26 ++++++++++++--- .../scripts/README.md | 13 +++++--- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 2fc9725a..1799c447 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -41,8 +41,8 @@ Then collect and confirm: 1. Courses and notebook resources. 2. Controller hostname, static IP, subnet, gateway, and DNS. -3. For SSH, every managed hostname and IP. Don't ask for a GPU host list or a - shared GPU group ID. Generation discovers both over SSH. +3. For SSH, every managed hostname and IP. Don't ask for a GPU host list; + generation discovers GPU hosts over SSH. 4. For PXE, the controller NIC, web port, rootfs SSH public key, and whether diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU policy input because agent hardware can't be inferred from the controller. @@ -57,16 +57,16 @@ Create a fresh schema and fill only its current fields. Run the generator rather than writing inventory or GPU policy by hand. For SSH, generation performs read-only discovery on every managed host and -publishes canonical artifacts only after GPU evidence and group IDs agree. +publishes canonical artifacts after GPU evidence is consistent. -For PXE with GPU agents, initial generation creates private bootstrap inventory -and vars. Run the PXE controller playbook with those private files. A successful -rootfs build finalizes generation automatically and publishes the canonical -inventory, PXE vars, runtime overlay, and GPU resolution report. +For PXE, generation writes the canonical inventory, PXE vars, runtime overlay, +and GPU resolution report directly as desired deployment inputs. Their existence +does not prove rootfs provisioning succeeded. Review, install, and validate those +files, then run the controller playbook with the canonical inventory and PXE +vars; the playbook must complete successfully before proceeding. Follow the exact generation, installation, and playbook commands in -[deploy/README.md](../../deploy/README.md). Don't invent a separate completion -step. +[deploy/README.md](../../deploy/README.md). ## Phase 3: Validate and execute @@ -82,7 +82,19 @@ then run the validator with the arguments shown in the deployment guide: Stop on validation failure. After a clean result, follow the topology's Ansible, storage, device plugin, and Helm sequence in -[deploy/README.md](../../deploy/README.md). +[deploy/README.md](../../deploy/README.md). Treat the AMD device plugin and ROCm +node labeller as infrastructure prerequisites owned outside AUPLC. Verify both +existing DaemonSets and advertised GPU capacity before Helm; do not install +these privileged components as part of the AUPLC procedure. + +Keep the GPU contract distinct from storage configuration. GPU hosts use +`root:render 0666` for `/dev/kfd` and AMD `renderD*`, and `root:video 0666` for +AMD `card*`, so every injected GPU device node has mode `0666`. Device-plugin +allocation is the visibility boundary and only `amd.com/gpu` requests receive +GPU nodes. Container group membership does not participate in GPU permissions; +host provisioning owns device-node discretionary access control. AUPLC Hub adds +no GPU supplemental group, and the plugin does not change Unix inode permissions. +`singleuser.fsGid: 100` is for shared storage only. ## Phase 4: Verify diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index 4aafa0fa..ff7d5bd5 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -8,11 +8,11 @@ commands into this reference. | Topology | Generator behavior | | --- | --- | -| `ssh-preinstalled` | Connects to every managed host, discovers GPU hosts and their shared `render` group ID, and publishes canonical files only when discovery is consistent. | -| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input. GPU-enabled first generation emits private bootstrap files; the PXE controller playbook finalizes canonical files after a successful rootfs build. | +| `ssh-preinstalled` | Connects to every managed host, discovers GPU hardware, and publishes canonical files when discovery is consistent. | +| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input and publishes canonical desired-input files before the controller playbook runs. Their existence does not prove rootfs provisioning succeeded. | -Don't hand-author generated GPU policy. Old unshipped specs should be recreated -from the current `--print-schema` output. +Don't hand-author generated GPU policy. Create deployment specs from the current +`--print-schema` output. ## Canonical validation inputs @@ -28,6 +28,24 @@ passes: Generation and validation must finish before Ansible or Helm changes are made. +## GPU permission contract + +- Host `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render 0666`. +- Host AMD `/dev/dri/card*` nodes are `root:video 0666`. +- Every GPU device node injected into a Pod has mode `0666`. +- Container group membership does not participate in GPU permissions; host + provisioning owns device-node discretionary access control. +- AUPLC Hub adds no GPU supplemental group to user Pods. +- AMD device-plugin allocation is the visibility boundary. Only Pods requesting + `amd.com/gpu` receive GPU device nodes; the plugin does not change host inode + ownership or mode. +- `singleuser.fsGid: 100` controls shared storage ownership only. + +The infrastructure owner deploys and maintains the AMD device plugin and ROCm +node labeller outside AUPLC. Before Helm, use the readiness and capacity checks +in [deploy/README.md](../../deploy/README.md); do not install these privileged +components as part of the AUPLC procedure. + ## Operator gates Keep the topology choice explicit. Confirm network, node, storage, course, and diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 65bd71c1..db9fbfef 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -8,17 +8,20 @@ sequence and argument paths. | --- | --- | | `detect_hardware.sh` | Reports controller network details and local AMD PCI devices as JSON. | | `detect_cluster.sh` | Reports Kubernetes nodes, AMD GPU labels, storage classes, and GPU DaemonSet state as JSON. | -| `gen_configs.py` | Prints the current spec schema, discovers SSH GPU state, and generates topology-specific deployment artifacts. PXE GPU bootstrap files remain private until the controller playbook finalizes them automatically. | +| `gen_configs.py` | Prints the current spec schema, discovers live GPU state, and directly publishes canonical topology-specific deployment artifacts. | | `validate.py` | Checks the selected topology against canonical inventory, GPU resolution, values overlays, and PXE vars when applicable. | ## Generator contract -The SSH topology discovers GPU hosts and their shared `render` group ID. Users -don't provide either value. The PXE topology has one GPU policy input: +The SSH topology discovers GPU hosts from managed-host evidence. Users don't +provide a GPU host list. The PXE topology has one GPU policy input: `pxe.diskless_agents_have_amd_gpus`. -Generate specs from fresh `--print-schema` output. Don't hand-edit generated GPU -policy or add a separate PXE completion step. +Generate specs from fresh `--print-schema` output. Both topologies write their +canonical artifacts immediately. For PXE, review and validate those files, then +run the controller playbook with the generated `inventory.yml` and +`pb-pxe-controller.vars.yml`. The files express desired inputs; their existence +does not prove the PXE rootfs was provisioned successfully. ## Validator contract From c64b8cd36ad0abd6c33d547bc80ade99487d307c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:11:50 +0800 Subject: [PATCH 078/180] docs(deploy): restore GPU setup commands --- deploy/README.md | 6 ++++-- deploy/ansible/README.md | 6 ++++-- deploy/k8s/README.md | 13 +++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 23703560..b53d0a5d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -60,8 +60,10 @@ agents' hardware is not inferred from the controller. The AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must deploy and -maintain them according to AMD's official guidance. Before Helm, verify that the -existing DaemonSets are ready and that GPU capacity is advertised. +maintain them according to AMD's official guidance. If they are not installed, +follow the pinned manual installation commands in the +[Kubernetes components guide](k8s/README.md). Before Helm, verify that the +DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index d698183b..eda9b349 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -48,5 +48,7 @@ provisioning owns device-node discretionary access control. - **SSH**: Root login with key-based auth to all nodes - **Hosts**: Consistent `/etc/hosts` entries across all nodes - **GPU integration**: The infrastructure owner must deploy and maintain the AMD - device plugin and ROCm node labeller outside AUPLC. Before Helm, run the - readiness and capacity checks in the [deployment guide](../README.md). + device plugin and ROCm node labeller outside AUPLC. Use the pinned manual + installation in the [Kubernetes components guide](../k8s/README.md) when the + cluster does not already provide them. Before Helm, run the readiness and + capacity checks in the [deployment guide](../README.md). diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index ce25b6c2..da148f52 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -44,9 +44,18 @@ For multi-node deployments, the AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must select, deploy, and maintain them according to the [official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). -AUPLC documentation does not install these privileged components. -Before deploying the AUPLC Helm release, verify the existing infrastructure: +To install the same pinned manifests used by `auplc-installer`: + +```bash +ROCM_DEVICE_PLUGIN_COMMIT="dea1db13f05159e64d8114bca4c31f48c3cfcac6" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-dp.yaml" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-labeller.yaml" +``` + +Before deploying the AUPLC Helm release, verify the installation: ```bash kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m From be209858dc2ac064bfe4ad175bda8c0ba7cf522c Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Tue, 28 Jul 2026 10:07:04 +0100 Subject: [PATCH 079/180] remove unrelated changes --- runtime/hub/core/jupyterhub_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime/hub/core/jupyterhub_config.py b/runtime/hub/core/jupyterhub_config.py index ccc75c2e..0a2bd1f9 100644 --- a/runtime/hub/core/jupyterhub_config.py +++ b/runtime/hub/core/jupyterhub_config.py @@ -151,9 +151,7 @@ def _camel_case(s: str) -> str: # Inject platform identity into every Jinja template context so that # {{ powered_by }} is available in all Hub-rendered pages. -if not isinstance(c.JupyterHub.template_vars, dict): - c.JupyterHub.template_vars = {} -c.JupyterHub.template_vars.setdefault("powered_by", "AUP Learning Cloud") +c.JupyterHub.template_vars = {"powered_by": "AUP Learning Cloud"} # Database configuration db_type = z2jh.get_config("hub.db.type") From 43dc742b675c44fd6bb8d7e8f6b0f6bb5e8aa0a6 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Tue, 28 Jul 2026 11:47:33 +0100 Subject: [PATCH 080/180] Simplify text --- runtime/hub/frontend/apps/spawn/src/App.tsx | 2 +- runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/hub/frontend/apps/spawn/src/App.tsx b/runtime/hub/frontend/apps/spawn/src/App.tsx index fe4e9565..5e4d2443 100644 --- a/runtime/hub/frontend/apps/spawn/src/App.tsx +++ b/runtime/hub/frontend/apps/spawn/src/App.tsx @@ -194,7 +194,7 @@ function App() { const autoOption: Accelerator = { key: 'auto', displayName: 'Auto (Best Available)', - description: `Automatically selected based on availability. Rate: ${rateDesc}`, + description: 'Auto select', quotaRate: minRate, }; return [autoOption, ...real]; diff --git a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx index 6388564e..9239183d 100644 --- a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx +++ b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx @@ -95,7 +95,7 @@ export const CourseCard = memo(function CourseCard({ const autoOption: Accelerator = { key: 'auto', displayName: 'Auto (Best Available)', - description: `Automatically selected based on availability. Rate: ${rateDesc}`, + description: 'Auto select', quotaRate: minRate, }; return [autoOption, ...real]; From 77400edb4db79d706894444b108edf789740db7b Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Tue, 28 Jul 2026 12:04:21 +0100 Subject: [PATCH 081/180] Make text clearer --- runtime/hub/frontend/apps/spawn/src/App.tsx | 4 ++-- runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/hub/frontend/apps/spawn/src/App.tsx b/runtime/hub/frontend/apps/spawn/src/App.tsx index 5e4d2443..ef8081a9 100644 --- a/runtime/hub/frontend/apps/spawn/src/App.tsx +++ b/runtime/hub/frontend/apps/spawn/src/App.tsx @@ -193,8 +193,8 @@ function App() { const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; const autoOption: Accelerator = { key: 'auto', - displayName: 'Auto (Best Available)', - description: 'Auto select', + displayName: 'Auto', + description: 'Auto select best available GPU node', quotaRate: minRate, }; return [autoOption, ...real]; diff --git a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx index 9239183d..f79f2cb0 100644 --- a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx +++ b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx @@ -94,8 +94,8 @@ export const CourseCard = memo(function CourseCard({ const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; const autoOption: Accelerator = { key: 'auto', - displayName: 'Auto (Best Available)', - description: 'Auto select', + displayName: 'Auto', + description: 'Auto select best available GPU node', quotaRate: minRate, }; return [autoOption, ...real]; From 1ca3cb7fd85c9a2b2eb0c915e6b96c5bd2df0a7b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 082/180] refactor(installer): install AMD GPU udev package --- auplc_installer/gpu_access.py | 253 +++++++------- tests/installer/test_gpu_access.py | 359 +++++++------------- tests/installer/test_gpu_access_ordering.py | 164 +++++++++ 3 files changed, 411 insertions(+), 365 deletions(-) create mode 100644 tests/installer/test_gpu_access_ordering.py diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py index c3a946a7..534e87e2 100644 --- a/auplc_installer/gpu_access.py +++ b/auplc_installer/gpu_access.py @@ -1,15 +1,27 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Single-node AMD GPU device-access reconciler.""" - from __future__ import annotations +import contextlib +import tempfile from pathlib import Path from typing import Protocol -from auplc_installer.util import InstallerError, run, run_capture +from auplc_installer.util import InstallerError, run, run_capture, verify_sha256 + +AMD_GPU_UDEV_PACKAGE_NAME = "amdgpu-insecure-instinct-udev-rules" +AMD_GPU_UDEV_PACKAGE_VERSION = "30.30.4.0-2341068.24.04" +AMD_GPU_UDEV_PACKAGE_FILENAME = "amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb" +AMD_GPU_UDEV_PACKAGE_URL = ( + "https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/" + f"{AMD_GPU_UDEV_PACKAGE_FILENAME}" +) +AMD_GPU_UDEV_PACKAGE_SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +AMD_GPU_UDEV_PACKAGE_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +AMD_GPU_UDEV_PACKAGE_RULES = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) -GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") LEGACY_ROCM_DEVICES_RULES_PATH = Path("/etc/udev/rules.d/70-rocm-devices.rules") @@ -33,91 +45,35 @@ LEGACY_AMDGPU_RULES_PATH: frozenset((LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_PXE_RULES)), LEGACY_ROCM_DEVICES_RULES_PATH: frozenset((LEGACY_ROCM_DEVICES_RULES,)), } -UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." -CANONICAL_UDEV_RULES = ( - f"{UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' -) -_FSYNC_PATH_SCRIPT = ( - "import os\n" - "import sys\n" - "fd = os.open(sys.argv[1], os.O_RDONLY)\n" - "try:\n" - " os.fsync(fd)\n" - "finally:\n" - " os.close(fd)\n" -) -_VERIFY_DEVICE_ACCESS_SCRIPT = ( - "import grp, pathlib, stat\n" - "drm = pathlib.Path('/sys/class/drm')\n" - "devices = [(pathlib.Path('/dev/kfd'), 'render', 0o666)]\n" - "render_nodes = []\n" - "for node in drm.glob('renderD*'):\n" - " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu':\n" - " render_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" - "if not render_nodes: raise SystemExit('no AMD renderD device found')\n" - "devices.extend((path, 'render', 0o666) for path in render_nodes)\n" - "card_nodes = []\n" - "for node in drm.glob('card*'):\n" - " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu':\n" - " card_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" - "if not card_nodes: raise SystemExit('no AMD card device found')\n" - "devices.extend((path, 'video', 0o666) for path in card_nodes)\n" - "for path, expected_group, expected_mode in devices:\n" - " data = path.lstat()\n" - " try:\n" - " group_name = grp.getgrgid(data.st_gid).gr_name\n" - " except KeyError:\n" - " raise SystemExit(f'unknown GPU device group: {path}')\n" - " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or group_name != expected_group or stat.S_IMODE(data.st_mode) != expected_mode:\n" - " raise SystemExit(f'bad GPU device access: {path}')\n" -) class GpuAccessHost(Protocol): - """Privileged host-operation seam for GPU access provisioning.""" + def read_text(self, path: Path) -> str | None: ... - def read_text(self, path: Path) -> str | None: - """Return a privileged file's text, or ``None`` when it is absent.""" + def remove_udev_rule(self, path: Path) -> None: ... - def write_udev_rule(self, path: Path, text: str) -> None: - """Write a managed udev rule after reconciliation has authorized it.""" + def installed_package_version(self) -> str | None: ... - def reload_udev_rules(self) -> None: - """Reload host udev rules.""" + def package_owns_rule(self, path: Path) -> bool: ... - def trigger_udev(self) -> None: - """Apply reloaded udev rules to current devices.""" + def install_package(self, deb: Path) -> None: ... - def settle_udev(self) -> None: - """Wait until triggered udev events finish before inode verification.""" + def reload_udev_rules(self) -> None: ... - def remove_udev_rule(self, path: Path) -> None: - """Remove an explicitly recognized legacy udev rule.""" + def trigger_udev(self) -> None: ... - def verify_device_access(self) -> None: - """Verify the relevant GPU device inodes use the host access contract.""" + def settle_udev(self) -> None: ... - def is_symlink(self, path: Path) -> bool: - """Return whether ``path`` is a symlink without following it.""" + def is_symlink(self, path: Path) -> bool: ... - def is_regular_file(self, path: Path) -> bool: - """Return whether an existing ``path`` is a regular file.""" + def is_regular_file(self, path: Path) -> bool: ... - def path_exists(self, path: Path) -> bool: - """Return whether ``path`` exists after a separate symlink check.""" + def path_exists(self, path: Path) -> bool: ... - def is_directory(self, path: Path) -> bool: - """Return whether an existing ``path`` is a directory.""" + def is_directory(self, path: Path) -> bool: ... class SystemGpuAccessHost: - """Production host adapter using the installer's sudo-aware command helpers.""" - def read_text(self, path: Path) -> str | None: exists = run(["test", "-e", str(path)], sudo=True, check=False) if exists.returncode != 0: @@ -125,33 +81,32 @@ def read_text(self, path: Path) -> str | None: result = run_capture(["cat", str(path)], sudo=True) return result.stdout or "" - def write_udev_rule(self, path: Path, text: str) -> None: - self._write_text_atomically(path, text) + def remove_udev_rule(self, path: Path) -> None: + run(["rm", "-f", str(path)], sudo=True) - def _write_text_atomically(self, path: Path, text: str) -> None: - """Durably replace ``path`` after atomically renaming a temporary file.""" - _validate_parent_chain(self, path.parent) - run(["mkdir", "-p", str(path.parent)], sudo=True) - temporary_result = run_capture( - ["mktemp", str(path.parent / f".{path.name}.XXXXXX")], + def installed_package_version(self) -> str | None: + result = run_capture( + ["dpkg-query", "--show", "--showformat=${Status}\t${Version}", AMD_GPU_UDEV_PACKAGE_NAME], sudo=True, + check=False, ) - temporary_path = (temporary_result.stdout or "").strip() - if not temporary_path: - raise InstallerError(f"Could not create temporary GPU access rule beside {path}") - - try: - run(["tee", temporary_path], sudo=True, input_text=text) - run(["chmod", "0644", temporary_path], sudo=True) - self._fsync_path(temporary_path) - run(["mv", "-f", temporary_path, str(path)], sudo=True) - self._fsync_path(str(path.parent)) - except BaseException: - run(["rm", "-f", temporary_path], sudo=True, check=False) - raise - - def _fsync_path(self, path: str) -> None: - run(["python3", "-c", _FSYNC_PATH_SCRIPT, path], sudo=True) + if result.returncode != 0: + return None + status, separator, version = (result.stdout or "").strip().partition("\t") + if status != "install ok installed" or not separator or not version: + return None + return version + + def package_owns_rule(self, path: Path) -> bool: + result = run_capture( + ["dpkg-query", "--listfiles", AMD_GPU_UDEV_PACKAGE_NAME], + sudo=True, + check=False, + ) + return result.returncode == 0 and str(path) in (result.stdout or "").splitlines() + + def install_package(self, deb: Path) -> None: + run(["dpkg", "--force-confnew", "--install", str(deb)], sudo=True) def reload_udev_rules(self) -> None: run(["udevadm", "control", "--reload-rules"], sudo=True) @@ -162,12 +117,6 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: run(["udevadm", "settle"], sudo=True) - def remove_udev_rule(self, path: Path) -> None: - run(["rm", "-f", str(path)], sudo=True) - - def verify_device_access(self) -> None: - run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT], sudo=True) - def is_symlink(self, path: Path) -> bool: return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 @@ -181,35 +130,68 @@ def is_directory(self, path: Path) -> bool: return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 -def render_udev_rules() -> str: - """Return the canonical AMD GPU host-device udev rules.""" - return CANONICAL_UDEV_RULES - - -def provision_gpu_access(host: GpuAccessHost | None = None) -> None: - """Reconcile and verify the canonical AMD GPU host-device policy.""" +def provision_gpu_access( + host: GpuAccessHost | None = None, + *, + offline_mode: bool = False, + bundle_dir: Path | None = None, +) -> None: active_host = host if host is not None else SystemGpuAccessHost() - _validate_parent_chain(active_host, GPU_ACCESS_RULES_PATH.parent) + _validate_parent_chain(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH.parent) + installed_version = active_host.installed_package_version() legacy_paths = _legacy_rules_to_remove(active_host) - existing_rule = _read_regular_text(active_host, GPU_ACCESS_RULES_PATH) - - for path in legacy_paths: - active_host.remove_udev_rule(path) - if _should_rewrite_udev_rule(existing_rule): - active_host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) - active_host.reload_udev_rules() - active_host.trigger_udev() - active_host.settle_udev() - active_host.verify_device_access() + if installed_version == AMD_GPU_UDEV_PACKAGE_VERSION: + _verify_installed_package(active_host, installed_version) + else: + _install_package(active_host, offline_mode=offline_mode, bundle_dir=bundle_dir) + installed_version = active_host.installed_package_version() + if installed_version is None: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} was not installed") + _verify_installed_package(active_host, installed_version) + _remove_separate_legacy_rules(active_host, legacy_paths) + + +def _install_package(active_host: GpuAccessHost, *, offline_mode: bool, bundle_dir: Path | None) -> None: + if offline_mode: + if bundle_dir is None: + raise InstallerError("Offline GPU udev package installation requires a bundle directory") + deb = bundle_dir / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + if not deb.is_file(): + raise InstallerError(f"Offline GPU udev package is missing: {deb}") + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + return + + with tempfile.NamedTemporaryFile(prefix="auplc-amdgpu-udev-", suffix=".deb", delete=False) as temporary: + deb = Path(temporary.name) + try: + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + finally: + with contextlib.suppress(OSError): + deb.unlink() + + +def _verify_installed_package(active_host: GpuAccessHost, installed_version: str) -> None: + if installed_version != AMD_GPU_UDEV_PACKAGE_VERSION: + raise InstallerError( + f"{AMD_GPU_UDEV_PACKAGE_NAME} has version {installed_version}, expected {AMD_GPU_UDEV_PACKAGE_VERSION}" + ) + if not active_host.package_owns_rule(AMD_GPU_UDEV_PACKAGE_RULES_PATH): + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} does not own {AMD_GPU_UDEV_PACKAGE_RULES_PATH}") + rule = _read_regular_text(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH) + if rule != AMD_GPU_UDEV_PACKAGE_RULES: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} rule does not match the pinned package policy") def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: if host.is_symlink(path): - raise InstallerError(f"Refusing symlinked GPU access file: {path}") + raise InstallerError(f"Refusing symlinked GPU udev rule: {path}") if not host.path_exists(path): return None if not host.is_regular_file(path): - raise InstallerError(f"Refusing non-regular GPU access file: {path}") + raise InstallerError(f"Refusing non-regular GPU udev rule: {path}") return host.read_text(path) @@ -217,13 +199,13 @@ def _validate_parent_chain(host: GpuAccessHost, parent: Path) -> None: components = [*reversed(parent.parents), parent] for index, component in enumerate(components): if host.is_symlink(component): - raise InstallerError(f"Refusing symlinked GPU access directory: {component}") + raise InstallerError(f"Refusing symlinked GPU udev directory: {component}") if not host.path_exists(component): if index != len(components) - 1: - raise InstallerError(f"Missing parent GPU access directory: {component}") + raise InstallerError(f"Missing parent GPU udev directory: {component}") return if not host.is_directory(component): - raise InstallerError(f"Refusing non-directory GPU access parent: {component}") + raise InstallerError(f"Refusing non-directory GPU udev parent: {component}") def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: @@ -232,17 +214,22 @@ def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: content = _read_regular_text(host, path) if content is None: continue + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH and host.package_owns_rule(path): + continue if content not in expected_contents: raise InstallerError(f"Refusing to remove unexpected legacy GPU udev rule: {path}") removals.append(path) return removals -def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: - if existing_rule is None: - return True - if existing_rule == render_udev_rules(): - return False - if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: - raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") - raise InstallerError(f"Refusing to overwrite unrecognized managed GPU udev rule: {GPU_ACCESS_RULES_PATH}") +def _remove_separate_legacy_rules(host: GpuAccessHost, paths: list[Path]) -> None: + removed = False + for path in paths: + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH: + continue + host.remove_udev_rule(path) + removed = True + if removed: + host.reload_udev_rules() + host.trigger_udev() + host.settle_udev() diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index 230da4a1..159e6a83 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -1,6 +1,6 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Tests for the single-node AMD GPU host device-access reconciler.""" +"""Tests for AMD's packaged single-node GPU udev policy.""" from __future__ import annotations @@ -11,26 +11,29 @@ from auplc_installer import gpu_access from auplc_installer.gpu_access import ( - GPU_ACCESS_RULES_PATH, - LEGACY_AMDGPU_PXE_RULES, + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_RULES_PATH, - LEGACY_KFD_RULES, - LEGACY_KFD_RULES_PATH, - LEGACY_ROCM_DEVICES_RULES, - LEGACY_ROCM_DEVICES_RULES_PATH, SystemGpuAccessHost, provision_gpu_access, - render_udev_rules, ) from auplc_installer.util import InstallerError class FakeGpuAccessHost: - """In-memory adapter for the installer host-operation seam.""" - - def __init__(self, *, files: dict[Path, str] | None = None) -> None: + def __init__( + self, + *, + files: dict[Path, str] | None = None, + installed_version: str | None = None, + package_owns_rule: bool | None = None, + ) -> None: self.files = dict(files or {}) + self.installed_version = installed_version + self._package_owns_rule = installed_version is not None if package_owns_rule is None else package_owns_rule self.calls: list[str] = [] self.symlinks: set[Path] = set() self.nonregular_files: set[Path] = set() @@ -40,14 +43,24 @@ def read_text(self, path: Path) -> str | None: self.calls.append(f"read:{path}") return self.files.get(path) - def write_udev_rule(self, path: Path, text: str) -> None: - self.calls.append(f"write-rule:{path}") - self.files[path] = text - def remove_udev_rule(self, path: Path) -> None: self.calls.append(f"remove-rule:{path}") self.files.pop(path, None) + def installed_package_version(self) -> str | None: + self.calls.append("installed-version") + return self.installed_version + + def package_owns_rule(self, path: Path) -> bool: + self.calls.append(f"owns-rule:{path}") + return self._package_owns_rule + + def install_package(self, deb: Path) -> None: + self.calls.append(f"install-package:{deb}") + self.installed_version = AMD_GPU_UDEV_PACKAGE_VERSION + self._package_owns_rule = True + self.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] = AMD_GPU_UDEV_PACKAGE_RULES + def reload_udev_rules(self) -> None: self.calls.append("reload-udev") @@ -57,9 +70,6 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: self.calls.append("settle-udev") - def verify_device_access(self) -> None: - self.calls.append("verify-devices") - def is_symlink(self, path: Path) -> bool: return path in self.symlinks @@ -73,254 +83,139 @@ def is_directory(self, path: Path) -> bool: return path in self.directories -def test_render_udev_rules_is_the_canonical_host_device_policy() -> None: - rules = render_udev_rules() - - assert rules == ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' - ) - assert "chmod" not in rules +def test_offline_install_replaces_legacy_rule_at_the_package_owned_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: an offline bundle and a legacy rule from an earlier shipped installer. + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + host = FakeGpuAccessHost(files={LEGACY_AMDGPU_RULES_PATH: LEGACY_AMDGPU_RULES}) + verified: list[tuple[Path, str]] = [] + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + # When: GPU access is provisioned from the bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) -def test_device_verification_checks_kfd_and_amd_render_and_card_nodes_without_a_render_gid() -> None: - script = gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + # Then: package installation replaces the path without deleting the package-owned rule afterward. + assert not any(call == f"remove-rule:{LEGACY_AMDGPU_RULES_PATH}" for call in host.calls) + assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} + assert verified == [(deb, gpu_access.AMD_GPU_UDEV_PACKAGE_SHA256)] - assert "path.lstat()" in script - assert "stat.S_ISCHR(data.st_mode)" in script - assert "glob('renderD*')" in script - assert "glob('card*')" in script - assert "'render', 0o666" in script - assert "'video', 0o666" in script - assert "render_gid" not in script - assert "sys.argv[1]" not in script - -@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d")]) -def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: +def test_online_install_downloads_to_a_temporary_deb_then_removes_it(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: no installed package and a downloader that materializes its destination. host = FakeGpuAccessHost() - host.symlinks.add(unsafe_parent) + downloads: list[list[str]] = [] + verified: list[Path] = [] - with pytest.raises(InstallerError, match="symlinked GPU access directory"): - provision_gpu_access(host) - - assert not any(call.startswith(("read:", "write-", "remove-rule:")) for call in host.calls) - - -def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: - host = FakeGpuAccessHost() - host.nonregular_files.add(GPU_ACCESS_RULES_PATH) - - with pytest.raises(InstallerError, match="non-regular GPU access file"): - provision_gpu_access(host) - - assert f"read:{GPU_ACCESS_RULES_PATH}" not in host.calls - assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls - - -def test_provision_reconciles_the_canonical_rule_without_group_lookup_or_state() -> None: - host = FakeGpuAccessHost() - - result = provision_gpu_access(host) - - assert result is None - assert host.files == {GPU_ACCESS_RULES_PATH: render_udev_rules()} - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] - assert not any("group" in call or "state" in call for call in host.calls) - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_KFD_RULES_PATH, LEGACY_KFD_RULES), - (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_RULES), - (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_PXE_RULES), - (LEGACY_ROCM_DEVICES_RULES_PATH, LEGACY_ROCM_DEVICES_RULES), - ], -) -def test_provision_removes_only_exact_legacy_rules_before_verifying(path: Path, content: str) -> None: - host = FakeGpuAccessHost(files={path: content}) - - provision_gpu_access(host) - - assert path not in host.files - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls.index(f"remove-rule:{path}") < host.calls.index(f"write-rule:{GPU_ACCESS_RULES_PATH}") - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n'), - ( - LEGACY_ROCM_DEVICES_RULES_PATH, - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n', - ), - ], -) -def test_near_legacy_rule_fails_closed_without_removal(path: Path, content: str) -> None: - host = FakeGpuAccessHost(files={path: content}) - - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) - assert host.files[path] == content + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, _: verified.append(Path(path))) + + # When: GPU access is provisioned online. + provision_gpu_access(host, offline_mode=False, bundle_dir=None) + + # Then: the exact Radeon URL is downloaded, verified, installed, and cleaned up. + downloaded_path = Path(downloads[0][-1]) + assert downloads == [["wget", "-q", gpu_access.AMD_GPU_UDEV_PACKAGE_URL, "-O", str(downloaded_path)]] + assert verified == [downloaded_path] + assert not downloaded_path.exists() + assert host.calls == [ + "installed-version", + f"install-package:{downloaded_path}", + "installed-version", + f"owns-rule:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", + f"read:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", + ] -def test_matching_managed_rule_is_reapplied_and_verified_without_rewriting() -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: render_udev_rules()}) +def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> None: + # Given: the package is already present with the expected package-owned rule. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES}, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + # When: provisioning is repeated. provision_gpu_access(host) - assert not any(call.startswith("write-") for call in host.calls) - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] + # Then: no download, install, legacy removal, or device probe is performed. + assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) + assert not any(call in {"reload-udev", "trigger-udev", "settle-udev"} for call in host.calls) @pytest.mark.parametrize( - "unexpected_rule", + ("installed_version", "package_owns_rule", "rule"), [ - f"{gpu_access.UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n', - f'{gpu_access.UDEV_MANAGED_MARKER}\nKERNEL=="kfd", MODE="0666"\n', + (AMD_GPU_UDEV_PACKAGE_VERSION, False, AMD_GPU_UDEV_PACKAGE_RULES), + (AMD_GPU_UDEV_PACKAGE_VERSION, True, 'KERNEL=="kfd", MODE="0660"\n'), ], ) -def test_noncanonical_managed_rule_fails_closed_before_mutation(unexpected_rule: str) -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: unexpected_rule}) +def test_installed_package_fails_closed_when_its_version_or_rule_contract_is_wrong( + installed_version: str, package_owns_rule: bool, rule: str +) -> None: + # Given: an installed package that does not satisfy the pinned package contract. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: rule}, + installed_version=installed_version, + package_owns_rule=package_owns_rule, + ) - with pytest.raises(InstallerError, match="unrecognized managed"): + # When: provisioning checks the installed package. + with pytest.raises(InstallerError): provision_gpu_access(host) - assert host.files[GPU_ACCESS_RULES_PATH] == unexpected_rule - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) - assert "reload-udev" not in host.calls + # Then: it fails before installing or mutating any udev rule. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) -def test_unmanaged_rule_fails_before_any_mutation() -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}) +def test_symlinked_legacy_rule_fails_closed_before_installation(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a legacy-rule path replaced by a symlink. + host = FakeGpuAccessHost() + host.symlinks.add(LEGACY_AMDGPU_RULES_PATH) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) - with pytest.raises(InstallerError, match="unmanaged"): + # When: first-time provisioning inspects legacy rules. + with pytest.raises(InstallerError, match="symlinked GPU udev rule"): provision_gpu_access(host) - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) - assert "reload-udev" not in host.calls + # Then: no package installation is attempted. + assert not any(call.startswith("install-package:") for call in host.calls) -def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: - commands: list[list[str]] = [] - capture_commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - return SimpleNamespace(returncode=0) +def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() -> None: + # Given: the exact package verification constant. + rules = AMD_GPU_UDEV_PACKAGE_RULES + old_pxe_shape = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' - def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: - capture_commands.append(command) - return SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n") - - monkeypatch.setattr(gpu_access, "run", fake_run) - monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) - - SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") - - assert capture_commands == [["mktemp", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.XXXXXX"]] - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/etc/udev/rules.d"], - ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - [ - "mv", - "-f", - "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", - "/etc/udev/rules.d/70-auplc-gpu-access.rules", - ], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d"], - ] + # When: its policy is inspected. + # Then: it matches the extracted package rule rather than the former two-line PXE shape. + assert rules == ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' + ) + assert rules != old_pxe_shape + assert "card" not in rules -def test_system_adapter_removes_temporary_rule_when_durable_write_fails(monkeypatch) -> None: +def test_system_adapter_uses_dpkg_for_the_package_install(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: the production host adapter and a recorded command runner. commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - if command == [ - "python3", - "-c", - gpu_access._FSYNC_PATH_SCRIPT, - "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", - ]: - raise InstallerError("fsync failed") - return SimpleNamespace(returncode=0) - - monkeypatch.setattr(gpu_access, "run", fake_run) monkeypatch.setattr( gpu_access, - "run_capture", - lambda command, **kwargs: SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n"), + "run", + lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0), ) - with pytest.raises(InstallerError, match="fsync failed"): - SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") - - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/etc/udev/rules.d"], - ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["rm", "-f", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ] - - -@pytest.mark.parametrize("failing_method", ["write_udev_rule", "reload_udev_rules", "trigger_udev", "settle_udev"]) -def test_reconciliation_stops_when_udev_mutation_fails(monkeypatch, failing_method: str) -> None: - host = FakeGpuAccessHost() - original_method = getattr(host, failing_method) - - def fail_after_recording(*args: object) -> None: - original_method(*args) - raise InstallerError(f"{failing_method} failed") - - monkeypatch.setattr(host, failing_method, fail_after_recording) - - with pytest.raises(InstallerError, match=f"{failing_method} failed"): - provision_gpu_access(host) - - assert "verify-devices" not in host.calls - - -def test_failed_inode_verification_leaves_the_reconciled_rule_in_place(monkeypatch) -> None: - host = FakeGpuAccessHost() - - def fail_verification() -> None: - host.calls.append("verify-devices") - raise InstallerError("device ownership mismatch") - - monkeypatch.setattr(host, "verify_device_access", fail_verification) - - with pytest.raises(InstallerError, match="ownership mismatch"): - provision_gpu_access(host) - - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-1] == "verify-devices" - - -@pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) -def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: - host = FakeGpuAccessHost() - host.symlinks.add(path) - - with pytest.raises(InstallerError, match="symlinked"): - provision_gpu_access(host) + # When: it installs the verified package artifact. + SystemGpuAccessHost().install_package(Path("/tmp/package.deb")) - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) + # Then: installation is delegated to dpkg with sudo awareness. + assert commands == [["dpkg", "--force-confnew", "--install", "/tmp/package.deb"]] diff --git a/tests/installer/test_gpu_access_ordering.py b/tests/installer/test_gpu_access_ordering.py new file mode 100644 index 00000000..6363ffc8 --- /dev/null +++ b/tests/installer/test_gpu_access_ordering.py @@ -0,0 +1,164 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, + LEGACY_KFD_RULES, + LEGACY_KFD_RULES_PATH, + provision_gpu_access, +) +from auplc_installer.util import InstallerError +from tests.installer.test_gpu_access import FakeGpuAccessHost + + +def _offline_bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + return bundle + + +def test_wrong_installed_version_downloads_and_converges_to_the_pinned_package(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a different installed package version and an online downloader. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0660"\n'}, + installed_version="30.30.4.0-older", + ) + downloads: list[list[str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: GPU access is provisioned. + provision_gpu_access(host) + + # Then: the pinned deb is acquired and the installed rule converges to its exact content. + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + + +def test_exact_installed_package_skips_network_then_removes_separate_legacy_rule( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given: an exact package rule plus a separately shipped legacy KFD rule. + host = FakeGpuAccessHost( + files={ + AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES, + LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES, + }, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) + + # When: provisioning checks an otherwise already-correct installation. + provision_gpu_access(host) + + # Then: it removes only the separate legacy file and applies its removal to live udev state. + assert LEGACY_KFD_RULES_PATH not in host.files + assert not any(call.startswith("install-package:") for call in host.calls) + assert host.calls[-3:] == ["reload-udev", "trigger-udev", "settle-udev"] + + +def test_acquires_and_verifies_the_offline_deb_before_deleting_legacy_rules( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a first installation from a verified offline bundle and a legacy KFD rule. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: host.calls.append("verify-deb")) + + # When: the package is installed. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: package installation completes before the separate legacy rule is deleted. + install_index = next(index for index, call in enumerate(host.calls) if call.startswith("install-package:")) + removal_index = host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") + assert host.calls.index("verify-deb") < install_index < removal_index + + +def test_failed_installation_keeps_legacy_rules_intact(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given: a first offline installation whose package install fails. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + def fail_install(deb: Path) -> None: + host.calls.append(f"install-package:{deb}") + raise InstallerError("dpkg failed") + + monkeypatch.setattr(host, "install_package", fail_install) + + # When: package installation fails. + with pytest.raises(InstallerError, match="dpkg failed"): + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: the legacy rule remains and no udev refresh occurs. + assert host.files[LEGACY_KFD_RULES_PATH] == LEGACY_KFD_RULES + assert "reload-udev" not in host.calls + + +def test_wrong_version_package_owned_differing_conffile_converges( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a package-owned conffile from a different installed package version. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + installed_version="30.30.4.0-older", + package_owns_rule=True, + ) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: the pinned package is installed from an offline bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) + + # Then: forced installation replaces the differing conffile with the exact package rule. + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + assert not any(call.startswith("remove-rule:") for call in host.calls) + + +def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given: a config-files package state that still owns a differing conffile. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + package_owns_rule=True, + ) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: the pinned package is installed from an offline bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) + + # Then: ownership prevents legacy admission and the package converges to the exact rule. + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + assert not any(call.startswith("remove-rule:") for call in host.calls) + + +def test_unknown_unowned_amdgpu_rule_fails_closed() -> None: + # Given: an unowned, unrecognized rule at the AMD package path. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + package_owns_rule=False, + ) + + # When: provisioning admits legacy rules. + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + # Then: no package installation or rule deletion is attempted. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) From 2f87d335369fd421397a30b40ef58d2590582f01 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 083/180] refactor(installer): pass GPU package context --- auplc_installer/cli.py | 18 ++++++++--------- tests/installer/test_cli_gpu_access.py | 28 ++++++++++++++++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 609930a9..390f2b32 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -326,10 +326,10 @@ def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") -def _provision_gpu_access_for_local_hardware() -> None: +def _provision_gpu_access_for_local_hardware(*, offline_mode: bool, bundle_dir: Path | None) -> None: match classify_gpu_hardware(): case GpuHardware.GPU: - provision_gpu_access() + provision_gpu_access(offline_mode=offline_mode, bundle_dir=bundle_dir) case GpuHardware.CPU: return case GpuHardware.UNKNOWN: @@ -355,7 +355,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) with stage("Provisioning GPU device access", idx=2, total=total): - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) paths = state.runtime_paths() with stage("Generating values overlay (initial)", idx=3, total=total): @@ -604,7 +604,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -620,7 +620,7 @@ def cmd_dev_deploy(state: InstallerState) -> None: def cmd_dev_upgrade(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -637,7 +637,7 @@ def cmd_dev_upgrade(state: InstallerState) -> None: def cmd_dev_reinstall(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -648,7 +648,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -664,7 +664,7 @@ def cmd_rt_install(state: InstallerState) -> None: def cmd_rt_upgrade(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -705,7 +705,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index e4cdaaf9..b48210b4 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -39,7 +39,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) @@ -84,7 +84,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) @@ -116,7 +116,7 @@ def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) @@ -152,7 +152,7 @@ def test_reinstall_gates_host_access_before_removing_runtime( state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -169,7 +169,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) with pytest.raises(RuntimeError, match="hardware"): @@ -190,7 +190,7 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -204,3 +204,19 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( def test_cli_exposes_no_render_gid_reconciliation_api() -> None: assert not hasattr(cli, "_render_gid_for_local_hardware") assert not hasattr(cli, "load_existing_gpu_access") + + +def test_gpu_hardware_gate_passes_offline_bundle_context_to_package_provisioning( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a local GPU installation running from an offline bundle. + bundle = tmp_path / "bundle" + package_calls: list[dict[str, object]] = [] + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.GPU) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: package_calls.append(kwargs)) + + # When: the CLI's local-hardware gate provisions GPU access. + cli._provision_gpu_access_for_local_hardware(offline_mode=True, bundle_dir=bundle) + + # Then: package provisioning receives the bundle context unchanged. + assert package_calls == [{"offline_mode": True, "bundle_dir": bundle}] From 36dc762b82164b9c9e241853c4f2df45ee82b214 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 084/180] feat(installer): bundle AMD GPU udev package --- auplc_installer/pack.py | 14 ++++++++++++++ tests/installer/test_pack.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/installer/test_pack.py diff --git a/auplc_installer/pack.py b/auplc_installer/pack.py index c4a1b4d7..ef7531b4 100644 --- a/auplc_installer/pack.py +++ b/auplc_installer/pack.py @@ -19,6 +19,11 @@ from auplc_installer.catalog import HUB_IMAGE_NAME, CourseSelection from auplc_installer.gpu import GpuConfig, detect_and_configure_gpu +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) from auplc_installer.images import ( EXTERNAL_IMAGES, pull_and_tag, @@ -121,6 +126,14 @@ def pack_download_k3s_images(staging: Path) -> None: ) +def pack_download_gpu_access_package(staging: Path) -> None: + packages_dir = staging / "packages" + packages_dir.mkdir(parents=True, exist_ok=True) + deb = packages_dir / AMD_GPU_UDEV_PACKAGE_FILENAME + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + + def pack_save_manifests(staging: Path) -> None: log_step("Saving manifests") out_dir = staging / "manifests" @@ -461,6 +474,7 @@ def pack_bundle( pack_download_binaries(staging) pack_download_k3s_images(staging) + pack_download_gpu_access_package(staging) pack_save_manifests(staging) if local_build: diff --git a/tests/installer/test_pack.py b/tests/installer/test_pack.py new file mode 100644 index 00000000..0b984a52 --- /dev/null +++ b/tests/installer/test_pack.py @@ -0,0 +1,37 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for offline bundle package artifacts.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from auplc_installer import pack +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) + + +def test_pack_downloads_and_checksums_the_offline_gpu_udev_package(tmp_path: Path, monkeypatch) -> None: + # Given: an empty bundle staging directory and a recording downloader. + commands: list[list[str]] = [] + verified: list[tuple[Path, str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + commands.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(pack, "run", fake_run) + monkeypatch.setattr(pack, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + + # When: package artifacts are added to the offline bundle. + pack.pack_download_gpu_access_package(tmp_path) + + # Then: the pinned deb is placed in packages/ and verified before archiving. + deb = tmp_path / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + assert commands == [["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]] + assert verified == [(deb, AMD_GPU_UDEV_PACKAGE_SHA256)] From be284a601f82208fc6bf28674c2c6183c288f7fb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 085/180] refactor(ansible): install AMD GPU udev package --- .../roles/gpu_access/defaults/main.yml | 12 + .../roles/gpu_access/handlers/main.yml | 17 -- .../ansible/roles/gpu_access/tasks/apply.yml | 266 ++++++------------ .../roles/gpu_access/tasks/preflight.yml | 176 +++++++++--- .../ansible/roles/gpu_access/tasks/verify.yml | 162 +++++++++++ .../templates/70-auplc-gpu-access.rules.j2 | 4 - tests/skills/test_gpu_access_role.py | 244 ++++++++++------ 7 files changed, 556 insertions(+), 325 deletions(-) delete mode 100644 deploy/ansible/roles/gpu_access/handlers/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/verify.yml delete mode 100644 deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 8f5ff6c7..4a9d4e3b 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -7,3 +7,15 @@ auplc_rootfs_path: "" # Rootfs adapters must explicitly constrain their writable target below this # canonical directory. Live hosts leave this empty. auplc_rootfs_allowed_root: "" +auplc_gpu_udev_package_name: amdgpu-insecure-instinct-udev-rules +auplc_gpu_udev_package_version: 30.30.4.0-2341068.24.04 +auplc_gpu_udev_package_filename: amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_url: >- + https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_checksum: sha256:4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162 +auplc_gpu_udev_package_cache_path: >- + /var/cache/auplc/amdgpu-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_rule_path: /etc/udev/rules.d/70-amdgpu.rules +auplc_gpu_udev_rule_content: | + KERNEL=="kfd", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666" diff --git a/deploy/ansible/roles/gpu_access/handlers/main.yml b/deploy/ansible/roles/gpu_access/handlers/main.yml deleted file mode 100644 index 6afe9532..00000000 --- a/deploy/ansible/roles/gpu_access/handlers/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - ---- -- name: Reload udev rules - ansible.builtin.command: - argv: - - udevadm - - control - - --reload-rules - when: auplc_rootfs_path | length == 0 - -- name: Trigger udev rules - ansible.builtin.command: - argv: - - udevadm - - trigger - when: auplc_rootfs_path | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml index f830b935..261d8a35 100644 --- a/deploy/ansible/roles/gpu_access/tasks/apply.yml +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -1,54 +1,89 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -- name: Inspect canonical GPU access rule before apply - ansible.builtin.stat: - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - follow: false - register: _auplc_apply_destination_rule - -- name: Reject unsafe canonical GPU access rule before apply - ansible.builtin.assert: - that: - - not _auplc_apply_destination_rule.stat.exists or - (_auplc_apply_destination_rule.stat.isreg and not _auplc_apply_destination_rule.stat.islnk) - fail_msg: Unsafe canonical GPU access destination. - -- name: Read canonical GPU access rule before apply - ansible.builtin.slurp: - src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - register: _auplc_apply_existing_rule - when: _auplc_apply_destination_rule.stat.exists - -- name: Define canonical GPU access rule contents for apply - ansible.builtin.set_fact: - _auplc_apply_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - -- name: Recheck canonical GPU access rule before apply - ansible.builtin.assert: - that: (_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule - fail_msg: Unmanaged canonical GPU access rule. - when: _auplc_apply_destination_rule.stat.exists - -- name: Inspect recognized project-owned legacy GPU rules for apply +- name: Install AMD udev package when required + block: + - name: Create deterministic AMD udev package cache + ansible.builtin.file: + path: "{{ auplc_gpu_udev_package_cache_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Download checksummed AMD udev package + ansible.builtin.get_url: + url: "{{ auplc_gpu_udev_package_url }}" + dest: "{{ auplc_gpu_udev_package_cache_path }}" + checksum: "{{ auplc_gpu_udev_package_checksum }}" + owner: root + group: root + mode: "0644" + + - name: Install AMD udev package on live host + ansible.builtin.apt: + deb: "{{ auplc_gpu_udev_package_cache_path }}" + state: present + allow_downgrade: true + dpkg_options: force-confnew + when: _auplc_target_root | length == 0 + + - name: Copy AMD udev package into PXE rootfs + ansible.builtin.copy: + src: "{{ auplc_gpu_udev_package_cache_path }}" + dest: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + remote_src: true + owner: root + group: root + mode: "0644" + when: _auplc_target_root | length > 0 + + - name: Install AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - apt-get + - --option=Dpkg::Options::=--force-confnew + - install + - --yes + - --no-install-recommends + - "/tmp/{{ auplc_gpu_udev_package_filename }}" + environment: + DEBIAN_FRONTEND: noninteractive + changed_when: true + when: _auplc_target_root | length > 0 + + - name: Verify installed AMD udev package + ansible.builtin.import_tasks: verify.yml + + always: + - name: Remove temporary AMD udev package from PXE rootfs + ansible.builtin.file: + path: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + state: absent + when: _auplc_target_root | length > 0 + when: _auplc_gpu_udev_install_needed | bool + +- name: Verify installed AMD udev package without installation + ansible.builtin.import_tasks: verify.yml + when: not _auplc_gpu_udev_install_needed | bool + +- name: Recheck recognized project-owned legacy GPU rules before apply ansible.builtin.stat: path: "{{ item.path }}" follow: false loop: "{{ _auplc_legacy_gpu_rules }}" register: _auplc_apply_legacy_gpu_rule_stats -- name: Reject legacy GPU rule symlinks and non-regular files before apply +- name: Reject unsafe legacy GPU rules before apply ansible.builtin.assert: that: - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" -- name: Read recognized project-owned legacy GPU rules for apply +- name: Read recognized project-owned legacy GPU rules before apply ansible.builtin.slurp: src: "{{ item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" @@ -58,7 +93,10 @@ - name: Reject unexpected legacy GPU rule content before apply ansible.builtin.assert: that: - - (item.content | b64decode) in item.item.item.contents + - >- + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + (item.content | b64decode) == auplc_gpu_udev_rule_content) fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) @@ -68,153 +106,23 @@ path: "{{ item.item.item.path }}" state: absent loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" - when: not item.skipped | default(false) + when: >- + not item.skipped | default(false) and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + register: _auplc_removed_legacy_gpu_rules -- name: Create target udev rules directory - ansible.builtin.file: - path: "{{ _auplc_target_root }}/etc/udev/rules.d" - state: directory - owner: root - group: root - mode: "0755" - -- name: Install canonical AMD GPU udev rules - ansible.builtin.template: - src: 70-auplc-gpu-access.rules.j2 - dest: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - owner: root - group: root - mode: "0644" - -- name: Reload live udev rules on every apply +- name: Reload live udev rules after legacy cleanup ansible.builtin.command: argv: [udevadm, control, --reload-rules] changed_when: false - when: _auplc_target_root | length == 0 + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed -- name: Trigger live udev rules on every apply +- name: Trigger live udev rules after legacy cleanup ansible.builtin.command: argv: [udevadm, trigger] changed_when: false - when: _auplc_target_root | length == 0 - -- name: Settle live udev events before inode verification - ansible.builtin.command: - argv: [udevadm, settle] - changed_when: false - when: _auplc_target_root | length == 0 - -- name: Inspect /dev/kfd after live reconciliation - ansible.builtin.stat: - path: /dev/kfd - follow: false - register: _auplc_kfd - when: _auplc_target_root | length == 0 - -- name: Verify /dev/kfd ownership and mode - ansible.builtin.assert: - that: - - _auplc_kfd.stat.exists - - _auplc_kfd.stat.ischr - - _auplc_kfd.stat.uid == 0 - - _auplc_kfd.stat.gr_name == 'render' - - _auplc_kfd.stat.mode == '0666' - fail_msg: /dev/kfd is not root:render with mode 0666 after reconciliation. - when: _auplc_target_root | length == 0 - -- name: Find live DRM render nodes - ansible.builtin.find: - paths: /dev/dri - patterns: renderD* - file_type: any - recurse: false - register: _auplc_render_nodes - when: _auplc_target_root | length == 0 - -- name: Find live DRM card nodes - ansible.builtin.find: - paths: /dev/dri - patterns: card* - file_type: any - recurse: false - register: _auplc_card_nodes - when: _auplc_target_root | length == 0 - -- name: Resolve live DRM node driver symlinks - ansible.builtin.command: - argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] - loop: "{{ (_auplc_render_nodes.files | default([])) + (_auplc_card_nodes.files | default([])) }}" - register: _auplc_drm_node_drivers - changed_when: false - failed_when: false - when: _auplc_target_root | length == 0 - -- name: Select AMD live DRM render nodes - ansible.builtin.set_fact: - _auplc_amd_render_nodes: >- - {{ (_auplc_amd_render_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and - (item.item.path | basename) is match('^renderD') else []) }} - loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Select AMD live DRM card nodes - ansible.builtin.set_fact: - _auplc_amd_card_nodes: >- - {{ (_auplc_amd_card_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and - (item.item.path | basename) is match('^card') else []) }} - loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Require AMD live DRM render nodes - ansible.builtin.assert: - that: (_auplc_amd_render_nodes | default([])) | length > 0 - fail_msg: No AMD renderD node was available for GPU access verification. - when: _auplc_target_root | length == 0 - -- name: Require AMD live DRM card nodes - ansible.builtin.assert: - that: (_auplc_amd_card_nodes | default([])) | length > 0 - fail_msg: No AMD card node was available for GPU access verification. - when: _auplc_target_root | length == 0 - -- name: Inspect AMD live DRM render nodes - ansible.builtin.stat: - path: "{{ item }}" - follow: false - loop: "{{ _auplc_amd_render_nodes | default([]) }}" - register: _auplc_amd_render_node_stats - when: _auplc_target_root | length == 0 - -- name: Inspect AMD live DRM card nodes - ansible.builtin.stat: - path: "{{ item }}" - follow: false - loop: "{{ _auplc_amd_card_nodes | default([]) }}" - register: _auplc_amd_card_node_stats - when: _auplc_target_root | length == 0 - -- name: Verify AMD render node ownership and mode - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.ischr - - item.stat.uid == 0 - - item.stat.gr_name == 'render' - - item.stat.mode == '0666' - fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0666." - loop: "{{ _auplc_amd_render_node_stats.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Verify AMD card node ownership and mode - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.ischr - - item.stat.uid == 0 - - item.stat.gr_name == 'video' - - item.stat.mode == '0666' - fail_msg: "AMD card node {{ item.item }} is not root:video with mode 0666." - loop: "{{ _auplc_amd_card_node_stats.results | default([]) }}" - when: _auplc_target_root | length == 0 + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml index 23d448e4..59810a95 100644 --- a/deploy/ansible/roles/gpu_access/tasks/preflight.yml +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -19,7 +19,7 @@ fail_msg: GPU access rootfs must be an existing non-symlink directory. when: _auplc_target_root | length > 0 -- name: Inspect canonical GPU access destination parents +- name: Inspect AMD udev rule destination parents ansible.builtin.stat: path: "{{ _auplc_target_root }}{{ item }}" follow: false @@ -29,68 +29,141 @@ - /etc/udev/rules.d register: _auplc_destination_parent_stats -- name: Reject unsafe canonical GPU access destination parents +- name: Reject unsafe AMD udev rule destination parents ansible.builtin.assert: that: - not item.stat.exists or (item.stat.isdir and not item.stat.islnk) - fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" + fail_msg: "Unsafe AMD udev rule destination parent: {{ item.item }}" loop: "{{ _auplc_destination_parent_stats.results }}" -- name: Inspect canonical GPU access destination +- name: Define recognized project-owned legacy GPU rules + ansible.builtin.set_fact: + _auplc_legacy_gpu_rules: + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" + sha256: + - 79773871430cb63f5a28cf25666e0eccacf2bb27d4d9f48e10d0b05931650cf0 + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" + sha256: + - 678b6a1084576de785b47fcfa0c0b3048117a3add62c1fb8dcff83947004005b + - cc5e78a7861477ac5169a4b84edd4e687c1f14b9a88a9557b0c986479ebbaccd + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" + sha256: + - 951fb3d879d2d45b56cfd4cdb0f7ea061a4a0af77d93b9f2a4da9a8c36d20cad + +- name: Inspect AMD udev rule destination ansible.builtin.stat: - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" follow: false register: _auplc_destination_rule -- name: Reject unsafe canonical GPU access destination +- name: Reject unsafe AMD udev rule destination ansible.builtin.assert: that: - not _auplc_destination_rule.stat.exists or (_auplc_destination_rule.stat.isreg and not _auplc_destination_rule.stat.islnk) - fail_msg: Unsafe canonical GPU access destination. + fail_msg: Unsafe AMD udev rule destination. + +- name: Query installed AMD udev package on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record installed AMD udev package state on live host + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_live_package }}" + when: _auplc_target_root | length == 0 + +- name: Record installed AMD udev package state in PXE rootfs + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_rootfs_package }}" + when: _auplc_target_root | length > 0 + +- name: Record whether AMD udev package installation is needed + ansible.builtin.set_fact: + _auplc_gpu_udev_install_needed: >- + {{ _auplc_installed_package.rc != 0 or + _auplc_installed_package.stdout != 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version }} + +- name: Query AMD udev rule package ownership on live host before admission + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 -- name: Read existing canonical GPU access rule +- name: Query AMD udev rule package ownership in PXE rootfs before admission + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record AMD udev rule owner on live host + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_live_rule_owner }}" + when: _auplc_target_root | length == 0 + +- name: Record AMD udev rule owner in PXE rootfs + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_rootfs_rule_owner }}" + when: _auplc_target_root | length > 0 + +- name: Record whether the AMD udev rule is package-owned + ansible.builtin.set_fact: + _auplc_rule_owned_by_amd_package: >- + {{ _auplc_existing_rule_owner.rc == 0 and + _auplc_existing_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path }} + +- name: Read existing AMD udev rule ansible.builtin.slurp: - src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" register: _auplc_existing_rule when: _auplc_destination_rule.stat.exists -- name: Define canonical GPU access rule contents +- name: Allow package-owned AMD udev rule convergence ansible.builtin.set_fact: - _auplc_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" + _auplc_rule_content_admitted: >- + {{ (not _auplc_destination_rule.stat.exists) or + ((_auplc_existing_rule.content | b64decode) == auplc_gpu_udev_rule_content) or + ((_auplc_gpu_udev_install_needed | bool) and + (((_auplc_existing_rule.content | b64decode) | hash('sha256')) in _auplc_legacy_gpu_rules[1].sha256 or + (_auplc_rule_owned_by_amd_package | bool)) }} -- name: Reject unmanaged canonical GPU access rule +- name: Reject modified AMD udev rule before package installation ansible.builtin.assert: - that: (_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule - fail_msg: Unmanaged canonical GPU access rule. - when: _auplc_destination_rule.stat.exists - -- name: Define recognized project-owned legacy GPU rules - ansible.builtin.set_fact: - _auplc_legacy_gpu_rules: - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" - contents: - - "KERNEL==\"kfd\", MODE=\"0666\"\nSUBSYSTEM==\"drm\", KERNEL==\"renderD*\", MODE=\"0666\"\n" - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" - contents: - - | - # ROCm device permissions - # Grant render group access to AMD GPU devices - # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules - KERNEL=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" - - "KERNEL==\"kfd\", MODE=\"0666\"\nKERNEL==\"renderD[0-9]*\", MODE=\"0666\"\n" - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" - contents: - - | - # ROCm device permissions - # Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group - SUBSYSTEM=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + that: _auplc_rule_content_admitted | bool + fail_msg: Existing AMD udev rule is neither the package rule nor a recognized legacy rule. - name: Inspect recognized project-owned legacy GPU rules ansible.builtin.stat: @@ -116,7 +189,24 @@ - name: Reject unexpected legacy GPU rule content ansible.builtin.assert: that: - - (item.content | b64decode) in item.item.item.contents + - >- + ( + item.item.item.path != _auplc_target_root + auplc_gpu_udev_rule_path and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + ) or + ( + item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + ( + (item.content | b64decode) == auplc_gpu_udev_rule_content or + ( + (_auplc_gpu_udev_install_needed | bool) and + ( + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (_auplc_rule_owned_by_amd_package | bool) + ) + ) + ) + ) fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) diff --git a/deploy/ansible/roles/gpu_access/tasks/verify.yml b/deploy/ansible/roles/gpu_access/tasks/verify.yml new file mode 100644 index 00000000..5e6a3be5 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/verify.yml @@ -0,0 +1,162 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before package verification + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_verify_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs before package verification + ansible.builtin.assert: + that: + - _auplc_verify_rootfs.stat.isdir + - not _auplc_verify_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect AMD udev rule parents before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + register: _auplc_verify_parent_stats + +- name: Require safe AMD udev rule parents before package verification + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isdir + - not item.stat.islnk + fail_msg: "Unsafe AMD udev rule parent: {{ item.item }}" + loop: "{{ _auplc_verify_parent_stats.results }}" + +- name: Inspect retained PXE shipped legacy GPU rules + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-kfd.rules + - /etc/udev/rules.d/70-rocm-devices.rules + register: _auplc_retained_legacy_gpu_rule_stats + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Reject retained PXE shipped legacy GPU rules + ansible.builtin.assert: + that: not item.stat.exists + fail_msg: "Retained PXE rootfs has a shipped legacy GPU rule: {{ item.item }}" + loop: "{{ _auplc_retained_legacy_gpu_rule_stats.results | default([]) }}" + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Query installed AMD udev package version on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package version in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require installed AMD udev package status and exact version on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_package.rc == 0 + - _auplc_verify_live_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length == 0 + +- name: Require installed AMD udev package status and exact version in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_package.rc == 0 + - _auplc_verify_rootfs_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length > 0 + +- name: Query AMD udev rule package ownership on live host + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query AMD udev rule package ownership in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require package-owned AMD udev rule on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_rule_owner.rc == 0 + - "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length == 0 + +- name: Require package-owned AMD udev rule in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_rule_owner.rc == 0 + - "_auplc_verify_rootfs_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length > 0 + +- name: Inspect installed AMD udev rule + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + follow: false + register: _auplc_verify_rule + +- name: Require safe installed AMD udev rule + ansible.builtin.assert: + that: + - _auplc_verify_rule.stat.exists + - _auplc_verify_rule.stat.isreg + - not _auplc_verify_rule.stat.islnk + fail_msg: AMD udev rule has an unsafe filesystem type. + +- name: Read installed AMD udev rule + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rule_content + +- name: Require exact AMD udev rule content + ansible.builtin.assert: + that: (_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content + fail_msg: AMD udev rule is a modified package conffile. diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 deleted file mode 100644 index f57d023a..00000000 --- a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Managed by auplc-installer: AMD GPU device access. -KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" -SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" -SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index ccece9ce..3fe821ec 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,6 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Contract tests for the Ansible GPU device-mode role.""" +"""Contract tests for AMD's packaged GPU udev rules in Ansible.""" from pathlib import Path @@ -10,33 +10,48 @@ PXE_CONTROLLER_ROLE = ANSIBLE / "roles" / "pxe_controller" PXE_GPU_ACCESS_TASKS = PXE_CONTROLLER_ROLE / "tasks" / "gpu_access.yml" +PACKAGE = "amdgpu-insecure-instinct-udev-rules" +VERSION = "30.30.4.0-2341068.24.04" +FILENAME = f"{PACKAGE}_{VERSION}_all.deb" +URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{FILENAME}" +SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +RULE_PATH = "/etc/udev/rules.d/70-amdgpu.rules" +RULE_CONTENT = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) + def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_gpu_access_role_uses_shc_proven_device_mode_contract() -> None: +def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") - - assert "auplc_gpu_access_enabled: false" in defaults - assert 'auplc_rootfs_path: ""' in defaults - assert "auplc_render_gid" not in defaults - assert "normalize" not in defaults - assert "gpu-access.json" not in preflight - assert "groupmod" not in apply - assert "GID collision" not in preflight - assert rules == ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' - ) - - -def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert PACKAGE in defaults + assert VERSION in defaults + assert FILENAME in defaults + assert URL in defaults + assert f"sha256:{SHA256}" in defaults + assert RULE_PATH in defaults + assert " " + RULE_CONTENT.replace("\n", "\n ").rstrip() in defaults + assert "dpkg-query" in preflight + assert r"--showformat=${Status}\t${Version}" in preflight + assert "ansible.builtin.get_url" in apply + assert "ansible.builtin.apt" in apply + assert 'checksum: "{{ auplc_gpu_udev_package_checksum }}"' in apply + assert 'deb: "{{ auplc_gpu_udev_package_cache_path }}"' in apply + assert "dpkg-query" in verify + assert r"--showformat=${Status}\t${Version}" in verify + assert "--search" in verify + assert "package-owned" in verify + assert "modified package conffile" in verify + + +def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") @@ -46,91 +61,165 @@ def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() - assert "_auplc_canonical_allowed_root" in validation assert "Inspect GPU access rootfs target" in preflight assert "follow: false" in preflight - assert "Reject unsafe canonical GPU access destination parents" in preflight - assert "Reject unsafe canonical GPU access destination" in preflight + assert "Reject unsafe AMD udev rule destination parents" in preflight + assert "Reject unsafe AMD udev rule destination" in preflight assert "Define recognized project-owned legacy GPU rules" in preflight - assert "Reject unexpected legacy GPU rule content" in preflight + assert "hash('sha256')" in preflight assert "70-kfd.rules" in preflight - assert "70-amdgpu.rules" in preflight assert "70-rocm-devices.rules" in preflight + assert "70-auplc-gpu-access.rules" not in preflight + assert "Reject unexpected legacy GPU rule content" in preflight + assert "Recheck recognized project-owned legacy GPU rules before apply" in apply assert "Remove recognized project-owned legacy GPU rules" in apply - assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( + assert apply.index("Download checksummed AMD udev package") < apply.index( + "Remove recognized project-owned legacy GPU rules" + ) + assert apply.index("Verify installed AMD udev package") < apply.index( "Remove recognized project-owned legacy GPU rules" ) + assert "Reload live udev rules after legacy cleanup" in apply + assert "Trigger live udev rules after legacy cleanup" in apply -def test_gpu_access_role_reconciles_and_verifies_live_devices_only() -> None: +def test_gpu_access_role_skips_package_cache_and_download_when_exact_version_is_installed() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "Reload live udev rules on every apply" in apply - assert "Trigger live udev rules on every apply" in apply - assert "Settle live udev events before inode verification" in apply - assert "Inspect /dev/kfd after live reconciliation" in apply - assert "Verify /dev/kfd ownership and mode" in apply - assert "Find live DRM render nodes" in apply - assert "Verify AMD render node ownership and mode" in apply - assert "Find live DRM card nodes" in apply - assert "Verify AMD card node ownership and mode" in apply - assert "/sys/class/drm" in apply - assert "readlink" in apply - assert "basename" in apply - assert "_auplc_kfd.stat.mode == '0666'" in apply - assert "item.stat.mode == '0666'" in apply - assert "item.stat.mode == '0666'" in apply - assert "item.stat.gr_name == 'render'" in apply - assert "item.stat.gr_name == 'video'" in apply - assert ( - apply.index("Trigger live udev rules on every apply") - < apply.index("Settle live udev events before inode verification") - < apply.index("Inspect /dev/kfd after live reconciliation") - ) - assert "when: _auplc_target_root | length == 0" in apply - assert "gpu-access.json" not in apply + assert "_auplc_gpu_udev_install_needed" in preflight + assert "Install AMD udev package when required" in apply + install_block = apply.split("Install AMD udev package when required", maxsplit=1)[1] + assert "Create deterministic AMD udev package cache" in install_block + assert "Download checksummed AMD udev package" in install_block + assert "when: _auplc_gpu_udev_install_needed | bool" in install_block + assert "Verify installed AMD udev package without installation" in apply + assert "install ok installed" in preflight -def test_gpu_access_role_rejects_noncanonical_managed_rule_content() -> None: +def test_gpu_access_role_requires_installed_status_and_exact_version() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert "install ok installed" in verify + assert "Require installed AMD udev package status and exact version" in verify + + +def test_preflight_allows_package_owned_wrong_version_rule_for_convergence() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Query AMD udev rule package ownership on live host before admission" in preflight + assert "Query AMD udev rule package ownership in PXE rootfs before admission" in preflight + assert preflight.index("Query installed AMD udev package") < preflight.index("Read existing AMD udev rule") + assert preflight.index("Query AMD udev rule package ownership") < preflight.index("Read existing AMD udev rule") + assert "_auplc_gpu_udev_install_needed | bool" in preflight + assert "_auplc_rule_owned_by_amd_package | bool" in preflight + + +def test_preflight_allows_package_owned_partial_state_rule_for_convergence() -> None: preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Record whether AMD udev package installation is needed" in preflight + assert "Allow package-owned AMD udev rule convergence" in preflight + assert "install ok installed" in preflight + + +def test_preflight_rejects_unknown_unowned_rule_content() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Reject modified AMD udev rule before package installation" in preflight + assert "_auplc_rule_owned_by_amd_package | bool" in preflight + assert "Existing AMD udev rule is neither the package rule nor a recognized legacy rule." in preflight + + +def test_preflight_legacy_admission_matches_package_owned_convergence_admission() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + primary_admission = preflight.split("Allow package-owned AMD udev rule convergence", maxsplit=1)[1].split( + "Reject modified AMD udev rule before package installation", maxsplit=1 + )[0] + legacy_admission = preflight.split("Reject unexpected legacy GPU rule content", maxsplit=1)[1].split( + "fail_msg:", maxsplit=1 + )[0] + + assert "_auplc_gpu_udev_install_needed | bool" in primary_admission + assert "_auplc_rule_owned_by_amd_package | bool" in primary_admission + assert "_auplc_gpu_udev_install_needed | bool" in legacy_admission + assert "_auplc_rule_owned_by_amd_package | bool" in legacy_admission + assert "auplc_gpu_udev_rule_path" in legacy_admission + assert "auplc_gpu_udev_rule_content" in legacy_admission + + +def test_gpu_access_role_installs_the_package_without_custom_rule_or_device_probes() -> None: apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - pxe_tasks = read(PXE_GPU_ACCESS_TASKS) - assert "_auplc_previous_canonical_rule" not in preflight - assert "(_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule" in preflight - assert "Reject unmanaged canonical GPU access rule" in preflight - assert "_auplc_apply_previous_canonical_rule" not in apply - assert "Recheck canonical GPU access rule before apply" in apply - assert "(_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule" in apply - assert "Unmanaged canonical GPU access rule." in apply - assert "_pxe_retained_previous_canonical_rule" not in pxe_tasks - assert "(_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule" in pxe_tasks - assert "non-canonical GPU access rule" in pxe_tasks + assert not (GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2").exists() + assert "ansible.builtin.template" not in apply + assert "70-auplc-gpu-access.rules.j2" not in apply + assert "udevadm settle" not in apply + assert "/dev/kfd" not in apply + assert "/dev/dri" not in apply + assert "/sys/class/drm" not in apply + assert "card" not in apply -def test_pxe_gpu_access_installs_rules_without_gid_or_state_contract() -> None: +def test_gpu_access_role_verifies_exact_installed_package_version_and_rule() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert "auplc_gpu_udev_package_version" in verify + assert "auplc_gpu_udev_rule_path" in verify + assert "auplc_gpu_udev_rule_content" in verify + assert "Require installed AMD udev package status and exact version" in verify + assert "Require package-owned AMD udev rule" in verify + assert "Require exact AMD udev rule content" in verify + assert "follow: false" in verify + + +def test_pxe_gpu_access_uses_safe_chroot_install_and_strict_retained_admission() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") tasks = read(PXE_GPU_ACCESS_TASKS) + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main assert "Re-preflight PXE GPU rootfs before TFTP" in main assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( "Stop NFS before rootfs rebuild" ) - assert "Inspect retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU rule" in tasks + assert "tasks_from: verify" in tasks assert "tasks_from: preflight" in tasks assert "tasks_from: apply" in tasks assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks - assert "auplc_render_gid" not in tasks - assert "render_gid" not in tasks - assert "groupadd" not in tasks - assert "groupmod" not in tasks - assert "collision" not in tasks.lower() - assert "gpu-access.json" not in tasks + assert "auplc_reject_legacy_gpu_rules: true" in tasks + assert "Reject retained PXE shipped legacy GPU rules" in verify + assert "chroot" in apply + assert "apt-get" in apply + assert "Copy AMD udev package into PXE rootfs" in apply + assert "Mount virtual filesystems for AMD udev package installation" not in apply + assert "mount --bind" not in apply + assert "Unmount virtual filesystems after AMD udev package installation" not in apply + assert apply.index("Verify installed AMD udev package") < apply.index( + "Remove temporary AMD udev package from PXE rootfs" + ) + assert main.index("Re-preflight PXE GPU rootfs before TFTP") < main.index("Find latest kernel in rootfs") + assert RULE_CONTENT not in tasks assert "/dev/kfd" not in tasks assert "/dev/dri" not in tasks +def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + rootfs_removal = main.split("Remove existing rootfs (force rebuild)", maxsplit=1)[1].split( + "Check if NFS rootfs already exists", maxsplit=1 + )[0] + chroot_unmount = main.split("Unmount virtual filesystems from chroot", maxsplit=1)[1].split( + "Remove chroot setup script", maxsplit=1 + )[0] + for task in (rootfs_removal, chroot_unmount): + assert "set -e" in task + assert "if mountpoint -q" in task + assert "&& umount" not in task + assert "|| true" not in task + + def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") @@ -145,15 +234,7 @@ def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: assert "render_gid" not in pxe_playbook -def test_pxe_controller_playbook_has_no_obsolete_finalizer_post_tasks() -> None: - pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") - - assert "post_tasks:" not in pxe_playbook - assert "pxe_finalizer_" not in pxe_playbook - assert "--finalize-pxe" not in pxe_playbook - - -def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() -> None: +def test_deploy_ansible_has_no_obsolete_gpu_access_policy_or_state_contract() -> None: forbidden = ( "auplc_render_gid", "auplc_normalize_render_gid", @@ -162,7 +243,6 @@ def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() "groupmod", "render GID collision", ) - ansible_text = "\n".join( path.read_text(encoding="utf-8") for path in ANSIBLE.rglob("*") From 4e15da9e7cc2b9af9c29a289f534a1077b2a74d1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 086/180] fix(pxe): install GPU udev package safely --- .../roles/pxe_controller/tasks/gpu_access.yml | 80 +++---------------- .../roles/pxe_controller/tasks/main.yml | 26 ++++-- 2 files changed, 31 insertions(+), 75 deletions(-) diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml index d0a0bd5b..adec7028 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -10,75 +10,17 @@ that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] fail_msg: PXE GPU admission phase is invalid. -- name: Inspect retained PXE canonical GPU access parents - ansible.builtin.stat: - path: "{{ pxe_nfs_root }}{{ item }}" - follow: false - loop: - - /etc - - /etc/udev - - /etc/udev/rules.d - register: _pxe_retained_canonical_gpu_parent_stats - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU access parents - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.isdir - - not item.stat.islnk - fail_msg: "Retained PXE rootfs has an unsafe canonical GPU access parent: {{ item.item }}" - loop: "{{ _pxe_retained_canonical_gpu_parent_stats.results }}" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Inspect retained PXE GPU policy paths - ansible.builtin.stat: - path: "{{ pxe_nfs_root }}{{ item }}" - follow: false - loop: - - /etc/udev/rules.d/70-kfd.rules - - /etc/udev/rules.d/70-amdgpu.rules - - /etc/udev/rules.d/70-rocm-devices.rules - - /etc/udev/rules.d/70-auplc-gpu-access.rules - register: _pxe_retained_gpu_policy_stats - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE legacy GPU rules absent - ansible.builtin.assert: - that: not item.stat.exists - fail_msg: "Retained PXE rootfs has a legacy GPU rule requiring a separate migration: {{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU rule destination - ansible.builtin.assert: - that: - - _pxe_retained_gpu_policy_stats.results[3].stat.exists - - _pxe_retained_gpu_policy_stats.results[3].stat.isreg - - not _pxe_retained_gpu_policy_stats.results[3].stat.islnk - fail_msg: Retained PXE rootfs requires an exact canonical GPU access rule. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Read retained PXE canonical GPU rule - ansible.builtin.slurp: - src: "{{ _pxe_retained_gpu_policy_stats.results[3].item }}" - register: _pxe_retained_canonical_gpu_rule - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Define retained PXE canonical GPU rules - ansible.builtin.set_fact: - _pxe_retained_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU rule - ansible.builtin.assert: - that: (_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule - fail_msg: Retained PXE rootfs has a non-canonical GPU access rule. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' +- name: Verify retained PXE AMD udev package before lifecycle changes + ansible.builtin.include_role: + name: gpu_access + tasks_from: verify + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_reject_legacy_gpu_rules: true + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' - name: Preflight GPU access after final PXE re-preflight ansible.builtin.include_role: diff --git a/deploy/ansible/roles/pxe_controller/tasks/main.yml b/deploy/ansible/roles/pxe_controller/tasks/main.yml index b5217f60..1a34b0e1 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/main.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/main.yml @@ -172,9 +172,16 @@ - name: Remove existing rootfs (force rebuild) when: pxe_rootfs_force_rebuild | bool ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi rm -rf {{ pxe_nfs_root }} changed_when: true @@ -299,9 +306,16 @@ always: - name: Unmount virtual filesystems from chroot ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi changed_when: true - name: Remove chroot setup script From 04d8ee3baf544ade9fb0d1bee6d1a8d5c07be8aa Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 087/180] docs: describe AMD GPU udev package --- README.md | 7 +++++++ runtime/values.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f19aa4b8..566443c1 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,13 @@ This operation needs root privileges. Requesting sudo password... kubectl is configured at $HOME/.kube/config; try `kubectl get nodes` ``` +The GPU access stage installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`. It sets mode `0666` only on +`/dev/kfd` and DRM `renderD*` nodes; `card*` keeps the normal system policy. The +device plugin remains a separate allocation layer, and the tested ROCm compute +path needs no supplemental GPU group. The offline `pack` bundle carries the +pinned deb for installation without network access. + See the full guide at [Quick Start](https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html) and [Single-Node Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/single-node.html). ### Uninstall diff --git a/runtime/values.yaml b/runtime/values.yaml index 3dd59180..2a942e02 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -669,7 +669,7 @@ monitoring: singleuser: # Storage ownership only. AUPLC runtime does not inject GPU groups. - # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. + # amd.com/gpu requests allocate devices; host udev policy controls node modes. fsGid: 100 storage: From 5e898d9f43be03d667d85380d40f0e0bc5309953 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 088/180] docs(deploy): document AMD GPU udev package --- deploy/README.md | 34 +++++++++++++++++++--------------- deploy/ansible/README.md | 18 ++++++++++-------- deploy/k8s/README.md | 7 +++++++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index b53d0a5d..7edbffc8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -105,16 +105,20 @@ records under `/sys/bus/pci/devices`; it does not require the devices to be attached to `amdgpu` before ROCm installation. The resulting GPU resolution report records which managed hosts have AMD display hardware. -The GPU permission contract is fixed across GPU hosts and PXE root filesystems: - -- `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render` with mode `0666`. -- AMD `/dev/dri/card*` nodes are `root:video` with mode `0666`. -- Every GPU device node injected into a Pod therefore has mode `0666`. -- Host provisioning owns device-node discretionary access control. -- AUPLC Hub adds no GPU supplemental group to user Pods. -- AMD device-plugin allocation is the visibility boundary: only Pods that - request `amd.com/gpu` receive GPU device nodes. The plugin does not set Unix - ownership or modes on host device nodes. +The installer, Ansible GPU access role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package, pinned to version +`30.30.4.0-2341068.24.04`. Its package-owned rule sets mode `0666` only on +`/dev/kfd` and DRM `/dev/dri/renderD*` nodes. It does not match +`/dev/dri/card*`; card nodes retain the normal system policy, observed as +`root:video 0660`. + +This host permission policy is separate from Kubernetes allocation. The AMD +device plugin remains the visibility boundary: only Pods that request +`amd.com/gpu` receive allocated GPU devices, and the plugin does not change +host inode ownership or mode. AUPLC Hub adds no GPU supplemental group. The +tested ROCm compute path needs none: on both SHC GPU nodes, `rocminfo` succeeded +as UID `12345` with only supplemental GID `100`, while card nodes remained +inaccessible at mode `0660`. The reported agents were `gfx1151` and `gfx1200`. `singleuser.fsGid: 100` controls shared notebook storage ownership only. It is not part of GPU access and must not be treated as a GPU group setting. @@ -157,10 +161,10 @@ helm upgrade --install jupyterhub ./runtime/chart \ -f runtime/values-basic-example.yaml ``` -A fresh PXE rootfs receives the fixed udev rule during the controller playbook. -A retained rootfs is accepted only when it already contains that exact canonical -rule and no conflicting legacy GPU rule. Rebuild or correct a retained rootfs -separately if that safety check fails. +A fresh PXE rootfs receives the pinned AMD udev package during the controller +playbook. A retained rootfs is accepted only when that exact package version and +its unmodified package-owned rule are present, with no conflicting legacy GPU +rule. Rebuild or correct a retained rootfs separately if that safety check fails. #### Discovery failures @@ -169,7 +173,7 @@ separately if that safety check fails. | Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | | `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | | Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | -| Retained PXE rootfs has a legacy or non-canonical GPU rule | Rebuild the rootfs, or replace the conflicting rule through a separate reviewed maintenance action before rerunning the playbook. | +| Retained PXE rootfs has the wrong AMD udev package version, a modified package rule, or a conflicting legacy GPU rule | Rebuild the rootfs, or correct the package state through a separate reviewed maintenance action before rerunning the playbook. | ## Deployment branch boundary diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index eda9b349..cbd02ad1 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -32,14 +32,16 @@ hosts from managed-host evidence. PXE generation uses only `pxe.diskless_agents_have_amd_gpus` and writes canonical files before the controller playbook runs. -The GPU access role sets AMD device-node policy on GPU hosts and GPU-enabled PXE -root filesystems. `/dev/kfd` and AMD `renderD*` nodes are `root:render 0666`; -AMD `card*` nodes are `root:video 0666`. All injected GPU device nodes therefore -use mode `0666`. Device-plugin allocation is the visibility boundary: only Pods -requesting `amd.com/gpu` receive the nodes. The plugin does not change host inode -permissions, and AUPLC Hub adds no GPU supplemental group to user Pods. Ordinary -container group membership does not participate in GPU permissions; host -provisioning owns device-node discretionary access control. +The GPU access role installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`, on GPU hosts and GPU-enabled PXE +root filesystems. The package sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*` nodes, which retain normal system +policy, observed as `root:video 0660`. + +Device-plugin allocation is a separate layer and remains the visibility +boundary for Pods requesting `amd.com/gpu`; it does not change host inode +permissions. AUPLC Hub adds no GPU supplemental group. No GPU group was needed +for the tested ROCm compute path. ## Prerequisites diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index da148f52..7334c198 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -45,6 +45,13 @@ cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must select, deploy, and maintain them according to the [official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). +The device plugin allocates devices to Pods; it does not set host device-node +permissions. Host provisioning separately installs the pinned +`amdgpu-insecure-instinct-udev-rules` package at version +`30.30.4.0-2341068.24.04`. That package sets mode `0666` only on `/dev/kfd` and +DRM `renderD*` nodes and leaves `card*` under normal system policy. AUPLC adds +no supplemental GPU group; none is required for the tested ROCm compute path. + To install the same pinned manifests used by `auplc-installer`: ```bash From 3cf219b707e63a8b015fd8e99a7f1b945d749e18 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 089/180] docs(skills): align AMD GPU package workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 20 +++++++++++-------- skills/deploy-aup-learning-cloud/reference.md | 19 ++++++++++++------ .../reference.md | 4 ++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 1799c447..2c88018a 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -87,14 +87,18 @@ node labeller as infrastructure prerequisites owned outside AUPLC. Verify both existing DaemonSets and advertised GPU capacity before Helm; do not install these privileged components as part of the AUPLC procedure. -Keep the GPU contract distinct from storage configuration. GPU hosts use -`root:render 0666` for `/dev/kfd` and AMD `renderD*`, and `root:video 0666` for -AMD `card*`, so every injected GPU device node has mode `0666`. Device-plugin -allocation is the visibility boundary and only `amd.com/gpu` requests receive -GPU nodes. Container group membership does not participate in GPU permissions; -host provisioning owns device-node discretionary access control. AUPLC Hub adds -no GPU supplemental group, and the plugin does not change Unix inode permissions. -`singleuser.fsGid: 100` is for shared storage only. +Keep the GPU contract distinct from storage configuration. The installer, +Ansible role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package at the pinned version +`30.30.4.0-2341068.24.04`. Its rule sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*`, which retains normal system policy, +observed as `root:video 0660`. + +Device-plugin allocation is a separate visibility layer. Only `amd.com/gpu` +requests receive allocated GPU devices, and the plugin does not change Unix +inode permissions. AUPLC Hub adds no GPU supplemental group; none is required +for the tested ROCm compute path. `singleuser.fsGid: 100` is for shared storage +only. ## Phase 4: Verify diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index ff7d5bd5..9c07ed68 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -30,17 +30,24 @@ Generation and validation must finish before Ansible or Helm changes are made. ## GPU permission contract -- Host `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render 0666`. -- Host AMD `/dev/dri/card*` nodes are `root:video 0666`. -- Every GPU device node injected into a Pod has mode `0666`. -- Container group membership does not participate in GPU permissions; host - provisioning owns device-node discretionary access control. -- AUPLC Hub adds no GPU supplemental group to user Pods. +- Installer, Ansible, and PXE provisioning install AMD's + `amdgpu-insecure-instinct-udev-rules` package at version + `30.30.4.0-2341068.24.04`. +- The package sets mode `0666` only on `/dev/kfd` and DRM + `/dev/dri/renderD*` nodes. +- The package does not change `/dev/dri/card*`. Card nodes retain normal system + policy, observed as `root:video 0660`. +- AUPLC Hub adds no GPU supplemental group. No GPU group is required for the + tested ROCm compute path. - AMD device-plugin allocation is the visibility boundary. Only Pods requesting `amd.com/gpu` receive GPU device nodes; the plugin does not change host inode ownership or mode. - `singleuser.fsGid: 100` controls shared storage ownership only. +Operator evidence from SHC showed `rocminfo` reporting `gfx1151` and `gfx1200` +on the two GPU nodes from UID `12345` Pods with only supplemental GID `100`. +Their `card*` nodes remained inaccessible at mode `0660`. + The infrastructure owner deploys and maintains the AMD device plugin and ROCm node labeller outside AUPLC. Before Helm, use the readiness and capacity checks in [deploy/README.md](../../deploy/README.md); do not install these privileged diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 514da9f9..656aac83 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -109,6 +109,10 @@ cd auplc-bundle-gfx1151-* sudo ./auplc-installer install ``` +The bundle includes the pinned +`amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb`; offline +installation verifies and installs it from the bundle. + ## Troubleshooting | Symptom | Likely cause | First checks | From 9a78201e6335bfbff5e2611ebf6be2b89a1ba317 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Tue, 28 Jul 2026 12:24:06 +0100 Subject: [PATCH 090/180] Remove not use code --- runtime/hub/frontend/apps/spawn/src/App.tsx | 5 +---- .../hub/frontend/apps/spawn/src/components/CourseCard.tsx | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/runtime/hub/frontend/apps/spawn/src/App.tsx b/runtime/hub/frontend/apps/spawn/src/App.tsx index ef8081a9..347711e1 100644 --- a/runtime/hub/frontend/apps/spawn/src/App.tsx +++ b/runtime/hub/frontend/apps/spawn/src/App.tsx @@ -187,10 +187,7 @@ function App() { if (!selectedResource?.metadata?.acceleratorKeys) return []; const real = accelerators.filter(acc => selectedResource.metadata?.acceleratorKeys?.includes(acc.key)); if (real.length <= 1) return real; - const rates = real.map(a => a.quotaRate); - const minRate = Math.min(...rates); - const maxRate = Math.max(...rates); - const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; + const minRate = Math.min(...real.map(a => a.quotaRate)); const autoOption: Accelerator = { key: 'auto', displayName: 'Auto', diff --git a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx index f79f2cb0..0504a953 100644 --- a/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx +++ b/runtime/hub/frontend/apps/spawn/src/components/CourseCard.tsx @@ -88,10 +88,7 @@ export const CourseCard = memo(function CourseCard({ } const real = accelerators.filter(acc => acceleratorKeys.includes(acc.key)); if (real.length <= 1) return real; - const rates = real.map(a => a.quotaRate); - const minRate = Math.min(...rates); - const maxRate = Math.max(...rates); - const rateDesc = minRate === maxRate ? `${minRate} credits/min` : `${minRate}–${maxRate} credits/min`; + const minRate = Math.min(...real.map(a => a.quotaRate)); const autoOption: Accelerator = { key: 'auto', displayName: 'Auto', From 269849dd21fbba92e99db079a20c5e8d29d36299 Mon Sep 17 00:00:00 2001 From: Mario Ruiz <mruiznog@amd.com> Date: Tue, 28 Jul 2026 13:06:18 +0100 Subject: [PATCH 091/180] Fix ruff issues --- runtime/hub/core/spawner/kubernetes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index 6c759a49..66b6a1f8 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -785,8 +785,7 @@ async def _resolve_auto_accelerator(self, resource_type: str, eligible_keys: lis pods = await v1.list_pod_for_all_namespaces(field_selector="status.phase=Running") node_labels = { - node.metadata.name: (node.metadata.labels or {}, node.status.allocatable or {}) - for node in nodes.items + node.metadata.name: (node.metadata.labels or {}, node.status.allocatable or {}) for node in nodes.items } used_gpus: dict[str, int] = {} From 544e29c722d67d57cf7f84af3af1dc54afcaee43 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:23:54 +0800 Subject: [PATCH 092/180] feat(deploy): validate direct SSH inventory --- .../scripts/gpu_resolution_validation.py | 8 ++ .../scripts/helm_validation.py | 37 ++++++ .../scripts/validate.py | 55 ++++----- .../test_direct_inventory_validation.py | 113 ++++++++++++++++++ 4 files changed, 181 insertions(+), 32 deletions(-) create mode 100644 skills/deploy-aup-learning-cloud/scripts/helm_validation.py create mode 100644 tests/skills/test_direct_inventory_validation.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index f33c5cc8..6d36271b 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -34,6 +34,14 @@ class AcceleratorValidationResult: passed: list[str] +def check_gpu_inventory(repo: Path, inventory_path: str) -> GpuArtifactValidationResult: + inventory_file = configured_path(repo, inventory_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"inventory not found: {inventory_file}"], []) + _, errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + return GpuArtifactValidationResult(errors, [] if errors else ["GPU access inventory is valid"]) + + def check_accelerator_labels( accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None ) -> AcceleratorValidationResult: diff --git a/skills/deploy-aup-learning-cloud/scripts/helm_validation.py b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py new file mode 100644 index 00000000..cd1c4e39 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +CHART = "runtime/chart" + + +@dataclass(frozen=True, slots=True) +class HelmValidationReporter: + ok: Callable[[str], None] + warn: Callable[[str], None] + fail: Callable[[str], None] + + +def check_helm(repo: Path, values: list[str], reporter: HelmValidationReporter) -> None: + if not shutil.which("helm"): + reporter.warn("helm not on PATH; skipped chart dry-run") + return + chart = repo / CHART + if not chart.exists(): + reporter.warn(f"chart not found at {CHART}; skipped dry-run") + return + cmd = ["helm", "template", "jupyterhub", str(chart)] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if path.exists(): + cmd += ["-f", str(path)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + reporter.ok("helm template rendered the chart successfully") + else: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] + reporter.fail("helm template failed:\n " + "\n ".join(tail)) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 8e3982c1..22b9514d 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,8 +12,9 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * generated inventory, GPU-resolution manifest, and PXE rootfs policy agree - when generated artifacts are supplied; + * direct inventory GPU access booleans are valid, or generated inventory, + GPU-resolution manifest, and PXE rootfs policy agree when both artifacts + are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -36,18 +37,21 @@ import argparse import json import re -import shutil -import subprocess import sys from pathlib import Path from config_common import DuplicateJsonKeyError, strict_json_loads -from gpu_resolution_validation import GpuArtifactValidationRequest, check_accelerator_labels, check_gpu_artifacts +from gpu_resolution_validation import ( + GpuArtifactValidationRequest, + check_accelerator_labels, + check_gpu_artifacts, + check_gpu_inventory, +) +from helm_validation import HelmValidationReporter, check_helm from values_resolution_parsing import collect_effective_values PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" INVENTORY = "deploy/ansible/inventory.yml" -CHART = "runtime/chart" errors: list[str] = [] warnings: list[str] = [] @@ -168,27 +172,6 @@ def check_version_sync(repo: Path, configured_path: str | None = None) -> None: ) -def check_helm(repo: Path, values: list[str]) -> None: - if not shutil.which("helm"): - warn("helm not on PATH; skipped chart dry-run") - return - chart = repo / CHART - if not chart.exists(): - warn(f"chart not found at {CHART}; skipped dry-run") - return - cmd = ["helm", "template", "jupyterhub", str(chart)] - for rel in values or ["runtime/values.yaml"]: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - cmd += ["-f", str(p)] - proc = subprocess.run(cmd, capture_output=True, text=True) - if proc.returncode == 0: - ok("helm template rendered the chart successfully") - else: - tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] - fail("helm template failed:\n " + "\n ".join(tail)) - - def main(argv=None) -> int: global errors, passed, warnings errors = [] @@ -209,8 +192,10 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) - ap.add_argument("--inventory", help="generated inventory.yml to cross-check with GPU resolution") - ap.add_argument("--gpu-resolution", help="generated gpu-access-resolution.json to cross-check") + ap.add_argument("--inventory", help="inventory.yml to validate directly or cross-check with GPU resolution") + ap.add_argument( + "--gpu-resolution", help="generated gpu-access-resolution.json; requires --inventory for consistency checks" + ) ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -246,8 +231,14 @@ def main(argv=None) -> int: warn(message) for message in accelerator_result.passed: ok(message) - if bool(args.inventory) != bool(args.gpu_resolution): - fail("--inventory and --gpu-resolution must be supplied together") + if args.gpu_resolution and not args.inventory: + fail("--gpu-resolution requires --inventory") + elif args.inventory and not args.gpu_resolution: + inventory_result = check_gpu_inventory(repo, args.inventory) + for message in inventory_result.errors: + fail(message) + for message in inventory_result.passed: + ok(message) elif args.inventory and args.gpu_resolution: artifact_result = check_gpu_artifacts( GpuArtifactValidationRequest( @@ -264,7 +255,7 @@ def main(argv=None) -> int: for message in artifact_result.passed: ok(message) if args.helm_dry_run: - check_helm(repo, args.values) + check_helm(repo, args.values, HelmValidationReporter(ok=ok, warn=warn, fail=fail)) if args.json: print( diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py new file mode 100644 index 00000000..2bceb295 --- /dev/null +++ b/tests/skills/test_direct_inventory_validation.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI tests for direct SSH inventory validation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +VALIDATE = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "validate.py" + + +def run_validate(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(VALIDATE), *args], capture_output=True, text=True, check=False) + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def valid_inventory() -> str: + return """k3s_cluster: + children: + server: + hosts: + server: + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + auplc_gpu_access_enabled: false +""" + + +def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", valid_inventory()) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access inventory is valid" in result.stdout + + +@pytest.mark.parametrize( + ("inventory_content", "expected_error"), + [ + (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), + ( + valid_inventory().replace( + " auplc_gpu_access_enabled: true\n", + " auplc_gpu_access_enabled: true\n auplc_gpu_access_enabled: false\n", + ), + "must define exactly one", + ), + ], +) +def test_validator_rejects_invalid_direct_inventory( + tmp_path: Path, inventory_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", inventory_content) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_reports_direct_inventory_not_found(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", "missing.yml", "--values", str(values) + ) + + assert result.returncode == 1 + assert "inventory not found" in result.stdout + assert "generated inventory not found" not in result.stdout + + +def test_validator_rejects_gpu_resolution_without_inventory(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write(repo / "resolution.json", "{}\n") + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "--gpu-resolution requires --inventory" in result.stdout From 0e513f4b27c28d1dc3c0fdde181842a9e1a018e8 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:24:46 +0800 Subject: [PATCH 093/180] fix(ansible): require explicit GPU access flags --- deploy/ansible/inventory.yml | 3 +++ tests/skills/test_gpu_access_role.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/deploy/ansible/inventory.yml b/deploy/ansible/inventory.yml index a210de23..cbda4db7 100644 --- a/deploy/ansible/inventory.yml +++ b/deploy/ansible/inventory.yml @@ -24,10 +24,13 @@ k3s_cluster: hosts: # suggested: aup-SHC1-395-1 # You need to config the hostname in /etc/hosts + # Set true on hosts that provide AMD GPU access. <YOUR-SERVER-HOSTNAME>: + auplc_gpu_access_enabled: false agent: hosts: <YOUR-AGENT-HOSTNAME>: + auplc_gpu_access_enabled: false # strix-5: # phx-1: # phx-64g: diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index 3fe821ec..fb11fefe 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -4,6 +4,8 @@ from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] ANSIBLE = ROOT / "deploy" / "ansible" GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" @@ -51,6 +53,20 @@ def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: assert "modified package conffile" in verify +def test_inventory_placeholders_define_boolean_gpu_access() -> None: + inventory_text = read(ANSIBLE / "inventory.yml") + inventory = yaml.safe_load(inventory_text) + raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) + hosts = inventory["k3s_cluster"]["children"] + raw_hosts = raw_inventory["k3s_cluster"]["children"] + + for group_name in ("server", "agent"): + for host_name, host in hosts[group_name]["hosts"].items(): + value = host["auplc_gpu_access_enabled"] + assert type(value) is bool + assert raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] in {"true", "false"} + + def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") From 61a49a5f2313571bb9f96493bb8280fa32f17ccd Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:25:24 +0800 Subject: [PATCH 094/180] docs(deploy): restore direct SSH workflow --- deploy/README.md | 73 ++++++++++++++++++++++++++++------------ deploy/ansible/README.md | 20 +++++++---- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 7edbffc8..23c29e8c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,10 +53,10 @@ sudo ./auplc-installer install ### Multi-Node Cluster -Generate the spec and fill in the network and node details. The SSH flow needs -only the managed host details. A PXE spec asks one extra GPU question: -`pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless -agents' hardware is not inferred from the controller. +For SSH-preinstalled nodes, edit the Ansible inventory and multi-node values +file directly. PXE remains generator-based because the controller inventory, +rootfs settings, runtime overlay, and GPU policy must be generated as one +consistent artifact set. The AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must deploy and @@ -67,21 +67,44 @@ DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled +Edit `deploy/ansible/inventory.yml` with the server and agent hostnames, IPs, +k3s token, and other site settings. Every host entry must set +`auplc_gpu_access_enabled` to the YAML boolean `true` or `false`. Use `true` +only for hosts where the AMD GPU access package and ROCm should be installed. +Don't quote the boolean or use alternatives such as `yes` and `no`. + +For example: + +```yaml +k3s_cluster: + children: + server: + hosts: + controller-1: + ansible_host: 192.0.2.10 + auplc_gpu_access_enabled: false + agent: + hosts: + gpu-worker-1: + ansible_host: 192.0.2.11 + auplc_gpu_access_enabled: true +``` + +Copy the human-maintained multi-node values example, then edit the copy for the +site's authentication, storage, images, accelerators, and network access: + ```bash cd .. REPO_ROOT="$(pwd)" DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json -# Edit spec.json: choose ssh-preinstalled and fill the node/network fields. -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +cp runtime/values-multi-nodes.yaml.example runtime/values-multi-nodes.yaml +# Edit deploy/ansible/inventory.yml and runtime/values-multi-nodes.yaml. + python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ - --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ --values "$REPO_ROOT/runtime/values.yaml" \ - --values "$REPO_ROOT/runtime/values-basic-example.yaml" + --values "$REPO_ROOT/runtime/values-multi-nodes.yaml" \ + --helm-dry-run cd "$REPO_ROOT/deploy/ansible" sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml @@ -96,14 +119,14 @@ cd "$REPO_ROOT" helm upgrade --install jupyterhub ./runtime/chart \ --namespace jupyterhub --create-namespace \ -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml + -f runtime/values-multi-nodes.yaml ``` -Generation runs read-only Ansible discovery against every managed host. It -cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class -records under `/sys/bus/pci/devices`; it does not require the devices to be -attached to `amdgpu` before ROCm installation. The resulting GPU resolution -report records which managed hosts have AMD display hardware. +The validator checks an inventory supplied by itself for exactly one explicit +YAML boolean `auplc_gpu_access_enabled` on every managed host. This validates +the direct-edit workflow without a generated GPU resolution report. +`--gpu-resolution` may be supplied only with `--inventory`; when both are +supplied, the validator also checks generated-artifact consistency. The installer, Ansible GPU access role, and PXE controller install AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to version @@ -125,14 +148,22 @@ not part of GPU access and must not be treated as a GPU group setting. #### PXE-diskless -After setting `topology` to `pxe-diskless`, fill the PXE network fields and set -`pxe.diskless_agents_have_amd_gpus` explicitly. Generation writes the canonical +Create a fresh spec, set `topology` to `pxe-diskless`, fill the PXE network +fields, and set `pxe.diskless_agents_have_amd_gpus` explicitly. Diskless agent +hardware can't be inferred from the controller. Generation writes the canonical inventory, controller vars, runtime overlay, and GPU resolution report directly. These artifacts express the desired deployment inputs; their existence is not proof that rootfs provisioning succeeded. Review and install them before running the controller playbook, whose successful completion provisions the rootfs. ```bash +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. +GENERATED_DIR="$REPO_ROOT/generated" + cd "$REPO_ROOT" python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" @@ -166,7 +197,7 @@ playbook. A retained rootfs is accepted only when that exact package version and its unmodified package-owned rule are present, with no conflicting legacy GPU rule. Rebuild or correct a retained rootfs separately if that safety check fails. -#### Discovery failures +#### Generator discovery failures | Error | Action | | --- | --- | diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index cbd02ad1..fefe0fea 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -24,13 +24,21 @@ SOFTWARE. K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible). -For the generator, canonical inventory, validator arguments, and topology-specific -playbook commands, see the authoritative [deployment guide](../README.md). +For the human SSH-preinstalled workflow, edit `inventory.yml` directly and use +the playbook commands in the [deployment guide](../README.md). Every server and +agent host entry must define `auplc_gpu_access_enabled` as the unquoted YAML +boolean `true` or `false`. Set it to `true` only on hosts where the GPU access +package and ROCm should be installed. Pass `--inventory` to the deployment +validator to check this explicit per-host policy. A generated +`--gpu-resolution` report is not required for the human workflow; if supplied, +it requires `--inventory`, and the validator checks the two generated artifacts +for consistency. -Don't write GPU policy into the inventory by hand. SSH generation discovers GPU -hosts from managed-host evidence. PXE generation uses only -`pxe.diskless_agents_have_amd_gpus` and writes canonical files before the -controller playbook runs. +The deploy skill has a separate generator-first SSH workflow that discovers GPU +hosts from managed-host evidence. PXE is always generator-based and uses only +`pxe.diskless_agents_have_amd_gpus` as its GPU policy input. See the +[skill scripts guide](../../skills/deploy-aup-learning-cloud/scripts/README.md) +for the complete generator-first skill command sequences. The GPU access role installs AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to `30.30.4.0-2341068.24.04`, on GPU hosts and GPU-enabled PXE From dae1f1efdebfedf4c830883b4354677d6fcf3097 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:26:02 +0800 Subject: [PATCH 095/180] docs(skills): retain generated deployment workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 40 ++++--- skills/deploy-aup-learning-cloud/reference.md | 21 ++-- .../scripts/README.md | 109 ++++++++++++++++-- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 2c88018a..9af9e4c4 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -13,9 +13,11 @@ description: >- Stand up a multi-node AUP Learning Cloud cluster with Ansible, AMD GPU access, shared storage, and the JupyterHub Helm chart. -Use [deploy/README.md](../../deploy/README.md) as the source of truth for the -generator schema, commands, generated files, validation, and troubleshooting. -This skill defines the interview and safety gates around that procedure. +Use the [skill scripts guide](scripts/README.md) as the source of truth for the +complete generator-first command sequences and generated files. Use +[deploy/README.md](../../deploy/README.md) for the human direct-edit workflow, +operational background, and troubleshooting. This skill defines the interview +and safety gates around the generated procedure. ## Prerequisites @@ -65,27 +67,34 @@ does not prove rootfs provisioning succeeded. Review, install, and validate thos files, then run the controller playbook with the canonical inventory and PXE vars; the playbook must complete successfully before proceeding. -Follow the exact generation, installation, and playbook commands in -[deploy/README.md](../../deploy/README.md). +Follow the complete topology command sequence in the +[skill scripts guide](scripts/README.md). Don't substitute the human direct-edit +SSH workflow from `deploy/README.md`; the skill's SSH path remains +generator-first and discovers GPU policy from managed-host evidence. ## Phase 3: Validate and execute Install the canonical generated inventory and runtime overlay into the checkout, -then run the validator with the arguments shown in the deployment guide: +then run the topology's exact validator command from the +[skill scripts guide](scripts/README.md). The validator inputs are: - `--repo` - `--topology` -- `--inventory` -- `--gpu-resolution` +- `--inventory` to validate explicit host booleans +- `--gpu-resolution` with `--inventory` for generated-artifact consistency - both `--values` files - `--pxe-vars` for PXE only -Stop on validation failure. After a clean result, follow the topology's Ansible, -storage, device plugin, and Helm sequence in -[deploy/README.md](../../deploy/README.md). Treat the AMD device plugin and ROCm -node labeller as infrastructure prerequisites owned outside AUPLC. Verify both -existing DaemonSets and advertised GPU capacity before Helm; do not install -these privileged components as part of the AUPLC procedure. +An inventory can be validated without a GPU resolution report. A resolution +report requires an inventory. Supply both in this generator-first workflow so +the validator also checks their consistency. + +Stop on validation failure. After a clean result, continue with the topology's +Ansible, device plugin, and Helm commands in the skill scripts guide. Treat the +AMD device plugin and ROCm node labeller as infrastructure prerequisites owned +outside AUPLC. Verify both existing DaemonSets and advertised GPU capacity +before Helm; do not install these privileged components as part of the AUPLC +procedure. Keep the GPU contract distinct from storage configuration. The installer, Ansible role, and PXE controller install AMD's @@ -119,5 +128,6 @@ are changed. ## Reference -- [Deployment commands and troubleshooting](../../deploy/README.md) +- [Complete skill command sequences](scripts/README.md) +- [Human deployment and troubleshooting](../../deploy/README.md) - [Skill-specific summary](reference.md) diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index 9c07ed68..e10c8e9c 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -1,8 +1,10 @@ # Deploy AUP Learning Cloud Reference -The authoritative procedure, command lines, generated file list, and failure -guidance live in [deploy/README.md](../../deploy/README.md). Don't copy those -commands into this reference. +The complete generator-first command sequences and generated file list live in +the [skill scripts guide](scripts/README.md). Human direct-edit deployment, +operational background, and failure guidance live in +[deploy/README.md](../../deploy/README.md). Don't copy those commands into this +reference. ## Topology contract @@ -11,13 +13,14 @@ commands into this reference. | `ssh-preinstalled` | Connects to every managed host, discovers GPU hardware, and publishes canonical files when discovery is consistent. | | `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input and publishes canonical desired-input files before the controller playbook runs. Their existence does not prove rootfs provisioning succeeded. | -Don't hand-author generated GPU policy. Create deployment specs from the current +The skill is generator-first for both topologies. Don't hand-author generated +GPU policy, including for SSH. Create deployment specs from the current `--print-schema` output. ## Canonical validation inputs -Use the validator command from [deploy/README.md](../../deploy/README.md). It -passes: +Use the topology's validator command from the +[skill scripts guide](scripts/README.md). It passes: - repository root with `--repo` - selected topology with `--topology` @@ -26,7 +29,11 @@ passes: - base and generated overlays as two `--values` arguments - canonical PXE vars with `--pxe-vars` for PXE only -Generation and validation must finish before Ansible or Helm changes are made. +`--inventory` alone validates that every managed host has exactly one explicit +YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires +`--inventory`; supplying both enables generated-artifact consistency checks. +The skill supplies both because its workflow is generator-first. Generation +and validation must finish before Ansible or Helm changes are made. ## GPU permission contract diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index db9fbfef..dad1fd59 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -1,8 +1,10 @@ # Helper scripts -These dependency-light helpers support the multi-node deployment skill. See -[deploy/README.md](../../../deploy/README.md) for the authoritative command -sequence and argument paths. +These dependency-light helpers support the multi-node deployment skill. This +file is the source of truth for the complete skill command sequences: artifact +generation and installation, validation, Ansible, device plugin readiness, and +Helm. For human direct-edit deployment, operational background, and +troubleshooting, see [deploy/README.md](../../../deploy/README.md). | Script | Purpose | | --- | --- | @@ -23,12 +25,105 @@ run the controller playbook with the generated `inventory.yml` and `pb-pxe-controller.vars.yml`. The files express desired inputs; their existence does not prove the PXE rootfs was provisioned successfully. +## SSH-preinstalled commands + +Run these commands from a clean checkout. Fill the generated `spec.json` with +the SSH topology, network settings, and every managed host. Don't add a GPU host +list. The generator discovers GPU policy over passwordless root SSH. + +```bash +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose ssh-preinstalled and fill the node and network fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" +``` + +After validation passes, run Ansible, check the infrastructure-owned GPU +components, and install the chart with the generated overlay: + +```bash +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +## PXE-diskless commands + +Fill the generated `spec.json` with the PXE topology and all controller, +network, and rootfs fields. Set `pxe.diskless_agents_have_amd_gpus` explicitly. + +```bash +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +The controller playbook must finish successfully before the remaining cluster +and Helm steps begin. A fresh rootfs receives the pinned GPU access package. A +retained rootfs must pass the package version, package-owned rule, and legacy +rule safety checks described in the deployment guide. + ## Validator contract -Use the exact validator command in -[deploy/README.md](../../../deploy/README.md). Its canonical inputs are -`--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` -arguments, and `--pxe-vars` for PXE only. +The exact topology commands above pass `--repo`, `--topology`, `--inventory`, +`--gpu-resolution`, two `--values` arguments, and `--pxe-vars` for PXE only. +`--inventory` alone validates that every managed host defines exactly one +explicit YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires +`--inventory`; supplying both performs generated-artifact consistency checks. +The generator-first skill workflow supplies both. ## Conventions From ee7ac6e3bc5ed8cac7215b85d49a4133149f1195 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:15:32 +0800 Subject: [PATCH 096/180] fix(validation): allow auto only for direct SSH inventory --- .../scripts/gpu_resolution_parsing.py | 35 +++++- .../scripts/gpu_resolution_validation.py | 3 +- .../scripts/validate.py | 24 +++-- .../test_direct_inventory_validation.py | 100 ++++++++++++++++++ 4 files changed, 148 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py index 6fe4c6d1..eb5a38ec 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -13,6 +13,11 @@ class GpuInventory: hosts: dict[str, bool] +@dataclass(frozen=True, slots=True) +class GpuInventoryHostScalars: + hosts: dict[str, str] + + @dataclass(frozen=True, slots=True) class GpuResolution: status: str @@ -43,7 +48,7 @@ def yaml_indent(line: str) -> int: return len(line) - len(line.lstrip()) -def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: +def scan_gpu_inventory_host_scalars(text: str) -> tuple[GpuInventoryHostScalars | None, list[str]]: host_values: dict[str, list[str]] = {} host_names: list[str] = [] stack: list[tuple[int, str]] = [] @@ -85,20 +90,42 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: parse_errors.append("inventory has no generated k3s server or agent hosts") if len(set(host_names)) != len(host_names): parse_errors.append("inventory has duplicate generated host names") - hosts: dict[str, bool] = {} + hosts: dict[str, str] = {} for host in host_names: values = host_values[host] if len(values) != 1: parse_errors.append(f"inventory host '{host}' must define exactly one auplc_gpu_access_enabled") continue - enabled = parse_gpu_boolean(values[0]) + hosts[host] = values[0] + if parse_errors: + return None, parse_errors + return GpuInventoryHostScalars(hosts=hosts), [] + + +def validate_direct_gpu_inventory(text: str) -> list[str]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return parse_errors + for host, value in host_scalars.hosts.items(): + if value not in {"auto", "true", "false"}: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + return parse_errors + + +def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return None, parse_errors + hosts: dict[str, bool] = {} + for host, value in host_scalars.hosts.items(): + enabled = parse_gpu_boolean(value) if enabled is None: parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") continue hosts[host] = enabled if parse_errors: return None, parse_errors - return GpuInventory(hosts=hosts), parse_errors + return GpuInventory(hosts=hosts), [] def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index 6d36271b..b3c3061f 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -8,6 +8,7 @@ parse_gpu_inventory, parse_gpu_resolution, parse_pxe_gpu_policy, + validate_direct_gpu_inventory, ) @@ -38,7 +39,7 @@ def check_gpu_inventory(repo: Path, inventory_path: str) -> GpuArtifactValidatio inventory_file = configured_path(repo, inventory_path) if not inventory_file.exists(): return GpuArtifactValidationResult([f"inventory not found: {inventory_file}"], []) - _, errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + errors = validate_direct_gpu_inventory(inventory_file.read_text(encoding="utf-8")) return GpuArtifactValidationResult(errors, [] if errors else ["GPU access inventory is valid"]) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 22b9514d..934e3ea7 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,9 +12,9 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * direct inventory GPU access booleans are valid, or generated inventory, - GPU-resolution manifest, and PXE rootfs policy agree when both artifacts - are supplied; + * direct SSH inventory GPU access values are exact unquoted `auto`, `true`, + or `false`; generated inventory, GPU-resolution manifest, and PXE rootfs + policy agree when both artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -192,7 +192,10 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) - ap.add_argument("--inventory", help="inventory.yml to validate directly or cross-check with GPU resolution") + ap.add_argument( + "--inventory", + help="inventory.yml for direct ssh-preinstalled validation (auto/true/false) or generated checks; pxe requires --gpu-resolution", + ) ap.add_argument( "--gpu-resolution", help="generated gpu-access-resolution.json; requires --inventory for consistency checks" ) @@ -234,11 +237,14 @@ def main(argv=None) -> int: if args.gpu_resolution and not args.inventory: fail("--gpu-resolution requires --inventory") elif args.inventory and not args.gpu_resolution: - inventory_result = check_gpu_inventory(repo, args.inventory) - for message in inventory_result.errors: - fail(message) - for message in inventory_result.passed: - ok(message) + if args.topology == "pxe-diskless": + fail("pxe-diskless inventory validation requires --gpu-resolution") + else: + inventory_result = check_gpu_inventory(repo, args.inventory) + for message in inventory_result.errors: + fail(message) + for message in inventory_result.passed: + ok(message) elif args.inventory and args.gpu_resolution: artifact_result = check_gpu_artifacts( GpuArtifactValidationRequest( diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py index 2bceb295..370f3e37 100644 --- a/tests/skills/test_direct_inventory_validation.py +++ b/tests/skills/test_direct_inventory_validation.py @@ -51,11 +51,111 @@ def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path assert "GPU access inventory is valid" in result.stdout +def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + write(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + pxe_vars = write( + repo / "pxe-vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +pxe_gpu_access_enabled: false +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe-diskless inventory validation requires --gpu-resolution" in result.stdout + + +@pytest.mark.parametrize("value", ("auto", "true", "false")) +def test_validator_accepts_supported_direct_inventory_values(tmp_path: Path, value: str) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", valid_inventory().replace("true", value).replace("false", value)) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access inventory is valid" in result.stdout + + +def test_validator_rejects_auto_when_inventory_is_cross_checked_with_gpu_resolution(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write( + repo / "gpu-access-resolution.json", + """{ + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": false, "server": true} +} +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "inventory host 'server' has malformed auplc_gpu_access_enabled" in result.stdout + + @pytest.mark.parametrize( ("inventory_content", "expected_error"), [ (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "auto"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'auto'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "true"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'true'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "false"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'false'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: AUTO"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: TRUE"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: FALSE"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: no"), "malformed"), + ( + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: malformed"), + "malformed", + ), ( valid_inventory().replace( " auplc_gpu_access_enabled: true\n", From 6875615325c332cad07607e38cb691022df3ef7e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:16:23 +0800 Subject: [PATCH 097/180] feat(ansible): resolve automatic GPU access --- deploy/ansible/inventory.yml | 6 +-- deploy/ansible/playbooks/pb-rocm.yml | 18 ++++---- deploy/ansible/playbooks/pb-udev.yml | 16 +++---- .../roles/gpu_access/defaults/main.yml | 2 +- .../ansible/roles/gpu_access/tasks/detect.yml | 16 +++++++ .../ansible/roles/gpu_access/tasks/main.yml | 9 ++-- .../roles/gpu_access/tasks/resolve.yml | 42 +++++++++++++++++++ tests/skills/test_gpu_access_role.py | 40 ++++++++++++++++-- 8 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 deploy/ansible/roles/gpu_access/tasks/detect.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/resolve.yml diff --git a/deploy/ansible/inventory.yml b/deploy/ansible/inventory.yml index cbda4db7..9be9f9eb 100644 --- a/deploy/ansible/inventory.yml +++ b/deploy/ansible/inventory.yml @@ -24,13 +24,13 @@ k3s_cluster: hosts: # suggested: aup-SHC1-395-1 # You need to config the hostname in /etc/hosts - # Set true on hosts that provide AMD GPU access. + # Set auto to detect AMD display hardware, or true/false to override it. <YOUR-SERVER-HOSTNAME>: - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto agent: hosts: <YOUR-AGENT-HOSTNAME>: - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto # strix-5: # phx-1: # phx-64g: diff --git a/deploy/ansible/playbooks/pb-rocm.yml b/deploy/ansible/playbooks/pb-rocm.yml index 6788bf3d..7fb43ba8 100644 --- a/deploy/ansible/playbooks/pb-rocm.yml +++ b/deploy/ansible/playbooks/pb-rocm.yml @@ -22,26 +22,22 @@ any_errors_fatal: true become: yes pre_tasks: - - name: Assert explicit GPU access enablement - ansible.builtin.assert: - that: - - auplc_gpu_access_enabled is defined - - auplc_gpu_access_enabled is boolean - fail_msg: >- - Set auplc_gpu_access_enabled to true or false for every host in the - inventory before running pb-rocm.yml. + - name: Resolve GPU access enablement before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve - name: Preflight enabled GPU access hosts before ROCm mutation ansible.builtin.include_role: name: gpu_access tasks_from: preflight - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved roles: - role: rocm - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved tasks: - name: Apply GPU access after ROCm installation ansible.builtin.include_role: name: gpu_access tasks_from: apply - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/playbooks/pb-udev.yml b/deploy/ansible/playbooks/pb-udev.yml index 7e77eb80..53826ff1 100644 --- a/deploy/ansible/playbooks/pb-udev.yml +++ b/deploy/ansible/playbooks/pb-udev.yml @@ -22,23 +22,19 @@ any_errors_fatal: true become: yes pre_tasks: - - name: Assert explicit GPU access enablement - ansible.builtin.assert: - that: - - auplc_gpu_access_enabled is defined - - auplc_gpu_access_enabled is boolean - fail_msg: >- - Set auplc_gpu_access_enabled to true or false for every host in the - inventory before running pb-udev.yml. + - name: Resolve GPU access enablement before GPU access mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve - name: Preflight enabled GPU access hosts ansible.builtin.include_role: name: gpu_access tasks_from: preflight - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved tasks: - name: Apply GPU access on enabled hosts ansible.builtin.include_role: name: gpu_access tasks_from: apply - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 4a9d4e3b..65aa625c 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -1,7 +1,7 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -auplc_gpu_access_enabled: false +auplc_gpu_access_enabled: auto # Set for a PXE rootfs. Leave empty to configure the live host. auplc_rootfs_path: "" # Rootfs adapters must explicitly constrain their writable target below this diff --git a/deploy/ansible/roles/gpu_access/tasks/detect.yml b/deploy/ansible/roles/gpu_access/tasks/detect.yml new file mode 100644 index 00000000..939b4a95 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/detect.yml @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Detect AMD display BDFs through sysfs + ansible.builtin.command: + argv: + - python3 + - -c + - >- + from pathlib import Path; devices = Path('/sys/bus/pci/devices'); + print('\n'.join(sorted(device.name for device in devices.iterdir() + if (device / 'vendor').read_text().strip() == '0x1002' and + (device / 'class').read_text().strip().startswith('0x03')))) + register: _auplc_gpu_access_sysfs + changed_when: false + failed_when: false diff --git a/deploy/ansible/roles/gpu_access/tasks/main.yml b/deploy/ansible/roles/gpu_access/tasks/main.yml index 4826393b..c3317bc1 100644 --- a/deploy/ansible/roles/gpu_access/tasks/main.yml +++ b/deploy/ansible/roles/gpu_access/tasks/main.yml @@ -1,14 +1,17 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- +- name: Resolve GPU access enablement + ansible.builtin.import_tasks: resolve.yml + - name: Validate GPU access configuration ansible.builtin.import_tasks: validate.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved - name: Preflight GPU access target ansible.builtin.import_tasks: preflight.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved - name: Apply GPU access configuration ansible.builtin.import_tasks: apply.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/tasks/resolve.yml b/deploy/ansible/roles/gpu_access/tasks/resolve.yml new file mode 100644 index 00000000..b7f36282 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/resolve.yml @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access enablement policy + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - >- + auplc_gpu_access_enabled is boolean or + (auplc_gpu_access_enabled is string and auplc_gpu_access_enabled == 'auto') + fail_msg: >- + Set auplc_gpu_access_enabled to true, false, or unquoted auto for every + host before running GPU access tasks. + changed_when: false + +- name: Detect GPU access hardware for auto policy + ansible.builtin.import_tasks: detect.yml + when: auplc_gpu_access_enabled == 'auto' + +- name: Require successful GPU access hardware detection for auto policy + ansible.builtin.assert: + that: + - _auplc_gpu_access_sysfs.rc == 0 + fail_msg: >- + GPU access hardware detection failed for auto policy; no GPU access + mutation was attempted. + when: auplc_gpu_access_enabled == 'auto' + changed_when: false + +- name: Resolve GPU access enablement + ansible.builtin.set_fact: + _auplc_gpu_access_enabled_resolved: >- + {{ auplc_gpu_access_enabled if auplc_gpu_access_enabled is boolean + else (_auplc_gpu_access_sysfs.stdout | trim | length > 0) }} + changed_when: false + +- name: Require resolved GPU access enablement boolean + ansible.builtin.assert: + that: + - _auplc_gpu_access_enabled_resolved is boolean + fail_msg: GPU access enablement did not resolve to a boolean. + changed_when: false diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index fb11fefe..f67ec67d 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -53,18 +53,26 @@ def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: assert "modified package conffile" in verify -def test_inventory_placeholders_define_boolean_gpu_access() -> None: +def test_gpu_access_defaults_and_inventory_placeholders_use_unquoted_auto() -> None: + defaults_text = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + defaults = yaml.safe_load(defaults_text) inventory_text = read(ANSIBLE / "inventory.yml") inventory = yaml.safe_load(inventory_text) raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) hosts = inventory["k3s_cluster"]["children"] raw_hosts = raw_inventory["k3s_cluster"]["children"] + assert defaults["auplc_gpu_access_enabled"] == "auto" + assert "auplc_gpu_access_enabled: auto" in defaults_text + assert inventory_text.count("auplc_gpu_access_enabled: auto") == 2 + assert all( + quoted not in inventory_text + for quoted in ('auplc_gpu_access_enabled: "auto"', "auplc_gpu_access_enabled: 'auto'") + ) for group_name in ("server", "agent"): for host_name, host in hosts[group_name]["hosts"].items(): value = host["auplc_gpu_access_enabled"] - assert type(value) is bool - assert raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] in {"true", "false"} + assert value == raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] == "auto" def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: @@ -237,12 +245,38 @@ def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> Non def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") + detect = read(GPU_ACCESS_ROLE / "tasks" / "detect.yml") rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + assert "ansible.builtin.import_tasks: resolve.yml" in role_main + assert "auplc_gpu_access_enabled | bool" not in role_main + assert "when: _auplc_gpu_access_enabled_resolved" in role_main + assert "python3" in detect + assert "/sys/bus/pci/devices" in detect + assert "0x1002" in detect + assert "startswith('0x03')" in detect + assert "sorted(" in detect + assert "register: _auplc_gpu_access_sysfs" in detect + assert "changed_when: false" in detect + assert "failed_when: false" in detect + assert "auplc_gpu_access_enabled is boolean" in resolve + assert "auplc_gpu_access_enabled == 'auto'" in resolve + assert "ansible.builtin.import_tasks: detect.yml" in resolve + assert "_auplc_gpu_access_sysfs.rc == 0" in resolve + assert "_auplc_gpu_access_sysfs.stdout | trim | length > 0" in resolve + assert "_auplc_gpu_access_enabled_resolved is boolean" in resolve + assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") assert "any_errors_fatal: true" in rocm_playbook assert "any_errors_fatal: true" in udev_playbook + for playbook in (rocm_playbook, udev_playbook): + assert "tasks_from: resolve" in playbook + assert playbook.index("tasks_from: resolve") < playbook.index("tasks_from: preflight") + assert "auplc_gpu_access_enabled | bool" not in playbook + assert "when: _auplc_gpu_access_enabled_resolved" in playbook assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") assert "tasks_from: apply" in rocm_playbook assert "tasks_from: preflight" in udev_playbook From 8853384bd146dc70e861d86ab83cc2574bd24adc Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:17:12 +0800 Subject: [PATCH 098/180] refactor(deploy): reuse GPU sysfs detection --- .../playbooks/pb-gpu-access-discovery.yml | 21 ++++++------------- tests/skills/test_gpu_access_resolution.py | 10 ++++++++- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml index f788e0d9..e12229b3 100644 --- a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -102,19 +102,10 @@ | reject('equalto', '') | join('\n') }} changed_when: false - - name: Discover AMD display BDFs through sysfs - ansible.builtin.command: - argv: - - python3 - - -c - - >- - from pathlib import Path; devices = Path('/sys/bus/pci/devices'); - print('\n'.join(sorted(device.name for device in devices.iterdir() - if (device / 'vendor').read_text().strip() == '0x1002' and - (device / 'class').read_text().strip().startswith('0x03')))) - register: _auplc_discovery_sysfs - changed_when: false - failed_when: false + - name: Discover AMD display BDFs through shared sysfs detector + ansible.builtin.include_role: + name: gpu_access + tasks_from: detect - name: Record machine-readable GPU access discovery evidence ansible.builtin.set_fact: @@ -125,8 +116,8 @@ rc: "{{ _auplc_discovery_lspci.rc }}" stdout: "{{ _auplc_discovery_lspci.stdout | default('') }}" sysfs: - rc: "{{ _auplc_discovery_sysfs.rc }}" - stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" + rc: "{{ _auplc_gpu_access_sysfs.rc }}" + stdout: "{{ _auplc_gpu_access_sysfs.stdout | default('') }}" changed_when: false - name: Write machine-readable GPU access discovery evidence locally diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index 37003f28..fdd802ea 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -66,7 +66,7 @@ def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) -def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> None: +def test_discovery_playbook_preserves_lspci_agreement_and_exact_v1_host_evidence_shape() -> None: playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( " changed_when:", maxsplit=1 @@ -83,6 +83,14 @@ def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> Non assert '{"version":1,"hosts":[' in playbook assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook assert "| to_json" in playbook + assert "name: gpu_access" in playbook + assert "tasks_from: detect" in playbook + assert "_auplc_gpu_access_sysfs.rc" in playbook + assert "_auplc_gpu_access_sysfs.stdout" in playbook + assert "/sys/bus/pci/devices" not in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0300"]' in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0302"]' in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0380"]' in playbook def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: From f39f890924feb437fc45dc88d0cc6c42efa5ac7c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:18:00 +0800 Subject: [PATCH 099/180] test(deploy): preserve generated boolean policies --- tests/skills/test_gpu_artifact_generation.py | 3 +++ tests/skills/test_pxe_finalization.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index 4e40a624..fe629147 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -154,6 +154,7 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { @@ -191,6 +192,7 @@ def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_art values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert inventory.count("auplc_gpu_access_enabled: true") == 2 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in values assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} @@ -209,6 +211,7 @@ def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index 554d6049..dac1fb0a 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -97,8 +97,10 @@ def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") assert "auplc_render_gid" not in inventory + assert "auplc_gpu_access_enabled: auto" not in inventory assert "gpuAccess" not in values assert "pxe_gpu_access_enabled: true" in pxe_vars + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest == { "version": 1, "status": "cpu_only", @@ -121,6 +123,7 @@ def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeyp pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "pxe_gpu_access_enabled: false" in pxe_vars + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert "auplc_render_gid" not in pxe_vars assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} @@ -137,6 +140,9 @@ def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "auplc_gpu_access_enabled: true" in inventory + assert "auplc_gpu_access_enabled: auto" not in inventory + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest["status"] == "gpu_resolved" assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} From 5431365958636be9356e26499b9ab30919c2a06a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:18:52 +0800 Subject: [PATCH 100/180] docs(deploy): document automatic GPU detection --- deploy/README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 23c29e8c..29dd70f4 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -68,10 +68,18 @@ DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled Edit `deploy/ansible/inventory.yml` with the server and agent hostnames, IPs, -k3s token, and other site settings. Every host entry must set -`auplc_gpu_access_enabled` to the YAML boolean `true` or `false`. Use `true` -only for hosts where the AMD GPU access package and ROCm should be installed. -Don't quote the boolean or use alternatives such as `yes` and `no`. +k3s token, and other site settings. Keep the human template default, +`auplc_gpu_access_enabled: auto`, unquoted on each host. `auto` runs Python 3 on +that host to scan `/sys/bus/pci/devices` for vendor `0x1002` devices whose PCI +class starts with `0x03`. It does not use `lspci` or require `pciutils`. A match +enables ROCm and the AMD GPU access package; a successful scan with no match +skips both. If a scan fails, the play aborts before either is changed, and +`any_errors_fatal` stops the play for all hosts. + +Set an unquoted YAML boolean `true` or `false` only when you need to override +detection. `true` forces ROCm and package installation, while `false` forces +both to be skipped. Either boolean bypasses the scan. Don't quote any of these +values or use alternatives such as `yes` and `no`. For example: @@ -82,12 +90,12 @@ k3s_cluster: hosts: controller-1: ansible_host: 192.0.2.10 - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto agent: hosts: gpu-worker-1: ansible_host: 192.0.2.11 - auplc_gpu_access_enabled: true + auplc_gpu_access_enabled: auto ``` Copy the human-maintained multi-node values example, then edit the copy for the @@ -122,11 +130,12 @@ helm upgrade --install jupyterhub ./runtime/chart \ -f runtime/values-multi-nodes.yaml ``` -The validator checks an inventory supplied by itself for exactly one explicit -YAML boolean `auplc_gpu_access_enabled` on every managed host. This validates -the direct-edit workflow without a generated GPU resolution report. -`--gpu-resolution` may be supplied only with `--inventory`; when both are -supplied, the validator also checks generated-artifact consistency. +With `--inventory` alone, the validator accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +This validates the direct-edit workflow without a generated GPU resolution +report. `--gpu-resolution` may be supplied only with `--inventory`; that pair +is for generated artifacts, whose inventory values and resolution entries must +remain strict booleans. The generator never writes `auto`. The installer, Ansible GPU access role, and PXE controller install AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to version From f47b5b29cad2089f603718caf759d534a5dd86c5 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:19:41 +0800 Subject: [PATCH 101/180] docs(ansible): document automatic GPU policy --- deploy/ansible/README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index fefe0fea..0c989e0a 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -26,13 +26,18 @@ K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s For the human SSH-preinstalled workflow, edit `inventory.yml` directly and use the playbook commands in the [deployment guide](../README.md). Every server and -agent host entry must define `auplc_gpu_access_enabled` as the unquoted YAML -boolean `true` or `false`. Set it to `true` only on hosts where the GPU access -package and ROCm should be installed. Pass `--inventory` to the deployment -validator to check this explicit per-host policy. A generated -`--gpu-resolution` report is not required for the human workflow; if supplied, -it requires `--inventory`, and the validator checks the two generated artifacts -for consistency. +agent host entry defaults to unquoted `auplc_gpu_access_enabled: auto`. On each +host, `auto` uses Python 3 to scan `/sys/bus/pci/devices` for vendor `0x1002` +and PCI class `0x03*`; it has no `lspci` or `pciutils` dependency. A match +enables ROCm and the GPU access package, while a successful empty scan skips +both. A scan failure aborts before mutation, and `any_errors_fatal` stops the +play. Unquoted `true` and `false` force enablement or disablement and bypass +detection. + +Pass `--inventory` to validate direct values of `auto`, `true`, or `false`. A +generated `--gpu-resolution` report is not required for the human workflow. If +supplied, it requires `--inventory`, and both generated artifacts must use +strict booleans. The deploy skill never generates `auto`. The deploy skill has a separate generator-first SSH workflow that discovers GPU hosts from managed-host evidence. PXE is always generator-based and uses only From e38dd7d37e452f2bd9f3e4cfb283ec2a0637711f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:20:42 +0800 Subject: [PATCH 102/180] docs(skills): distinguish direct and generated GPU policy --- skills/deploy-aup-learning-cloud/SKILL.md | 10 ++++++---- skills/deploy-aup-learning-cloud/reference.md | 14 ++++++++------ skills/deploy-aup-learning-cloud/scripts/README.md | 14 ++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 9af9e4c4..cc73af5f 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -80,14 +80,16 @@ then run the topology's exact validator command from the - `--repo` - `--topology` -- `--inventory` to validate explicit host booleans +- `--inventory` to validate generated host booleans - `--gpu-resolution` with `--inventory` for generated-artifact consistency - both `--values` files - `--pxe-vars` for PXE only -An inventory can be validated without a GPU resolution report. A resolution -report requires an inventory. Supply both in this generator-first workflow so -the validator also checks their consistency. +A human direct inventory can be validated by itself with unquoted `auto`, +`true`, or `false`. A resolution report requires an inventory, and that pairing +accepts only generated boolean values. Supply both in this generator-first +workflow so the validator checks their consistency. The skill resolves every +host to `true` or `false` and never generates `auto`. Stop on validation failure. After a clean result, continue with the topology's Ansible, device plugin, and Helm commands in the skill scripts guide. Treat the diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index e10c8e9c..b1da7733 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -15,7 +15,8 @@ reference. The skill is generator-first for both topologies. Don't hand-author generated GPU policy, including for SSH. Create deployment specs from the current -`--print-schema` output. +`--print-schema` output. Generation resolves hosts to strict `true` or `false` +values and never writes `auto`. ## Canonical validation inputs @@ -29,11 +30,12 @@ Use the topology's validator command from the - base and generated overlays as two `--values` arguments - canonical PXE vars with `--pxe-vars` for PXE only -`--inventory` alone validates that every managed host has exactly one explicit -YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires -`--inventory`; supplying both enables generated-artifact consistency checks. -The skill supplies both because its workflow is generator-first. Generation -and validation must finish before Ansible or Helm changes are made. +For a human direct inventory, `--inventory` alone accepts exactly one unquoted +`auto`, `true`, or `false` value for `auplc_gpu_access_enabled` on every managed +host. `--gpu-resolution` requires `--inventory`; supplying both checks generated +artifacts and requires strict booleans in the inventory and resolution report. +The skill supplies both because its workflow is generator-first. Generation and +validation must finish before Ansible or Helm changes are made. ## GPU permission contract diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index dad1fd59..83946594 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -19,6 +19,10 @@ The SSH topology discovers GPU hosts from managed-host evidence. Users don't provide a GPU host list. The PXE topology has one GPU policy input: `pxe.diskless_agents_have_amd_gpus`. +Generation resolves every host to `true` or `false`; it never writes `auto`. +Generated inventory and GPU resolution entries are strict booleans so their +consistency can be checked. + Generate specs from fresh `--print-schema` output. Both topologies write their canonical artifacts immediately. For PXE, review and validate those files, then run the controller playbook with the generated `inventory.yml` and @@ -120,10 +124,12 @@ rule safety checks described in the deployment guide. The exact topology commands above pass `--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` arguments, and `--pxe-vars` for PXE only. -`--inventory` alone validates that every managed host defines exactly one -explicit YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires -`--inventory`; supplying both performs generated-artifact consistency checks. -The generator-first skill workflow supplies both. +For direct validation, `--inventory` alone accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +`--gpu-resolution` requires `--inventory`; supplying both switches to generated +consistency validation, where inventory and resolution values must be strict +booleans. The generator-first skill workflow supplies both and never generates +`auto`. ## Conventions From d64b48ddedbe44188a66c42acceea166d9d16331 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:56:13 +0800 Subject: [PATCH 103/180] test(installer): reduce GPU orchestration coverage --- tests/installer/test_cli_gpu_access.py | 84 +++----------------------- tests/installer/test_gpu_hardware.py | 24 -------- tests/installer/test_overlay.py | 19 +----- 3 files changed, 12 insertions(+), 115 deletions(-) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index b48210b4..51854223 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Callable -from contextlib import contextmanager from pathlib import Path import pytest @@ -21,22 +20,14 @@ def test_full_install_gates_gpu_access_without_passing_it_to_the_overlay( monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: events: list[str] = [] - stages: list[tuple[str, int, int]] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) - @contextmanager - def fake_stage(label: str, *, idx: int, total: int): - stages.append((label, idx, total)) - yield - def fake_overlay(*args: object, **kwargs: object) -> Path: assert "render_gid" not in kwargs - events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) - monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) @@ -55,18 +46,6 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: assert events.count("provision") == expected_provision_count if expected_provision_count: assert events.index("provision") < events.index("device-plugin") - assert events.count("overlay") == 2 - assert stages == [ - ("Detecting GPU", 1, 9), - ("Provisioning GPU device access", 2, 9), - ("Generating values overlay (initial)", 3, 9), - ("Installing helm + k9s", 4, 9), - ("Installing K3s (single-node)", 5, 9), - ("Pulling custom + external images", 6, 9), - ("Deploying ROCm GPU device plugin + node labeller", 7, 9), - ("Refreshing values overlay from node labels", 8, 9), - ("Deploying JupyterHub runtime (helm install + wait)", 9, 9), - ] @pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) @@ -79,7 +58,6 @@ def test_runtime_upgrade_gates_host_access_without_provisioning_helm_values( def fake_overlay(*args: object, **kwargs: object) -> Path: assert "render_gid" not in kwargs - events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) @@ -94,59 +72,19 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: cli.cmd_rt_upgrade(state) assert events.count("provision") == expected_provision_count - assert events[-5:] == ["detect", "refine", "preserve-courses", "overlay", "upgrade-runtime"] - - -@pytest.mark.parametrize( - ("command", "expected_events"), - [ - (cli.cmd_dev_deploy, ("detect", "refine", "overlay", "deploy-runtime")), - (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), - (cli.cmd_rt_install, ("detect", "refine", "overlay", "deploy-runtime")), - (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), - ], -) -def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( - monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] -) -> None: - events: list[str] = [] - state = InstallerState() - paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) - - monkeypatch.setattr(state, "runtime_paths", lambda: paths) - monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) - ) - monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) - monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) - monkeypatch.setattr( - cli, - "generate_values_overlay", - lambda *args, **kwargs: events.append("overlay") or paths.overlay_path, - ) - monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) - monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) - - command(state) - - assert events == list(expected_events) @pytest.mark.parametrize( ("reinstall", "delegate_name"), [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) -@pytest.mark.parametrize( - ("hardware", "expected_access_events"), [(GpuHardware.GPU, ["provision"]), (GpuHardware.CPU, [])] -) +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) def test_reinstall_gates_host_access_before_removing_runtime( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str, hardware: GpuHardware, - expected_access_events: list[str], + expected_provision_count: int, ) -> None: events: list[str] = [] state = InstallerState() @@ -159,15 +97,17 @@ def test_reinstall_gates_host_access_before_removing_runtime( reinstall(state) - assert events == [*expected_access_events, "remove-runtime", "sleep", "delegate"] + assert events.count("provision") == expected_provision_count + assert events.index("remove-runtime") < events.index("delegate") + if expected_provision_count: + assert events.index("provision") < events.index("remove-runtime") def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeypatch) -> None: - events: list[str] = [] state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) - monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: None) monkeypatch.setattr( cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) @@ -175,8 +115,6 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp with pytest.raises(RuntimeError, match="hardware"): cli._cmd_install_inner(state, pull=True) - assert events == ["detect"] - @pytest.mark.parametrize( ("reinstall", "delegate_name"), @@ -198,12 +136,8 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( with pytest.raises(RuntimeError, match="hardware"): reinstall(state) - assert events == [] - - -def test_cli_exposes_no_render_gid_reconciliation_api() -> None: - assert not hasattr(cli, "_render_gid_for_local_hardware") - assert not hasattr(cli, "load_existing_gpu_access") + assert "remove-runtime" not in events + assert "delegate" not in events def test_gpu_hardware_gate_passes_offline_bundle_context_to_package_provisioning( diff --git a/tests/installer/test_gpu_hardware.py b/tests/installer/test_gpu_hardware.py index 6a455e26..aa8bb15b 100644 --- a/tests/installer/test_gpu_hardware.py +++ b/tests/installer/test_gpu_hardware.py @@ -63,30 +63,6 @@ def test_classify_gpu_hardware_returns_unknown_for_incomplete_pci_evidence(tmp_p assert hardware is GpuHardware.UNKNOWN -def test_classify_gpu_hardware_returns_unknown_for_malformed_pci_evidence(tmp_path: Path) -> None: - pci_devices = tmp_path / "devices" - malformed_vendor = pci_devices / "0000:00:02.0" - malformed_vendor.mkdir(parents=True) - (malformed_vendor / "vendor").write_text("0xZZZZ\n", encoding="ascii") - (malformed_vendor / "class").write_text("0x030000\n", encoding="ascii") - - hardware = classify_gpu_hardware(pci_devices) - - assert hardware is GpuHardware.UNKNOWN - - -def test_classify_gpu_hardware_returns_unknown_for_unreadable_pci_attribute(tmp_path: Path) -> None: - pci_devices = tmp_path / "devices" - unreadable_class = pci_devices / "0000:00:02.0" - unreadable_class.mkdir(parents=True) - (unreadable_class / "vendor").write_text("0x8086\n", encoding="ascii") - (unreadable_class / "class").mkdir() - - hardware = classify_gpu_hardware(pci_devices) - - assert hardware is GpuHardware.UNKNOWN - - def test_classify_gpu_hardware_prefers_positive_amd_evidence_over_incomplete_sibling(tmp_path: Path) -> None: pci_devices = tmp_path / "devices" incomplete_device = pci_devices / "0000:00:02.0" diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index 677eb115..8e4ece63 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -107,29 +107,16 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] -def test_overlay_never_emits_gpu_access_contract() -> None: - text = emit_overlay( - _strix_halo_cfg(), - image_registry="ghcr.io/amdresearch", - image_tag="v1.0", - courses=CourseSelection.default(), - offline_mode=False, - ) - parsed = yaml.safe_load(text) - - assert "gpuAccess" not in parsed["custom"] - assert "renderGid" not in text - assert "supplementalGroups" not in text - - def test_overlay_keeps_gpu_resources_without_gpu_access_contract() -> None: - _, parsed = _render( + text, parsed = _render( _strix_halo_cfg(), courses=CourseSelection.default(), ) custom = parsed["custom"] assert "gpuAccess" not in custom + assert "renderGid" not in text + assert "supplementalGroups" not in text assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) assert "teams" not in custom From 31afcedf0a9e4f9162948984a5e2fbe6b2d09ec9 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:57:06 +0800 Subject: [PATCH 104/180] test(installer): consolidate GPU package safety coverage --- tests/installer/test_gpu_access.py | 14 +++-------- tests/installer/test_gpu_access_ordering.py | 27 +++++---------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index 159e6a83..ad3285a1 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -123,16 +123,11 @@ def fake_run(command: list[str], **_: object) -> SimpleNamespace: # Then: the exact Radeon URL is downloaded, verified, installed, and cleaned up. downloaded_path = Path(downloads[0][-1]) - assert downloads == [["wget", "-q", gpu_access.AMD_GPU_UDEV_PACKAGE_URL, "-O", str(downloaded_path)]] + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL assert verified == [downloaded_path] assert not downloaded_path.exists() - assert host.calls == [ - "installed-version", - f"install-package:{downloaded_path}", - "installed-version", - f"owns-rule:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", - f"read:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", - ] + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> None: @@ -146,7 +141,6 @@ def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> N provision_gpu_access(host) # Then: no download, install, legacy removal, or device probe is performed. - assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) assert not any(call in {"reload-udev", "trigger-udev", "settle-udev"} for call in host.calls) @@ -193,7 +187,6 @@ def test_symlinked_legacy_rule_fails_closed_before_installation(monkeypatch: pyt def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() -> None: # Given: the exact package verification constant. rules = AMD_GPU_UDEV_PACKAGE_RULES - old_pxe_shape = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' # When: its policy is inspected. # Then: it matches the extracted package rule rather than the former two-line PXE shape. @@ -201,7 +194,6 @@ def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() 'KERNEL=="kfd", GROUP="render", MODE="0666"\n' 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' ) - assert rules != old_pxe_shape assert "card" not in rules diff --git a/tests/installer/test_gpu_access_ordering.py b/tests/installer/test_gpu_access_ordering.py index 6363ffc8..93635798 100644 --- a/tests/installer/test_gpu_access_ordering.py +++ b/tests/installer/test_gpu_access_ordering.py @@ -114,29 +114,14 @@ def fail_install(deb: Path) -> None: assert "reload-udev" not in host.calls -def test_wrong_version_package_owned_differing_conffile_converges( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path +@pytest.mark.parametrize("installed_version", ["30.30.4.0-older", None]) +def test_package_owned_differing_conffile_converges( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, installed_version: str | None ) -> None: - # Given: a package-owned conffile from a different installed package version. - host = FakeGpuAccessHost( - files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, - installed_version="30.30.4.0-older", - package_owns_rule=True, - ) - monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) - - # When: the pinned package is installed from an offline bundle. - provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) - - # Then: forced installation replaces the differing conffile with the exact package rule. - assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES - assert not any(call.startswith("remove-rule:") for call in host.calls) - - -def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - # Given: a config-files package state that still owns a differing conffile. + # Given: a wrong-version or partial package state that owns a differing conffile. host = FakeGpuAccessHost( files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + installed_version=installed_version, package_owns_rule=True, ) monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) @@ -144,7 +129,7 @@ def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatc # When: the pinned package is installed from an offline bundle. provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) - # Then: ownership prevents legacy admission and the package converges to the exact rule. + # Then: forced installation converges to the exact package rule without legacy deletion. assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES assert not any(call.startswith("remove-rule:") for call in host.calls) From 5c1dbd971fa658802575b152a25b004e82fc3e67 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:58:07 +0800 Subject: [PATCH 105/180] test(deploy): trim generated config validation coverage --- .../skills/test_config_generation_security.py | 1 - tests/skills/test_deploy_scripts.py | 431 +----------------- .../test_direct_inventory_validation.py | 29 +- 3 files changed, 9 insertions(+), 452 deletions(-) diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py index adb6c282..66d5a49e 100644 --- a/tests/skills/test_config_generation_security.py +++ b/tests/skills/test_config_generation_security.py @@ -86,7 +86,6 @@ def test_generator_applies_the_normal_unknown_field_policy_to_draft_gpu_fields() "raw", [ '{"topology":"ssh-preinstalled","topology":"pxe-diskless"}', - '{"topology":"pxe-diskless","k3s_version":"v1.32.3+k3s1","server":{"name":"server","ip":"192.168.1.10"},"network":{"interface":"eno1","subnet":"192.168.1.0/24"},"pxe":{"authorized_keys":["ssh-ed25519 AAA"],"diskless_agents_have_amd_gpus":true,"diskless_agents_have_amd_gpus":false}}', ], ) def test_generator_rejects_duplicate_public_policy_keys_before_discovery(tmp_path: Path, raw: str) -> None: diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index c57f916d..c4901737 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -19,33 +19,7 @@ DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" VALIDATE = DEPLOY_SCRIPTS / "validate.py" GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" -CONFIG_GENERATION = DEPLOY_SCRIPTS / "config_generation.py" ARTIFACT_STORE = DEPLOY_SCRIPTS / "artifact_store.py" -VALUES_RESOLUTION_PARSING = DEPLOY_SCRIPTS / "values_resolution_parsing.py" - -EXPECTED_GENERATOR_SCHEMA = { - "topology": "pxe-diskless | ssh-preinstalled", - "k3s_version": "v1.32.3+k3s1", - "server": {"name": "aipc1", "ip": "192.168.0.140"}, - "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], - "network": { - "interface": "enp1s0", - "subnet": "192.168.0.0/24", - "gateway": "192.168.0.1", - "dns_servers": "8.8.8.8,8.8.4.4", - }, - "pxe": { - "authorized_keys": ["ssh-ed25519 AAAA... you@host"], - "rootfs_password": "", - "web_port": 8080, - "diskless_agents_have_amd_gpus": True, - }, - "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, - "storage": {"class": "nfs-client"}, - "proxy": {"node_port": 30890}, - "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, -} def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -278,43 +252,6 @@ def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: assert "AMD_Radeon_8060S_Graphics" in result.stdout -def test_values_resolution_parser_preserves_overlay_precedence_and_error_categories(tmp_path: Path) -> None: - parser = load_deploy_module("values_resolution_parsing", VALUES_RESOLUTION_PARSING) - repo = tmp_path / "checkout" - base = write_file( - repo / "base.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - partial_overlay = write_file( - repo / "partial.yaml", - """custom: - accelerators: - strix-halo: - displayName: Renamed -""", - ) - invalid_overlay = write_file(repo / "invalid.yaml", "custom: *defaults\n") - - result = parser.collect_effective_values( - repo, - [str(base), str(partial_overlay), "missing.yaml", str(invalid_overlay)], - ) - - assert result.accelerators == {"strix-halo": "AMD_Radeon_8060S_Graphics"} - assert result.metadata == {"gpu": ["strix-halo"]} - assert result.missing_files == ["values file not found: missing.yaml"] - assert result.parse_errors == ["unsupported YAML syntax at custom"] - - def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -608,69 +545,6 @@ def test_validator_uses_generated_pxe_vars_file_when_requested(tmp_path: Path) - assert "k3s_version == pxe_k3s_version" in result.stdout -def test_validator_preserves_explicit_selector_and_accelerator_key_clears(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - base = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - selector_clear = write_file( - repo / "selector-clear.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: null -""", - ) - keys_clear = write_file( - repo / "keys-clear.yaml", - """custom: - resources: - metadata: - gpu: - acceleratorKeys: ~ -""", - ) - - selector_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(selector_clear), - ) - keys_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(keys_clear), - ) - - assert selector_result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in selector_result.stdout - assert keys_result.returncode == 0, keys_result.stdout + keys_result.stderr - assert "no acceleratorKeys found" in keys_result.stdout - - def test_validator_honors_every_supported_explicit_clear_syntax(tmp_path: Path) -> None: repo = tmp_path / "checkout" base = write_file( @@ -764,36 +638,6 @@ def test_validator_main_resets_report_state_between_invocations(tmp_path: Path) assert second == 0 -def test_validator_requires_product_labels_under_active_accelerator_node_selectors(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - values = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - env: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(values), - ) - - assert result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout - - def test_validator_ignores_accelerators_and_metadata_outside_custom_resources(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -892,60 +736,6 @@ def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_ assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout -def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - inventory = write_file( - repo / "generated/inventory.yml", - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: false - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - ) - values = write_file( - repo / "generated/values-basic-example.yaml", - """custom: - resources: - metadata: {} -""", - ) - resolution = write_file( - repo / "generated/gpu-access-resolution.json", - json.dumps( - { - "version": 1, - "status": "cpu_only", - "hosts": {"agent": False, "server": False}, - } - ), - ) - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--inventory", - str(inventory), - "--values", - str(values), - "--gpu-resolution", - str(resolution), - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "GPU access artifacts agree" in result.stdout - - def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, resolution = write_resolved_gpu_artifacts(repo) @@ -1011,84 +801,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( assert expected_error in result.stdout -@pytest.mark.parametrize( - ("inventory_content", "expected_error"), - [ - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' must define exactly one auplc_gpu_access_enabled", - ), - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: yes - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' has malformed auplc_gpu_access_enabled", - ), - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: true - auplc_gpu_access_enabled: false - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' must define exactly one auplc_gpu_access_enabled", - ), - ], -) -def test_validator_rejects_missing_malformed_or_duplicate_inventory_host_booleans( - tmp_path: Path, inventory_content: str, expected_error: str -) -> None: - repo = tmp_path / "checkout" - inventory, values, resolution = write_resolved_gpu_artifacts(repo) - inventory.write_text(inventory_content, encoding="utf-8") - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--inventory", - str(inventory), - "--values", - str(values), - "--gpu-resolution", - str(resolution), - ) - - assert result.returncode == 1 - assert expected_error in result.stdout - - def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, _ = write_resolved_gpu_artifacts(repo) @@ -1409,56 +1121,27 @@ def fail_late_replace(source, destination): assert values_target.read_text(encoding="utf-8") == "old symlink target\n" -def test_artifact_store_rolls_back_destination_when_staged_unlink_fails( +def test_artifact_store_rolls_back_non_force_destination_after_post_link_fsync_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - module = load_deploy_module("deploy_artifact_store_unlink", ARTIFACT_STORE) - destination = tmp_path / "inventory.yml" - original_unlink = module.os.unlink - failed = False - - def fail_first_staged_unlink(path, *args, **kwargs): - nonlocal failed - if not failed and Path(path).name.startswith(".inventory.yml."): - failed = True - raise OSError("injected staged unlink failure") - return original_unlink(path, *args, **kwargs) - - monkeypatch.setattr(module.os, "unlink", fail_first_staged_unlink) - - with pytest.raises(SystemExit): - module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) - - assert not destination.exists() - - -@pytest.mark.parametrize("force", (False, True)) -def test_artifact_store_rolls_back_destination_when_parent_fsync_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, force: bool -) -> None: - module = load_deploy_module(f"deploy_artifact_store_fsync_{force}", ARTIFACT_STORE) + module = load_deploy_module("deploy_artifact_store_nonforce_fsync", ARTIFACT_STORE) destination = tmp_path / "inventory.yml" - if force: - destination.write_text("old inventory\n", encoding="utf-8") original_fsync_parent = module._fsync_parent calls = 0 - def fail_after_publication(path): + def fail_after_publication(path: Path) -> None: nonlocal calls calls += 1 - if calls == (2 if force else 1): + if calls == 1: raise OSError("injected parent fsync failure") - return original_fsync_parent(path) + original_fsync_parent(path) monkeypatch.setattr(module, "_fsync_parent", fail_after_publication) with pytest.raises(SystemExit): - module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=force) + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) - if force: - assert destination.read_text(encoding="utf-8") == "old inventory\n" - else: - assert not destination.exists() + assert not destination.exists() def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: @@ -1512,103 +1195,3 @@ def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: assert result.returncode == 0, result.stdout + result.stderr assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout - - -def test_generator_print_schema_is_byte_stable() -> None: - result = run_script(GEN_CONFIGS, "--print-schema") - - assert result.returncode == 0, result.stdout + result.stderr - assert result.stderr == "" - assert result.stdout == json.dumps(EXPECTED_GENERATOR_SCHEMA, indent=2) + "\n" - - -def test_generator_exits_with_usage_error_when_spec_is_omitted() -> None: - result = run_script(GEN_CONFIGS) - - assert result.returncode == 2 - assert result.stdout == "" - assert result.stderr == "gen_configs: --spec is required (or use --print-schema)\n" - - -def test_generator_replaces_colliding_artifacts_when_force_is_given(tmp_path: Path) -> None: - spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) - token_path = write_file(tmp_path / "token.txt", "characterization-token\n") - out_dir = tmp_path / "generated" - write_file(out_dir / "inventory.yml", "old inventory\n") - write_file(out_dir / "pb-pxe-controller.vars.yml", "old pxe vars\n") - write_file(out_dir / "values-basic-example.yaml", "old values\n") - - result = run_script( - GEN_CONFIGS, - "--spec", - str(spec_path), - "--out-dir", - str(out_dir), - "--token-file", - str(token_path), - "--force", - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "old inventory" not in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "old pxe vars" not in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "old values" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 - assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 - assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 - - -def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: - generation = load_deploy_module("deploy_config_generation", CONFIG_GENERATION) - artifacts = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) - - assert generation.SCHEMA == EXPECTED_GENERATOR_SCHEMA - assert generation.validate_spec(generator_spec()) == "ssh-preinstalled" - assert callable(generation.render_inventory) - assert callable(generation.render_pxe_vars) - assert callable(generation.render_values) - assert callable(artifacts.preflight_destinations) - assert callable(artifacts.publish_artifacts) - - -def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - fake_ansible = fake_bin / "ansible-playbook" - fake_ansible.write_text( - r"""#!/usr/bin/env python3 -import json -import pathlib -import sys -args = sys.argv[1:] -output = next(arg.split('=', 1)[1] for arg in args if arg.startswith('gpu_access_discovery_output_path=')) -def host(name, bdf): - return { - 'host': name, 'reachable': True, - 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, - } -pathlib.Path(output).write_text(json.dumps({'version': 1, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') -""", - encoding="utf-8", - ) - fake_ansible.chmod(0o755) - spec = generator_spec() - spec["agents"] = [{"name": "agent", "ip": "192.168.1.11"}] - spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) - out_dir = tmp_path / "generated" - result = subprocess.run( - [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir)], - capture_output=True, - check=False, - env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, - text=True, - ) - - assert result.returncode == 0, result.stdout + result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert inventory.count("auplc_gpu_access_enabled: true") == 1 - assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["hosts"] == {"agent": False, "server": True} diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py index 370f3e37..805ed26c 100644 --- a/tests/skills/test_direct_inventory_validation.py +++ b/tests/skills/test_direct_inventory_validation.py @@ -38,19 +38,6 @@ def valid_inventory() -> str: """ -def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - inventory = write(repo / "inventory.yml", valid_inventory()) - values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") - - result = run_validate( - "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "GPU access inventory is valid" in result.stdout - - def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory = write( @@ -90,7 +77,7 @@ def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path @pytest.mark.parametrize("value", ("auto", "true", "false")) -def test_validator_accepts_supported_direct_inventory_values(tmp_path: Path, value: str) -> None: +def test_validator_accepts_direct_inventory_values_without_resolution_manifest(tmp_path: Path, value: str) -> None: repo = tmp_path / "checkout" inventory = write(repo / "inventory.yml", valid_inventory().replace("true", value).replace("false", value)) values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") @@ -141,21 +128,9 @@ def test_validator_rejects_auto_when_inventory_is_cross_checked_with_gpu_resolut ("inventory_content", "expected_error"), [ (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "auto"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'auto'"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "true"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'true'"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "false"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'false'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: AUTO"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: TRUE"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: FALSE"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: no"), "malformed"), - ( - valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: malformed"), - "malformed", - ), ( valid_inventory().replace( " auplc_gpu_access_enabled: true\n", From 21e775aabf5b2a9a8be75b0976361f8aefd6f946 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:59:18 +0800 Subject: [PATCH 106/180] test(deploy): consolidate GPU artifact policy coverage --- tests/skills/test_gpu_access_resolution.py | 98 +++----------------- tests/skills/test_gpu_artifact_generation.py | 67 +------------ tests/skills/test_pxe_finalization.py | 82 ++++------------ 3 files changed, 36 insertions(+), 211 deletions(-) diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index fdd802ea..6cd29546 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -6,7 +6,6 @@ import importlib.util import json -import re import sys from pathlib import Path @@ -66,31 +65,12 @@ def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) -def test_discovery_playbook_preserves_lspci_agreement_and_exact_v1_host_evidence_shape() -> None: +def test_discovery_playbook_records_lspci_and_sysfs_evidence() -> None: playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") - evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( - " changed_when:", maxsplit=1 - )[0] - fallback_block = playbook.split("_auplc_gpu_access_unknown_evidence:", maxsplit=1)[1].split( - " pre_tasks:", maxsplit=1 - )[0] - evidence_keys = re.findall(r"^ ([a-z_]+):", evidence_block, re.MULTILINE) - fallback_keys = re.findall(r"^ ([a-z_]+):", fallback_block, re.MULTILINE) - - assert evidence_keys == ["host", "reachable", "lspci", "sysfs"] - assert fallback_keys == ["reachable", "lspci", "sysfs"] - assert "combine({'host': discovery_host})" in playbook - assert '{"version":1,"hosts":[' in playbook - assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook - assert "| to_json" in playbook - assert "name: gpu_access" in playbook - assert "tasks_from: detect" in playbook + assert "_auplc_discovery_lspci.rc" in playbook + assert "_auplc_discovery_lspci.stdout" in playbook assert "_auplc_gpu_access_sysfs.rc" in playbook assert "_auplc_gpu_access_sysfs.stdout" in playbook - assert "/sys/bus/pci/devices" not in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0300"]' in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0302"]' in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0380"]' in playbook def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: @@ -103,27 +83,11 @@ def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> Non assert evidence[0].sysfs.stdout == GPU_BDF -@pytest.mark.parametrize( - "replacement", - [ - {"version": True, "hosts": []}, - {"version": 1, "hosts": [], "unexpected": "field"}, - {"version": 1, "hosts": [{"host": "gpu-1"}]}, - {"version": 1, "hosts": [host_evidence("gpu-1", lspci_rc=True)]}, - ], -) -def test_parse_fleet_evidence_rejects_nonexact_or_boolean_integer_values(replacement: dict) -> None: +def test_parse_fleet_evidence_rejects_boolean_integer_values() -> None: module = load_resolution_module() with pytest.raises(module.EvidenceParseError): - module.parse_fleet_evidence(json.dumps(replacement)) - - -def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: - module = load_resolution_module() - - with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): - module.parse_fleet_evidence('{"version":1,"version":1,"hosts":[]}') + module.parse_fleet_evidence(json.dumps({"version": True, "hosts": []})) def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: @@ -146,17 +110,11 @@ def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() assert resolution.hosts[0].status is module.HostStatus.CPU -@pytest.mark.parametrize( - "evidence", - [ - host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"]), - host_evidence("host-1", lspci_bdfs=[GPU_BDF], lspci_rc=1), - host_evidence("host-1", lspci_bdfs=[GPU_BDF], reachable=False), - ], -) -def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: +def test_resolve_fleet_blocks_disagreeing_lspci_and_sysfs_evidence() -> None: module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(evidence)) + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"])) + ) resolution = module.resolve_fleet(expected_targets(module, "host-1"), parsed) @@ -164,22 +122,11 @@ def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: assert resolution.hosts[0].status is module.HostStatus.UNKNOWN -@pytest.mark.parametrize( - ("targets", "hosts"), - [ - (("gpu-1", "gpu-2"), ("gpu-1",)), - (("gpu-1",), ("gpu-1", "gpu-2")), - ], -) -def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( - targets: tuple[str, ...], hosts: tuple[str, ...] -) -> None: +def test_resolve_fleet_blocks_incomplete_host_evidence() -> None: module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(*(host_evidence(host, lspci_bdfs=[GPU_BDF]) for host in hosts)) - ) + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) - resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) assert resolution.status is module.FleetStatus.BLOCKED assert resolution.reason == "incomplete host coverage" @@ -218,18 +165,6 @@ def test_resolution_manifest_preserves_explicit_host_booleans() -> None: } -def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: - manifest = load_manifest_module().build_resolution_manifest( - status="gpu_resolved", - hosts={"zeta": True, "alpha": False}, - ) - - assert type(manifest) is dict - assert list(manifest) == ["version", "status", "hosts"] - assert list(manifest["hosts"]) == ["alpha", "zeta"] - assert set(manifest) == {"version", "status", "hosts"} - - def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: module = load_manifest_module() base = module.build_resolution_manifest( @@ -242,11 +177,6 @@ def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> gpu_access_enabled=True, ) - assert base == { - "version": 1, - "status": "gpu_resolved", - "hosts": {"gpu-1": True, "gpu-2": True}, - } - assert list(manifest) == ["version", "status", "hosts", "pxe_rootfs"] + assert base["hosts"] == {"gpu-1": True, "gpu-2": True} + assert "pxe_rootfs" not in base assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} - assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled"} diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index fe629147..25bec235 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -35,7 +35,7 @@ def ssh_spec() -> dict: } -def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> Path: +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> None: fake_bin = tmp_path / "bin" fake_bin.mkdir() fake_ansible = fake_bin / "ansible-playbook" @@ -60,11 +60,9 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document encoding="utf-8", ) fake_ansible.chmod(0o755) - record = tmp_path / "ansible-argv.json" - monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(record)) + monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(tmp_path / "ansible-argv.json")) monkeypatch.setenv("FAKE_ANSIBLE_EVIDENCE", json.dumps(document)) monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") - return record def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: @@ -141,7 +139,7 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - record = write_fake_ansible( + write_fake_ansible( tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, @@ -154,71 +152,12 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "gpu_resolved", "hosts": {"agent": False, "server": True}, } - discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" - discovery_evidence = out_dir / ".gpu-access-discovery-evidence.json" - assert discovery_inventory.stat().st_mode & 0o777 == 0o600 - assert discovery_evidence.stat().st_mode & 0o777 == 0o600 - assert json.loads(record.read_text(encoding="utf-8")) == [ - "-i", - str(discovery_inventory), - str(ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml"), - "-e", - f"gpu_access_discovery_output_path={discovery_evidence}", - ] - - -def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible( - tmp_path, - monkeypatch, - {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent", gpu=True)]}, - ) - out_dir = tmp_path / "generated" - - result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) - - assert result.returncode == 0, result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert inventory.count("auplc_gpu_access_enabled: true") == 2 - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in values - assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} - - -def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible( - tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} - ) - out_dir = tmp_path / "generated" - - result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) - - assert result.returncode == 0, result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert inventory.count("auplc_gpu_access_enabled: false") == 2 - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { - "version": 1, - "status": "cpu_only", - "hosts": {"agent": False, "server": False}, - } @pytest.mark.parametrize("failure", ["missing", "nonzero"]) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index dac1fb0a..06313b0a 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -74,60 +74,28 @@ def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: ) -def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: - return ( - out_dir / "inventory.yml", - out_dir / "pb-pxe-controller.vars.yml", - out_dir / "values-basic-example.yaml", - out_dir / "gpu-access-resolution.json", - ) - - -def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("policy", [(True, "true"), (False, "false")]) +def test_pxe_agents_publish_explicit_boolean_rootfs_policy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, policy: tuple[bool, str] ) -> None: + gpu_agents, expected_policy = policy write_fake_ansible(tmp_path, monkeypatch) out_dir = tmp_path / "generated" - result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) + result = run_generator( + "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(gpu_agents))), "--out-dir", str(out_dir) + ) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "auplc_render_gid" not in inventory - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "gpuAccess" not in values - assert "pxe_gpu_access_enabled: true" in pxe_vars - assert "pxe_gpu_access_enabled: auto" not in pxe_vars - assert manifest == { - "version": 1, - "status": "cpu_only", - "hosts": {"controller": False}, - "pxe_rootfs": {"gpu_access_enabled": True}, - } - assert not list(out_dir.glob(".pxe-finalizer-*")) + assert "auplc_render_gid" not in inventory + pxe_vars + assert f"pxe_gpu_access_enabled: {expected_policy}" in pxe_vars + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": gpu_agents} assert "do-not-print-this-secret" not in result.stdout + result.stderr -def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - out_dir = tmp_path / "generated" - - result = run_generator( - "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(False))), "--out-dir", str(out_dir) - ) - - assert result.returncode == 0, result.stderr - pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert "pxe_gpu_access_enabled: false" in pxe_vars - assert "pxe_gpu_access_enabled: auto" not in pxe_vars - assert "auplc_render_gid" not in pxe_vars - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} - - def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -140,30 +108,10 @@ def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "auplc_gpu_access_enabled: true" in inventory - assert "auplc_gpu_access_enabled: auto" not in inventory - pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest["status"] == "gpu_resolved" assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} -def test_pxe_generator_refuses_existing_canonical_artifacts_without_force( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - out_dir = tmp_path / "generated" - out_dir.mkdir() - existing = out_dir / "values-basic-example.yaml" - existing.write_text("existing\n", encoding="utf-8") - - result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "refusing to overwrite existing" in result.stderr - assert existing.read_text(encoding="utf-8") == "existing\n" - assert all(not path.exists() for path in canonical_artifacts(out_dir) if path != existing) - - def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -188,4 +136,12 @@ def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 - assert all(not path.exists() for path in canonical_artifacts(out_dir)) + assert not any( + (out_dir / name).exists() + for name in ( + "inventory.yml", + "pb-pxe-controller.vars.yml", + "values-basic-example.yaml", + "gpu-access-resolution.json", + ) + ) From 7259d4e71b2d68cfc14a739ffa47a03838f3a986 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:00:31 +0800 Subject: [PATCH 107/180] test(ansible): narrow GPU role contract coverage --- tests/skills/test_gpu_access_role.py | 345 +++++++++------------------ 1 file changed, 108 insertions(+), 237 deletions(-) diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index f67ec67d..f6efb872 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,5 +1,4 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - """Contract tests for AMD's packaged GPU udev rules in Ansible.""" from pathlib import Path @@ -14,8 +13,7 @@ PACKAGE = "amdgpu-insecure-instinct-udev-rules" VERSION = "30.30.4.0-2341068.24.04" -FILENAME = f"{PACKAGE}_{VERSION}_all.deb" -URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{FILENAME}" +URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{PACKAGE}_{VERSION}_all.deb" SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" RULE_PATH = "/etc/udev/rules.d/70-amdgpu.rules" RULE_CONTENT = ( @@ -27,277 +25,150 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: - defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") +def test_gpu_access_role_enforces_pinned_package_contract() -> None: + defaults = yaml.safe_load(read(GPU_ACCESS_ROLE / "defaults" / "main.yml")) apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - assert PACKAGE in defaults - assert VERSION in defaults - assert FILENAME in defaults - assert URL in defaults - assert f"sha256:{SHA256}" in defaults - assert RULE_PATH in defaults - assert " " + RULE_CONTENT.replace("\n", "\n ").rstrip() in defaults - assert "dpkg-query" in preflight - assert r"--showformat=${Status}\t${Version}" in preflight - assert "ansible.builtin.get_url" in apply - assert "ansible.builtin.apt" in apply - assert 'checksum: "{{ auplc_gpu_udev_package_checksum }}"' in apply - assert 'deb: "{{ auplc_gpu_udev_package_cache_path }}"' in apply - assert "dpkg-query" in verify - assert r"--showformat=${Status}\t${Version}" in verify - assert "--search" in verify - assert "package-owned" in verify - assert "modified package conffile" in verify + assert [ + defaults[key] + for key in ( + "auplc_gpu_udev_package_name", + "auplc_gpu_udev_package_version", + "auplc_gpu_udev_package_url", + "auplc_gpu_udev_package_checksum", + "auplc_gpu_udev_rule_path", + "auplc_gpu_udev_rule_content", + ) + ] == [PACKAGE, VERSION, URL, f"sha256:{SHA256}", RULE_PATH, RULE_CONTENT] + assert all(token in apply for token in ("ansible.builtin.get_url", "ansible.builtin.apt", "checksum:", "deb:")) + assert all( + token in verify + for token in ( + "dpkg-query", + r"--showformat=${Status}\t${Version}", + "--search", + "install ok installed", + "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path", + "(_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content", + ) + ) -def test_gpu_access_defaults_and_inventory_placeholders_use_unquoted_auto() -> None: - defaults_text = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - defaults = yaml.safe_load(defaults_text) - inventory_text = read(ANSIBLE / "inventory.yml") - inventory = yaml.safe_load(inventory_text) - raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) - hosts = inventory["k3s_cluster"]["children"] - raw_hosts = raw_inventory["k3s_cluster"]["children"] +def test_gpu_access_defaults_and_inventory_leave_auto_unquoted() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + inventory = read(ANSIBLE / "inventory.yml") - assert defaults["auplc_gpu_access_enabled"] == "auto" - assert "auplc_gpu_access_enabled: auto" in defaults_text - assert inventory_text.count("auplc_gpu_access_enabled: auto") == 2 - assert all( - quoted not in inventory_text - for quoted in ('auplc_gpu_access_enabled: "auto"', "auplc_gpu_access_enabled: 'auto'") - ) - for group_name in ("server", "agent"): - for host_name, host in hosts[group_name]["hosts"].items(): - value = host["auplc_gpu_access_enabled"] - assert value == raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] == "auto" + assert "auplc_gpu_access_enabled: auto" in defaults + assert inventory.count("auplc_gpu_access_enabled: auto") == 2 + assert 'auplc_gpu_access_enabled: "auto"' not in inventory + assert "auplc_gpu_access_enabled: 'auto'" not in inventory -def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: +def test_gpu_access_rootfs_and_legacy_cleanup_remain_contained_and_verified() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "realpath" in validation - assert "auplc_rootfs_path != '/'" in validation - assert "_auplc_canonical_allowed_root" in validation - assert "Inspect GPU access rootfs target" in preflight - assert "follow: false" in preflight - assert "Reject unsafe AMD udev rule destination parents" in preflight - assert "Reject unsafe AMD udev rule destination" in preflight - assert "Define recognized project-owned legacy GPU rules" in preflight - assert "hash('sha256')" in preflight - assert "70-kfd.rules" in preflight - assert "70-rocm-devices.rules" in preflight - assert "70-auplc-gpu-access.rules" not in preflight - assert "Reject unexpected legacy GPU rule content" in preflight - assert "Recheck recognized project-owned legacy GPU rules before apply" in apply - assert "Remove recognized project-owned legacy GPU rules" in apply - assert apply.index("Download checksummed AMD udev package") < apply.index( - "Remove recognized project-owned legacy GPU rules" + assert all( + token in validation + for token in ( + "realpath", + "auplc_rootfs_path != '/'", + "_auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/')", + ) ) - assert apply.index("Verify installed AMD udev package") < apply.index( - "Remove recognized project-owned legacy GPU rules" + assert all( + token in preflight + for token in ( + "follow: false", + "_auplc_legacy_gpu_rules", + "hash('sha256')", + "70-kfd.rules", + "70-rocm-devices.rules", + ) ) - assert "Reload live udev rules after legacy cleanup" in apply - assert "Trigger live udev rules after legacy cleanup" in apply - - -def test_gpu_access_role_skips_package_cache_and_download_when_exact_version_is_installed() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert "_auplc_gpu_udev_install_needed" in preflight - assert "Install AMD udev package when required" in apply - install_block = apply.split("Install AMD udev package when required", maxsplit=1)[1] - assert "Create deterministic AMD udev package cache" in install_block - assert "Download checksummed AMD udev package" in install_block - assert "when: _auplc_gpu_udev_install_needed | bool" in install_block - assert "Verify installed AMD udev package without installation" in apply - assert "install ok installed" in preflight - - -def test_gpu_access_role_requires_installed_status_and_exact_version() -> None: - verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - - assert "install ok installed" in verify - assert "Require installed AMD udev package status and exact version" in verify - - -def test_preflight_allows_package_owned_wrong_version_rule_for_convergence() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Query AMD udev rule package ownership on live host before admission" in preflight - assert "Query AMD udev rule package ownership in PXE rootfs before admission" in preflight - assert preflight.index("Query installed AMD udev package") < preflight.index("Read existing AMD udev rule") - assert preflight.index("Query AMD udev rule package ownership") < preflight.index("Read existing AMD udev rule") - assert "_auplc_gpu_udev_install_needed | bool" in preflight - assert "_auplc_rule_owned_by_amd_package | bool" in preflight - - -def test_preflight_allows_package_owned_partial_state_rule_for_convergence() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Record whether AMD udev package installation is needed" in preflight - assert "Allow package-owned AMD udev rule convergence" in preflight - assert "install ok installed" in preflight - - -def test_preflight_rejects_unknown_unowned_rule_content() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + assert apply.index("ansible.builtin.import_tasks: verify.yml") < apply.rindex("state: absent") + assert apply.index("item.content | b64decode") < apply.rindex("state: absent") - assert "Reject modified AMD udev rule before package installation" in preflight - assert "_auplc_rule_owned_by_amd_package | bool" in preflight - assert "Existing AMD udev rule is neither the package rule nor a recognized legacy rule." in preflight - -def test_preflight_legacy_admission_matches_package_owned_convergence_admission() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - primary_admission = preflight.split("Allow package-owned AMD udev rule convergence", maxsplit=1)[1].split( - "Reject modified AMD udev rule before package installation", maxsplit=1 - )[0] - legacy_admission = preflight.split("Reject unexpected legacy GPU rule content", maxsplit=1)[1].split( - "fail_msg:", maxsplit=1 - )[0] - - assert "_auplc_gpu_udev_install_needed | bool" in primary_admission - assert "_auplc_rule_owned_by_amd_package | bool" in primary_admission - assert "_auplc_gpu_udev_install_needed | bool" in legacy_admission - assert "_auplc_rule_owned_by_amd_package | bool" in legacy_admission - assert "auplc_gpu_udev_rule_path" in legacy_admission - assert "auplc_gpu_udev_rule_content" in legacy_admission - - -def test_gpu_access_role_installs_the_package_without_custom_rule_or_device_probes() -> None: - apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert not (GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2").exists() - assert "ansible.builtin.template" not in apply - assert "70-auplc-gpu-access.rules.j2" not in apply - assert "udevadm settle" not in apply - assert "/dev/kfd" not in apply - assert "/dev/dri" not in apply - assert "/sys/class/drm" not in apply - assert "card" not in apply - - -def test_gpu_access_role_verifies_exact_installed_package_version_and_rule() -> None: - verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - - assert "auplc_gpu_udev_package_version" in verify - assert "auplc_gpu_udev_rule_path" in verify - assert "auplc_gpu_udev_rule_content" in verify - assert "Require installed AMD udev package status and exact version" in verify - assert "Require package-owned AMD udev rule" in verify - assert "Require exact AMD udev rule content" in verify - assert "follow: false" in verify - - -def test_pxe_gpu_access_uses_safe_chroot_install_and_strict_retained_admission() -> None: +def test_pxe_gpu_access_chroots_without_bind_mounts_and_rejects_unsafe_retained_rules() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") tasks = read(PXE_GPU_ACCESS_TASKS) apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main - assert "Re-preflight PXE GPU rootfs before TFTP" in main - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Stop NFS before rootfs rebuild" + assert main.index("pxe_gpu_admission_phase: retained-read-only") < main.index("rm -rf {{ pxe_nfs_root }}") + assert main.index("pxe_gpu_admission_phase: final") < main.index("ls {{ pxe_nfs_root }}/boot/vmlinuz-") + assert all( + token in tasks + for token in ( + "tasks_from: verify", + "tasks_from: preflight", + "tasks_from: apply", + 'auplc_rootfs_path: "{{ pxe_nfs_root }}"', + 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"', + "auplc_reject_legacy_gpu_rules: true", + ) ) - assert "tasks_from: verify" in tasks - assert "tasks_from: preflight" in tasks - assert "tasks_from: apply" in tasks - assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks - assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks - assert "auplc_reject_legacy_gpu_rules: true" in tasks - assert "Reject retained PXE shipped legacy GPU rules" in verify + assert "not item.stat.exists" in verify assert "chroot" in apply assert "apt-get" in apply - assert "Copy AMD udev package into PXE rootfs" in apply - assert "Mount virtual filesystems for AMD udev package installation" not in apply assert "mount --bind" not in apply - assert "Unmount virtual filesystems after AMD udev package installation" not in apply - assert apply.index("Verify installed AMD udev package") < apply.index( - "Remove temporary AMD udev package from PXE rootfs" - ) - assert main.index("Re-preflight PXE GPU rootfs before TFTP") < main.index("Find latest kernel in rootfs") - assert RULE_CONTENT not in tasks - assert "/dev/kfd" not in tasks - assert "/dev/dri" not in tasks -def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> None: +def test_pxe_unmounts_only_when_present_and_propagates_failures() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - rootfs_removal = main.split("Remove existing rootfs (force rebuild)", maxsplit=1)[1].split( - "Check if NFS rootfs already exists", maxsplit=1 - )[0] - chroot_unmount = main.split("Unmount virtual filesystems from chroot", maxsplit=1)[1].split( - "Remove chroot setup script", maxsplit=1 - )[0] - for task in (rootfs_removal, chroot_unmount): - assert "set -e" in task - assert "if mountpoint -q" in task - assert "&& umount" not in task - assert "|| true" not in task + assert main.count("set -e") == 2 + for mount in ("dev", "sys", "proc"): + assert main.count("if mountpoint -q {{ pxe_nfs_root }}/" + mount + "; then") == 2 + assert main.count("umount {{ pxe_nfs_root }}/" + mount) == 2 + assert "&& umount" not in main + assert "|| true" not in main -def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: +def test_gpu_access_resolves_before_preflight_rocm_and_apply_fail_fatally() -> None: role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") - resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") - detect = read(GPU_ACCESS_ROLE / "tasks" / "detect.yml") - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + rocm = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev = read(ANSIBLE / "playbooks" / "pb-udev.yml") - assert "ansible.builtin.import_tasks: resolve.yml" in role_main - assert "auplc_gpu_access_enabled | bool" not in role_main - assert "when: _auplc_gpu_access_enabled_resolved" in role_main - assert "python3" in detect - assert "/sys/bus/pci/devices" in detect - assert "0x1002" in detect - assert "startswith('0x03')" in detect - assert "sorted(" in detect - assert "register: _auplc_gpu_access_sysfs" in detect - assert "changed_when: false" in detect - assert "failed_when: false" in detect - assert "auplc_gpu_access_enabled is boolean" in resolve - assert "auplc_gpu_access_enabled == 'auto'" in resolve - assert "ansible.builtin.import_tasks: detect.yml" in resolve - assert "_auplc_gpu_access_sysfs.rc == 0" in resolve - assert "_auplc_gpu_access_sysfs.stdout | trim | length > 0" in resolve - assert "_auplc_gpu_access_enabled_resolved is boolean" in resolve - assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") - assert "any_errors_fatal: true" in rocm_playbook - assert "any_errors_fatal: true" in udev_playbook - for playbook in (rocm_playbook, udev_playbook): - assert "tasks_from: resolve" in playbook + assert role_main.index("import_tasks: resolve.yml") < role_main.index("import_tasks: preflight.yml") + assert role_main.index("import_tasks: preflight.yml") < role_main.index("import_tasks: apply.yml") + for playbook in (rocm, udev): + assert "any_errors_fatal: true" in playbook assert playbook.index("tasks_from: resolve") < playbook.index("tasks_from: preflight") - assert "auplc_gpu_access_enabled | bool" not in playbook + assert playbook.index("tasks_from: preflight") < playbook.index("tasks_from: apply") assert "when: _auplc_gpu_access_enabled_resolved" in playbook - assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") - assert "tasks_from: apply" in rocm_playbook - assert "tasks_from: preflight" in udev_playbook - assert "tasks_from: apply" in udev_playbook - assert "render_gid" not in pxe_playbook + assert rocm.index("tasks_from: preflight") < rocm.index("- role: rocm") < rocm.index("tasks_from: apply") -def test_deploy_ansible_has_no_obsolete_gpu_access_policy_or_state_contract() -> None: - forbidden = ( - "auplc_render_gid", - "auplc_normalize_render_gid", - "gpu-access.json", - "auplc_from_json_strict", - "groupmod", - "render GID collision", +def test_gpu_access_auto_detection_requires_successful_boolean_resolution_before_preflight() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") + + assert "auplc_gpu_access_enabled == 'auto'" in resolve + assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") + assert resolve.index("_auplc_gpu_access_sysfs.rc == 0") < resolve.index("_auplc_gpu_access_enabled_resolved: >-") + assert resolve.index("_auplc_gpu_access_enabled_resolved: >-") < resolve.index( + "_auplc_gpu_access_enabled_resolved is boolean" ) - ansible_text = "\n".join( - path.read_text(encoding="utf-8") - for path in ANSIBLE.rglob("*") - if path.is_file() and "__pycache__" not in path.parts + assert role_main.index("ansible.builtin.import_tasks: resolve.yml") < role_main.index( + "ansible.builtin.import_tasks: preflight.yml" ) - for term in forbidden: - assert term not in ansible_text + +def test_gpu_access_rejects_unknown_unowned_udev_content_before_deletion() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + admission = preflight.split("_auplc_rule_content_admitted: >-", maxsplit=1)[1] + cleanup = apply.split("register: _auplc_apply_legacy_gpu_rule_contents", maxsplit=1)[1] + + assert "(_auplc_rule_owned_by_amd_package | bool)" in admission + assert "that: _auplc_rule_content_admitted | bool" in admission + assert role_main.index("ansible.builtin.import_tasks: preflight.yml") < role_main.index( + "ansible.builtin.import_tasks: apply.yml" + ) + assert cleanup.index("item.content | b64decode") < cleanup.index("state: absent") + assert "in item.item.item.sha256" in cleanup From 8f1bf458b34453fc14d89862dca23b54d99583c2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:01:45 +0800 Subject: [PATCH 108/180] test(image): focus GPU permission ownership contract --- tests/scripts/test_gpu_image_permissions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py index b07047cc..b8885a4b 100644 --- a/tests/scripts/test_gpu_image_permissions.py +++ b/tests/scripts/test_gpu_image_permissions.py @@ -20,9 +20,3 @@ def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: ) for pattern in forbidden_patterns: assert re.search(pattern, dockerfile) is None, pattern - - assert "echo 'export USER=jovyan' >> /entrypoint.sh" in dockerfile - assert "echo 'export SHELL=/bin/bash' >> /entrypoint.sh" in dockerfile - assert 'CMD ["/bin/bash", "/entrypoint.sh"]' in dockerfile - assert "USER $NB_UID" in dockerfile - assert "WORKDIR /home/jovyan" in dockerfile From db2d73e1b33df844eb4529e41b21a86688de95a5 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:54:55 +0800 Subject: [PATCH 109/180] fix(ansible): repair GPU access preflight admission --- .../roles/gpu_access/tasks/preflight.yml | 3 ++- tests/skills/test_gpu_access_role.py | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml index 59810a95..dede942c 100644 --- a/deploy/ansible/roles/gpu_access/tasks/preflight.yml +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -46,6 +46,7 @@ sha256: - 678b6a1084576de785b47fcfa0c0b3048117a3add62c1fb8dcff83947004005b - cc5e78a7861477ac5169a4b84edd4e687c1f14b9a88a9557b0c986479ebbaccd + - a9782dc222d43fdeaa4df0dfb0cfa6898ff973309f6affb1327cda4b1e63f347 - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" sha256: - 951fb3d879d2d45b56cfd4cdb0f7ea061a4a0af77d93b9f2a4da9a8c36d20cad @@ -158,7 +159,7 @@ ((_auplc_existing_rule.content | b64decode) == auplc_gpu_udev_rule_content) or ((_auplc_gpu_udev_install_needed | bool) and (((_auplc_existing_rule.content | b64decode) | hash('sha256')) in _auplc_legacy_gpu_rules[1].sha256 or - (_auplc_rule_owned_by_amd_package | bool)) }} + (_auplc_rule_owned_by_amd_package | bool))) }} - name: Reject modified AMD udev rule before package installation ansible.builtin.assert: diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index f6efb872..3b5c7a26 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,6 +1,7 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. """Contract tests for AMD's packaged GPU udev rules in Ansible.""" +import hashlib from pathlib import Path import yaml @@ -19,6 +20,9 @@ RULE_CONTENT = ( 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' ) +SHC_LEGACY_RULE_CONTENT = ( + 'KERNEL=="kfd", GROUP="render", MODE="0660"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) def read(path: Path) -> str: @@ -172,3 +176,21 @@ def test_gpu_access_rejects_unknown_unowned_udev_content_before_deletion() -> No ) assert cleanup.index("item.content | b64decode") < cleanup.index("state: absent") assert "in item.item.item.sha256" in cleanup + + +def test_gpu_access_rule_admission_expression_has_balanced_parentheses() -> None: + tasks = yaml.safe_load(read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml")) + admission_task = next(task for task in tasks if task["name"] == "Allow package-owned AMD udev rule convergence") + expression = admission_task["ansible.builtin.set_fact"]["_auplc_rule_content_admitted"] + + assert expression.count("(") == expression.count(")") + + +def test_gpu_access_admits_shc_legacy_render_group_rule() -> None: + tasks = yaml.safe_load(read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml")) + legacy_task = next(task for task in tasks if task["name"] == "Define recognized project-owned legacy GPU rules") + rules = legacy_task["ansible.builtin.set_fact"]["_auplc_legacy_gpu_rules"] + amdgpu_rule = next(rule for rule in rules if rule["path"].endswith("/etc/udev/rules.d/70-amdgpu.rules")) + shc_rule_sha256 = hashlib.sha256(SHC_LEGACY_RULE_CONTENT.encode()).hexdigest() + + assert shc_rule_sha256 in amdgpu_rule["sha256"] From 3e5779b383d2cb655101202d1a6e2ed4cfcfa3e7 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:09:56 +0800 Subject: [PATCH 110/180] test(ansible): generalize legacy rule naming --- tests/skills/test_gpu_access_role.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index 3b5c7a26..ababb790 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -20,7 +20,7 @@ RULE_CONTENT = ( 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' ) -SHC_LEGACY_RULE_CONTENT = ( +LEGACY_RENDER_GROUP_RULE_CONTENT = ( 'KERNEL=="kfd", GROUP="render", MODE="0660"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' ) @@ -186,11 +186,11 @@ def test_gpu_access_rule_admission_expression_has_balanced_parentheses() -> None assert expression.count("(") == expression.count(")") -def test_gpu_access_admits_shc_legacy_render_group_rule() -> None: +def test_gpu_access_admits_legacy_render_group_rule() -> None: tasks = yaml.safe_load(read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml")) legacy_task = next(task for task in tasks if task["name"] == "Define recognized project-owned legacy GPU rules") rules = legacy_task["ansible.builtin.set_fact"]["_auplc_legacy_gpu_rules"] amdgpu_rule = next(rule for rule in rules if rule["path"].endswith("/etc/udev/rules.d/70-amdgpu.rules")) - shc_rule_sha256 = hashlib.sha256(SHC_LEGACY_RULE_CONTENT.encode()).hexdigest() + legacy_rule_sha256 = hashlib.sha256(LEGACY_RENDER_GROUP_RULE_CONTENT.encode()).hexdigest() - assert shc_rule_sha256 in amdgpu_rule["sha256"] + assert legacy_rule_sha256 in amdgpu_rule["sha256"] From 9b2416ae7735738809f4f84c944d450ef18da564 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:09:56 +0800 Subject: [PATCH 111/180] docs(deploy): remove environment-specific references --- deploy/README.md | 8 ++++---- skills/deploy-aup-learning-cloud/reference.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 29dd70f4..0a7d0e1c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -148,7 +148,7 @@ This host permission policy is separate from Kubernetes allocation. The AMD device plugin remains the visibility boundary: only Pods that request `amd.com/gpu` receive allocated GPU devices, and the plugin does not change host inode ownership or mode. AUPLC Hub adds no GPU supplemental group. The -tested ROCm compute path needs none: on both SHC GPU nodes, `rocminfo` succeeded +tested ROCm compute path needs none: on representative GPU nodes, `rocminfo` succeeded as UID `12345` with only supplemental GID `100`, while card nodes remained inaccessible at mode `0660`. The reported agents were `gfx1151` and `gfx1200`. @@ -218,6 +218,6 @@ rule. Rebuild or correct a retained rootfs separately if that safety check fails ## Deployment branch boundary This branch and these instructions do not modify or roll out any live -deployment. SHC, FET, and other deployment branches or environments must -backport the host permission and immediate artifact publication changes before -their own reviewed rollout. +deployment. Environment-specific deployment branches must backport the host +permission and immediate artifact publication changes before their own reviewed +rollout. diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index b1da7733..b1df45e6 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -53,8 +53,8 @@ validation must finish before Ansible or Helm changes are made. ownership or mode. - `singleuser.fsGid: 100` controls shared storage ownership only. -Operator evidence from SHC showed `rocminfo` reporting `gfx1151` and `gfx1200` -on the two GPU nodes from UID `12345` Pods with only supplemental GID `100`. +Operator evidence from representative GPU nodes showed `rocminfo` reporting +`gfx1151` and `gfx1200` from UID `12345` Pods with only supplemental GID `100`. Their `card*` nodes remained inaccessible at mode `0660`. The infrastructure owner deploys and maintains the AMD device plugin and ROCm From 6d12b4051e65d57a146a4e842cec429130923947 Mon Sep 17 00:00:00 2001 From: Sonya <195730002+sonyyang-tw@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:56:07 +0800 Subject: [PATCH 112/180] fix(physim): align Genesis ROCm runtime Upgrade the course image to Genesis 1.3.1 while ensuring Quadrants and PyTorch share one HIP runtime for stable zero-copy interop. Co-authored-by: Cursor <cursoragent@cursor.com> --- dockerfiles/Courses/PhySim/Dockerfile | 38 ++++++++++----------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/dockerfiles/Courses/PhySim/Dockerfile b/dockerfiles/Courses/PhySim/Dockerfile index 69d123bf..198362ee 100644 --- a/dockerfiles/Courses/PhySim/Dockerfile +++ b/dockerfiles/Courses/PhySim/Dockerfile @@ -17,41 +17,31 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Use a verified ROCm PyTorch image as base ARG BASE_IMAGE=ghcr.io/amdresearch/auplc-base:latest FROM ${BASE_IMAGE} +ARG GENESIS_WORLD_VERSION=1.3.1 + +ENV ROCM_PYTHON_LIB=/opt/rocm-python/lib +ENV LD_LIBRARY_PATH="${ROCM_PYTHON_LIB}:${LD_LIBRARY_PATH}" + USER root -# Vulkan backend and GUI +RUN SDK_LIB=$(python3 -c "import _rocm_sdk_core, os; print(os.path.join(os.path.dirname(_rocm_sdk_core.__file__), 'lib'))") && \ + mkdir -p /opt/rocm-python && \ + ln -s "${SDK_LIB}" "${ROCM_PYTHON_LIB}" && \ + ln -sf libamdhip64.so.7 "${SDK_LIB}/libamdhip64.so" && \ + ln -sf /opt/rocm/lib/llvm/bin/ld.lld /usr/local/bin/ld.lld + RUN apt-get update && apt-get install -y --no-install-recommends \ - curl wget git ca-certificates locales \ ffmpeg \ libgl1 libglx-mesa0 libgl1-mesa-dri libegl1 libgbm1 libglib2.0-0 \ - libvulkan1 mesa-vulkan-drivers vulkan-tools \ - && rm -rf /var/lib/apt/lists/* - -RUN ln -sf /opt/rocm/lib/llvm/bin/ld.lld /usr/local/bin/ld.lld + && rm -rf /var/lib/apt/lists/* -# IMPORTANT: must remain under USER root here. pip install as jovyan silently -# falls back to ~/.local/, which gets masked by the PVC mount at runtime. - -# Pinning to <2.4 restores the expected behaviour. RUN pip3 install --no-cache-dir \ "numpy>=1.26.4,<2.4" \ - "numba>=0.61" - -# Install Genesis related packages -RUN pip3 install --no-cache-dir \ - loguru \ - omegaconf \ - gstaichi \ - "genesis-world==0.4.6" + "genesis-world==${GENESIS_WORLD_VERSION}" -# Copy related rocm notebooks into the docker -RUN mkdir -p /opt/workspace/PhySim -COPY ./course_data /opt/workspace/PhySim +COPY --chown=jovyan:1000 ./course_data /opt/workspace/PhySim -USER root -RUN chown -R jovyan:1000 /opt/workspace USER jovyan WORKDIR /opt/workspace/PhySim From 21d6aba3d097094c3f6a77edc287bb1d76d80cbc Mon Sep 17 00:00:00 2001 From: Sonya <195730002+sonyyang-tw@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:56:13 +0800 Subject: [PATCH 113/180] fix(physim): update notebooks for Genesis 1.3 Adapt camera recording calls to the current Genesis API and remove stale execution output from the course notebooks. Co-authored-by: Cursor <cursoragent@cursor.com> --- projects/PhySim/PhySim01_hello_genesis.ipynb | 4 +- .../PhySim/PhySim02_control_your_robot.ipynb | 117 ++------------ .../PhySim/PhySim03_motion_planning.ipynb | 146 +++--------------- .../PhySim/PhySim04_parallel_simulation.ipynb | 94 ++--------- 4 files changed, 47 insertions(+), 314 deletions(-) diff --git a/projects/PhySim/PhySim01_hello_genesis.ipynb b/projects/PhySim/PhySim01_hello_genesis.ipynb index 7b1eb956..8b90bf45 100644 --- a/projects/PhySim/PhySim01_hello_genesis.ipynb +++ b/projects/PhySim/PhySim01_hello_genesis.ipynb @@ -230,13 +230,13 @@ "source": [ "# render rgb, depth, segmentation, normal\n", "rgb, depth, segmentation, normal = cam.render(rgb=True, depth=True, segmentation=True, normal=True)\n", - "cam.start_recording()\n", + "cam.start_recording(save_to_filename=\"Videos/video_01.mp4\", fps=60)\n", "\n", "for _ in range(100):\n", " scene.step()\n", " cam.render()\n", "\n", - "cam.stop_recording(save_to_filename=\"Videos/video_01.mp4\", fps=60)" + "cam.stop_recording()" ] }, { diff --git a/projects/PhySim/PhySim02_control_your_robot.ipynb b/projects/PhySim/PhySim02_control_your_robot.ipynb index 51895392..d7c5cf9a 100644 --- a/projects/PhySim/PhySim02_control_your_robot.ipynb +++ b/projects/PhySim/PhySim02_control_your_robot.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "77699ce4-db80-4b47-bfa2-bbbda26ab3f0", "metadata": {}, "outputs": [], @@ -66,21 +66,7 @@ "execution_count": null, "id": "f16475de", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] \u001b[38;5;23m╭───────────────────────────────────────────────╮\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] \u001b[38;5;23m│┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈\u001b[0m\u001b[38;5;17m \u001b[38;5;23m\u001b[1m\u001b[3mGenesis\u001b[0m\u001b[38;5;17m \u001b[38;5;23m┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈│\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] \u001b[38;5;23m╰───────────────────────────────────────────────╯\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] Consider setting 'performance_mode=True' in production to maximise runtime speed, if significantly increasing compilation time is not a concern.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] Running on \u001b[38;5;23m\u001b[4m[AMD Radeon Graphics]\u001b[0m\u001b[38;5;17m with backend \u001b[38;5;23m\u001b[4mgs.vulkan\u001b[0m\u001b[38;5;17m. Device memory: \u001b[38;5;23m\u001b[4m60.75\u001b[0m\u001b[38;5;17m GB.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] 🚀 Genesis initialized. 🔖 version: \u001b[38;5;23m\u001b[4m0.3.3\u001b[0m\u001b[38;5;17m, 🌱 seed: \u001b[38;5;23m\u001b[4mNone\u001b[0m\u001b[38;5;17m, 📏 precision: '\u001b[38;5;23m\u001b[4m32\u001b[0m\u001b[38;5;17m', 🐛 debug: \u001b[38;5;23m\u001b[4mFalse\u001b[0m\u001b[38;5;17m, 🎨 theme: '\u001b[38;5;23m\u001b[4mlight\u001b[0m\u001b[38;5;17m'.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [09:57:03] [INFO] Scene \u001b[38;5;23m\u001b[3m<bb06885>\u001b[0m\u001b[38;5;17m created.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "import genesis as gs\n", "import numpy as np\n", @@ -115,38 +101,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "0eaa9cca", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [10:02:08] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m0\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<1a7f5ec>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.Plane>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:02:08] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m1\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<84432a7>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.MJCF(file='/opt/conda/envs/py_3.12/lib/python3.12/site-packages/genesis/assets/xml/franka_emika_panda/panda.xml')>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint1`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint1`.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint2`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint2`.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:02:09] [INFO] Applying offset to base link's pose with user provided value in morph.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:02:09] [INFO] Building scene \u001b[38;5;23m\u001b[3m<bb06885>\u001b[0m\u001b[38;5;17m...\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] Reference robot position exceeds joint limits.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:02:09] [WARNING] Constraint solver time constant should be greater than 2*substep_dt. timeconst is changed from `0.005` to `0.02`). Decrease simulation timestep or increase timeconst to avoid altering the original value.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:02:10] [INFO] Compiling simulation kernels...\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:02:14] [INFO] Building visualizer...\u001b[0m\n", - "Successfully built the scene.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "amdgpu: os_same_file_description couldn't determine if two DRM fds reference the same file description.\n", - "If they do, bad things may happen!\n" - ] - } - ], + "outputs": [], "source": [ "########################## entities ##########################\n", "plane = scene.add_entity(\n", @@ -171,11 +129,6 @@ ] }, { - "attachments": { - "267b2468-7d14-45fd-a20d-3eb514f2c857.png": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABBAAAAF1CAIAAADeBo7pAAAAAXNSR0IArs4c6QAAIABJREFUeAHsvQdcVGe+/+/+9v/733tXo0lUirvZ3dzdze7du5tsysYUG1Y6AvaK0vtQpYOxa+woKApE6U1RqqAgIL0qRYr0mQGmMMP0cs75ZebRk5EAUWwDfh/nNZ459Xne5zBzPufbZhDQgAAQAAJAAAgAASAABIAAEAAC4xCYMc58mA0EgAAQAAJAAAgAASAABIAAECBAMMBFAASAABAAAkAACAABIAAEgMC4BEAwjIsGFgABIAAEgAAQAAJAAAgAASAAggGuASAABIAAEAACQAAIAAEgAATGJQCCYVw0sAAIAAEgAASAABAAAkAACAABEAxwDQABIAAEgAAQAAJAAAgAASAwLgEQDOOigQVAAAgAASAABIAAEAACQAAIgGCAawAIAAEgAASAABAAAkAACACBcQmAYBgXDSwAAkAACAABIAAEgAAQAAJAAAQDXANAAAgAASAABIAAEAACQAAIjEsABMO4aGABEAACQAAIAAEgAASAABAAAiAY4BoAAkAACAABIAAEgAAQAAJAYFwCIBjGRQMLgAAQAAJAAAgAASAABIAAEADBANcAEAACQAAIAAEgAASAABAAAuMSAMEwLhpYAASAABAAAkAACAABIAAEgAAIBrgGgAAQAAJAAAgAASAABIAAEBiXAAiGcdHAAiAABIAAEAACQAAIAAEgAARAMMA1AASAABAAAkAACAABIAAEgMC4BEAwjIsGFgABIAAEgAAQAAJAAAgAASAAggGuASAABIAAEAACQAAIAAEgAATGJQCCYVw0sAAIAAEgAASAABAAAkAACAABEAxwDQABIAAEgAAQAAJAYDoQWLeO2Lz5qYFUVxNz5hAPHjw1Ez4AgeclAILheYnB+kAACAABIAAEgAAQUEcCEwsGiUTRZ6mUYDIJmUwd+w99UlsCIBjU9tRAx4AAEAAC04fA0NBQaWlpeXl5dXV1TU1NnUqrqamprKwsLS2l0+nTZ8AwEiDwJgiMJxiamwmhkNi4kbh0idi0ifjiC2L5cqK29k10EY45NQmAYJia5w16DQSAABCYOgTkcvnly5e3bt1qZ2dHoVA8PT19fHz8/Px8fX337Nnj7u5uo2w1NTU4jk+dYUFPgYDaEZhAMPD5xAcfEJ99RpSVEd3dhJkZsXAhIRKp3RCgQ+pJAASDep4X6BUQAAJAYPoQePjwoYuLi42NjYeHh6+vb2BgYHBwcEhISFBQkJ+fn6enp6Wl5eHDh3k83vQZM4wECLwJAuMJhqYmQiAg/vhH4uDBx90qKiLeeYfo6HgTvYRjTkECIBim4EmDLgMBIAAEpg4BqVR66dIlCwsLFxcXb29vf3//oKCgYGULCAjw8fFxdna2srK6c+cOmBemzlmFnqopgXXriC1bnupbTY0i6BkJhg8/JJKTHy998EAxv6rqqZXhAxAYjwAIhvHIwHwgAASAABB4CQSam5sdHR2tra3d3d19fHyQWhhlXjh48CCLxXoJB4NdAIG3m4CFBaGr+xSC/HzivfeI3l6Czyc+/JCIiXm8tK6OmD2baGh4amX4AATGIwCCYTwyMB8IAAEgAARelACGYZcvX96+fbuzszNpXggJCQkODg4ICNizZ4+Li4u1tXVBQQGYF16UNWwPBAgiNpZ4912iru4xC7GYMDZWxDfLZASXq3BJcnB4vCghgdDUJAYHgRoQeCYCIBieCROsBASAABAAApMg0N/f7+PjY2lp6e7u7ufnFxAQEKJsQUFBvr6+np6eNjY2pHkBNMMkCMMmQECVgFhMWFsrLAkWFgpt8PnnxD//Sdy/r1iFwyE++ohYupTw8yOOHiV+/3vCx0d1U5gGAhMRAMEwER1YBgSAABB4BQRwjMBwghgzHxCaP+aiV9CTV7tLHMdTU1N37tzp5OTk7e0dEBCAQheCgoL8/f2RecHGxgbMC6/2NMDe3z4C+fnEvn2Evz8RHa0ouYAah0P86U9EQoLi5eensEWgsgxvHx4Y8WQIgGCYDDXYBggAASDwIgQwqRhPPIuf9CDOeI964ae98e/d5DejcLn0RQ6hDtv29fV5eHgg8wKZHCk4OBglR/Ly8rKysjp8+PDw8LA69Bb6AASmNwEOR+GSlJ4+vUcJo3tVBEAwvCqysF8gAASAwJgEcILARDx8w6fEf/+K+MuMn14fzSD+PgP/3xn4n2dgLsa4eMonSE9KStqxYwcyL6DkSCh6ITAw0MfHx9XVFaIXxrxCYCYQeBUEOBxFHYZr117FvmGf058ACIbpf45hhEAACKgbAUzCww87EFZLCZvlj1+2K3D7VfLl72H/moH9bQZGMcHFQnXr9nP1p6enx8PDY9euXSg5kmrtBX9/fy8vLxsbm0OHDkFypOeiCisDgUkTkMmI8nKCwZj0DmDDt5oACIa3+vTD4IEAEHgjBHBcjon4mHCEEPIULxEfl0vkqeHYN7Owf8zAlmtgd69P9QjgrKys7du3Ozo6enl5keaFkJAQZF5wcXGxs7OD6IU3cvnBQYEAEAACz0sABMPzEoP1gQAQAAIvSkBOYBiOYcq458dRzrnxmM572D9nYMvexwtSMRwn8Ckc+TwyMnL06NFdu3a5ubn5+Pgg8wKKXiDNCxC98KKXEWwPBIAAEHhdBEAwvC7ScBwgAASAwBMCuCJDkuKFEQrZgDVWY8vm4v+cQXzxayIvGZ/qxgWCKCoqsrGxcXBw8PLy8vPzIwVDYGCgr6+vq6urra1tYWHh1B/okzMK/wMBIAAEpjUBEAzT+vTC4IAAEFBvAooA6N52bPdS/JNfEZ/MwA45Y3KZend5ot4hAcDhcA4ePEiaF0ZlU/X29raxsTly5AibzZ5oX7AMCAABIAAE1IYACAa1ORXQESAABN4+AnjfI3z3t8Q/fkV8PAM/YKfInjT1IRQVFVlZWdnb23t5ealmU1U1L+Tl5YF5YeqfahjB1CMwMjKFH0lMPdzTqMcgGKbRyYShAAEgMDUIKOwKCockwQjurIf981fyj2fIgy3kQi6m6P9jyaBYQXFPjaFoBhyXT4nBcTicw4cP79y5k0Kh+Pj4IPNCSEgIKtbm5eVlZ2fn5+dHp9OnxHCgk0BgOhGgUoWmppXNzSPTaVAwltdDAATD6+EMRwECQAAIPCaAwhcwTI6FBeJf/Fr2vzOIDf+U97fLCRzDFPIAradQCwqVgMtlYjkmU0ZIKwWFeoOsrq62tra2t7f39PT08/MLCgoKUTaUHIlCoVhbW1+/fh1TxG5AAwJA4LUScPd4oKVdvHVrlUQCf4Cvlfw0OBgIhmlwEmEIQAAITDECuFypFj7/v/J/zMA3/BNrrZMrtQGuCIN+3BTTIr488ZzcfiXuZY6X5qq/D49EIjl//ryFhQWFQtmzZ88o84K3t7ednZ2/v39/f/+TUcL/QAAIvCYCd+4M/fZ3tzQ0szU0b8XE9L6mo8JhpgsBEAzT5UzCOIAAEJgiBBQWhqvfE5/9Gv94htzkz1htASYV4sIRXMDDBVxcOEIIRnDhCEYQ8vTL+Mf/BzP+M/7te/iaD/CuJtJhST3HWldXZ29vb2tri6IXkHkhODh4lHlB/ZWPeuKFXgGBSRPgcmXLV5RoauVpaGZpaN769NM7ff1TuzTkpFHAhpMjAIJhctxgKyAABIDACxAI3E787//B/zUDM/4zYb9abq1DWC8jbBQv3HoZbrlU5rAK72nFHHSxRe9gXc1Ychj2918RsaeUWVhf4LivZlMkACQSSWho6I4dO1xdXVXNC8HBwQEBAd7e3vb29n5+fn19fYpADWV7Nd2BvQIBIDCawIGDDzW18pVqIUtDM0tT67ab+/3RK8FnIDA+ARAM47OBJUAACACBV0TAfxvxl/+D/2MG9j8zsI9myP86A/9oBv4XxYv4q3Lik/8Py0vAl8/Ftn0uE45g9cX4wv/AnXQJuTqGPiPBUF9fb2tjY2tr6+HhgaIXgpUtMDDQz8/Pzc1NNXoBjAyv6MqC3QKBnxOoreX895/yNTRzSMGgoZn92w9uFRYyfr4yzAECYxIAwTAmFpgJBIAAEHiVBPKTiVNexFlfItQPC/VTTvgq3pUv/KwvcWk/XnEb+3Ymvu3fmGBEXl+Mf/UfuOUSQiZ9ld2a/L5lMlno2bM7tm93cXb29vYmoxeCg4P9/f2RecHf359Go6FjgGCYPGvYEgg8DwGhUG5mVqFqXkCyQVMrb82aewKBOj6DeJ7xwbqviQAIhtcEGg4DBIAAEHguAnjHA/zr3xDbPseEPHn9PfnC/8Cs1VcwdHd3u7u7W1tbe3h4kLUXUDZVX19fCoViY2OTnp4OOuG5rgFYGQi8OIGIiO4noQsKZyTVl6ZW3sWL3S9+CNjD20AABMPbcJZhjEAACExBAo8e4N/MxLd+hgu48voibOH/T1BM1NMlCcOw5ORkS0tL5/HNC0FBQVQqdQqeBugyEJjCBDra+f/z93wNzVxVnfDTtFbuJ5/c6e8TTeERQtdfFwEQDK+LNBwHCAABIPA8BHABF9v1Nfblf2KNZVjkIfyvM4jUCyppV59nX69sXWQx6O3t9fT0VDUvhISEoORIqtELYF54ZecBdgwExiAglxO7d9f83BnpJ8GgiH7O9/FpGmNjmAUEniYAguFpHvAJCAABIKAeBHCCkN9OlS78T3zxu/Iv/q9s3d8waqd6dG10L5KSknbu3EmaF8hsqig5koODQ0BAADIvgGYYzQ4+A4FXRiA1laqlnassvPCUJ5KqYNDQzPn9H25VVbFfWS9gx9OEAAiGaXIiYRhAAAhMMwIKwSAXY4XXsL27seNu2MNq9bzb7u3t9fLysrS0dHd3HxW9gMwLtra2EL0wzS5OGI76E6DTxP/+slBTaxxnJJVgBk2t29t31Mhlj2vMq//QoIdvhAAIhqewo9TgcrkcTajnz/NTPYYPQAAITFMCOIEpX4Si0JviheE/lYFWozEnJiZaWFg4OTl5e3v7+/sHBQWhbKoBAQF79uxxcHAICgqi0+lq1GPoChB4CwgcPdauqXX7aWPCeHaG7AW/zb17F1KsvgWXxQsMEQTDU/AwZSMIAsMwJBueWgwfgAAQAAJAQIUAm80OCQnZvXu3u7u7j48PyqYaHBwcFBTk5+fn7u6+a9eu6OhomUymshFMAgEg8MoJnDrVMXH0gqqW0NTKD7/Q9cr7BAeYygTedsEgUjah8HGBdCQYWlpahoeHMQybwMIgEAjkallBaSpfjdB3IAAEphiBvLy8Xbt2OTo6kuYFFO6MzAuOjo5OTk7379+f4Lt0ig0YugsEpgiBhw95f/ww9+libeNZGHL+8MfclpaRKTIy6OabIfC2C4aCggJ9ff27d++q4i8tLR0aGlKdQ6VSpVIp8lPCMCwmJubkyZMsFgt+BVUpwTQQAAJvFQEWixUcHIzMC76+vmSxNmRe8PDwsLS0DA8PF4kgaeNbdV3AYNWFwDGFV1K+hmbe+HHPOZpaeVrahf7+zTiEMKjLeVPTfrzVggHHcTabvXv3bqlUSqfTk5KSSktLMQy7d+8eg8EoKCgoLCzMzMykUqkbN24sLi5G5/Dy5ctHjx5F0xiGqemJhW4BASAwLQi0tbWNen7xZoel+pQkNzfXwsLCwcFhPPOCi4vLgwcPVDd5s52HowOBt4qAVIpFRHQvWlysqZU9pmb4y1/zjE0qzod1joyA0+BbdWlMZrBvtWAgCILFYtnb2yM33B9/mB0cHGpra11dXe/fv29lZVVRUWFjY/Pw4UMXFxcUtCcQCAwNDSMjI/fv38/hcOCHcDIXHWwDBIDAsxGQSCQGBgY/FjRAHpJq9YXDZDKDg4MtLCzc3Nz27NkTGBgYHBwcEhISGBjo5+fn4eGxe/fuiIgIiUTybGOFtYAAEHglBAQCeXx83x8/vPW0Zsj+3Qc5JSXMV3JI2Ol0JDDlBQPpJkSeHTSH/DjxBI/Hs7e3b2xs3L17N0EQYWFhsbGxBw4caGtrCw4O5vF4FAqlrq7Ox8eHz+cTBEGj0Xbu3EkQREhISGpq6sQ7h6VAAAgAgRchEBsX9+tf/3rBggW1tbXP9c32Igd9xm0LCwt37dpFmhdQZqTg4OCAgAAfHx8HBwcXFxeIXnhGmLAaEHilBPr7RX/9W97T8QzZv/9Dbl/f4wDOV3p02Pn0IDDlBcOLnIaWlpZz584FBQVxuVwLC4v79++fOXOmubnZ2dm5tLTUzs6us7Nz+/btpaWlFAqlqakJwzCxWOzt7V1fX3/69OmampoXOTpsCwSAABCYgMDQ0NDnn3/+q1/9asaMGdu2bROL1SgSQCwWh4aGWlpaUigUHx8f0ryAohc8PT0tLS0jIiLEYvEEA4RFQAAIvB4Czc0jv//DqApu2R98kNPZKXg9HYCjTAMCU1swiGUiJovR0dFRW1tXXFx879692tranp4eqVT6LOdmaGjowoULPT09BEF0dHRcuXKlvr5eIBCEhYWlp6eHhobm5eWFhoaWl5cXFBTU1NQgfwAajRYTE1NWVqZW7gHPMl5YBwgAgSlE4MCBA3/5y1/+9a9/LVq0SFtbOyUlRX2+c+pqa+3t7R0cHDw9Pf38/FBpZ+SPtGfPHicnJ4hemEJXGnR12hNoaOBoLxgVxpD9299lP3qkcJ2ABgSehYC6C4ZRVngcx2k0WmFh4dWYHw4fOer/nZ+Hj5uPj8/hA0cuXLhw7fq14uLitrY2Ho8Hab+f5fTDOkAACKgngaampg8//PDMmTNGRkYBAQEODg5fLVxIpVLVobdisfjcuXO7du1SNS+g2gu+vr4eHh5WVlaXLl2C5EjqcLKgD0CAIIjaWs7TAQxZGprZ2guyOzpAMMAF8qwE1FowkI/TRkZGioqKwsLC9u/f7+Li4u3tffr0qbS067XVdb29vQKBgMxWhOO4UChkMplDQ0MQbPesVwGsBwSAgJoROHbsmL6+PoPBMDY29vX17enpWbhwYUZGBvmt+Ab7W1dX5+DgYGdnp2peIKMXnJycKBRKY2OjOnT1DVKCQwMB9SFQVTWsqXVLtVKbhma2plZWOwgG9TlJat8TNRUM5C+NQCC8kX4T1RCNiooqKSnp6+sTjeXLi+O4XC4XiUQjIyMMBqO/v5/BYChrqymsFMoToSjEppzGCQK9lElRIfew2l+m0EEg8LYR6Ovro/b3EwRhbm7u4uKCYRiVSuVyuW+cg0gkCg0NtbCwoFAoe/bsGbP2wqVLlyB64Y2fKegAECAJlJezNbXyfiYYslvbeOQ6MAEEJiag1oKhvr7B29t37979tbX1v/jzg+O4TCZDgoHJZPb391OpVA6HgynqMcs6+4YGWByCkOGYFMMxnMBwHCNwDMMxOVQrmfgagaVAAAi8RgJPHnA8PiQSDOpTV76x8YGdnZ2trS0yLwQGBqLSzoGBgb6+vsi8gGovjBrIa0QIhwICQOApAiUlrJ8LBg3NrIcPQTA8BQo+TEBAfQVDUVGRu7t7RkamUKjIDfLEODDuWEjBwOPxmEwmlUrt7+9nsViNDxo5HOaxyMJ953MIQi6TYVyeUCASyeRSnJAqTQ3j7hMWAAEgAATeLAEzMzMXFxe5XP5m77/lcrlYLB4eHo6Ojt61a5erqytpXkDRC/7+/p6entbW1pcvXxYKFbka32yH3+xZg6MDAbUicOfO0JiCoanpzRst1QoUdGYCAmoqGDgczu3bt7u6ugQCgVAolEqlqG7RBCNRFQxsNptKpfb29rLZ7JamptaOR5u9khduj3I6lL5jT7IZJW69R+Jm70S7falnYovvlLeyOGRmsZ/8k5C9guzAqEMjAQO/iKOwwEcgAAReLgFTU1MkGF7ubifeG47jKIs0h8NBHp5tbW11dXWJiYm2yubh4UEmRyKjF5ydnR0cHOrq6uCLcWK8sBQIvGYCeXmDYwqGB40gGF7zqZjCh1M7wYB+aTgcztDQEIPBGB4eRoLhF3+BxhQMQwyGWMivbOxYZBG31LXwa7ucJc6FS1xKlrgWLXYp/Nbx1leWGQst4vWcYo9dKmhq65fJJLTe3txbORcuXAgJCfHy8nJzc/Pw8PD39w8NDc3MzOzs7EQB1iAYpvBVD10HAlOHQGJSYm5u7i9+Ab7ggDAMk0gkfD5/eHh4YIDe3d3d3t7e1NRUU1NTpWwVFRV5eXmBgYHbt293cXFRNS8EBwf7+fkh88KJEyc4HM4LdgY2BwJA4OUSyMoa+JlgyNLQzKpvgL/Wl0t6Ou9N7QQDQRAymayvr6+/v5/JZLJYLIFAIJPJfvH3khQM6DePRqN1d3fT6QNiEe9cfMmXu9JXepSt9Li3yrNspWf5cvdSHfdiHfei5e73VniWL3MtXmiZscQy1tI3coe1/cbNmzZu3Lh58+YtW7Zs2rRp8+bN69evNzU1Xbt27ZYtW0JCQkpKSlCpBzI703S+RmBsQAAIvCECv/i9N7l+oW9LpBCQA2dXV1dTU1N9fX1tbW1VVVVlZWWFsqHp6urqysrKK1euWFpa2traenh4+Pr6omJtZDZVFxcXBweHe/fu/aL76OT6DFsBASAwaQLpN+hjCoa6ehAMk4b61m2oXoIB/TryeLyuri46nf7gwYMbN260tbU9S4JU9BMoFouRYKDT6T09PXQ6ncVmbfWK/deOVB2Xuys8K5a55H9jnbxkV9xKx8TVdgk61nFLrZO/tb251CV/BaVsoXXWlxvP6m9w3rZ9y7atW7dv27Z169YtW7Zs3Lhx/fr15ubmJiYmenp6+vr6Pj4+Dx48mFrXC3IzwJSNtJCQv+6qtybkUuQJRn5UXWdqjR16CwSmHAHyb/Ol9BzHcZFINDw8PDQ01N3d/fDhw6ampoaGhurq6oqKikplQ/IAmRTQe3V1dVVVVXV1dUFBQWBg4I4dO5B5wd/fP1jZgoKCAgICvL29bWxsTpw4MTIy8lJ6CzsBAkDgJRJISaWCYHiJPN/OXamjYGCxWJ2dnTQa7f79+9nZ2Y8ePXoWCwOGYVKpVCQS8Xg8NptNp9N7e3tpNBqDycgqfBB4MsvENfFri9hPNsessbv8oK17iDkyyOL2D7DCIpNWrLP7Zl3AV1vCv7G+ruNcvHB3ss6GPVu27ti+bfuWLVs2b968cePGdevWmZqaGhkZ6evr6+npLV++3MjIKDIyks9X1D2ZEnfSqolWSCWAuGEYNiqqEt2sqM5EeuPt/DuBUQOBN0VAoGwveHQcx6uqqvLy8mpqapC5ABkQKioqkEggBcMozYAEQ01NzbVr12yUDUUvjEqO5OrqamdnV1paOiW+CV8QJmwOBKYcgZQUEAxT7qSpXYfVSzCgO28qldrV1UWlUplMJofDEQqFzyIY5HK5RCIRCoUcDofJZNJotJ6ent7eXsbQEE/IlUqEDDanvKHzdGzJVq+Y6wUPBDx2V3d7eHjoOnNzMyMjMzMTk/WbV6+z/3bj4a+trn1jeX3J+kMbt23fvk1hYTA3N0dSQVdXd82aNatXr165cuXy5cuXLVvm4uLS3t6udid2rA5hGHb37t0TJ07U1taSyzEMu3Llyii3Y/LRJoZhPB4vNDQ0Li4OhZ6TG8IEEAACr4HAnj17/P39VdX+JA7KGBo6fvx4YGDgzZs3kcWA1Amq9gQ0XVlZqTqzurq6vLz89OnTu3fvVjUvhISEBAcHo+RINjY2x48fJ79GQDZM4hzBJkDg1REAwfDq2L49e1Y7wYBhWE9PT3d3N41GGxoaekbBgKq2IX8kJBioVGqPstFpVCabxRsRiBSZ/uQEQfT09eUVlN4tKPT23rNGd42+voGRsbGp6VpzM7ON6802bNpusMFl4Zaz/7ZIWrb+iPmmjSZGJps2bXJ2dnZzc7OzszM3N1+5cuUyZdPR0Vm0aJGZmdmdO3fQRfPGfymRHQB/XJlu9JXc0tKiq6vL5XKlUmlPT49MJiMIgslkSqVSPp/PZDK5XC6DwThz5gyDwUBjSU1NTUpKam5uVrU2jN4vfAYCQODVEMjPzy8oKJjcFwu51d27d93c3FxdXb/77ruMjAxkZ1BVBeNNV1ZW1tTUZGRkODs7W1tbj0qOFBgY6OPj4+LiYmNjk5aWhsytU8Xi+mpOF+wVCKgjARAM6nhWplqf1Esw4DgulUo7OztVBcOzBD2TEc88Ho8UDN3d3V1dXSh4mscbEYlEfD6/ubm5rra2oqLcy8tLT09PV1dXX1/f0NBw7dq1ZmZm69ev27Jly45tWzdts1y6cd/fzM9/YxqSkpZG7e/nKhuDwWhubk5ISLCzs1uxYsVSZVuyZMmKFSvCwsIEgsfpWcnf6dd/PeAKUYThmLKI9c8OT6PRbGxsJBJJeHh4TEzM3r17fzTjUCiUH1PQuri4xMfHe3h4VFZWrl+/vqurC2mPw4cPb9mypaKigiAICPL+GVGYAQReFYEX/xpBexAIBBERERQKxd3dnUKhIM2AfI3G0wnk/GplO336tIWFhZOTE0qOFBQURJoXvLy87OzsgoODi4qKWlpa1KEW9as6H7BfIDBlCSQng0vSlD15atNx9RIMBEGIRKKOjo6enh7SwvCMgoEMYCAFQ2dn56NHj3p6ehgMBo/HGxkZuX//fm1tbU1Njb+/v7Gx8cqVK1evXq2np2dgYGBiYmJubr5hw/qtWzfv3Llzt8VOc/P19n6hn6w7eyq2RHm+cLky8yASHj09PVFRUebm5qRm+Oabb1xdXVtbW9HJffEf+8ldJGK5jM5gK4/+U00Jcld0Ot3W1rasrOzHDCcEQdjZ2ZWUlLi7uw8ODnp6enK5XBcXl6qqqh+jGdFjQjSK5uZmBwcHkUj0pgZF9h8mgAAQeF4CdXV1/v7+bm5u7sqmqhl+UTbU1tbm5eVRKBRLS0t3d3fV5EiotDOFQrGzs4uNja2pqamsrGxububxoHbs854iWB8IvFoCIBheLd+3Y+9qJxgEAkF7e/vjeGUGg8PhCAQCqVQ68a2bQwhZAAAgAElEQVQqitwVCoWqFoaOjo62trauri5Uz6G1tbWurq6+vv7EiRMbNmxAcQhIMBgaGiLBsHHjxu3bt+3atct07doLF8Jlcll8Tu3npidzyx4SBCHHMblMJpZIBALByMjI8PBwaWmpk5PT0qVLlyjbN998Y2xsHB8fr2pqIFMMvdIrCsMxgsCEEmnA2cy4jCqCGMPCwGaz6+vrnZ2da2pqdu/eTRBESEhIfX29u7s7m8328fHhcrnOzs4VFRU+Pj4oYkGubARBoOqtE5+FVzpA2DkQeDsJMBiMoaGhSY9dKBRGR0e7uroiteDu7q7qm/SLgqGiouL48eM7d+50cnLy8vJCyZGQeQElR7KzswsKCiouLkYVGyoqKpqbmyFX0qTPF2wIBF4FgRcXDFJCJsAFXJzLJtgsgsUhOFycy5FzuHIuX86XEQr3ZmjTm4DaCYaRkZFJCwZ0H09aGDo6OlpaWjo6OgYHB1taWsrKylCuj+3btxsbG+vo6KxYsWLVqlW6urpIMKxbt27Tps0WFhYbNmzw8fERiUTKcy+/mlGzcld4Z/8ggWMyuVQikaBgCS6XOzw8/OjRo4MHD65cuXLJkiXLli1bsmTJt99+a21tnZubS3r0vh6nXvoQ23Ffxl/XXoi7UaPs+WgLQ3l5ubW1dX5+vlwuP3ny5Llz59LS0qhU6oYNG9LT07dt25aVlbVhw4bc3NyAgIDa2locx5lM5sGDB7Oysrq7u8ElaXp/F8Do1JNAYGCgs7PzpIOe29vbg4KCkD+SqmagUCj79u3LyckhS7ORbkjkRG1tbX5+vqOj4+7du93d3X18fFDthZCQEFXzQlxcHBIeKNUS0gxgZ1DPywl69XYSSErqHzOtam3duHUYuDi3QdoQL0g4xD3kMOxoSjdb1LP4Hw//+ae6P/+h7MMP7/7pT/kf/Tnnr3/N+ftnt75YU6jnUOF4sPFQWt+1XkGfFJO+nZyn96jVTjAMDw9PTjBInjz4VxUMTU1NSDbcuXOnuLi4tLTUz8/P3Nx81apVpGBYs2YNcklav379li1btm/f/mOZtoaGBmRSUAYDYIcu5jvuS8EwTKbMxSQSiQQCAY/HQ5phYGAgOjra2NgYuSctXbr022+/XbJkiYWFxZUrV1pbW5+ljsSkrzOFBYMgGtto5u6xC60zl9rnGbhcib5WxeYq8r0qtMrjN8X/QkXkN4HCl1FKE6lUOvKk8Xg8Pp8vEokkEolMJkOpV7lcLp/PR2HlEMOAkMI7EHhtBGxtbdevXz85wSCXyxMTE3+uFtyUjUKhnDhxorCwcDw7Q01NTVFRUUhIiL29vbe3t5+fX3BwsGr0gr29fUhISFFREUrVipQGkg1NTU2gGV7bRQIHAgITE0hKGjuGYZRg4GLccnH5Ke5pC5bF5/TPf9/7B81OrfnNGu9XzJuT9/471+bMSpozM2H2zDjlK372zPjZio8Js2cmzZ6ZMmdW6pyZ8bO1U36rV2B4vPnEQ67CNQPatCGgdoKByWQiwUCn08m0qhKJZOJbVZRTlbQwsFgsGo3W0dHR2NjY3NxcVFSUl5dXUlKSlpa2detWMzMzHR2d5cuXkxYGfX19ExOTdevWoaLOP+YxROmDlLmGFOdaJJHt8o/LuNtMEJhIIhKJxAIBH8VFcDgcNpvNZDLz8vIsLS11dHRQAiXS2qCrq+vg4BAWFnbv3r2BgYFRlw6ZwHTU/Gf8iBMYjmN9NIae1eWv7HJWelas8Li31PnulxYpZq7xJdXtBIFjmByTyzBMkSEKGhAAAlOLgKurq6mp6eQEw8DAwMGDB11cXEjbwqgJNze3U6dOIc0wKpsquvuvrq6+du2an5+ft7e3amlnPz8/CoXi6OgYFxc3po0C4hmm1mUGvZ3eBBITx7YwPKhXPENkY+wsYbY72+Mb+re/7f+d1sACzS6t9yvmz8l5b1bynJmxs2fGKF+xsxXTSC2M944kRPLsmcmzNZK1NxdvzaXekuNw+zEdri+1EwwMBuPFBQOTyaRSqR0dHQ8ePKiqqsrOzs7Pzy8uLj5+/LiZouCCyc8Fg7GxsZmZ2aZNm1AQgmrILzrP5XWda52iGKwRmUQoEAr5SgvDyMgIR9nYytbU1LR//35DQ0O0f6RJli9fvmTJksWLFy9btszU1NTLyys6Orquro7MWU5eR5OJEMBxOSbHMHl4culnOxNWuZet8ihZ5Vm2wrN8keOdryyuXIi/J5VLCBwJhtFOSuShYQIIAAH1JODp6WlkZPRcggF9k+A4np+f7+XlRYY7j1ILKJ4BaYaCgoIx7/urqqoqKytTUlIOHz4cpGyo9oK3t7etra23t3dhYeGYG5J2huHh4cl8s6nnyYBeAYGpSWBMwaClkRtdVRjI8/837UstqrY2Y4Fml/bcqvmzs95TGA1URcITzfCbq7N/c1VFPKjMf2p90gSRMntW/Jy1hWaFA4VTkxz0+icCaicYBgcH29raent7J21hGB4eJgVDfX397du3MzIy8vLyCgoK3N3dzc3NDQwMxhQMpqamPzojmZqa1tfXjxIMuByXyCUWPolHom5jckwg4JOCgcvlcjic4eFhNpvNYrGoVGp2dra7u7uhoeHKlStXKdtKZUOF3hYvXvztt98uX75806ZNgYGBqampE+QV+eUfWpyQY3JlcIUs+FzOQotUHUrRUufbq9zLV7qX6biVfLEj4WjUXQyXEiDxf7rsYQoITBkCkxAMaGwMBuP7779XDXf+uWBAmoFCoZw5c6akpGRM36RqZbtz584PP/xw5MiR4OBgX19fV1dXKyurEydOlJeXj7kV6Z5UX1/PZrOnDG7oKBCYjgQSfx7DMC9H4+/x2jX/o83S0qIt0Hio+V7h3FkpT+wJyJiANEPC7HdS58zJfO/d/PffK5r7ftm8uZXz59bMn1c7f17N/LnV8+dWzn//3rx378ydk/WewispYc5P4iFO6baUPPu9pHn2FY49/N7pSPdtGZPaCQY6na4qGLhcrkAgeC6XpOHhYQaD0d/f39HRUV1dffPmzczMzPz8/OzsbEtLS3Nzcz09PRTAgG7o16xZo6+vb2xsvFbZNmzYgAJ8yUtAUY5ALicIPOXWg39vONXaRZdJBHz+Y5ekUYIBpTTp7u6+desWhULR0dFBNaHJkImVK1euULZly5YtUrbVq1fv3Lnz8OHDmZmZjx49ehJsrTg+qoTwy7JB2VeeUOh8OP3jTT/o213+1ipxkV3uKvd7K9xLvtgeFxpzF3+SN+kZ96bcJbwBASDwhgkoBIOh4XNZGFCPi4qKfsyVPIF5gdQPbsoWGhpKJjtCt/uq7yhr6u3bt69du3blypXz589fuXLl7t27E6gFtHllZWVDQwNohjd8GcHh324CCT93SXo3T3NnkBZdQ6NZ49289x/f5ZM6IVYhEt7Ne39u1fz5zZpaPdratAXaAwu0hxZoM5SvoQXag8o59AXa1AVafdpavdpaPdrKmAfNucidKemJckCyIWX23zP+kdid9Hafiik8erUTDDQabXKCQSwWCwQCFIWMBEN7e3tZWVlqampmZubt27fT09O3bt1qbm6ur6+PohdWrVq1evVqVLvNyMjIRNk2b97c399PnlJ0yy6XS3FcTh1kf7XxzHfh2XKplMdXFHZAagFZGIaVjcViMZnMoaEhBoNRW1u7bt06HR2d9evXOzo6GhkZrVixAvkprVa2lcqGCsAtWrRo8eLF+vr61tbWJ06cuH37NpVKnThyg+ykQloo9AA2wOYaOkRdSi2taey2+S7tC4ukZW5lOu6ln22Lzi5pQgoEBIMqN5gGAmpOwMPD43ldkgiC4I2MhIWFTRC9QKoFNOHm5kahUEJDQ4uKisZ0MUJ3/zU1NbW1tSjE+dnLRVdUVNTV1YFmUPMrDbo3jQmkJAxqauZraGb99Jqfoxm7+t17s2fFq7gYxcyelThnzq3359VrKETC4AJt5oLf9X3wT+rHS+hLTQfMbBg2FCbFmeliw7AxH1y3hLb04/5Pftv7O4WQGFigRVXKhl5tpB80WjXfL5s3O/3dx1EQcbNnJs6elTDHtdptRDoyjWlP16GpnWCgUqmqMQzPbmEgU52y2eyhoaG+vr6Ojo67d+8mJiZmZGTk5+enpqZu2rTJzMzMwMAAlWxbs2aNrq6unp4eKvZsrGzr1q1rb28n760xDFOUIpBJpHIZJhPZhiR/szWsu38IBViT5gWkFpByQL5JQ0NDTCbz/PnzixYtQukLExMTv//++40bN27atGmU2WHFihXI8rBs2TLks7R48WJjY2MKhRIWFnb79u1Hjx5xOByRSCQUCpFxQyAQKIKvxeLHGY3kmAyTEATR1E4tKGtRBGpLxReTShfvil/mcnepc6GefUz/AEs5rjFKNEzX6xvGBQSmOgFfX19dXd3nzbRWW1u7Z8+eUfmRRomEUR9JzTCmnQHFJCDZQL6POZNcSk5UVlZWVFSAnWGqX4rQ/ylKQEpID16/pamRrzE/+7FgeO+W5vLz79/Vnhn3juJuXhmKMCt5znsl8zQfaWkzFvyu/4OF1K9smDY/8H4oFZdS5VQ5MUbgshyX0+S0ElHJ5ZHInYyd/6J+qk3VViiHfoW1QWFz6NN+HBeR/q7CTylW6aGUMmf1bb32EcWNFrQpREB9BcPAwACLxSIFw8QWeblcTgoGFos1ODjY29vb1tZ2+/ZtJBhu3bqVkpKyefNmU1NTQ0PD1atXI7WABIOBgYGhoaGRkZGxsbGBgUFxcTHKPYrKlslkMlR7gcBl4Umlf17zfeyNSplYyOUqLAxIKqAYBtVpJpPJYrFaWlrWrVvn4eERHBycnZ198+bNlJSU2NjYiIiIwMBABweHDRs2IPFAviPlgAIeFi1ahDK0Ghoa2traHj9+/ObNmw0NDd3d3b29vT3K1tfXR6PRBgYGhgYVEkVRUY7DEQhFEqmEIGQ5Ja1LdkcvpRQvtLzhezJLESOtKPEGDQgAgalBwM/Pb/ny5aqeir/Yb5lUGhcX91xqgbQzuLm5kXaGMfMmkTJg4olR2yJpAZrhF88drAAEXi6BOnHdFuYW7a7faXo6KwTDe7c05uVqaObMO7Vm1vXfPJYKSXPeK52r2aP1P/S/b2BsDOWeKxeXczHu8/ZkQD54jX/NgrHrr9S/aQ8+kQ09StnQqfX+vXmzEpVOSnGKHEr/zPjk3lDZ8x4C1n+DBKa8YMCVTSaTicViVOaZxWINDAz09PS0tLTk5eUlJiZmZmbeunUrPT19586da9euNTExIW0LyLxACgYjI6M1a9acP3+eIAipVCqTyaRSKVILIpEIk0vyypr/bnTWKjiFz+eT5gWUIonNZiPBwFE2UkKkpKRs2rRp9erVW7du9fX1PX/+fFpaWn5+fklJSU1Nzblz53x9fX18fFauXLljxw4DA4Ply5eTIRajxMPixYt1dHTWrVvn4+Pzww8/lJWV9fT00Gg0Op0+ODjIQI3JGGIxhphDDCZnmMUS8Ueu59d9uytGx+3e1xZxja0/eVu9wcsODg0EgMAzEoiKivLx8XkuC0N7e3tISMgkBAOKgUaaYUw7w8QiQXXpKONDdXU1cmFqaGgYGhp6dmfLZ6QEqwEBIDCKgBgXn+Kc/ivtb4pH/n1aWlRNrWgjTZ0wjT+lzKVsm3X9v2bGzpqZMPvdwvd/9/AD0yHTKF5Uu/TlPPVvk7YdGD7wcf8nCtnQp7Q29Clkw/xmzdkZSlODUjP8Pu3DW7S8Ud2Gj2pLQO0EQ39/f3t7e09PD51OZ7FYHA5HIBCIxWKpVDrmbwxyGZJIJEKhUPFwXZkiaWBgoLu7u6mpKTc3Nz4+/ubNmzk5OdnZ2Q4ODkZGRmvXrkWGBT1lM1A2ZF4wMjIyMDCwsrJiMpnIsIAEA/IFkkrE9x/2frnx3OIdF9u7qPwRhXmBzVbkR0KNtDCoxjYwGIwHDx5kZmaePHnSyclp7dq1qOBDYGBgREREWFhYfn5+XFychYVFQUFBaGior6+vra2tmZnZSmVbvnw5kg0o/gGVd1i0aNHSpUtNTEy8vLyysrJ6e3v7+/vpdLoidoLJZLOHORwuhzPMHeGOjIxIhIKwuLtfWaR8bZvlcyJHIBRIpVKIZFDbv0noGBAgCeA4jr7c0JMRcv4EEzKZLDk5mUKhPEu48yivpFF2BvRQQ1UGPPt0tbKhgIeqqqqysrL8/Pxr166hAvOogiR8C01wHmEREHgRAh3Sjs2MLUqp8Ng7SLNDU6Nz7rw6zbkR384L2jj38jdzct7/S8NHnmzPGmk19iQtyoscdNS2vbLewOGgP/X/eX6TxvxGDYWHUr+2ZpfWe4VzH/tBJc7WSv1tDjV31IbwUT0JqJ1g6OvrU41hYLPZPB5PKBSKxWJUexiVH0axyIrSy0p/IZFIRJoXhoaGqFTqo0eP6uvrMzMzY2Jibt68mZ2dnZubGxwcrKenZ2xsrK/SSPMCimEwMTHR1dVNTU1VhAEogwTEYjESDGKRqKd/aMnOi5+Yns0qqhfyRpQ6YQzBwOFwuFzFzTqXy0Vl3ZhM5uDgYHd3d21tbVpa2rFjx+zt7VHlh507d3p4eNjb22dkZMTFxVVVVRUXF6empl66dMna2nrbtm26uroooRNyW1qubDo6OkuWLFm6dKmBgcGJEyfa2tq6u7tpNBoyNbBYLDabzeFwRkZG+Hw+XzDitO/6QpucZZZXG1oecRS2B4W7l0gkmtjXSz2vWugVEHhLCKjeUj+jZujv79+3b98vZlMdUyqQM5HeOHfu3L1798aMgUZ6YJR+QGaEGmWrrKy8e/duRkZGVFTUuXPnDh8+7OXl5ejo6ObmVl9fj8alOrq35ITCMIHAayCQK8j9jPa5Qi2gQIJe7ffL572TMmdmym/mXP6f+YvOabx7W+N3GZqrz+a2Nrzq/lRKKg37jd4tev/dvPc1mjW1laaGueXzFVWiYxVh0L9N+33hwN1X3Q3Y/4sTUDvBQGZJotFoKG6Yy+Xy+XyhUIiie2VPN4lEUXgZ5UdisVhDQ0N0Oh35I5WXl6elpYWHh6enp6PabREREcbGxkbKhiQDUgtkAANKrmpoaLhly5aOjg6ZTCZSNoGi8gJfKBD00wZXWEb879rQ01cLhPwRtvK+HJkX0D06ckxCgoGnbGScA0ulDQ0NdXV1VVVVpaamHjlyxNbWdv369du2bbO2tvb09ERuS7W1tQEBAaWlpZaWlkuXLkUZWpVWB8Ubkg3I4KCjo3P06NGOjo7e3l6kGVAEBZvN5vN5fJ5AKuE3tXUvs4z53CLth+sVcrmUx+NxuVyU00m5Gh+Uw4v/OcEegMDLJfDct9Q4npmZ6aZs5N3/5CbQTsLDw8vKysbLnVpdXV37pFVXV9+7dy83NzcpKSk6OvrEiROBgYFubm52dnbWT9ru3bsjIyPJeIznHt3LhQt7AwLTjgCGY+e55/9I+29FqlOlWtDs0nq3QPlEP37mnKsfanx1UeNdZbokDUU8Q0MN/zUwGMFHgphB79+ePzNp9nuFczXbtbSpC+bVaSjKwyk1w4fX/1zPfuXS5TWMdHof4kUFA3ro9RK/95lMZmtrq+qNL5fLRUYGibLJZDJF1qInTSwWC4VCUjAMDg5SqdSurq779+8XFBQkJCQcOXIkNjY2Kyvr1q1b165d27Vrl66urpGRkeGThvQDckkyNjY2MTFBPksUCoVGo0mlUjIxEZ/P6+sfWG4Z8Q/z83uOZyoMDEoNgAQDepzP4ynSrY6MjCC1gDIaIWsDmYYViQqUTAlFXHR0dJSXlycmJh44cMDKysrMzGz9+vU7d+48cuTIlStXAgICli5dGhIScu7cuR07dqCUrKtWrUKyQUdHZ+nSpStWrEhISOjr6+vv7x8YGGAwGEieDA8P8/k8gUCE49Jjl/M/3pJkG5Iu4POECmwKbsiPi8FgDA0NcTgcsVj8Es/m9P7jgdEBgddAoKioKCoqCsUw/OLfJofDOXHixLNnU51YSyCnpvDwcFU7AzIjIF+j8vLy27dvZ2ZmJiYmnjt37sCBA56eno6Ojvb29jbKZmtra/ek2djYeHl5NTc3/+IoXgNVOAQQmH4EhLjQj+33uCqCUi1otGvNyX5PmZ7onXeS39UwOaCwLagkV62t5bw2Dkn85A+Kfj8zfvY7KXPmVs3X7lswr15jVoKyVFzy7G9yFg2Khl5bZ+BAkyDwooKBdBB6Wb8BQqGwtbW1q6sLOeUzGAwOh8Pj8VAkw+MUoijVqVzRkL8Qn8/ncDhMJnNgYKC3t7ejo6OmpiY3NzcmJubEiRMXLlzIzs6+detWbm7u8ePHUQDDE72gSI6EGumSpCzgtlZPT8/JyamxsRH5EPN4itIL7Z30b3dc+Hh9uHVQMo+viBEYHh5GvkbNzc11dXW1tbVtbW10Ol0RPKBsKH0T0g9IS6hGOJCygcViMRgMOp3+6NGj0tLSmJiYoKCgHTt26OnpmZiYHDhwoL29fWBgoKGh4eTJk8bGxmvWrFmlbCjCYcmSJZaWlq2trX19fSiYgakIZmA33G8YGBhQRmxLe+iM5TYxyyzjWjv7OFxFhANytRIIBMihi/mkgsTIyIhUKp3E9QSbAAEg8HIJpKamBgUFCQSCCXZLfv2Wl5f7+PhMLtx5lHhAaoGibOHh4aWlpbW1tVVVVSUlJXfu3Ll27VpUVNT333/v4+Pj5ubm5OSErAg/1wlIL9ja2lpbW/8YwC0WiycYCCwCAkBgcgSGMfZuhqUiyFhZQE2rT1ujRVNRAwElM037r/d9zTXm5qmqBQ3NrNcpGAiCuCMs+HPxR8iwMCf3fc12rbnV82fGKe0MKbM3FW8RY/D9MLnz/zq2mqRgIH+fcBwXCARUKvVldVYul3d0dLS2tnZ3d6OH5Ww2Gznio9BnuVyOPWlyuRy5JPF4vOHh4aGhIRqN1tPT8/Dhw4qKiuzs7KtXr547dy4qKiozMzM3NxcZGaytrVetWqXqjPRzwYB8k/T09NavX3/mzJm6ujoWiyUWCSsbOr7YdP7TjRHrXKMKi+5e/eHK/v37nJyctm/fbm5ujgwXZmZmO3bs8PT0PHfu3K1bt1pbW4eHh1HnRSKRIqJApUo0V9lQAQeUWAnd6DOZTBqN1traiiIF29rakBFgcHCQRqNdvXrVwMBgjbKtXr161apVK5SVHLKysqhUKunNhUQUi8USiURSiZQgpIFnbn287lJBdYdUIiElDSkbkA0EeXahtLZCoZA81y/rFMN+gAAQeHYC6Ntu4vXRHymPxzt//vwLRi+Mkg0ob5K7u/u5c+cSEhJOnz4dEhLi5+fn4uJia2trY2NjbW1tY2Njq2xPbAlj/I+cLZuamuD7ZOJTCUuBwCQI0OQ00wFThVroeZySaN59DUXQQoyiKNs7N9+Zn/UPjb8naMzLebOCgSCIMlHZX+7+dWbi7JlXZ89Smhrm5CptIMq8ScebT0xi+LDJ6yHwfILhx9QWwcHBTk5OTCaT7B+dTs/JySE/KkuD4RzOYztXb2+vq6urs7NzUFAQaY5QXVl1Gv2WdHd3379//9GjR+hhOQrP5fF4IpFIIpGMKRhGRhTxx0gwdHV1NTU1lZSUZGRkxMTEREREIJekrKwslCvp4sWLZmZmq1evRppBVS0glyRU8hm9GxkZrV69eq2Jsasr5cDB7xy9D3++Iezf26K+NN+ra6C3RlHPYZWenp7qrgwNDQ0MDHR1dVEZaXNzc0dHx+PHj9+4caOpqYnJZKIAblSaGukHdKeOxINqtlYmk4l0AuliNDQ0NDg42NXVZWdnt2rVKl1dXVI2LFu27PTp06qCYXh4WGEYUUaAiCVigpAX13T8wzQ0IlWR/Jj07JJKpWRgN5/PR50ZHh5GFo/BwUEejzdmiirVcwfTQAAIvDoC6I8UGS3R1yCubKpHrK2tRc/7f37T/4Jz3NzcnJ2dHRwcbGxsrKysSJFgZ2dnb28/sVpAXklWVlaRkZFgXlA9XzD99hBoHGpkCRWFU19F65H16A7o/aQWerXn1cx/XPEgZvacW+8t6f/aJ+6GxvtPV3pWOibV1Ay/ii5NvM9iQfHvcv/wOIAhbvaspDkKI0Pc7JkJs+cna5YxoDjDxPze2NLnEwwEQfzwww/Hjh0jCKKhoSE/P18kEjEYjM7OzqGhocbGxjt37gwMDNy4cWPPnj0sluLPA6XuaWxsvHTpEtISEzxhQov6+voaGhrIXEksFmt4eJjL5aJcSSi/KvqxRBYG5FFDCobOzs7Gxsbi4uL09PSYmJirV6+Gh4enpaXdvHkzMzMzJycnMzPz6NGjRkZGurq6yDGJDGD4uWBAssHA2HCNrq6u7vKFeq7/3hz15baohaYBRsYGSi8mI+TLNN67kZGRvr4+qvxgZmZmY2Nz8ODBlJSU+vr6wcFBoVCIbgXQKEifJVI2DCsbSnk0PKzIbsRgMGg0mre3NxIMin4pZYOOjo6vry+qzDA4OMhkMoeHh1E0hdKhSySXyVgcno7FBZ/jCoGHyZ+KBlGVDaSTEpvNZjAYVCq1v7+fy+WCbHhjf6lw4LeGAEqlyufz+/v7GxoaUMBxXl5eTk4OMpPm5+eXlZXV1tY2NDQ8fPhwcHAQVaG5cuWKi4vLpLOpjicqPDw83NzckDCwm1Sztrb28PCA6IW35hKGgT5FQI7J9WP1V15ZOSIZeWrBy/jwSNa5cmCVqlqYWz7vcQKi2NnvFs1dT9tAJ6gpsUOaWqP9kTQ0s96IYCAIIo2dNvemxmOdEKtUC0gzJM3+NmfxiOzlg3oZsN/2fTy3YIiPj7948WJpaWlkZGRiYuKhQ4dKSkpCQkJKSkooFEpiYuKRI0dyc3O/++47hd889riocF5eXm1tLRIMEyBHgqG/v7+urq6trQ3d+yIXHSQYpFKpXC7HnzQMw6RSqUgkQgl/kIXh0btpUp0AACAASURBVKNH9+/fLyoqSk9Pj42NjY+P37t3b3h4+PXr12/cuIE0Q3p6+sGDB83MzHR1dQ0MDCYQDEgGGJkYmRgZ6RubfWl28N/bor/cHPXNWh9jYz1jYxNj44kEg4mJCdqDiYkJclhC4mHlypVGRka7du0KCQmJi4urrKyk0WgCgQAViUNh1iMjIyjbEnrkjz6yleXhenp6HBwckAhBgkGhZpYvp1Aojx49IhMlsdlsFG/N5/PFIpFULpFKxFb+sVYBqTguw+WK0hbKSBC5aok6sbKRIdEoyxOKJqfT6fCMcIILGBYBgckRkMlkPB6PRqU2Njbm5OTExMRcvnz55MmTR44csbGxMTY23rdvX2hoaHh4+MWLFyMjI3/44YcrV65cvXo1NjY2PT39zp07ycnJvr6+LyV6QVU5IPnh4uIyKaWg2AhFL0RHR5PJkSaHCLYCAlOOAI7jvdxeDMfaWG0fnf1IP1afJ+G9xFG0ydqWDehokZ5Ivdrv35unuAuPUdyCv1c6z4HpMIIrbr5jYnvVSjAQBHG6/8ystCe2BaQW0HvK7H0P9r9ESrCrl0XgWQUD6UeLBMP+/fuLioqkUunWrVurqqqOHDnS0tJy+vTprq6uPXv2FBcXX7x4UfEMW3k/iuN4VFQUj/fLfyekhQGFDvf29vb19TEYDPSkXCQSyWQyJEKeSAYcxT2TLklUKrWzs/PBgwfIwhAfH5+SkhIREXHy5MkLFy4kJSWlp6dnZmZmZ2dfu3bt+++/37p1K7rhNjQ0VDUvqJoLkM+SiZH+SmPLf20MXbgt8otNlxYZOxnq6q5es3rVqpWrFY5Ja1DyIuSGhPK0ks5OyExB7h+5Oa1SNlSOzcDAYNu2bX5+ftHR0cXFxd3d3aMCDJCnEErBJBQKa2pqNm3apFp+Tk9Pb9WqVRQKpbW1tb+/H1VjYD8pxSAQCEQioVgilYpFJy/d2ugRK5dJMBxXFQykZkCiBaWf4nA4qMYFMm6glLWo6NLLugRhP0Dg7SSAYxifz29tbb19+3ZycvKFCxd+fM6yZ88eCoXiomxubm7e3t46OjqamppOTk5BQUF79+7dt2/f6dOnL1y4EBkZGR8fn5SUlJiYmJCQcOzYsZduW3B3d/fw8KBQKA4ODpMWDNbW1r6+vu3t7RPYlt/OCwBGPe0JsIXsT8M/3Vu4lyCIdlb7n07/ySTehC99OZlMm6XNiwcWaw8ogxaUOZHeK1amT42ZPSth9tzKeQHDATJChiCroWDACdyq0WZm6s80Q7yimlv9MGRZVbu/j+cQDDiO5+fnf//996mpqVevXj127Fhvb++hQ4eqq6v9/Pxqamr27t374MEDJyenoqKiAwcOIAsDjuPd3d2RkZHPPnSUFLWzs7OtrS0tLa2urg5lLBWLxci8oLorDMMkEgkKeka+Oo8ePWpsbCwpKblx40ZiYmJaWlpycnJkZOTBgwdPnTqVmJiYmpqKZsbGxp47d87FxcXY2HjFihUo3Sp5c09qBsV9v4mRsZHxV2bBn22NWLjth8/Nw5Yb7dhtYXHw4MGrV6+mpqZmZWVlZGRER0fv27fP0tLS2NgY6Qc9ZXgDuSu0c/IQyFtJV1cXiQcUjWBoaLhp0yYKhXLixImUlJR79+61trbSaDRU54HJZLa0tISEhBgaGurp6ek/aUgw+Pn5PXz4sLe3F2VWfVowiKRSmVDIzbxdbeYaLRGLcRXzAspSi8pak8mdhEIhCphGgRAoASsqlIGSPKqeCJgGAkBgYgJSqRR5TnZ3d7e0tNy7dy8+Pv706dP79+/39/f38PDw9PT08vJyd3dHuYnc3Ny8vLx0dHQWLFjg4OCA4o/RTG9v76CgoGPHjkVERCQnJ8fExAQEBCDB4OHhoWoiePFpZ2fnSasFZF5ITU1FD3om5gNLgcA0I4ATeHJT8juH3zlScoQgiIfMh3889cd1SeuEMuELjvSh5OG39G8fl2ZTqoX3i+cpahrEzJ6ZOGdejcYR7hGcwMmjxMT2jWNheH1pVcnOkBMD8oF/Fy5UBECrWhgU0c9z1haaSTAJuSZMqAOBZxUM6OFQSkpKeHg4ij+OjIy8evUqk8nMz8/fv39/UlLSoUOHUlNTAwIC2trawsPDGQwG2qq1tbWlpeUZRyuXy1tbW5ubm7u6urq7uysrKzs6OlDtNolE8vNfHRzHpVIpn89HXkl0Or27u7upqamsrCwzMzM5Ofn69espKSlXr169cOHC4cNHzp49e+XKlaioqEjULl8+e/ZsSEiInZ2dmZkZumUnEygZGBjo6euuWb1GX2/NMmOHzzZe+Gpr5GebI1fsCistr+Zyx44W4vF4LS0t6enpqCIbcnxavXo1Eg+k2YE0OKxduxaVk0NuS8bGxoaGhigsYfXq1fr6+qamptu3b7e3t3d0dLS1tV23bt0TmfDU/2vWrDl27Fhra2tPT89YgkEskYp5I9zy6rZ1blf4QgEulyn+KZv0SSOjKpGdoaenh06ns1gsshjF8PBwZ2dnX1/fM55QWA0IvJ0EkMMkh8Pp6+trbm4m0zBERkZevHjx/PnzYeGKdunSpaioqIiIiLAn7ejRo/v27du/f7+fnx8SDNra2g4ODqQSQCXV0Lu3t/eBAweOHDlCpjN6cYWguocXNy8EBAT09PS8ndcAjBoIEASR1JQ089DME2WK/D+Ng40fnPhgU8omkUw0aTgd0o7FA0ueUgv3ntgWkuZo1GudGTkzaufqKRgIgijmFs+/oaUIuhilGRLnpPVdGzUK+PhmCTyrYHhtveTz+c3NzahoMZ1OR85IyLl/PIs2juMymUwgEKAsol1dXa2trZWVlbm5uSkpKRcvXvwxQVNgYMDpM2ezsm9V19Tl37lzIeLi9yeOH//++PdHjh4+eOjQoUMHDhwIDg52d3e3sbHZtm3bunXrNmzYsGvXbk9Pr+PfH/EKPvjFulNfbrv81ZbL/zANCwnNfUYgPB7v4cOHN27cOHr0qL29PdIkKFiZjJ1QjaAgbRHkBBIY+vr6yHVqzZo1SHgYPGkobltfX9/IyOjq1asoIy3KrErGi/P5fEVmVamUzWI2tXZb+CeP8IXYY7GgUAxP9IJUVTBIpdLh4eFHjx6x2WxUThsVehsYGGhsbOTxeLiyPSMKWA0ITHsCOI7zeDwqlVpTU5OTk5OcnBwaGrp///6QkJCAgAA/P7/AwMCgoKCQkJC9e/eeOnXqgrJFR0fHxcUlPGnxT9rly5dDQ0MNDQ0XLFhgZWXl7Ozs6uqKjA+q4sHBwcHR0fGlRy8g2fDi5oWUlBSZ7LFfxLS/AGCAQGBMAnH3435z8DdnKhT38Q8GHyw4vmB72vbJ+Sb1yft06MufUgtlj20Ls1LmaD7QPsc79/M+XI0ZL4bhTVoYUD/9Wv1n/dwxKUlRyk0gf1FTzM9RwJxJE1A7wcBgMFD1saGhIWRYEIlEYrEYPfAWKUsGoDoGyE9G6Z2vmM3j8ZCRoa+vr6ur68GDB4WFhdevX7979+6tW7ciIiKCg0P27PG5HBlVVl5ZVVN77Xr66dNn9u3bHxQUHBgYFBAQ5O8f4O/vHxAQsH///sjISGXtBTaByx90DBg7/vDZpoiF2y4t3BL1702hD1onU3diZGSkra0tIyPj5MmTzs7O69evJ7MnIfGAtAEpFSaeQDqBfNfX19+xY0dBQUFLS0tXVxcpGFDYNFkqm0aj9vQP2oYkc0Z40nF0Agp6Jt/RtqhWA7IzsFgs5Pj0i1Hsk74uYUMgMLUIjIyMtLa2ZmZkhIWF/eiQuWfPHtcnDd3io/vvUcYBX2Xbv3//6dOno6Ki4uPjk5OT09LSUpXt2rVrGRkZtra2H374YVBQkLe3t4ODg729vYODAxIPbm5urq6udnZ2NjY2r0IzuLm5vWD0ApgXptZlDL19uQQKugrOV52vp9cTBHGl4cp/HfivsKowgiBqaDVax7UOlRx63sMx5AyjAeOfciL1ac+tfBzlPCt5jkaj5umR02PuU50FA1PO/Oz2F2M5Js2O6FBk14SmJgTURzA8drbr7e0tLy9H1cpQqpCoqKgffvghIiLiwoULqampxcXFtbW19+/fb2xsrK2tzc7JuXL1ytmzZw8cOLB///5Tp05dvHjx0qVL0dHRV65cjYmJYTIel4wQCQWPOtozMjIuRlyKjr5642ZmRmZ2Suq1y5HRp06dPXz46JkzobGxscXFxVQqVS6Xo6DtlLwHOrvCP1l/aeHWyK+2Rv/d9HxIaA6h4ho4wYmc4AG8QCDo6OjIzc09efKkg4ODqakpimDW19cnLQ8TCwa0FGkMFAtx4MCBmpqalpaWzs5OKpU6ODjIYrE4yoZu+sUScX9fb+uj/s1ecdwRoewpc4LCtECKBIUCe9JQ9DP6JFQ2DofT1dXV2NiIKE1AABYBgWlJgLR24jjOYDAKCwvDwsICAgIoFAqSCc/yvJ8UDyi2GD2qOH/+/NWrV9PS0tKVLSsry8nJ6aOPPoqJiUlISDhx4oSXlxdKPeSgbPZPyiDY2to6ODi83LjnF0yOZGNjA9EL0/L6h0H9IgEZJnPPdV8YsfDry1/POTxnf5Ei7U9kXeR/7PuPizWKlDANAw2tzNZf3I/qClyMu2Voq6pamFevMStBUZ1tVvKc+U0ah0bGVSDjCYbq6rE9q1WP+xqm04fSZ6e99zOvpNmfZHzKkb55G8hrIDAlDvEmBQNfKGjtond0D3XSGHK5DMcxHMevX7+O0n5TKBRPT889e/YEBATs3bv3yJEjFy9evH2ngEYfJMmKRKLG5qaU1JSjR4+6ublZWVk5Ojp+9913hw8fPnDgwMGDh0+fDh0cGED39yNC4eCwIjuBSCKi0+n379+vqKgoLy9vbGzs7e1lsVhSqZTcM5oQS2UW/gn/WBv25bbLX2yL+Nf6S0aOl6lDHAL/KZZo1CaT+Mjlcqurq8+dO2dra7t+/XoTExMkG1CqpYllg5GRESoSt2PHjps3b9bV1ZGCYWBgAJViQIYaoUAkEI70dnVVPegycr7M54tkUoXpRrU90Qhj/I+kAsq1yuVy+/r66urqBALBJMYLmwCB6UGAyWQWFhYeP37c3d0dOQuh+/Vnv2snPYtQBAL60vP39z9+/HhMTMzNmzfz8vJcXFw++uij+Pj43NzcrKys5OTkU6dOeXl52dvb2ykbqpuG3h0cHJ5FqyBzx8TvLxi9YGVlBeaF6XGdwygmQeBk+UmDWAOWkCWRSy7VXPqvQ/+FciVdqL7wn/v/82brzefdpwyXuTBdtYd+quU8v0lzVvIcRb3k5DnzmzV9h/3kuOJB55jt6tWxXZLURDBgBGZaYT4zeXQkw6zkOZc7nyNlzphjh5kvi8CbFAylDZ0fm574dP15PduLXB6fwBVFG27fLti797v9+/d/9913Icr23XffHTp06NSpU5cvX87Lu93bS5VKZDKZXCqVslis+oaGlNSUU6dOonSEgYGBZ8+ejYiICA+/EB5+MTz8orIoNdYzxN7pG3smtkjhRUM8rg4xJsRRZoH7D3u+2RL27y2XP9lwacn28OqmHoLAcTn2jEaGMQ9BzlQ9Fo7jXV1d0dHRKJTCwsLC3NwcmQ5I5UBWdUBB0qik9OrVq83NzS9dulRaWlpfX//w4cPOzs7+/n46nY4EA4fDUVgYBEIOh03t60nMaTB0uiwUCiRiMakMkGxAfkfkTNUJgUrjcrk0Gq2+vh7V5iOHAxNA4C0hIJFIqqqqjh8/7uHh4erq+uwKYeJ7dCQbkOXB19cX5UlDgiEhISFX2VAFt9TU1JMnT7q5uamqBTTt5OT0i0d5lhWcnZ1JTYKUybO/29raWllZ/ZgDA4yQb8lfBAxTlQCO459e+FRVFUTVRf3mwG8q+isUNcta0tpYbarrP8v04eEj2vQFWr3KJKq92hqtmu9ce1ehFhLnzG/WcB/2kOKjn3iq7lbNBQNBEKXcsrnX54+Ofk6avTD7a74cHk2qnsw3Nv0mBUN71+DCzeGfb7r09dbznb1MdAt+505hcPDeoKCgffv2nTp1KiEhoaSkpKWlpa+vr6+3t7mlpa+fimGPH/CTFobjx497eno6OzsjwXD58uXIyMiIS5ePHz9Fo/aWN3QaOcd9uiN9V9A1hSlDxaEI3bKr3rirngrkeHAmvujPeifXWF+obOhVeu0rChi8RMGgevTBwcGLFy9GRERERUWFhYUdPHjQ3d3dwsLC7P+x9x5gTWR7/zjP+/zbT1fUdVdl3b2v9969e/febbp9LWBDRRGQKh2kC0gXUURUkCYgvfcaktB7L9JLIPQaSkIoAQIkAVL/mzkyhiJrwxU3nyfPcGbmzJlzvmdCzme+TVr64sWLFyCIQwBu0GfPnlVSUoqMjCwvL6+rq2ttbe3s7Ozv7x8eHh4dHQUpLMhk8tzcHJVKHSMSx4iEu3558pZxtAXK4sIzwgArEJ5XgPkCCEhFJBKbm5snJyd5xcUv8yXwV5DA2NhYTEzMrVu3jI2N39Tr/LUreFNTUzMzMzs7Ozk5uS+//BKJRBYWFuZDKCgoKCkpycrKun37to6ODi9n0NXV1dfXf30O88rqBT09PX19fW1t7YcPHxIIr+Lr9Vd4hPhjfP8kgCFiEtsScTM4MLSj4UdNckx4h3ku5pxtsS3vkRcvR8/HHCB8un/kKVvYh9svmLWbyxYSBD9q36szrUNj/4Fz8LtPGDgcjmaT1lolw/aknYnDiBeXFb/m5kngzyQMi3SGonX8Ifngr6V9I5NruG/u2eza2tqEhITy8vKBgQEQh4d38HQGY2JqksFigvU6lbrQ2t6ORqM93D2sLC2vX79+7949f3//iIiI6KjoqPBwN3d3B5/kU9pxIoalZ8yrRHTih/ETvA1uXObqItjs6VmKY3BBz9A4Z0PVxMZNveDZ+fn56Ohob2/vqKio1NTUnJyc7OzslJSUyMhIT0/PW7duXbt2TV1dXUlJSUNDQ1dX18bGBoVC1dTU1NbWNjU1tba2dnV1AcJAIBAmJibgQEk02sIgbpA4RpQ0irzumLJEX6BRaTANWMUT4ONrCxQKhUwmA5suvobhBaeVX21LSwD2WKDTlxobG318fEwhrF3lv8EjwFrJ1NRUR0dHU1MzMDAwOzu7CEJhYWFJSUloaKihoaGuri4vYQBlAwOD69evv05ngC+1/itBV1fXyMiotLQUltuWnn1+5/kS2FgCdBbdPM9cyF3o08efCnkI5fVxgygmtiX+3w/+78S2RPha2SRZr5r1PZLhOusWKhcq/0X4Yj/+WYK2XcV7tscIbo8T/Kjl40sTElOsqXUv5D0YHb2uSVLOO2KSBLpaN1u3J2WNkgG582KJ+Mb6E96R8subJ4E/kzBwOJyskvZDMl6HFELPawf14EY5HA6DwaBQKLAOAXqj/8xhgMVmT81M05e1BCsIg5WViYmJg4NDUFBQVFRUTEyUb1CotMHjn9VQJ80qzlhWilpW/qSRlFr8UukD2Uwmiw05QHP5DGsjW6Y3MkksFis5Odnb2/vx48exsbGZmZlFRUVPnjypq6traGiora2tqKgoLCzMyckpLi4G28bGxqampubmZiwW297e3tPTMzAwMDw8DPI9g3Rvs7Oz8/OUgb6equaBryU8XcMK2Ew6jUqlLGMVMVg+vM5fkCOPQCDwTZLeyIzzG9kqEqAvLeXn5d28edPExGSTkh6su8S3gGBlZeXs7JycnFxSUlJaWlpQUAByRK5LGHR1dQ0MDEA/123zeQdhn4pXjqaqp6enpaXl5uY2M/NOOFNulaeL388tKoFF5qJqsuovIb9gx7EzCzPKycpf+X01vzTP4XBuFNz4P07/x7HCsX+6363STSRChER7GoLlxQc7xBj6dfQ3rjHS0FP1wh4oiOq2WMEP6z76lfgbjvFUp7Fxm1uCMLA4LMmqy6uVDAmCOxEf1k3XbTxA/tm3IIE/lTBwfQEY9j6Fh+T8v5ELNLyHTk5Nd3R8YHL9+p27dzDNmLXjZ7PZU9PTdG5Uby6LoFBo2LZWFArF1TBYWZmamjo6OoaFhcTGxjg/Dj2j+fhXrbSTJsVnzCtETMuPGBYeUk+2fJS5sQ/DipuyOUw2C3LGZnPNoN6or/OKG/HsYDAYNzc3T0/PoKCgpKSk9PT0wsLCysrKxsbG9vb2rq6uvr6+fgg9PT2dnZ3t7e1tbW3tELq6unp7e4EPw+joKPB7BrGSxscncAM99wMK/33RI7kAw1ykUSmU+WWsYgbLh5/+5T0Lx66Fws7+8YuNp0Zc0IRBImTxWITxDJtf5EvgHZYAg8HIy8uztLR8fWuf5y3Wn3ccZIA2MzMzNTW9c+dOcHBwQUFBXl6eh4fHtWvX1iUMsJ7hFTgDcOB+JdUC9yJdXd1r167x1Qvv8LPM79obkwCNTlNGK5+IPDFBeWq50DDacMDjAH4Oz/3h47ADGwK/9PvyoPdBaYR0/3T/y96YyqbKjcs/S7kwIvQxlhsWaVuM4O7yPf8d/W/t0osuo6OihtbL9PxuaRg4HE7GROYO9K7V4ZJQO00azV5Wevz6b1wCfyZhYEE+Oh3d3WeUnQ5fCT0sG/D9pRui5yUkL4lJSV4yNro+M7NOOC1uOKOlpwnDKRRaSysWJgwmJiZOTg8jI8L8gqPUTPwuaPv8pOh7RDPxN72c0/pxRg5ZBg8y7H3yFxY38g164yJ+2QapVKq/v7+7u7u3t3dMTAwajc7KyiosLHzy5ElDQ0Nra2t3d/cABBwO19fX17WMnp4ewCWGhoZqa2srKyvHx8cnJydJJNLc3NxAf19LR6eoVuhPV/xbuoYXqdT5+fm5FwNvTTKZPDk5OTg42Nzc/IJRkljA6YPD4LCZLCabb6jwso8Ev/6fKwEajZaVlXXjxg2w/oZfwz9vif9mj+vp6SkpKYFbA5cJT0/P6OjohISER48ebcwZ9PX1X8o2CXhvGxkZvTJh0NLSevTo0dTUS79J/XOnmH93vgReVgI0Ok0uSe7H4B/nlubga++V3vst9DfeLM40Om2c+iy0I1zzRQr3px88C6I6LLSvb/+OlF3bonfszN39v/iDGbSXCLW0VQgDlUX9oeDn1TkZEgX/kfqvySW+z+SLPDWbWOetEgaQkhmYy7O5Bj6sWQrV7vZt8YsSJ6QMv5f3OXwl+HtZd1EpXVl5eVUlOUwLZmlpCdQHNjPz85SxsTFu/FPoZT9Xw9DK1TB4eniAl39Ozg9jomIQCQg331Dpaz6/XkX8pIHSvIvqHSSyWHQ2i0FnMlibb1n0ajPGXUpD46qvr3/06NHjx4/Dw8MRCERaWlpOTk5JSUl1dXVTU1N7e3tvb29/fz8OhxsYGOjr6+vt7QVUYWBgYGhoqL6+3s/PLz8/fwICiUTiplrrbAtCVP5X0kfWLGZ6emZunjo3Oze7DEAclvc2+jszMzM+Pt7f39/a2voiCVzZHA4wMOvBjYGItC+h4Xk1OfKv4kvgzUmATqdnZ2dbWlqCxTowEHqzlGCD1qysrCQkJA4ePKivrw97NZiamt6+fTswMDAhIcHZ2Rl4MjxP1fCynMHU1NTA4GnA1pelDXp6eoaGhmVlZfyXAm/uAeS39I5KgEqnKqIV97nvqx6pBl2MxcbucdtzwP3A0fCjehl6oU2hzWPNTK7L5asglZr2GeFvT8MiQfZIO/M/3Ba9Y0fyzn0D+/3nuAngXhxRUev7MNTXv3Omg869LmsTP3+QtDNh+JlDyIsPnF/zDUrgbRAGsA6mUCiTk5N4PB6Hw83MzADC0NPfIysjI37poqTEpXMSKr9dtj8sH/C9nP9vsnYXFIyaGjGQJQuLyaQzGEtQYuJF0hSJzqBznZE5bAqV1tLaikKjPD09blhZWJqbuDk7e/qE6t4OOaISdlgl4bhKaGBiBfOZE8T6ogM9fBd+5EBPFhcX4+PjPTw8QkJC4uLi0Gh0RkZGfn5+RUVFXV1dS0tLR0dHd3d3f38/UDXACofBwUEcDldRUVFZWdnf308kEsfHx6anSKMjo0XldWe1w/8j7e0eVkKjzM6QZ8kzG4EMYW2NqampsbGxrq6unh5uYLgNhMadYujD4bB7hycu6YU8aeJaW7K4VJG7gcy81p8R/lG+BN4FCdDp9NzcvBs3rDcvGtIGbAGcMjQ01NTUXNUBMzOz27dvgyxvDx8+BLZAenp6q2gD2AX+DC8Y+/WV1QvAewFSL7yQpeK7ML/8PvAl8DoSoDFo8kj5j1w/qsXXxmBjDj4+mNWbVT1S7V3jLY+UP/j4oEaqBo3+B/GL1u3AAH3gO8IhIcKy68KI0J7qj7fFCm5PEPy4e6/JtCmT83I8JDJyfZOkd5AwdFI796ceWB1fFSmoWKXM2vzAM+tOB/8gkMBbIgxkMrmvr6+jo6OtrQ2LxRKJxIWFBQqFUldbJyUlJSl5SYoLSSmpy2KXtY/J2x+W9fpa2lP8WoRbZGkNdpA8R4UjFFEoc2w2A/SeTl/q6GpLSUF5uLsbmtnIad85q+nxnbzvd3KBZ7VCtW762tjdT4yNysvLLSkpATnaRkeJFAplaWlpcnKyp6cHh8NRKNxsbtw17LukeRgaGgoICAgKCgK2B2g0OjMzs7i4uLKysr6+HovFdnR09PX1DQwM4HC4wWUMQejp6RkaGiIQCBBhGJ8ikdqxTUYOCd/IBB5RCmhs75+bm53mwdQaAKXEqsPT09NTU1OTk5MEAgGDwfT19YFZWHfL5nCYXErAjWeFI5BkzBK+kY8MQJRSaAscDp3NZjLYrA2yzKzbJv8gXwJvUwL0paW8vDyQ4OUPl/WbWmFdIygzM7MbN244OTmFhoba2dmtogp6K/GHcZPMzMwsLCxeR72gq6vLVS/wgyO9zWeUf68/WwKUJYp0orSgq+C/vP9VNVLF2x3KEmWJ+dR8mvf4H5YX2UuK40orXBfa9nEz1DmKbwAAIABJREFUOscIflj/kcSE5Cx79g8bWVVhCxEGDocjWy23PWllErdEQSHUpyO0kVXj4u++TQm8JcKAw+FaWlqwWCx4Mw0H0Oju7paWlpaSkrx8+bKMjKyM7GVlBRlNzavyqjoSKpbX7iWJ6oQcVfY9qxOieRvpGFgUkVKTkFVTUtdb2Ywrru6Nzai745miaBosrOz5g4z7zzIeZ9S89G2CH/nFRkREhwUG6GhrffnlV19++eU333zz3XffHT58+Pjx4xISEjIyMqdPnz506NCPP/54/vz5zMzMtyn0P7wXeG3f1tbm4+Pj5+cXFRWVkJCARqOzs7OLi4urqqoaGxvhlAs4HG6IB9yEFSMjeDweJgzE0RHvsNQf5P2/UwiWNw4YHOybmZnhJQMkHkxCAAdW1ZmamiKRSGNjY4ODg8XFxY2NjRsPhMVmcdiskQmyknXCr3r5J03KjmjESpvF+8ZWEEkv/f9u43vxz/Il8GYlwGazS0tLQSiFTSUDf9i4hYWFlZXVqmpAXQC2jo6OwcHBICfDBrTBwMBglZpibZuvo17Q1NR8/PgxmbyO49mbnRp+a3wJvFMSmF+al0iQ2OOyBzuOfSMd8531exYWaUiIm3UhYzfXdaHgw+8Ih7rona9wl61FGJAE5Hb0zlWuzx+gdoYMhL7C2PmXvCkJvA3CwGKxhoeHYbP7wcFBEL+fzWbPzs6am5uLiYlJSUlJS0tLX5aWk1dQVlGUviz50PEBi81YZDCHiXPVLSPIXKx3XLmlK9raLf2Bf6FDQOGDgKK7PgXmrkkm90ONbj7SM75tZGzl6uKcEBeblpIaExPr5+8vJyv/9Vf/PXTo0PcQDh069PXXX/8bwldfffXtt99+8803n3/++c8//2xhYdHU1ARsbDYws3lTcn/Bdnp7exEIhI+PT0BAQFRUFAKBSE9PB4FW6+vrgQM0cGYAOobh4WEowR03piqBMDpKJMxMT6Tklh1X9v1BMeyQrE9IQi5pYhws/XlowtMiYAswZ+AlDKA8MTExMjLS0dGRkpLS0dHxRzoZFm2Rrmef9KM66rRVtahF5RmLGuHr5b+oocSN41B5jXQG1/scikL1VB7sZXBtlniwdl7ASVCNtQxg5/aCsuVX40tgXQmArz8eP+Lk5GRsbLxqVf3iuyBhs7m5+eu4PVhYWOjp6cnJyT2vJ2ZmZiYmJvfv3w8KCrp586a2tva6nAEkdDMyMgIcY12VhYmJycbeC2vP6kLQ09MzMDC4ceNGfX39u/PPc93J5R/kS2AzJDC3OHch7sInHp80E5tfs33MEuZz/L+e5mgbEto/IrS7/KNtMTt2JO/6pO9A+sJLODrz9iRi65gkcTiccfr45xn/3p64UsmAFFSoUuRbJfFO61suvw3CwGazx8bGQKxPAoEwNDTU2dk5PDxMJpMXFhZ6e3uTkpLi4uLi4+NjYmJioqMjIyPj4uLweG5gMiaTsbS0QJocb8U2V1aWVVWWj47iQUxVDoeztEjr7GzLSE/18/GyvWVzw8ry0SP3+AREalpGXHyCj6/PlStXfvnp56NHjx6HcOLEiXPnzklLS8vIyJw6deoHCMePH/fw8HBzcztx4gQCwU0o+E795jGZTBwOl5GRERAQ4OvrGx4ejkKhgA90bW0tBoPp7OyEbZOGhoaGIYyM4Al4/CxporS2/ZSG3yHFoG/lA3XuJU2OTU5PkXiJweTkJPCNhrfwWV5GAc6Ojo7icLji4mIkEpmdnT039yw6xNoHl+vAwGJml7ee0I4SNnpyxqIKyoZRddayVvh62Y+q8dYemeR5GofHFhOWPJvNZjKZgA/Q6XQymQxTCPhGgDMwmUzgew3YAtwCXI1f4EvgZSUwPT0dGBgIr/hfnCTw1lzXZ2DdlTrvVavKVlZWly5d2rVrl7a29vOuhTlDYGCgtbW1trb2SnOkFXuGhobrdszc3PwP1Qt6etwszjBJuHbtGnC/fvDggaenp6+vb3t7O/8L+LIPG7/++yGBucU50RjRv3n+bWT21c1mqCyqxJjkM/XCiNBH2L3gRftHHfscZh1eWVZbizBwsz43rMn6nCh4IOVvhEV+/vhXfgpe98K3QRg4HA6VSm1vby8qKoqLi3NwcJCSkhIXF5eRkVFSUtLU1ASxNUxNTW/cuHHnzp0bN25cv34dhAtUU1OzsrJ65O4eGhaWnZNTW9fQ1NJWVlFV14jp6O5taeuormsoLinJy8/LzSvIzs3PyMxBoVMys3LQySkxsTHOTs6+3j4JCQnJycmpqakZGRklJSW1EDIyMnx9ff39/VNTUzEYTFdXl5+fn4qKysam+a8r75e8nvfXd3JysrKyMjw83NvbGwRIyczMBMNpaWnh5mfo7cPhBgFlII7ix8fHUooaT2sGHVYI+lEp7JhyYG1z18w04AfjMD2YmJgYXwn41Pj4s2rj4+Ojo6NDQ0ONjY3JyckoFOru3bvd3d28PVw5ODYDWvJzOJwnTX1ndCKPGRQc1UQd0UoVNi4VtagSNa/5ST1FzSZxmPjMRZLJZEZGRpqbm/f3PwtZPT8/n5CQsEqbQaPReJUMZWVldXV14MjKbvD3+BJ4OQmw2ezc3FxTU9PnLaxXremftwvW96ampiYmJsBDAGgbLC0tYZ3DH97ixo0bkpKSe/bs2YAwgCxypqamQM9gbW2to6Ozrp4BUAcjI6O1fd5YvQCTBGDXZGNj4+Tk5OHh4e3t7eHhce/ePSsrq9u3b3d0dDz/H8LLzQK/Nl8CW04CE5QJ71pvkLXt1TrvSX78jC0MCe0b2L8jjRtHdXflnssT0lQ29dWa5XA4W44wpI6mrZOQAb3Tf/jlwkO9ssQ2uJC+0EUaVKMvtNMXmsnEe2zuS+aFDeq/N6feBmEAPyFEIhGJRDo7Oz948CAgICA8PDwoKMjPz8/Hx8fb29vLywveurq62tvb29nZ2dra6unpffPNN599+tnFCxcfP/bKzMopr6hsa+8cGsYPDuHrG5rQKWmBQaHe3gFIdGpp+ZMkVLKxidm33x4+evS4rKysmZmZj48POjk5Ozs7Pz+/oKAgPz8/Ozs7E0J6enpycjICgYiKigoPD4+KigoMDCwsLHyXZ3dhYaGrqystLS0gIMDHxyciIgKNRBXk5VdVVbc0N3d3teMGevHDOExbj71Pyk8K3t9fCf1JJfSby/6PIwtJk2OEUa4n9NgaAMoADsP0gXeXSCSOjo52dHSEhYUFBQWFhIS4ubk1NjZuvD5YDk/FamofOW8Qe0wlyCmsUMEq/mc1hLBxiahl7U+6KSo3EKSZOUivww37UFpaqqenx+FwiERiQ0PDEgQ8Hr+wsDAyMtLe3j4+Pv67NdSdO3fGx58Gt56dndXV1UUike/yxPH7tlUk0Nvbe//+/Y3N/dcuuOGlPyADpqamhoaG165ds7S0dHd3DwwMtLGxMTY2NjQ0BBoAAwMDYGVkAmEtPwENWllZSUlJ7dmzR0tLy8LCAr7L2g6AhGv3798PDAy8ceMGvMRfoV9Y3oFtk+B2VqV2Xq7I/auvr29sbHzjxg0HBwcnJydPCI6OjlZWVteuXdPQ0FBSUlJWVkahUNyA13zwJcCXwCtJoG2p7Qv8v58ZIw0L7S7fsy1G8IOUnV8NfNPF6HqlVp9eFB4xuG7itncwShLo8djS2D/TvlidkCFB8Lvqw1j6m/EVeWV5jnX/Njlwhc2iMpYG8K0HJvrF5ycDX7m1LXThWyIMTCZzfn4erEQXFhbYbPb8/PzU1BR4vT0xMQHyiwEbmImJCSKRSCAQQHYwNzc3cXFxeXl5Ozu7gMCAwuIC0jQ3fweLw+zt70lMSrh7766JmYmnp3syGpWUhLhjd/fHn34SFT2roKBga2ubkpICwoxWVlaWl5cXFxcXFhbm5eVlZmampaWh0WgkEpmQkBAfHx8bGxseHp6SkvIi6QX+rDmGF+hkMrm9vT07KzsiKiowODgmKgyJRmZk5ScmF9k+Tj6lFfKtTPCPqqG/qkR8LeNvdD9xeBCyUsJzQeD6N4wSIIxCIEIYHR0FxAAchLdjY2MjIyO5ubkGBgY//PDD4cOHv/3224sXL5aUlJBIpK6urtLS0sTExPDw8NDQ0Li4uIKCgra2NhKJBHrLgmIldQ2MX7GKGiKSFhYXk7KbJIxjfr2aetqq5hfNdCvnNOrSAsitUVNTc/v27dHRUU9PTzQafffu3d7eXiMjo+7ubm1tbSQSeevWrbq6Oi0tLdA+k8ksKSkJDg5OTU39syaFf9+tKwE6nT4/Pw+vdBcXF+Pi4q5fv77x0hxeZ/MWLC0tzc3NDAwM1NXVJCUlZWVl7e3tY2Nji4qKQkJCrhsb6+nqysnJnYYgKnrGycmpoqIiKSnJy8vrzp07ZmZm169fB8zBDIK5ubmVlZWsrOzu3buvXr36PJMkuA8gG/T9+/e9vb2trKw2UDIApS7Qe/AGRwKBWWGHB0tLy7t37zo6OrpBePDggbW1tYGBgYaGhoqKijIEFRUVRUXF390ngBHp1n0S+D3nS+BPlACDw1gRGWlY6GNuZCTB7bGCH9XvTaAmvGbfnkcY6uqmX7PlzbtctPjc6lhJsYKCubv/S/jKmewyxnzFdHiv3+GJvotTQ9qLlKoZgtVE186xnhNMxjNDiddv/51t4W0QBjqdPjExQSAQ4BUkk8kEYVXnePC7A/Tc3NwsD2ZmZggEAgqFAu/qAgIC4hMSiopL+wcGJyZJo2PjTc0tSFSyu8djB0en8Mjo9MxsFDrl7j17JWXlBw8e+Pj4IBCIqqqq1tbWdgitra0YDKahoaGmpubJkydlZWVFRUV5eXkZGRkwbUhJSYFXD+/gtMF2OHDfFhepTe09knqeohqPhVV8f5Dz+VYu8HulsJ9Uw35WDv9WNkD2euiT6vruro6uzk7get7b29vT0wPSOAwODg5BgHwfuD7TIyMj4Ah8ClAFYWHhS5cumZube3p6IhCI2NhY4BWqqqoKlg6qEJSVlZWUlFRUVAwMDJydnfPycmFVwDB+nDg5BZwWJmfmb3tm/agSf9qi6nvVpABEFXBNqa2tvXPnTkJCQmgoNx6CkpJSU1PTjRs3xsfHbWxs5ubmrl27hsViHRyeWnOWlZVlZGT4+fkFBAS8y0wPni9+4Z2SwODgYH5+fmVlJQ6Hm5+fr6+vv3Xr1suqF0BaN21tbTk5OVFR0XPnzuno6Pj5+aWlpWVnZ/v5+RkbG6upqUlKSpw6dUpEROTYsWM2NjbDw8McDodOp8/OznZ0dOTm5np7e9va2ppCADSAlzBYWlq+CI0xNTV98OCBt7e3mZmZjo4Or66At6yvrw/0DGZmZnC6aGCtZGdn5+Li4g7h/v37lpaW+vr6mpqagCSoqKioqqqCLSgoKyunpqbC7zLeqfnld4YvgS0hgbj5uP0Eof1QgjbudlBoZ9bubTE7BPN2G4wZvGzWhbVDDo9YPw/DO0sY2By2aOW51RqGOMEPEDv39e8XmvjkF+KvhbQ/wR6EzV6awd8ca/uf8b7Ti3OFlKlYfOt+xtLAWpm/f0c2kTCApS2TyRwfHweBPmk0bgYTFovFZDJnZ2eBVmFmZoZMJsM0AZTJZPI0BBKJVFJS4vjw4X2HB+ER4SFhoUUlJROkSSaTuUSn9/T1IpKSHB0dbWxuBQaFpKZlJKemP3BwvHnzVlRUNAKBKCgoaGxs7Ozs7O7u7uzsbG9vb21tbW5ubmxsrKurq6qqKi8vLy0tLSoqysnJQSKRCAQiMzPzXSYMqx5BYPPTPzx1WM73sGLoD8qhP6uE/qIc+otK2A9Xgr+W8Ze55vvIw9vf3zc4JDg+Ph6FQiUlJcXHx+fm5qZAyMzMzIaQn59fCCEvL6+wsLCoqKisrCwlJUVfX19ERERJScnV1RWJRJaVldXX10dEROjo6EhLS8vJyV25ckVZWRmwBRUVlStXrsjKyl6+fPnChQuioqJnzpy+cuVKaGjoMm3gPhdgFEwm0z+h8hf1WBGTCuGrUd2DY7293UlJiS4urgUFBZaWlgsLC9bW1r29vSYmJkQi0dLScmpqSktLC4vFWltbA1VVcXGxn5+fmpqagYEBlfrqJp6rBMvffY8lAD+Bs7OzxcXFiYmJSUlJqampaWlpDx8+fJFFOVjNg5pmZmZ6enry8vKnT58+duzYpUuXXF1dk5OT8/LyciH4+/tramqeP39OREREWFj46NGjt27dIhBWu+6xWKzp6en29va4uDg7OzsTExNTU1MrKysZWZndu3cDkyRYmbBxwczMzMnJycXFBSSB5uUJq8pGRkbgLnfv3nVycvL29vb09HR1dQUd0NLSgtUIMEMA33R4q6ioaGNjMzKy2tETFvJ7/CDxh8aXwBuRAJE59iP+p2eEYUTow+qPtscKbk/a+UPrTwTW6v8Vr3DTsPD1TZLeWcIQQYnYXy/EFUI8zycOSl3Xunf/sJDQ+Ce/EY7MsTeKvPIKgvrDS9gsyvSICbHj79OEm1CMHDqx64dpvPkfXvgeVNh0wjA1NYWDMouNj4+DnxA2m00mk0G4pMHBwdHR0bGxMWCVNDExMTY2BvxrYcuZurq6sPDw6LiY2Pi4uPj40rLSwaEh0tTU2Ph4M7YFiUI9evTowYMHIaHhGZnZmVk58QmIx4+909Mz8/LygHoBvE3v6+vr6enp7OwEyeOAqqG2tvbJkyelpaXFxcWZmZlIJDInJ2fLEYa+ockfFXx/Ugn7WSXsF+XwH5WDv5XxP6oaFIyqmZ6Zwg30FRQU+Pn5GRoaysnJ3bt3r7S0tLu7u62traSkJCUlBYFAJCUlIRAIJBKZmJiYkJCQmJiYnJzs4+MjKioqIyPj5OSEQCDy8/Nramqqq6sfPnwoA0FeXl4BgoqKipqaGnjdqKioKC8vLy0tLSEhceHChbNnz548efLo0aOysrJpaWkrfZdZHA47EFH9k2rcr9p5lm5ZGVmZt2/dwuEG6XS6v79/QEAAFottaWnR19dPTk42MjLKyMjQ0dFpbm52dnbmdbkGVmeAjr4HX0v+EN6CBFgsVmNjIxICCoVCIpEeHh4vrluwsLCwtLS8fv26goLC2bOiIiIiJ06cUFFV8fPzy8vLy87ORqFQ4eHhXl5eJqYmYmJioIKIiIilpSXQLTxvjHQ6vaurKz4+3s7OzsLCQlpaGpgkQSZPGzOFp2eBbZKzs7Orq+vGnEFXV9fS0tLLy8vNzc3Ozs7MzExLS0tNTQ22OIKJwboFwCJSU1PB95rFZC4uLpLJ5N/pPUiI+bwx8o/zJcCXACwBu2k7ofHlpM7DQnt79n+A2rktVnBP0d4M6ivGUYUbB4WtRRhSqan/O3pwb/d+bmTVuJWEIV7w47Z9+4eF9uOFvsZ/M8GcWDXSzdhl0sfIRIeJvvMzIwZM+iCHw6FMxRBa9zPpXC43PxWDx+4F5c24+7vT5iYSBg6Hw2AwhoeHcRDgEJxUKhWPx+NwuO7ubvC+v7m5uaWlpbm5GYPBNDU1NTY2NkBoamrCYrENDQ1FRUXxCQl37toZXDNQUlKSkpK6cPHCebHzFy+Ji4uLX7x4UVJS8soVJS1tHQtLKw9Pr/j4xLKy8qqqqubm5q6uroGBgUEIAwMDvb29QNXQ0tKCwWB4VQ1FRUVoNDovL2/LEYbewYnvZHy+Uwj6VibgOxl/YbWgO95ZnQNj0HO27HjM4czNzQGDH39//4yMjMbGxoGBgZ6entra2rKyspycHDc3t6ysrPJyruiSkpKUlZUtLCzCwsJAdKmampqqqiobGxtJSUkZGRlZWVk5CGpqalpaWlevXtXQ0FBXV1dQUJCQkOBODIRz586dOnXqxIkTx44dO3r0qJOT0/z8PPgCsNksrr6JzXYKzvtJI/lX9fiqRhAciRuRFQqqy3WDfroWgfJwM5ncI8uXs6Grn6kswO7yef5fvgQ2ksDAwEB6enpSUhIKhUKj0bGxscCR4IWW5FAlQ0NDWRmZEydOiIiInD592tjYODw8PCIiwtXV1draWl1dTVxc/OzZsyIQTp48eezYMQ0NDV6iu0H/lpaWurq6EhISlJSUdu3apamp+Yc+DLw9NzExsbS09Pf3d3NzMzAw2MCfQReCurq6MoTfI8XBygS48Dy2oKioaG1tPTAwMD8/PzY2hsPhOjo6sFhsc3Pz0NDQyrcDG4yVf4ovgb+uBDBLmH+M/JPX13lX8YfbYwR3pO4ywl1nc579gr+OjLYQYSCxSD8TftlP/GT/sNDusj1cJQP8iRPcVfjh/kGu7ZYQ8RPxMYm3kJaBSR8e6z46NXyNRk6dGpQZ7fiaScezWbTR9v/MjN7lKhlYVGLHV9N4C/pCO5M+9DrT9I5fu7mEgUql4nC4/v7+wcHBxcVFSH3DnpycHBkZ6e/v7+7uLi8vj4uLi4EQDSEmJiY2NhbKxxANynFcvQK3TkBAgLOz8927d21sbIDRsKGhoYGBgR6UNsjQ0NDExMTe/m5YWFhycnJ6enp2dnZhYWFZWVllZWVNTU1tbS14QV5VVVVTU4OFADhDfX19VVVVRUVFenp6SUnJllKmc5fLg4RxedMIzVsIO+/c+BzMIIH0vMcOiUQqKSnl5eXdvHnT398/Ly+vvb19YGCgq6urqKjI3t4+JyenpaWlrKzMyMjIzs4uKioqPT29uLgYyPDu3bvi4uLSy5CRkVFXVwcLDm1tbTU1tcuXL2tpaT1+/Dg7O7u6urqysjIjI8Pd3V1DQ+P06dMiIiK//PLL76bYs7MrMj3TFhdVb6EPq6BNnNJYbMab+hf5PCHwj/MlQKFQ8vPzEQgECgLQp/EuuNctm0EApzQ0NC5cuHACgoiIiKSkBOTurC4pKXny5Mnjx48LCwuLiIicPHny9OnTZ86cOX369KVLl17W1p/FYvn7++/Zs0dDQ2PdLm1w0NTU1NraOigoyMXFBeRP0FsJfX19PT09DQ0NXpNCXm4ACANQHoLjsOZBUVFRTU1NV1c3LCysu7sbi8U2NTVhlgHe9czMzPCfNL4E+BLYQAJMDlN1QlVo7Jl64WPsvu0JO7cnCn5bfojIIm5w7UudCg3bMiZJE8zJQ/hDTw20BoU+rP1oZ+6HO7N37yr88CPMx4AtcAnD+CePyY9fSggbVGaz2YylQSZ9HUdqMsF6ckAGXLtExQw3/z8zBCsOhzM36Y9v+5TF4Mbgoc4kEVp3THR/tTCbv8FdtvqpzSUMU1NTfX19vb29w8PD4N0wnU4fGxsjEAjAszYlJSUsLCwyMjJiGeEQwsLCwsPDw8LCQpcREhISFhYGaq2tDyqHh4eHhIQEBgYGBAQEQghYA2DoEhAQkJeX19bWBvQMTU1N9fX11dXVOTk5nZ2vknf9z3oO2GwWg8lkMZhLzCU2m8mNHcX9MLnpk9f0iclkKigo3Lx58+HDh6dOnXJ0dPTx8cnKysJisX0Qmpqa2traOjo6Hj58eOfOndjY2LS0tPz8/LKystra2oCAAHFxcSkpqcvLUFFR0YGgp6enqqqqoKAQFRVFIj2jK9wvIYOxsLAwNDQUEREhLS0NOIO9vf1KNQ4b24s/rhH7m1p0c89qY+g14+Af4EvgtSRAp9Pr6+sBVQDqBQQC4eDg8ILeC2bm5hqammfPnhUWFj5x4sRJCKdPc9VogCScOnUKkARRUdGzZ8+eO3fu7NmzoqKiXl5eCwsvEa4bvLno7+9/9OhRTEzM7du3QUoH8xeGiYmJjY1NWFiYi4vLWj2Dvr4+4Pm8JGHdMswTVFVVdXR0TE1NHz586OzsnJaWVltb27weQHKblxrva00q/2K+BLagBPJoeZ/gD3ANbIC786DQzszd22IFd6TsSpzkppF9U9hChIHD4dyfuf/MRmtE6Kl8Rrh5r58KakToHyP/bFtqe1PyYbOXxnvPTg0brG1wckBuatiATmubwV8b6/5plmgHrI/YrFlC+z9mx5zAJUu0Fsbie+76vLmEYXx8HBAGIpEIfvwoFMr4+DhIBdDf3x8bGxsRERG5jGXWwP0LlPuAP4SEhISGhoZBAAwiZBlBEIIhgHJAQEBQUBA4EhwcDA6CLcwi/P39UShUGwSgQMdgMHV1dRUVFdPT726UsbWPMofNYbIZHK65DhtQBTabyeKw1lWSTE1NCQsLHzx48MCBAwcPHrSysvL29g4ODs7Pz8disT09PTgcbmhoKD093draOiIiIi0tLTc3t7Cw8MmTJ5mZmVeuXBEXF5eEICEhoaCgoKWlBVJEqaqqqqmpZWdnU6lUGo3GaynEYrEYDMbi4iKVSn3y5MnVq1eFhYV/++23pKQkeDgsrgES+3Fs6ZfSoY+jy+Hj/AJfApshgcHBweTkZCQSCTgDGo0OCgqysrJ6QcKgoaEhKsp1WgBUAd7ykoTz58+LQbgIQUxMzMTEpLe3l6sQXHb6f6mhMRiM6upqJycnU1PTF+wnoBUgaBICgXj48KHeSujq6mpqaqo+B7A9kpqamoaGhoGBwc2bN52cnEJDQ9FodFVVVWpqakFBQWtrKzAobV4JDAbT3Nw8NgYMI19qrPzKW1ICDDyescaVf0uO5G11msaiXSBefJapbURoT/3HXPMblKBsndzrR0biHcfWIgxkFll8XJzLGYYhkgATquUoUkLET+TG5d+sPdL8ZPBIsyBj6VnSWCDAuXE3Yvu2sZ5js8R7zCXuC80lah1jkftmeW780eSADJvN4BX1e1zeRMLAZrPxeHxvb29fX9/UFDdILXB3HocA4oGEhIQAYgC0B4AeAHUBoAfgeFlZGRaLBTZFVVVVT548KS4uLigoAM6FWRAyMzNBISsrC6RgA7SBlzDAtMHf3z8iIqKxsbGrq6ujo6O1tRWLxWIwmKqqqvf4F25qaur48eOffvrpgQMH9u/fr6am5unp6efnFxERUVJSgsVicTgcBoNxcHDw9/dPTk7OycnJz88Cev9IAAAgAElEQVQvKSmpqKiwtbU9e/as+DIkJSXV1NQ0NTWvXr2qrq6uqqra3NxMo9FAJFwikchrvsxms+l0OpVKnZ+fb2xsVFFROXbsmJSUFBxZhcVmcthswgTphFbEFYuExaWl9/grxx/anyUBsFKfmZnJz89PSkqCCUNSUpKzszNwd153LQ4ftLCw0NLSAmzhFKRGOH2Ga24ENAmAJFy4IHbhwgVx8YuXLolLSFySkpKUkLgkKyubn5//alSB9yocDhcUFPRSnAF03tvbOzEx8f79+8AMCRCH56kXlJWVVVRUNJZhZWUVEBAQGxubnp4OgqcB97CampqsrKy6ujqYKQCSAHaBdVJXVxcIjvdnTTr/vm9NAgQ9Pdz+/cNiYpMuLottb+zV71vr/9u/Udx8vNAotCaG1sH7+vfvSN21PV5QKO0Adv4N5yYLCV3fJKm29h19Q9pL7/0Z/+vHHXs/xu7d17ufq1jgoQ1Co5/Ez79ubopVM85mzRPaP5/GW6w6zlwawGN3zI49jeTOZlFG2/9Dm02H1rSLLOYK++pV175nu5tIGJhM5tDQELB1AR7PDAaDRCKBgEggkkZ2djYaAohSgkAgQJSeuLg42JMhNzcXBGAF7hADEHp7e7u6ukB2hcbGRiyWG0sHaAz6+vpqa2uDgoICeODv7++3EsHBwdXV1SBuUkdHB4i4WldXNzT03vqsLC0tSUtLf/LJJwcOHPjss88uX77s7Ozs4eEBFhPFxcUhISHa2trOzs5JSUkgNzZwAklJSZGSkjp37pwYhAsXLsjJyalB0NTUlJGRSU5O5nA4TChGColEamtrgz2boS8V1zBpcXGRQqHMz89nZWWJiYkdOXIkIICb453NZrPYTAbkzewVX/a1tGdH3zp2hO/ZF48/nD9FAkwmE0RGAv9wUCgUGo0G1j6mpqZ/6FV8/fp14KIASIKoKNfcCPCEixcviotzPxISlyQlLwHLPWkIEhIStra2vKZ6LzV2BoNBoVBgBk4ikeLi4kC2NZjJbGyjZGpqamlpGRwcjEQi7ezsQHIGXV1dDQ0NENkMWBwBlYKGhoaRkZGtre3vydSBFZO1tXV6enpdXV1NTU1jYyPMClpbW8vLy8vKylqgkBUwbQAF+ODw8DA/QcpLzfhWrMyam+s+eHDc2nomNHRIWrrn4EGiiQmbH+f6+XM5x5o7ThDmEgbw1hwOpYreeavt9vOve8UzW44wcDicuoX6z0u+2J4k+AFy587s3XvboeBIQ0L7CUKH8IcnWVzngTeLuQmvEeweJh2/qtnZcc/hlp3zE49oM0kTfRdIg+psNtcp96+GTSQMdDodRCXq6+sDlqyLi4uALUxPT8Np2mZmZqYhAC4Bcj/D+YYJBMLExASZTJ6CAJJAj0JZioeHh4eGhgCL6O/v7+vrA1scFMW1urq6DEJpaWlJSQlI8Jyfn5+Xl5eVlZWRkVFUVNTV1dXT09Pd3d3V1QVCJzU1NfX19b3HD4Gent7+/fs/++yzgwcPSkhIODg4uLq6enp6+vj4GBkZXbp0yc7OLj4+Pj09HagXioqKSkpKnJ2dT548CRtkX7hwAeReAFkXfo/jDhIgQNYWXG4AUufCb0ZB5o3FxUUajTY7O/t7gF07O7tjx44pKyuTyWSIM7C4hlUcTv/w2Hcy3ogczHs8Bfyh/YkS6OvrS0lJgXULgDB4enr+4cobLNAVFRXPQxATE7twQezixWeaBElJCSkpSSkpKUASZGRk5EAYMVlZVVXV0tJS+OvwssNvbW39fSnf398PtzA7O4tEIm1sbIBWZGO2AM6amJjcuXMHgUDExMRYWVnp6OhoaWmpqKgoKSmpqqpevXpVV1dXC4KhoaG1tbWxsbG6urqaqqqBgcH169cDAgLq6+thDtAMoaWlpbGxMScnp6qqatUpEPIOVMNisXzv55ed9C1Xn1pe3iEoCCsWaDU1Hbt2zYSEbLmBvLUOh86FPTNGGhba28sNpbo9UfCLjC+Ji2/M1xkezlYkDBwOp3i2+EDu37jBVaMFd+bsBuRKaPyTm9M28NDeYIHFmMK3fjpLtF/bJmUqfAp3YWpQjkIK/+vYIK2SwyYShoWFhf7+/t7eXhwOtwQZmVAolImJCRKJNDs7Oz8/PwcBvHWeX8aqXVCNTCYDXjE1NUUikSYnJ0HGBiKRSFgGfhkgSRyRSBwbG4Osn8YnIADfidHRUeBvPTg4ODAwAFwseiB0dXU1Nzd3dnbCP8yrhPUe7N67d09ISOif//znwYMH5eXlHzx44ODg4OLi4uvr6+bm5uTkFBMTA2JMZWVl5eXlFRQU5ObmamlpCQsLnz7Ntb44c+aMuLi4AgRFRUUpKSmgXgBsYa3o2Gw2k8mE3Rjm5ubIZHJxcTGIMFNbW8srVRabqX47yd6ngPcgv8yXwBuRAJlMLiws5DVGQqPRCATi3r175ubmphA2YA46OjoSEpcucUM5A3MjCQkJCUlJSRACQFpaWmY51rC8vLyCgsKVK1cUFRVlZWUfPnwIB5V+hYHg8fjIyMjJyRWv0xYWFp5UPLG/a29iYgIowR9uzczMXF1ds7KyIiIizM3NNTU19fX1LS0tXVxcQkNDHRwcrl69CrIuSkpKcLUlFy9euXLF0NDQ2NjYysoqOTm5BQKgATAZyM/PR6FQjY2NazkDqNPU1DQw0M9XMrzC1G+hS8bv3On9xz+Y0AsgbreZzJ5//nPiwYMtNIS32dUp5tRvhCNChGfqhd1le7ZB3guPu702oyfBIbh9+wv27ste+cl5Z02SYCFkTWd9kvUplzMkCu7t2rcfL/TZ8N9qFlesHODKr18gEx3wWCEmFPuIw+HQFzun8caLlEqo5bWhZF7/hluphU0kDBQKBQ6RBPTpILszmUyen59fSwyWKQP3L+ASc3NzvBmgyRBg5jAJYWwZxGWMjo6CIkwYxsfHx8bGRiEQCAQ8Hj80NDQ4OAi0E0A1AbhNW1tbe3s7b7D/rTSZL9DXoKCgAwcO/Pvf//7iiy+0tLTuQXj48KG3t3dgYGBYWBgKhUpNTc3IyMjOzs7NzQWhJy9evHj8+PGTJ0+egiAhISELAaR5xuFwG9wZOD2vIgy/J+zT1dX97bffoqOjgVUS3EJ8ZoOGTeI6MZ7gGvwCXwIvLwEGg1FbWwuyLqCWgUajQ0NDXVxcQMjmW7du/R7z1wyCOQ8sLCyMjIyuXLmyHCJM6vLly9IQnpIEWVkuSZCXv7LME5QhKCkpaWhoFBYWriXSfzQCNhTGgPf3ibfMvZrFYtXWVt+7d8/MzNTc3Az68HR6TdHMzOzGjRthYWGZmZkRERHe3t7R0dHp6en5+fnl5eVeXl5SUlKAC128eOHCBTEpKSldXd2bN28+evQoMjIyPz8fUALYJKm5uRkoGdLT0ysqKsBZXjoBl7FY7BYLJvFH08M/v0ICTCbu5MneXbtwx46NGhlNBwaOGhv3/Otfi93d4Eml1dVRS0sZE28jx9aKjr2rO4GzQbyhVPd27/8gaed2hOChrO9nljYlGPHWJQwcDidjMkMo89Pt8YJ7aj8SmvhEelyGwY0J+Tyw5uerCQS/Efzjqel4BqOLw6E8r+ra4ywGcaTl49kxtyUahjSoTmg7SBpUoS9AT/La2n+xI5tIGObm5nogEAgE8PoZ6Afm5uZgwvA82gATBlCYnZ0lk8lgC7QNMzMzsLYBMIdxHsAqhfHxZ+oFEM4VEAbYnGlgYKC/v38AQl9fX1dXV2trK9CHvJdPgru7+9///vcvvvji119/NTY2trOzs7e3d3R09PLyCgsLS0hISE5OzszMzM7OzsvLy4cQFBR06tQpOCbMqVOn4LeqkpKS165d2yClK5vNBoRhaWlpYWGBQqHMzc3NzMxMTEw4OjoeOXLEyelpSLJlaTM6cWPi14LJ81wDQdYbylmz3Dj/719UAtxcJYODaWlpqwgDEon09fUNCQlJSUlJSkqKjo729/d3cnK6desWr5OAmZmZtrY2MDECxEAOgrycHCAJileuKEEA7sIg8bmampqioqK9vT0I+cBkMmk0GolEwuPxg9DbCvDCAphQjoyMkEgkMpm8sLDAYDJWEQw2940tm7awNDtHmZuljo2RBoeH8XjC0NBwcnLy7du3N1CMwMQBjMjW1jY5ObmoqKi8vLyioqK2traxsbG5ubmiouLGjRtiYucvXRKXlZXV0dF2c3NDo9FFRUV1dXVYLBZEoAZsoaWlBYvFtra2tre3t7W1VVRUFBQU8BIJmCqAAgaD6ezs5Hs/v69fv6WBgc7du8nx8dTy8jFr674vv+w/fJi+bNw7ZmnZfeDAsLh4/6FDEw4OHMZzQ8qwmczF9nYWjfa+CgqMi8wiHyEc4fVe2F0KpSdDCob1hW/S2INDtpjT8yo5pE2k7884sKt0jxDxkyTKsxCLq6oxmZOtrYp5+bsKCrcVFm3LzdtWUvpxY9PXAwNqU1MeS/QyDmcSiie56roVuzMEW0LL/4x2/AuKptq64txfe2cTCQOZTO7p6ent7QVxh5hM5tTUFKxeoGwIWNsA2MXz9AzT09OQa8NTOyXAHMAWWC6BMuAPY2NjwDUCj8fDVkk4CIAwgLTH7e3t7+sPG5vNtra2/s9//vPvf/9bSkrKysrKzs7uwYMHbm5uAQEBUVFRSCQSJLzLzc0FhCEvL8/T0xNElwfhI0+fPi0hISEFQVxc/ObNm3Q6fdX6Bv5OAe8FOp0OHBjm5+dnZ2cB2QsMDDx+/LitrS1cGSow5mlLSjfi+ka4yRy42Z75+MtIgM1mg/SOb3zEU1NT2dnZgC3wOjAgkci4uDiYRaAhIJHIqKgof3//+/fvW1tbm5ubGxoaqigrA78dQAkUFRWvKCgoQQBHVCGoqampL0NVVVVdXT0+Pn50dLSzs7OhoaGioiI3NzczMzMjIyMjPT0jHUZGdnZuVVUNFtva0dHR29PV3orpaK/p7i7DYDJKS6MyMgPzcgMzMh6jUR4paB8EwgeRGJGUFItCJaFQKUFBYbdv33kRzmBubm5mZubp6fnkyZOWlpZmCMDQqLW1NTk5+c6dO76+vggEori4uKmpqRUCXBOQhLa2ttbW1qamptLSUhQKFRgYmJWVVVxczNsmaBluH7g04PH45/2jeOMzzm/wbUpgJiKia98+xnIGHub09JCExOTDh6APlNLSRSw35s9ia2vX3r1zKSngOHN+fqm7m8lja0fv7+/+7DPaSjvVtzmQt3Ov8LmIFd4L3fuAeuGXnN+oTOom9SEoeKuaJMECySBl/qP1nydGT8yz5+GDKwvsvj7DwuJtGZnb0tK3paZxtxmZ27Kyt+UXbCsq3l5SKljf+PcBnPQM2YfDeW4wRubS8Ny4A2ORr1VYKV0OZxMJw/T0NCAMIDwInU6fmZkBugUKhUKlUuHt87gDL22AVQ2APJAhzCwDZg5A7UDiAcwZgGESL2cAeoahoafmSYOQV0NHR8frGByvFvC7tL+0tGRgYPDdd98dPnxYW1vb1tbW0dHx0aNHfn5+kZGRiYmJqampwHUhPz+/EEJ+fr6rq+vx48dFREROQXEkz5w5w7XdhnDhwgVbW1s6nc5kMtcuBdiQ9wKdTl9aWqLRaFQqFdiYTU1NTU9Px8TEHD9+/NatW7wXsrkJGTgmD1OLG7jfVX7K53fp8dn0vvz+TdTQ0Ghvb2dDAPcDjwfvQ/Li/QBX0en0uro6OKnzsjnSs79IJBI4NsBcIhlCYmJieHi4p6enpaXl1atXVSCoq6uD4EIwTwAkQQOCpqamhoaGpqamgYHB3bt3w8PDCwsLc3NzU1NTU1JSUrlIS0tLT0vPSE9LT0tLy8jIzMjISktLR6NikxA+iCTbuHjjiAglL69f3R79y8VVyNX1Q1Oz7f/977brpttcXLe7Ou9wcdnl6rbf0/M/3t5H/P0lQ0L1IyMcXJxvcY2SzC0szMzMzS1grcLagpmZmYWFBRKJbIHAu7JvampqaGgAegPe7ApYLBbYara2ttbV1eXm5vr6+trY2ChDJMrd3R1kvUQgEBt4P2MwmPb2dir1JWwDXnyi+TX/XAkMS0kNiYnx9mHC3r7v0CGuL8PkJDkqaioggPrkCWt6uu+rryY9PDgcDq2ysv/773GHD/d99x3X1QFSO0yHhfX84x+suTnept6z8jx7Xpgg8ky9MCy0q+SpeiFqIGbzBhsYtOUJA4fDqVmqqV6sfp6UaLTO/II9GZnbSkq3Y5q3t3d8UFu3PTNrW3oGlzaAT2bWtqLibTW137PZm8XNnte99+D4JhKGqakp3iQMi4uLs7OzgCRQqVQaD6hrsIpCwF4NsKoBNk8iLwNwBrAF77Bh1rCWMwDDJKBqGBkZ4WUOnZ2ds7PvZ2Dd+fl5TU3NH3744ejRo6ampk5OTh4eHn5+fmFhYfHx8SkpKYAtAKpQXFxcUlJSUFDg6urKyxbOnj17aRliYmKWlpY0Go0JuTUzIbCWwYBAp9MXFhZoNBqwRwIBr6anp8PDw48fP25vvyIcAZurVGBbuGSklnDfSLFYG9gpvgffPv4QVkhgdHTUxcVlcHAQLPQXFhbgdOCvQBjgS7q7u9FoNEwGnhEFFCqJB8hloFAouDIajUahUDExMQEBAcA3GsQX0tTUVIegCeHqMkBM0kePHqWnp1dXV7e0tNTU1OTk5KQ/RUYaV7GQmpmRmZ6Zk5qWjkSFRUXfDQhUcHM79NBx/737/9e9+wIODgJOzgKuLgJubgKeXgJm5gIffyxgai7g7SPg9fjpx/OxgIengLu7gLv7/3h67nFz+dzmloSZuYWZhbm5meXG/gympqb29vYg51ozD3gpBEwS2trampqaSkpK4uLiXFxcDA0N5eXlRUREjhw5IiYmFhwcjMFgALsAykme9lYXMRgMHj8CXgqsmHj+zhaXwMDPP/f9618EXd1ZJJI+OLjQ1NTz978T9PTYFMrAkSODv/46Zm09oqAwcOxYx7ZttJoa5sxMzz//OWZpyaJSF1pbe/72NxBPaVhBgaCqusWF8QfdR1AQz3ydh4X2du37IElwO0Lwh+yf5xmbSKcDA98HwrCxcCcnM+vqtvf2fUCa2kGe3TE7x912d3+Qlb2CM+Tlbxse9t+4Kf7ZdSWwiYSBRCJ1d3f39fUBdzc6nT43N8dLFagvAMAcYMIA+zbAzIHMg5mZGTKZPL0MWNsACiC2Eqxn4OUMw8sYGhrq6up6XwnD7Oysurr6zz//fO7cudu3b3t6evr7+4eFhcXGxiYlJaWnp+dBYZGKi4tLS0tLSkrKysqKioq8vLxOnTp15swZOKyquLg4oAwXL178/WXq+Pg4g8Gg0+mAIcBbOgTgvUCj0cAkAsJAIpHc3d2FhYU9PT15n0s2h8nhsK3csqMzGrjHuRms+fhrSQBe6Ht4eCgoKOTk5PDm9HhZWZBIJJCmjZcngDISiUQgEEkrkZiYiEAg0BAAg0ChUEDhgEQiY2Njvby87O3tTU1NtbW1NTQ0rl69CvKda2tr6+vr29vbo1CohoYGLI/Rf01NTW5ubjqEjIzMzPTMFHRUYoJzUNAVD4/P3R5tc/f4Hy8vAV8/Af9A7tbHR8DbW8DHl1sIDBC4dUtg3z4BC0uBoGCBwCDoEyAQGCgQECjgHyDg6y/gC9X08Nhxx/aimbmpGaRrWKtbgI+YmZldv37dz88PhDaCLY4AYWiD0NLSAvK7BwQE3Lp1S0ND4+zZsydOnDh27Njx48eFhYXFxMT09PTQaDRgC3CIVUCTmtcDBoNpa2vjh1h92Wf43a/Pmp2lFBYSb97ECQv3fv5579/+NigmxiAS5/PzO3bupA8OgiGQ3N27Dx5kLy6So6M7P/oI9oEmeXpOPnzIXljoOXiQHMN9y85eWJgJDaWWlr77Y3+pHi6xly4QL+4nCj3NvTAs9DQ4ElIwuHdzQ9D6+/dv0ShJLy5hNnuRQlWk0nZMzzz9zJB3zFF2NGG2A9ukjMxtObnbyisOMxib4ln+4l3dojU3kTBMT0/DhIHFYrHZbGCXQqVSFyDw6BhWF2EqARMGYJ40twZrmQMvbQAeDqsIA+AMIJkDfiUGBwe7u7tBcoAtOqMbdHt+fv7q1atHjx69fPkySOccGhoaExOTlJSERqOzs7MLCgoATyiHHCLLy8tLSkoCAgLOnz8vKip6DspRdf78eZgwXLp0SVJSsqGhgcViLUEAJAGmCouLi0C9ANsjzczMkEgkAoFgYmJy6tSphIQVyRpZXJMkltF9tFdsGfdnA7JQ2mBE/FPvsQQKCvIlJC59/PHHIiIiwcHBwBUKHi8bwvN2wXEGg1FdXb3WGAkwgaSkJAQEUADEISAg4MGDB76+vrGxsWg0Ojk5GSgckEgk2E1OTkYikZGRkR4eHr+nYzMyMtLT09PR0bl582ZUVFRlZSVYdmO4aIY+mObmpprqmpycvIyMHDQ6OiLc2MPzkNujD909/sfXRyAoUCAsTCAqSiA6RiAuTiAxQSAhQSAmViAqWiAkRCAsVMDeXkBISMDCQiAinFszNIy75X5CuZ/QUG614GCB4CABb6+P79hegYiBGUwP1i2YmpreunUrOzsbEBtghtTe3t7Y2FhWVhYXF/fo0SM9PT2Qpe7o0aPAKPEEBBERkYsXL+rq6pqbm9vb2xcVFWGxWAwGg8ViKyoqiouLYQayljVgMJienh5YcQRPH7/wnkiAxWIQifThYTAcSlFRx44d1PJy1uwsra6u+x//wGtocDicEWXlQXHxFUNmsylFRZ379zPGxiilpbjjx4dOnqSWl6+os/V38mh5BwifPs1YPCy0r3f/B0hucKRvMw+R6dyURJsHP/+B954wMBhVc/On5+a5bGGGzFUvEMd2tLV/UFL6VMOQmcUlDITR+M2T8/vd8iYSBgqF0tvbOzQ0NDExsbCwACzawbJy1XZxGYBIgCUmTCp4yQMcXgk4Q/NqHniZA2ywNANhenoaNlLi1TOMQiBAwOPxBAJhaGiop6dng7A/W/ppWFhYMDAwOHbsmIyMjJubW0hISHR0NAKBSE5OTktLy87OTk5OLisrq6ioePLkSWVl5ZMnT0pLS3+3OpCUlDx37tyFZYiLiy97MUiIiYn5+fmBqKnL07gI5peXLVAoFDApwIHhd1NpGRmZ39urq6tbIVIuQ2Cr30Q4BufzCcMKyfwld0AsVENDw88///w///nP3bt329raQNRjWBGxSjB0Op1EIvX19TU3N9fV1aWnp8P2RSgeIBCIhIQEQBiAngEciYqKunfvnqKiooqqyh27O35+fiB0GBqN5vVzACoI4OTg4OBgZ2eXnZ0NFsoQVXi6aeKiGdOMxTRjC/PTw8Ovu7t/5+T8/z3y4GoPgoIEwsMFoiIFYmIE4uIFEpMEkCgBNFogNU0gPYP7QScLoJACTg8FhD4RuGnNrRYVtfoTGSUQEcFtJzxCIDDwfywtT5qYmFps6MYAKISpqamzs3NlZSUWi62pqcnMzAwLC7OwsFBVVT137pyIiIiwsPDx48dPnDhxchkiIiLHjh27evVqYGAgULOYmpoGBQVxBwmhqakpLS2tpKRkA87Q3Nw8wQ+vueqpfU932UtLEw4O/T/+OHL5MtHConPvXpK7O4fDwZ09SzQzWzXocVvbvv/+d+z69b5vv53y82NDuZtW1XnuLoPBJhIp9fWTEeGzJSXvpmqaxWEpTSg/i6Y6IrT7yUfbYwU/QAl6dfk8d2hv6ISv3/usYWAyGyhUbfLsXgqVSxVmyDtG8DsaGrfn5XH9nmEfhrz8bdU1Z9ns57o7vyFhv7fNbCJhYLPZExMTOByOSCROT0+Dn3kWi8WEwIYCbrIggCPAlIUJWcMzGAy4AMxdwGIUZhRwAdZNUNYAphMgqzRsrcSrcIATS4MccIODg0NDQyBrxPs35ywW6+bNm8eOHbt8+bKLi0tkZGRCQgIwRsrOzs7JyUGj0eXl5dXV1TUQqqqqysrK4uPjlZWVz507d3EZvIRBXFxcWVl5YGCAwWDQaDR4XnjZApiI2dnZ6elpEok0MzPj5+d37tw5NTW1VemoOBzWwuLSJf1IBz5heP+ev9cYUX9/v6ur66FDh/73f/9XXV29trZ2XcJAIpHq6+sB9U2CsC5bQCKRiYmJCQkJiTxISEiIi4tLSEjw9/e/cuXKkSNHTp48eeHCBUNDw0ePHkVERMCEASgogM4BiUTGxMSkpKQ0NDQ0Nzc/JQo8f1paWjCYxrR0Vx/fo46O/+8jV665UUCQQEioQEQkV5OQkCCAQAggkQLJyRBVSBfIzBTIyhbIzhXIyxMoLhbw8xc4cEDAxoarf4iJXecTHcPVTsTECji57FFTldQ3MNzYhwEQBjMzM0tLS3d3dwcHh2vXrklJSZ05cwZYHIlAWKYJT/+eOHHi6NGjV69ezcrKampqQiAQt2/fvn79uq2tbX5+PhaLBWkZioqK0Gh0Q0PD8zgDBoPp7u5+j0NXv8Zj/n5eypqboxMIHA5nwt5+UFSUOTU1Zms7cPQoe5EbO3sWgZjPzuZwOAPHjvUICXXt2TNuZ/cigmCTyYzOzoXExHkry2kpqYovvkgS3JmwQxC5c1eH/T0Oe3Xekhdpc1PrNC41HsT//Zl6oW//B8nc1M5fpv93cnFFWsbN6IaP73tIGNhsFp3xhErVJM/upS3s4OoWyDuGhj+oqeX6LfBShYzMbZB6YffU9JPNEO9fpM1NJAzcCAlM5sjISGtra1dXV3d3d08PN8pqf38/DocbGhoCL/XHxsYmIJBIJJCMeXx8fGJiYnJykgQB6AfIy5ibmwPZG2D/aV6/CLBg5WUXYOUKkgDMQwCxeshk8szMzDQEoH8gEomDg4Pvq3oBPNB+fn7Hjx8XExO7efNmdHR0YmJiZGRkZmZmfn5+Xl5eenp6WVlZbW1t3TIqKyszMzONjY3PnUz8ED0AACAASURBVDsnvoxLly7BGgYJCa6SwcXFBeiFYP4GWyIBX2dYvUAmk+vr6xUUFM6fP+/o6AhoJM+XjT4+NfvLFT/n8CK+hoFHLH/F4lpKQCaT4+Pjz549GxQUBKwceeVCIpFKSkqWgxGlAn9l1HpAIBDx8fGrCEN8fHwchKCgoLt378rISANjfWFhYVFR0du3b/MSBlBOSkoCVyUmJubl5TU2Nj5lCk3NmKYmDKalpaWjrCw5Okrr/2fvPOCautf/f+L93f+9VwQEBLXt7bi3Uzu0ra2tC8VRQBy4cFWtLW5luEcduGWvsAlkEAJhhrAhkIQEskiYQaYMZe9Ncv73e74QEZWqtY42z+u8wslZOec5h+S8z/N8nufKFb0rVxAnZ8QDD9KHAgJBxhGJjFCpIKoAUCEKiYlG4hgIk4nExyOJSUhyCpKSgmRkgoXffgd38QKOFoZQQsAqIG2JCkgjJARMoVDACCHobwcPf7Vl64979+2xtrJ6ZCaSChWwgkq2+/btW7lypUqWoOq1snjxYkNDQ9VbOD5//vydO3cyGAyJRCISiQQCAZVKhcxw69YtGKmQSCRisTghIYHFYkGEyH2USaXSe/fuPXx+R59N9fif0gNDTU3o0NBgY2OVqWnVwoV3TEyK9fXbiMSBysoiPb1uLrclKKhs9mxFb++jD39wYLC4uC84qH2PZcv8+Q1vvNFqoN851SBLRydIU4ugqRWIvZJ19Vokkkdv4eVNPdp8bHr9cGvn6dXTdfkgvKARrn01//oL2ClXtz8VMCiVvf39zK6uda1tOsOo0KJZUTEpiwfKIo1GhZhYgAqxDFBZNb/g2Atw9Z/4I/5YYACNtQcGqqqqYHori8XKxIzNZnO53LCwsLNnzzo5OQUFBVEoFCKR6OPj4+7u7uHh4YaZh4cHHjNPT088Hu+JGR6P98LMx8cHzvX29vZ50PwxCwwMJBAIgYEEP/9AKjUsJQX0KuJwODA1n4MZFzMej5ebm3vnzp329vY/989Yamrq4sWLjYyMDh065OfnFxkZSaFQGAxGenp6CmZsNluImUgkEgqF2dnZLBbLzc1t1apVK1euNDMzW4kZBIbVmMFZISEhqoJIUKwCuU6FZy0tLe3t7eXl5UeOHDE2NjYzM8vIgEKF0Y+ClDl5FR+vdPCgZqmB4U/8vfN7Dg1CKdxCWVmZi4sLVNKnpqbCJuWxsbFRUVGPIoVwWC4pJCQEAkPIiMH7fgqFQiAQ3N3d3dzc7Ozsfvzxx6VLly5cuHD1qlVubm7ho6oqwSADlUqFKxKJRAqFkpCQIBQKh5khV5qTzWUwXBydZ128OOHmTcTVBfH0BHqDwEBMroAFFsLDkchIJDoaiY1F4uKQBCyqkJQMaCE1FUlPR9hcwBXvvYe7dRNJTUMiIpEwGkLDhjAaGAcDFqBwdp6ybeuan376+eCBg1jv53GQAczas2fPypUrFy5cODrpSBVVgHIFQ0PDhSO2e/duSAtCoVAgEMBvCSKRCHtjh4SEwABLbm4un89PSkqCimqg5HjIYGGlh7XsT/jdq8Ts91xC6nVfugeUAwOdCQntAQG9YjGKos0eHiXvvafs6xtsapK/805XcvLDezhQX1+2fl3z2/9uM9DvmGrQaqDfaKDfZKB/e4oeEeMEpvZksZ4uV1cnUktbYmf38BZe4pQ7g3dmVM+cVjMid66YrhUzWYOi9W/6u1VdVS9gx1xc/yTAoER7BgbCOjqXtrZp9vSCUkjNLZq3Syex2Rqw/YKqgiocYXNAHVVm/MT09M/7+tS9xn/XtfYHAoPq2181MmZPc3NzDx48ePbsWSKRGBISQiAQeDwebAwM05NgMlJ/f78qSgCrc6ruRKGCtqmpCWYW3bt3r66uDpZJraysrKioKC8vFwpFZ8+dP3HytDtWP9TDwyMwMLCsrKyqqgp2Wi0pKSkrK2toaFA1jXrcDo/Z/9fxbW1t7aZNmxYtWrR7925XV1d4axUVFZWZmclisbhcruqOB/78C4VCDocTHR1taWkJ7/JheGH16tVr1qxZvXo1JAczzIhEYmdnZ39/f3d3tyoNqW3E2tvbi4uLjx07ZmJiYmZmdujQIRjMGeNtv/Ds9364GcIUAfeqqyS9jhfZH7PPY64T+CFZWVnbt28vLCxMT0+n0+mwsmdsbOzjwgswGYkyYiGYwZt+MplMJBK9vb1dMHNzc3NwcNi/f//y5ctPnjypUkirCiuFhoaSMSONGIVCSUxMFAtEUlmBUJgV4L/r1/OTLl9CHB2wekdeQKBMICAkIggRhNEwuUIUEhuDBRbiAS0kJyMpqQAM0tMRFgvJyEC4PIAH77yD3LiJZAvB9JgYJDwcDHQ6NoQjdOztmbMfb9u2Zc+efYcPHbaxAc0WHkcMtra2e/bsMTU1hajwMDAsXLgQS0paZGRktGXLluPHjzs4OCQkJEgkEsEoE4lEXC7X29vbysrq8uXLXC43NzcX6hlSU1OTkpIeIoX7EyQSSXl5+Z07VbW1tR0dHYOP7/6LPTgAzxSUSmV+fj6JRIKds/+Yq0y91ZfjgV6RqJ1Oh59ds2PHHQuLh3OKBtva0r76qlp/SrOBfsPI0GignzpZx3+SFktHp95AvxlDiFr9KSWmpsquP7BK6dO6ybXNTRVemFY9fYpEX4MCwgtWQpun3dSzLe/sUvq6i56Vyq6+flJHp2F7h2Z3D0CFpmbNktuTMjI1YmKHe7TBOkgJiRPj4kFUAaqc45gTmfHa9fUg801tv8cDfyAwqHbrcQ+EsrKy9uzZe/Xq1WAikUql+vv78/l81c+DavXfOdLY2Hj12o1Ldle8fXyDgoK9vb1DQ0MfqVJ45B3J7/z0V2p1qBu5du2aoaGhmZnZ9evXIyMjExMTo6OjYewlIyNDJBKpftUlEolQKOTz+enp6Tdv3oSxBRhkMDIyMjY2XrNmzapVq2CcwczMzMTE5Ny5c2KxuLOzEzJeN2ZdXV11dXXR0dE7d+40NjY2NzdfuXJlbGys6lyrqiEplIqfz9I/Wu2cllMCeEFdJemVuoBe6s6M+feEbxUKRXt7e1ZWFpVKjYiI2L9//+HDh4OCgmCjNFWQQaVkUCURwQjDCDiAZCQSieTv7+/p6enq6gqZwdXV1dnZ+cqVKwEBAaOBIRQzqHkgkUjkURYSEpKUlJSUHOLltebXX/9x+TLi4IBzcwe1UAEtBGF1kKj3FQuMGIQZhyTEI0lJgBZS05C0dCQdQ4VMNggv8HlIdBRuyRIE74UTCHB8PpKZicQygB46IhLkMkHlQ4Cf5i8/L9/1088HDx60sjpia3v0ccBga2trabnH1NQEZBwBQTPgBRU5wDSk5cuXr1mz5tChQ76+vgkJCdnZ2WLMcnJyBAIBfFWBA5fL9fT0tLW19fLygkKO3NxcHo9HpVJhnpLq+2TMiEgkYjAYUVFRycnJQqGwoaFBiT0gGHOiVRfd4ODg3r17v//+e7lc/rhlVAurR15fD/RKJHX79ike1Q2pgkaLnKxTqKfXhIFBo4F+pf4UsqZWuLZ2LRZwgCAB5k6f1rrUqPPkif74+KGRqk0vyyfdim7DOsNpdcP5SNOqpmsn6GiQtaaET5O2Sl/MXjk5v8bAoFS29vX7dXTO6+waRoXGJs1i+aS0dCBoVmmaExJB3pFI/F1RsSk36+OUVI2ExImJSaAy0u3Smy/Gz3/uT3kRwPA4D5aUlHrivckUKplMCQ8Pj46OLioqUt1EPm6tp53e1Nzs7unh5u7p4+sfFAyynmDyzNNu50+wPMSk/6VMrF692tDQ0Nramk6nJyUlpaWlsdlsPp/PYrGys7OheBFqFsVicU5OTmZmJpVK3blzp6mpKSSEOXPmvP/++6amppAWVK/GxsZr166FJSaTkpJYLFZsbKy7u/u+fftMTU1Xrlxpbm6+atWqAwcOjG52oVAqsH1TFpXXzbXw+GYrvqSqHkWVildPuPYnuAz+ZIdQW1sLMSA0NHTjxo1T9PTefffdzZs3u7u70+n0qKj7YgYYXhhzi08eMRheCAwM9Pb2dnZ2dnFxcXZ29vDwIJPJNBoNCh5geAEKpkfWe+AviUSmUkO9vG3Pnde8cgUB1ZBcQc8EP38stkBGQqmg8FFEBJaGhCkWEhMALaSkAn0zKx1EFTIzEQ4b4XKRrCyEx0O4WUgKC4QahDkTcgSIQISwOQgjDsgewBCNRMcg12++u23b1j179h4+fBj2craxsbG2th4dZLDFzNLS0sTE1HDREkPDJYaLFhsuWowlJRkuWbLExMRk7dq127Zt27Nnz7Fjx1xdXTMzM3Nzc2EOkooQxozAOIObm9upU6cYDAYm8gZpWVAT9ciUJEgOEomEw+EkJCQkJiampqZmZWXV1NTA7yjVMyaRSPS/TtsqXVlFRUVNTc1DwidUbX9CDzzqy185NMTdfyBUUyt5sk6enl6Dgb5AV9d/khZbR2d02AFiQ9tULHNpqkHTjE/aLTb1eHoMSiTKl9FGOr474Y3aN1VyZ/0CAw2qlkaYlgV76ws7cY5OryUwKJXNfX2e7R1fdXVrdnWDSqkNjZqFRZNS00BUAaICIw5QQXKKVmHhqo4OBooCAYxCca+5JaS07Ke8vE13qgmqh5IvzOF/yg96mcDQ2dWdK82T5MqYzPi4OAass676qXhe7u7t7ZXl52VkciKjYuh0kLLPymA9MsLwvD7xld0O9K1CoXBxcTEyMlqxYoWbm1t8fHxqKlB38Hg8Lpebnp4Of+NlmMEkBA6HEx8ff+3aNZh69L/eCytXrpwxY8bXX38NE5NgbtLoUMMPP/xgbGxsYmJibGz8ww8/mJqarlmzZu3atebm5qtXr05+MEVVqUSVStDU+YpP8oer3FYfCurq7cdyEF5ZX6p37JXwwMDAQFpaWkBAQFBQEIlECgkJcXNzs7CwePfdd/X09JYvX3758uWwsDCIDTCJiEgkkkaMPMoCMYMyBjc3N2dnZ1dXV19fX5XagTrKRq338CiFSAzE47c6OExyhbQAM5FIgBbCaCO0MCJaGJ2GlJGBsNkIhwsgIYuH8PkILxvJyUHEIkQkxImEiEgIxsViwBIJCSA9KToG6B9u2X+wa9fPBw4ctMLkzra2trA725EjR1TUYGtru3fvXlNTk4ULDBfMX7pw4WLDxYtWLJtntnr55s2bd+3adejQISvMDh8+DAsohYeH83i88YEhJydHJBJxOBwHBwdXV9ecnByo4sjOzo6MjISNKcbEFuBbiBbZ2dkcDicrK4vP50skkqampu7ublUNpZSUlAMHDoyppaYOL6B/YRvo6Eg0MaFg+uYcXZ1obe1ATS2Brm7LSMxBla2kGmk20G+fatAx1aBh+rTWBfM7rI70RUUOlZW9MC/ubNh5Px/pznSddD0NkpZWqE7yvZQXtg+Ojq8ZMCiUd3v7nNvbZ3X3jKBCg2ZB4aSU1PuaZkYcCCmkphnI5Tu7u7kvzJl/2Q96mcCAomhjY3NRcUkWjx8VHc3j8cevsP7MJ0mJKuUlZZlsblRUNIPBgJ2nn3lrr++KKhhrbm62srJasGDB+vXrg4ODk5KS0tPTORwOj8djsVhsNlv1iy6VSsViMY/HS05ODgwM/Omnn1RBhqVLly5evFgVW1AxgypJSTWyZsTWrl27evXqh9ULWCrCUN291sU7fD5Z53HKPg44GVDEaD306+t49Z7/UR5oamoikUh4PN7Pzy8oKIhGo0Vi5u/vf+jQoS+++EJfX3/27Nk2NjawiDARsxFeuJ9NFBwcDJsYUigUEonk5eXl7Ozs6ekZFBQEdQ7wFfaBhslI5McZ2GoImRTs77fDy0sXqpyJRCSEAtQIoM1C1LDEOT4eSU4C4mZVGhIbCyxwsxAeHwzZOUiOAEBCrgwnycWJxYhEhEjEiESCSHMBTiQkItGxoAwrhax95jTQBVlZDUcVrKysDxw4+PMvuw8e3APyk2xtf/llr7HxygULDI2WLFht9vXmzZ8dPvT26bOap07POnHC6tix47a2R62srA8dOrR//34rK6tbt265u7sHBwdzOJzHMYNqOuz45unpGRsbC2slicViNpudkpICv0zGeRWLxVKplMfjEYnE/fv3z5s3LykpEf7vKxSKMV3eVF9if9Qlpd7uK++B7pqauAULQzS1iJpawZpaQZpakVraPF1dmZ5elf4UqIRuHBE5qLChActZasUE0+1TDZo/+rB1zZoeZ+cBQY6ys/OPO2j5oPz96g+mVWNy5zvTp5ZO0wwHzdoWJC7qV7y4hgAODq8NMCgUNb19N9o7PunpHUaF+gbNvPxJySn3USGOCVAhLX16Scm+nh7ZA6dvqB+tKxjITemTsoea6x6YpX7z+zzwcoFBOTQ0VFFRKcsrSExKiYlhVFWBcgHP8TYRbqruXl2OQBzHjA8LDysqBFlPaisvL7e0tPz22283b94cFBSUnJyclpaWkZHB4XCSk5PZbDZ8TCiVSkUiEZvNjo+PJxAIp06dMjMzMzU1haEG+KoCAxU8qPTQo6esXbt2zZo1pqamN2+OzSbEwgtDzsEZM9d5fGbuHpteoD5Bag88iQeKi4vd3d1dXFzweDyRSFSJFiIjI2NiYv7XR/z8+fMLFy40MDCwsjpCJpODg4NJJBKsyQZDDWTMAgICAgMD4bgqyODj40Mmk1U6BxU5wMV+6xV0dQgmWBICdIKIoPIpLRTQQmQU6MgWxwDxgeFMpNRRmUhYbIHHR/hYYEEoAHiQmYE7eQIXRkOkMlyuBCeVIlIpkpuLyGRgscQkrG9DHEKhTL10aY21ta2N9XFra9uDh47s3r3P0tLs8tXvThzfb/nzvjVrl60z//KXXz45d266u/u/vL0neHsj7p7IjVt/O3vGyMbWxsra5siRI4cOHTpw4MDBgwcvXLjg4uICe1BkZWWp2GBMSpLqrUgkYrFYoaGhGRkZUPMgEomSk5N5PB7MbxzDDHl5eQUFBTKZLDo6+vjx43PnzjUwMJgxY8bevXsLCwuf468AqrY/nQc6ysvjvp8XggEDZAZYUJWqpR2vPVmkq1sxZUoDFnNoehQ5NGDyaFXYoXn+vC4b677Y2KGa6ufuKvs2h/vhherpejn6GmQtjTBtfInXc/+scTZo73D71Rc9KxRVvX127R0f9fRqdnYNt2qW5U1KSh6LCumst2+X2vb2ye8fcl8nWpGDpjn1uW+vtl5dfWjDnYOb7pzc3Zn1iIpb99dSjz2NB14yMKAo2tXVXVxckiMQxcQw0tLSVaWKnuYoHr0s/Mnp7euTyvLT0jPoEREsFmvgqfpHPnrDr/1UmJFVWVlpZWX11VdfmZmZwYpJTCYTKDaTkxkMBpvNzs7OhklKTCYzPDycQCBcvnzZwsICphvBKqsqZhjdmUFVPWkMMKxdu9bIyCgiIuJhLJSVVM/dgp+1yWfZbv/GlrbX3sXqA3ghHuByuVeuXLlx44a3tzeNRlMBAxyh0+nR0dFhYWGurq4+Pj7BwcFEItHZ2fnixYtBQUFEIpGMWVBQkL+/v+otGRNAw6LMqnKrKmyAIQi44vivJMAnBwhBWsEkrN/CiHSBEQuaLSQmDtdESmcB0QLIROKANKTRtCASIdLcCWlpiPlaxNcfyc/DyaQAGGSy+0NODpKcgmMycMw4JDxM0939uwsXth09Zml15Jf9+zdfufIxNUTj2vVvzp35zNlFJ4z2T0YMjskEOxBKQwL8/+ntPRmPf8/dzczR6Ya9vYOjo4Ozs7O9vf2NGzdgipG7u/u1a9f8/Px4PB5sv6AihIdHIDOkpqaKQRsKUFA1MzMzLCwsOzt7DDOIxeL4+PirV68aGRm99dZbH3300aZNm7y9vdlsdl5eXn19Pao2tQfG9UBHeXn84sXUEWYIHok2kDS1yJpaIVraDO3JfF3dkil69/RB9dWmUaroR4YdWqfqt3z+WdumjT0+3oN5ec+lRl832r2wbtH00XLnOB0Nita7kf+923tv3ON7zjNv2b/SwKBQlPb2nu3ofF+FCnV3J0lyNRKSxqICK+O9isqzff0VwEED/WhRKsoLRsOPoS7G6MUZinNzqw6uL95jUbRnY/GejaX7LMoOWvQVC5+zN/+qm3vpwAByTu7VNxQUFiWnpNFoYRKJBLZ5/j2h59FPp6qra7J4/MioqOiY6IYGdRXeB6701tZWPB4/b968WbNmHT58ODAwkE6nR0ZG0un0iIiIhISEuLi46Ojo8PDwkJCQwMDAGzduYKJJkx8wMzExgaWTRtPCmHHIDDC8sGzZsqVLl+bn54/shFKBKlAUbe3s2WJD/mIj/vO1ntcD07C56mSkESep/z7eA2lpaWfOnLl27VpQUNAYWoBvYT9mOp1OIpEIBAKZTP7ll1/mzp3r5+dHwgocwd6FqvACecQgGMBl4DQVM4wsMv5fSgiFQKZsIFOmUkL+Fjo6GQkLL0DpQlo6woK6BQ7QLdynBUyugGUf4SRioGQQixHpCCfk5SH5+Qh8zc8HcYb4eKw5dDwSH4ejhWsHEd8ODHwzmGBAj/h7VDRCC/sbPQKJZ0J1BI7FmpSc/HEMwzQs7ASNdjks3DsiMjwikh4eHhYZGRUREQHV4cHBwf7+/rDO7Pnz5319fcfRMwiFQlg9CQYiRCIRDDIIhUI6nZ6YmCiVSvMxy83NlclkGRkZ33///VtvvWlsbGxvb5+SnFxQUJCfnw9zIAsKCnp6elC1qT0wrge66+pSTFeGamrD3CSSphZVU0u4Y0fmli20t/4NyYGMJSxl6oDaSrX6U8YhhwYD/ZYRtUPTe++2Gv/QbW8/kJ2N/o5LMbkn+QG5c/5UjRAtjXCtI0LrcY/s+c+8eesVBYahoaLevhMdne/29g1HFWrrJonEkxIS76MCMx4kIGVkflRReaW/f1QUKMUJvfopenkmajcTvTRj8Oys0n1riyw3FWO0AF8r9m++53wSHXpx2V/P/+S9Mlt8ucAw7AaFQpHOYvn5BxKJRCsrK0tLy6NHjz5zK1BIGhUVFdHR0X5+fufPnz937lxQUJBM9mCi2ytzDl7WjqiwKi8v7+jRo7NmzZo7d+6hQ4c8PDyCgoL8/PxgPgZ84ArvHhwcHGxsbMzNzU1MADOswAz2VRgdaoCQADs2rFq1as2aNbBF1Pfff29hYdHWNhxAwHZA0dvff8I+6vN1nl9v9ft+m09pddPDFbhflovUn/uKeyAhIcHa2trd3V1VOPWR2ADbvBAww+PxDg4OEB6IROKZM2du3LhBoVBUfRVgVEGlc1Axg6pdA3lcUy1Po4XFxFBjo72io/dHRn8VHf0mI2ZiXBzCjAfag+HWbFhNJCBdwFTOvGygWxBiugUobgZyBSlSkIfkyRBZHi4/72/5+YAW4FBQgBQUgmhDZiZgBjAwwZCQgMQnIPFxSFwsyICKjkaiov4WGTmZGT8nNXV3ctJlZlwwIzY6Li6JRosMCgL/6FRqSFgYLSYmJgWzxMREBoNBp9PJZLK/v7+9vf2JEycCAwOzs7MhEuTk5IyfpCQUCmHraz6fD1vakSnkQEKgBOuALRQK8Xh8bGws6IZdWJiXlwcbSKs2XlenTj5+xf/5Xond629o4G3dFjpFn6SpFfPRx/mXLsGfj+6KikoymbP9x4j3P4DkQNLUCtPSTp6sk6unW6X/GwlLTQb6bVMNOqcaNLwxvXne912nTvYlJSmamp72mPc3Hhidj6TDAnLnyWF6WU2gM+mLtBs3S161lKTBQWlPz8H2jrd6+zQ7OjVb2zRraicJhBrM+LGowGZ/WlXl2N9/d6zH6MfQy5+i52eA4cInd62XFVpaQE4o2rOpZN86+Z4NRZYWlYfNFfVYRGLs+ur3T+eBVwIYUBSNT0x0dfUgEAg//vjjvHnzVqxYAfUMT3c02NLwPtjFxWXWrFkLFiyYN2/ehg0bfHx8KisrH86EeYbt/5lWUSgU0F0oihYVFdnb269evXrhwoXbtm3zwNpse3l5eXt7+/n5BQQE+Pj4ODk5nT59evv27aampj/88MPy5cuXLVu2fPlyWAdp5cqVqvCCSgNtZmZmZGS0du1aExOTL774wt7efrQDm9s7j92M+Xwtfs72gJnmHuc9EoCGRd2vbbSP1OOP9wCHw7lw4QKRSHxcp7bw8PCwsDAikQiLIBEIhKCgoGDMKBSKr6/vF198MW3aNFNT08uXLxOJRFihlUQiYeroR9dTIj+B0Wjg5jsujhnPTI5PSExIpCUlBSelXExM3piUPDslRTc19f/S05HMDISdOZyMlMXDVM45QOUMYguYuDk3FwQW8gsm5BUgefm4ggIchISCQoAKBYVIYRFSUISIc5HUFAAMMN0IvDKAtoHB+FtsrF4s4zsGc1d8vGtCfFhsDNPPN+jKlWtnTp87fPjIjh0/mpubr1u3buPGDRYWFtu3b7eysrazs3N0dCSRSDC6SCKRfHx8Lly4cOLECSKRyOfzx0cF2AcagIFMJhAIxGJxeno6m80+fPjwzp07ITDk5ubm5+fLZDKVsAEWZMvJyeHz+VwuVy6Xq8unPv7CV88Z5YGhoc7bt9uLinrvPnRDiaK9dXW1TGbOgQPRn31OxLKVQCBidMLSiNRhHJ1051QD0BJu9qxOy196abQnbOzQqmidU/PNtNrh9gtTy6ZNogO589K05f3KF/3A+/qNVwgYBgdFPT172zumq1DhTvWkHMHDqKDB4X5eXe06MNA46nyPGs2moJdmYsDwyeC5z0v3mRftGQ4vNNkaDpz7ouf01zWHjcv3rxsoyx21mnr0GT3wqgBDdGys3eWrQUHBO3funD9/vqWlZcez1kuGd8AEAmHBggUrVqxYvHjxli1bvL29y8vLn9FJf97VVLSgOsT+/v7i4uLw8HAPDw9nZ2cnJydXV1cPDw8fHx88Hu/k5PTrr7/u2bNn9erVxsbGy5YtW7x4ASXc/AAAIABJREFU8cKFC5csWbJs2bIVK1bAUqqmpqYmWEHV5cuXGxoaHjlypKCgwM7OzsRkZUVFBah/hJkgv3KTNeEzc89vtvrMsvBdvjuwtqEdQJ1qb9Qjag+M64HKysqQkJBHRhVUE2E2HZQ1EwgEKE4gYflIJBLJ0dHxxx9//M9//qOrq/vtt98ePXrUz8+PSqWqhNGPK8NKHmvDNZcwzXTQtavXPDw8IiOjkpISklOSUlLTU1Mz09LZLBYrgxWTmeGWyT6VyV7N5sxgcyZzuDgepl7IFiBCrHCqRIJIcpFcKS5PCjKOrGwQMmVCcREuvwAHIaGwCKACHIqKkPxCJFsARBHDcYY4JIGpwWR+FhdvyYy/mZQUk5ySRqcz/PwCjx49aWa2dsFCwwULDOfNW/T9vIULFixcAG3h/AUL5i9cANo8Yz2eN1+8eDEwMJBKpQYFBTk7O5/AjEajQSQY08ENShpgyaPsnOyoqKjjx46tX78+JSUlJycnKioqLi6Ox+OpCGHMiEQiycnJ4XK5bDbwUnZ2dnd397gnXz1T7YGn8EB/Y2N9Rqb45MmERYuIk3XIWJElkqYWXUs7XUdHpqd3Z6TC0iN10o1YwlLHVIO2qfrNMz5p37Sxx9d3qLhonHg4p5fzZs1bw+0XqqdPEUO5s5ZHmedT7PdzWvTqNfkjIwx8fstz+oQn2szQEK+nd3d7hwFEhZZWzao7k/jZGnHMsVGFLN4XNTWug4PN4223pQa9Phe9AMIL3Se/gclIRZabGmwWoxc+Qc9/gr3O6Dkzd0gQOd521POezAOvCjDQIyLsLl8NDibu2rVr/vz527Zta24e90J5/OHBm+D//fAYGRnB0p8WFhbe3t7YrerjV1PPGeWB6upqZ2fnG5g5Ojq6urri8XgPDw9HR8dLly5ZW1tv3rx55cqVxsbGixcvhjcb8+bNMzQ0NDIyWozZokWL5s2bN3/+/K1bt6anp9vY2q5ZvZrLzURRtKevV1hQecqF8e1m1y82en27LXCOhd/sDW4JmeoCVqPOgXr0tzygVCgKCgpgzr0KD8aMhIWFkUikAMxgkCEgIIBAIEBggDBApVIDAwNtbGzmzJkzefLkDz74YOfOnS4uLjAQoQo1kMc1bIOAGSgUyuVLF4xXLF++zGT/voOenvjIiIiUlBQWi5Wenp6RAVqesDMFXG4OLyuTnxXP47ll84/kZK/Ozv5IINASiYBoQSJBxBIkVwpSj9gc5IP3ETs7nPw2UoiFFFSoMDwCJuIKiwBaJCThklPezuJuY7HsExOoaamsjAx+YmK6k6Pbjh27TE1+MDX5brXZnDVr5qxZ882aNd8aG39vaGi4cMHiRQuWYN3csNbPhoaLFhkuXLgQYsO5c+dg4dqrV6/a2NhcuHAhLi4OCqBVoQbYIV4oFEIp84oVK6ZPn/7OO+9YWFgkJydLJJL4+Hg6nQ6V0PB1DDDADnGZmZmpqamwlRusfw2/z3/rWlDPV3vgST0w1N3dmpdXcP1G6g/GFH0DMhZ2CNbUCtXSZk6eLNDVLZ2id2/csIOqsUPz+/9tNTHusrcfEAiUD9VTudZ67X4+0p3p2kmgu/P0iLeqekA1yBdsV64Wv1xgGBxkdXdvbe+YokKFispJWTwNRtxYVODxZtXWug8NPQHJKJVo6BEgYLjwSevx+UWWm4r2bCo/sHro188ALcBUpfMz0EufoK4/oDXqpPTfe9G9KsAQn5jo7eP3v2afP+/e/d133/3www81NTXPdnCwBJCHh8ecOXOWLFmycOHC7du3x8bGqpNin8Sf8OdZIBDY29tfx8zBwcHFxcUdMycnp6tXr546dernn39eu3btypUrTUxMli9fvmTJknnz5i1YsGDx4sXzMJs/f/7XX389f/78xYsXL1269NixY+VlZQWl947djF13hPDlRvdP1+G/2hLwzfaAOVv8vljrFhAuAKEFdeOFJzlJ6mUwDzQ2NjIYjIeLI41mBhqNRiAQYE6d/4gFYWn7ME8JFkcKCQmhUqkkEunixYsrVqwwMDB49713L9ldIpFIwVhtJajkIY9nFDKZRCZTg4Jdr1z/co/lm6tXz1pq9J2JsbGFxdaLF+zCwmgwM4fNBp3KuNwsHi8nmyfMyZEIhRKhkCcURYrFDhLxvlzJMnHue7m5/5LJQDyBw0E+/hi5cgUBwDASVSgsQooKkaIipLgIkWMjhcWa+Xlzs3N+lkrD5MWFEml+Fk8kEIjpdNqpUz8eODTr+Mk3r1/X8/HRIhI1KCGTyBQNIknDy1frypVp1lYfbLb4avmyBYaLlixatNjQEED/EswWL168fPnybdu23rhxw9XV9fTp00eOHLl8+XJ8fLyqaFJOTk5qaqqHh4e5ufmHH344/Y03li9bdvXqVbiMSCSCkmgmk8nhcEanIY1mBkgRWVlZTCYzOho0zBnTr0191as98Hw9oOzv7yovL/Px4fy4I+K/7xM1tShYqSWyllaUtnamjk6+nl4NFnZofkyFpeYRqUPTv99qWbSw69dz/fFMldTB5J7J9LtYPtKd6QYl0ybRtDVoWhb8LQoUdCl9wXb5yssChqHBwaTung1t7ToQFZpbNMvKJ3G4D6OCRnb27Lt3PRWK1qdwTh4TvTgTvfBxo60hAAbLTU22i2Bg4T4wAGaYiToYooVJT7Fl9aIPeeBVAYaioiI2m1NYWHjz5s1Tp075+fn19vY+27MluFZERISpqemmTZvWrl17+fLlgoKC3l7QMFxtv+kBpVIZFxfn5OR069Yte3t7JycnNzc3KGlwc3NzcHC4dOmSjY3Nzp0716xZs2rVKjMzMxMTkxUrVixdutTIyMgQu91YsmSJkZHR8uXLDx8+zOFwYPelJHbRx2auszf5frPV79utft9s85lt4TPL3MM1JFOBKpRKhVKdjvSbp0e9AOaBnu5uNptNo9HGlztTKBR/f38/P78RWPAPCAiAkEAkEgMCAkgjppL4E4lER0fHX375xcXFhUgkBgcH+/j4BAQEkH/TSCEkEsXTc/0t+wlOjsiNm38/c07np13/WbVq1pIl8zds2HrhwqXw8PBMNuiqzsvi8fi87Gx+Tg5fIBCKhBKxWCaWyCS5edJcgUwWIZPdkMl2FhR+weFM/uTjCZevICWjgGEYFTBgKCqeVCRfLJffkhcLSuS3S+Tl8qKSkpJymSwrIsLW2/tTP79/xMWB9nBZWUg2H8nhIwIBwueBt2wOkp4ORNgU6j/sb+ke2Pe+qcm8hQsXLzZcYmQE/p2XgsJmS5cYLdm4cePly5evXLliY2NjZWV19erVlJQUGFgIDg7+/PPPdXR05s6de/z48aioKDhdIpGoqieJRKKsrCwYbRhTYhViAwQGoVDIZDJhmtm9ey+07qT6H+uv7IG++oa6OKbo6FHmnG9IOrqqrnChWtqJkyeL9HTLpkypf3zYoclAv9VAv3OqAajC9PlnPbt+kdLt/1vzoapfmy5/igZJa1KYNqmW/FL8bHf5xQNDf/9ATHePWVu7dk+vZnuHZnOLZmnZJDZHI5bxQFQhOUUjR/DlvXt4hfLpK6p3NaOOS9BLn9yzNoLA0HXym0cAw/kZgCsufY4mOaD96gpsz3gNvhLAoAIDJWbwUEaPP9XBwRX7+/sbMKuvr+/s7Ozv71d9ylNt7S+1MHRRW1sbmUx2d3d3xczT09Pb29vX19fPz8/Hx8fT09PJycnOzs7W1nbXrl0WFhbr1q1TdWozMzOD0mdzc3MLC4udO3e6urqqUC1PXvvVBo9vtvrP2Rrw9Rb/T8w9F+zERyZJgZOVg+DE/aXcrT7Y3+EBuVwOBc2j4wljxsPCwoKDg+Gl64eZv78/gUCAwAAzjmDogIRJGoKDg6HCAUYVgjAjkUjr1q0zMTEJDg4mYxYSEvKoiklEMik0kHDd2fkNe3vEyQlxdUM8PBF3jwmOzv86fnz6po2fLl22YP26TecvXIykR2RxuTk5Odk5OfzsbIFACGsKicVi7O45TyotlMmK8vJyCwszOFzPGR//+8qVj0tuv19Y9M9CDBKKYWyhWFNevKSk2EkuF8hLyuXykmJ5sVxeJi8qEkt8WKwlMdF/T04BYCAWgTQnqKUWCZHsbNDzgcMF5ZXS0pCUVNBFLg50csDdste2sPjCaInhksVGRksAMywbsY0bN546derEiRNWVlY7duw4e/ZsUlJSbm5uRETEsWPHyGRyVlYWLHYE9QxjXoVCIcw1gsAgwWx0kAGOp6WlBQQEhIWFwZSk33GNqFdVe+CpPTDY0dEiFBZeu5ZuYkqdNp2CJSwFaWpRtLRitCdzdHWKpujVYQGHR4YdoNShb7JBwJ6puvew7s5V4FUrbrIGWesd5vv1isfod596T59uBTu74qnTUgymMh8c4v8IDYNS2dPfH9bds6KtXROiQlOzZsntSRmZw6gQy5gYy5jIjJ+YnDJRJP66ocEbRYF88Rkt9gJ6ecZdq2XDwHBqzqOBAVRSmgHyl4i/oC2jarM+46f+FVd7VYBhzL0ivOkfM/EJz88jSeORE59wg3+1xQoLC2FZVV/MAgMDg7H+uPBpK4FA8PHxcXNzu3bt2okTJ/bv379r164ff/xxx44d27dvhyM7d+60tLTcv3+/lZXVyZMnw8LC+rH8zjt1zd9t9Zq53uuztR7fb/U46RRXXgU6NCmUCoXyfr2mv5rD1cf7tB6ora1lMBgwtvC4CAOdTg8NDYXVveCV7Ovr6+/vD/ORYK4R5AQyZiQSKSgoCGqjodoBlmENDg62s7M7d+4cXJhIJOLxeBKJFBoaSqFQVFsgkSkUEtE/YK2XN87bC8HjEU9PMHh7I34+SEAA4oH/+4ULUywt3zMz/XbD+g0XLl6Mjo7m8/kgXwcz2LsgN1eSmyuRSnNlMhlohVxUlJMjnjnjs4vnT5WWJhYVXS4uWlVU/GFxsZ682FAud5HLRSXFFcXyUrm8BAwlZfKiAqHgWlKSQWoyhgoS0LFBJgPlWSEzAGDIAT0fOFzQMC6dBYAhEQOGmGgkLAzB4//5i+VHy1cYGpuYmK9ds3XrVktLywMHDhw+fPjChQsXL148dOjQf//738VLFl+/fj0lJQVyDownjIGE0W9hz/iQkBA+n//IIENubq5UKs3OziYQCKGhoZ2dnU97YaiXV3vguXlgYAAkLHn78Hbtivzv+yRNrRBNrSBMKh2upZ06qjwrKKD0YM5Sq47+VsI0nbsYMNyZblA0VSNU619hWlsSV6D1LydudulS0QsABqWys7+f2NVl2NGp2Y1FFRqbNOXySeksjZhYEFVQoUJK6kSJ5Jumpt+HCvBk18jQy5832CyFwFBvswS98PED+UgqMQMcsZuJeqxEq7Enlc/tcvlLbOiVAIa/hKdfk4Ps7+9PTU2lUCjw+SuJRKJSqfCeLCIiIjw8nEajhYSEBAcHe3t7Ozo62tnZnT59+uTJk8eOHbO2tj506NDhw4dtbW1Pnjx55syZ8+fP37hxw8nJKTs7G0XR5o4uy7M0ywvhPuE8ecUD35vq2MJrcoG8/N3s6+vjcDihoaFj4glj3sJqqj6jDAIDiUSCFZBUEQMyZrDTyGhgCMQsODiYSqWGhISQyWQqlero6PjBBx98//33p06dCgoKCg0NhbMw9cI1P9/p/v4IIQghkpBgIkIkIgQCEhiIBPgjgYG4YCISHIy4OGsePPDeKrNvNqzbdPHSlVhGrEAgkEgkYszgU3apVIoBQ15hYaFIJJg589Nzv/5aWlZTLC8tlueXyJNLbhNLSoQlJZUlt8tAFhKILRTJS8ry85O4WVtSUiezWIhIhOQXgCEvfxgYcnNBkEEkRHIEQCHNzQLAwGIhqWkgwsBkgo4N9HDQl9rD8x+X7TbY29v7+vgGBQXh8fh9mJFIJGdn56NHj+7cufPAgQNWVlY3b95MT0+HeoZH1k0azQwCgSAuLi4xMVFVXPXhCINUKk1OTubxeLCsqlIdd0TV9pI9MNDcfDcqSnzsOPPLryh6U6gYNgRjmoc47ck80E96Cgw7NBnot+jpF3+kPyt72pQaDBiqp+vypkwkaf6DruWye1rXBzM6du3sCQwcul2CDgy8sAO7+AcDg0LR2t8f2NU9v7NLs7sHJCA1NGkWFk1KTR+LCqlpGlLpNy0tfr8rqjDGcXGX+k4bFgPR88aSvevbT3yPXpgJWjQAPTTWpWEMM1yaidovQktAFRa1PbkH1MDw5L76SyxZVVUVExMThll4eHhERAQUIDKZzPgRYzKZMTExNBoNYoOrq6uTk5O9vf2NGzeuXr0KE51v3brl4ODg6urq4+NDIBCCg4Nra2uVSkV3bz+KDkJXqm8F/hKX1HM9SIVCkZubS6fTHxdYgNgQFhYWGhoaGBgIG4l4Y+bj4+Pv7w9jAiSs0wJ5lAUFBfn7A4WDylTAoFqKQqEEBgZaWVnNmjVLV1d35syZe/fuxePxGFHQvLy3uLv/LWAEGKgh4Gk9PQIMtDAklIqQSQAkKCEIiTTB3V3Dyurf5mvmbtxocdnuShyDKRAIxWLRSIRhGBgKCgqEQuHMmTN//fXX8rLyktvy27dLS0vLy8qqysoqy8rKSktv375deruktOR2RV4ek5XxXUYGTigAkACE0cWgV0Me1vdNKkVyc0HHaJEQ0zDArCQOkpEBZAwpqaDdW2zshAg6jkrFEQHbzCEEeZ6/cMnY2Pjtt9/W1dNdaWZGIpECAwPPnj1rixnUMzg7O6enp6vkCoJxLTs7m8lkjhNkyM3NFUskz1z04rlebuqNqT3wgAeGurpa+dlFN2+mGi0NnTadqqVFxsIOsDxrGlaetV57Cn2VgX719KlYMhLIR2Lo/JOkaRCuI/jKoF1PH6vNatDw7jutxsZd164NcDnKPz6YduHiHxVhUCga+/vxnV3fdHUPo0J9g2Z+waTUtAdQIT5hYmraxLy8b9ra/FG04wG3/v43ve295NNVBzdWHdhSeWBLxQGLlsub0dgLKPUQemnWSK+GB8nh0kz0xly0IPn3f/hfZwtqYPjrnOvfPtKenp6srCwGgxETE8NgMOLi4uLj45OSklJTU7GKkBlsrMILh8PJyMhISUmJi4uLiIgIDQ0lEolBQUEEAgFmgHh5efn6+gYGBsLMDRiXSExMGAQPVJSgMZtySK1Y+O3zoV7iIQ/cu3fvNysjweuNTCb7+Ph4jTJvb29VQVWIDeQRg6VXVToHf8xGF2CFC8LoRGhoKIlEOn/+/NKlS/X09N566y1z8/XXb5xzdJrh7ITzDwARBjIZoYUi4eFIZCQSEwN6qMXFIbGxSEQEoIjQECSMhlBpCN5Tw/rIv1eafbV9+47YGGZuLsjrxxJzADDIZLLCwkKhUPjJJ5/8+uuvlZWAECoqKipHrKKioqK8vLyssrS0Mj+Pnp75DSsTJ5MBTlDVUwLAkA+YAQIDKNgqAs2k+Q8BQ2Ii2MmoaFxIKHLhAmK2cuJ7772tqak9c+bM7du3Ozo6wn7YJBLp6tWr1tbWNjY2tra2KmZgsViqKqvjIINQKExPT4fq54fDC3CKRCIpKSnp6+tD1ab2wCvpAeVAf6dcXubtzd2yhf6f/1BAwpJ2MFZkif4P7e12+nr1o/KRqFp/D9Va5q53902DRn39BgMwYM2kATw0vTG9Ze63nTY2fVGRirt3/6DnaL+ef/7AoFTe7e1z7Oya3d2j2dUNogr1DZp5+ZOSUzWiYybGxIIEpJjYifEJE9PTJxYWftfRQXj+qKC6PAb7u/jJzWEBjVT/Tk7SUMdIqaWKbNRrHQg1jAkyQBn0jW/RohTVNtQj43tADQzj++cvMVcl8CgtLU1ISEhKSkrGLC0tjcVisdnsrKys7OxsoVAIK5nIZDKxWCwQCLhcbnp6elJSEpPJZDAYUVFRkZGRdDodVrqMiIhgMBjx8fGJiYkJCQl0On1Us211CtJf4tJ6vgfZ3d2dlpY2fh1VqISmUqkEAsHLywuPGaQGHx8fmIYE0+3Ioyw4ONgPq6QEX1XAAJcfteD90dDQUDKZ7OTktGnTpvfe+4+enu63c//v2nWQgBQU/AhgYDKRxEQkOQUMzHgkMgoJD8eFhyM0GoLH/+PkqQ9DqB4SsUwiAf9lUswgMIhEotmzZ9vZ2d25c6eysrKmpubu3bv19fX37t2rra2trqmrrJBLJNcyMz/hcHCyPKS4GPRkUA3DwIBpGEBKEgQGTMaQxUO4mIwhMxNkJSUmgXJJ537FffopoquHe+89xMT0v5evXAwODg4PByGdUMyoVKqbmxtMQbTBDMKDn58f/wmaQAsEguzs7LCwsNTU1McpGaAH6uuBwEltag+84h7oa6ivoYWJDh+O+fQzirYOSVf7y4Sp+nUj+Uj8KROJmn+P0D573KBbe5gWIDPAV6iTHm4mPeOTju3bu73wQ0WFyuda1/HU6YLnqGFQKKr7+q53dM7s6QWo0Nauea9eUyqblJxyv/wRRAVWhkZR8XcdnQQUfXmSpI56lLL/scxw8zu0lPuKX2OvyO6pgeEVOREvczfgI422tjYOh5OampqRMRxJ4PF4fD5fIBDA7q2FhYVyubwEM7lcXlRUlJeXJxaLYX/WzMxMFouVlpaWkpICeQM2q4JBCTabnZCQkJGRoVAoXuahqj/7NfQAvD4VT9CmTUULsJqqh4eHp6cnZAY8Hg/zkUgjRh5lgYGBPj4+Km00DDWoCrCOWnDsKJUaQg0NxXs6bd/20cJ5OHsHHCEICQ7GqSIMERFYhCEWKAQSkwAtpKaBFKC0dCQpCRcTA2IOUZEInY7EJ/wkEgnF4lyxBNRKkmIyhnzMIuj0LC636s6du3fvtre39/b2Dg4ODvT3d3V2N9bXSKWn01laQiFSVIAUFyPFo5q7FRQiBQWg+xvUPefmIrkSRCQGEQaoe+ZykXQWLjZ2QlIyLjl5QlLShOPHcD/8gDt9GnFzQ3z93goJuU6j0UNDaaGhVAgMNBrN39//zJkzkBNUzHDixAkikZgNij79hgmFwpSUlMjISKFQCJkBPokYHXCQSCQFBQUdHc87dQFVm9oD43mgV6FI7+gceCblzFB3d1NySrSX7dsl06dWY8BQNV2bqTORqKkZp+NvPKVWG9RmfVgkrSKHZoPhhKXGf7/V9sMPXZcuDbDZyraRh+Xj7fhvzDt6LO+5AMPQUHlv34WOjo96ejU7uwAq3L2nmZurkZj0QFQhIXFiRqaG/Pb8rq7gl4kKKq/0d6GRpx4taYB6hrpC1bLqkcd5QA0Mj/PMn3H68GN9JWiR9qA1NjVmZWXBYAKfz8/JyVHFE/LygPKypKSkrKyscpSVl5eXlpYWFRXl5+dLpVKYjZCdnc0dMRiXgPcOOTk5mZmZiYmJ6juABx2vfvcbHoC0gKLo3bt3o6Ojx5cuwGQkKNnH4/Hu7u6QGTw9Pb28vAIDA8mPMiKRCEsGjwGGxy3/4DaIFArNP+C0g72Ouyfi6zeBEIgjEnGbt+CsjuDo4YAHYmKROCYSHw+e36uAgZWBZLKRjEwkJQWTGkcgCQkLBAKWSCIZAwyFhYUVFRVVVVV1dXVdXV0PILeit7T014yMiWIRUizHaAFr6DY6vFCA0cLolCRJLiKTIAIRKJTEz0ZIpL99/ukE34AJaWlIQjwSFYNFP+gIMRjxxCMB/ltpofQRWADIQKPRyGSynZ3daGCAiUlHjx4lEom/gQsjsxMTE9PT02Uy2cO0oEpMqqioGBp6CV2ufuOiVM/+83pA0tOzobbOv6n5mQ+RMEgebvB8Z7qBfOqkMO1/UrQ+iNDz/c/kkIlaDO3JfF2d21Om3H08OYxOWGqcatD8zZzOw4f6QkMVNTXP3NvUykr2O4FhaKi4p/dke8d/VahQd1dTJNZISByOKsTEggSkxKSJbI5GWdmCnh4SinY/sxuf/4qD/Wj8NaBneFgGbTcTxZujnS+n4u3zP9I/bItqYPjDXPvKbHhIMahUDqKgu+SDT/ehmgBV1jc2ZGVl8Xg8IWawSKJMJisARR2L5HI5zJyurq6uqampw6waM5hUXVJSAqMNQK0oFkNygJsSiURYsXXwwufzU1JSqqvV9Y9fmSvj9dmRzs7OjIyMJ0xGgsDg6ekJgcEdMy8vr+Dg4Ef1TyDD5CVvb2+fEYPkQCAQyE9gFBLFxWWl/S0c3gvx88ERAnHBQciOHTgbawAMkZFIZDRghqRkUIkIAEMqiDBksAAwcLhIFhekKtEBV7zB59PEYqlYLIF3zDKZLD8/v6ioqLy8vLq6uqWlRakc/S/c39DgnJ2tK5Pibt9G5CWg67MKFeBIQeFweAFGGPLykPx8HIeD8/JFQmkAGLhZAGOuXMFFx0xITsXFA90zQo/EhVJBYpWXN+Lj9U1ISGAobTgfCQYZYLUoKGCAEQYbGxtra+v9+/efOXMGNnQb4YLH/uVyuQkJCQKBAB7sw68SiUQmk7W2PofHq6/Pla7e05fsAe+mpg13762/Ux3V9iydAZRK5baGbdPvYQ2eq6frCfUnkrT+SdcypX4cN3s2eRJoJk3U1CJpgmbSbB1d+ZQpdfpTQLu3BwuzqnKWVAlLLQb6TR9+2L7ZosfNbTAvT9nzdN3HDhyUPjMwDA3l9fRYtXf8u7dvOKpQW6cpFGnEJzyACknJE7lcjcoqw74+Koo+3e69oLOuVKDJjlhn6AcF0OdnoJdnouFHUYX68cR4p0INDON557Wep3o0i6Job/9AbUObrKSOn1vOFZaJCqvu3G1q6+xUooNtLc1ioUAgyIHP+aRSaR4o/w5QoaSkBBRkKS+HTzfvjbK7d+/W1dXV1NRUVlaqQg0FBQV5eXkymQxmU8gwyxsxsVickZFRWlr6WntVvfMvxQMSiSQ0NJROp4/frI1Go0Ek8Pf3d3d3dxsxd3d3PB7/SEEChUIJCAjA4/EP11MiEomwrRt5HKPGA5DNAAAgAElEQVSEBhFcbt5638kZAIOvL1Y+NRihUIGsOSwMRA+cnJGlS3DHj+Mi6KCGaVoGkpaOY7FwmZmg0XIWD2FlIFFRSETkPzPZ10WiPAlG2bm5uRAYCgsLYX+0ngdvERSKpJramaWl/6qo+H+lpROK5fe1zkVYVhKgBaysKuj1VgzUC+FhuEMHcV/OnqCrg9jY4kBlVS7C5uC4XByWIgUiDHFQmU0DBZ18fRF3d90g4kVaKKhtoDIajebn53fy5EnrEYPAsG/fvl9++cXPzw8+NXgsK2AzhEJhXFwcg8EYp8SqWCKpKC8fGhyuq/ZSrj31h/51PNCjUByqrll/p3pdZZXFneqszq6nPfbawdovamZNgwVV70yfnKQ7kailGTvZrdkd7e6uT0iQnj7D/ObbkCn6IRg2DJdXmqyTp6dXPS45wLBD+1SD9qkGjW+90bZ8Wdf58wNpaYrGJ3ou/oul+BmAYXBI3N29r73jzZ4+zY5OzdY2zeoazRzBJGb8A6iQkjqRx59UXbOifyACRXuf1mkvdnklYIZLj9JA232KSmNe7M68Zp+mBobX7IQ94e6qaCGv9O4tAmvLcfL8H32/2oT/eiP+yw34Lzfh52/3Wrk/aO8F+kWX8PDYNLFEUlJcKC8sKiwskhfLS0rkpaWlsB5LdXV1XV3d3bt3Gxoa6uvrVa+QGaqrqysrKysqKm7fvg2FDYWFhQUFBYWjLB8zqVSalZVVXl7+hIegXkztAegBhUIhEAhgA8GIiIjIyMjwRxnsEAKBwcvLy8XFxc3NzdXV1c3Nzd3d3dfXl/woI5FIfn5+UOcAtdGwBquqAOujVho9jeruvvva9b+7uSF4L8QHAAN4Nk8igzqq4WGABLy9cEuNkKnTcO+/j/tlDxIUBLCBzUbYmQiHA57xZ2RiWUmRSFraIaFILBaJYVxOKpUWYGZmZubs7Dz44H2zEo0eGFzQ0qpXdedfpWX/Jy+5Dwz5xUh+Ia6gACcvwRUUIrEM3IVfcfPn4954A5k5A7djBxLgh0tPxyIMWLNn2IohORnkTTFGSjkRSYifL+Lm9jdfv520Yf3CMDKEhYURCITTp0+rspKsra0PHjy4Z8+e3bt329raxsTEwM4Mj2MGmPTI5XIpFAqbzR5H/SyTyRoaGlC1qT3wx3ugsLd3Y9Ud84pK8/KKdXeqd92pvv2UpbpYvazptVh4oWr61LJpmvTJE0laOgl6wn7R/d0fHGzh84udnVOWLaO9+RYVIweiplaolnbS5Mm5erpV+lMaRxKWVKGG0SONBvptBvrtUw2aDPSbv5zdvmdPb0jIUGXlOAlLu34SPRUwDA7yu3t2t7VP7R1BhTvVk7JzNOKY91EhljExLV1DINS+e89scDAORV9cW4n7znyGMeUQGnvpERroSzNRV1O058HIkiQSJDKNDu0qFWj0OTSb8gyf/LqvogaG1/0Mjt1/BYoqFKB6aWVd02mH+O82e89c5z5rk/fXW/znbPWbs9X/620BX2/1/3qz/1ebfD9f7zXL3GvOJrz5kQCngHi+MK+6orSyorQcy5mGCUj19fVNTU11dXVNTU0NDQ01NTVNTU2NjY319fV3796trq6GxVtgnKFkxORyeTFmEBwKCgry8/MFAoE6JWnsCVO/fwIPDAwMiMXi69evX7x40cfHJywsLCIiAgYcIDuEhYVRqVQKZiQSydPT08nJycXFxRUzd3f3gIAA8kMGuzvj8XiojVaVYIUFWCF7kEikh9ZTTQghBfs7On1tb49zcwMZ/z6+iL8/QAISCaGGAGCg05GYWFxsLOLhjmzdjHv3XeTNt5DVqxAPD1xq6nBSUGYmSAQKD0eSksyFwhyhELRwgxE/GO6Lj48XiUTd3WMSgmtRdF9H55vVNRPLyv9PXoIbrqZajOQXIQUFE2R5iIcnztRkwn/+g3vnHWT9epy7K5KeBuquSiS4bAEIbnAfDwwkMug35+aGeOENQ6mU0ND7WUlQxnDt2jVVPtKRI0f27NljaWn5888/79ixw8HBASLB44BBIBDArMWYmBg6nf5IGQOkCLFYXFRUpC6xiqrtj/dARGvbhrq75uUVcFhfU3uktq7hQVAffy8c2hxU+Uj6UgMNipZGiNYX3C9b0Udn1nUWFZUTgtjrN0R++FGIJujqQNTUCtHSjteeLNDVLdcfTyQNEaIF00m3Gug3ffB+67p13S7Og1LpwwlLm7fkPBkwKAcGM7u6t7S1T+npBVGFllbNyqpJPP4DqMCIm5jJ1pDKpjQ2bR4aYo3Ndh7fR6/C3IFelLT3Ecxg9ynKJz2wg9W56JHJKCfg/sRsMmo1Ga35KzaKVgPD/cvgzzGGCROG2JJyE0vfmeauX271/XZbwDfbAC3M2uD7ubn77A3uX232/GYzfs4mj683eXyxweMzc4/P1+A/XeO+6CfPy17x0ryShnt1dXW19+4BVGhpaWlvb2/FrK2trR6zlpaWxsZGWNixpqZGxQwQG0qxblKQHYqLi4swKy4ulkgk6ozkP8dl9lKOQi6Xnzhxwtzc3Nra2tvbOzw8HJbxDQ8PDw0NhbQQEhISGBjo6urq7OzsMmJ4PB7mFz2cYgTbGHs8WE/Jx8dn/HwkEjAymUwN8L9685a+oyPi6org8YiPzzAwEElISAhoxUCnI5GRABjiE5DkZIRGxVlbI7O+wE3Rw333He7KVVw6CwQZEhMBMDAYs3n8RKFQLBKJIDNAKVFpaVlVZWVjY6NCoQAVC7CaBUq0WYlad3f9u6ZGo6z873I5rqgIKSxG8vJwBQVIYcEEsWTCxvWI4WLk6jUkLQWXDwQMiFSCCIUILwcHBQyckcZtqWlg92CEITICFHsFwBCAuLshrq4fkMm+WKGk4QgDlD47OzvDTCRra+sDBw5YWlr+gtnOnTttbGwSEhLGDzJAlsjKyoqOjuZyuRAPRpMDHIevdXV1qqjpS7n21B/6p/eAEkUv3L27vqZWBQzm5RXra2rPFOS39z9pS5BtDdtVwKDD1tMgaU0K1/6xYMfwP+3jndhXV1cbE5uzd2/MrNlkjBxIWCfpWO3JWTq6vymSHp2w1PDmG63LlnaePNmfkqK4dw9+5pq1/KnTkg2mMh8c4nk8lbx7aGAwtat7Q1u7bk8vaKrQ0qpZUTmJm6XBiBuOKkTHTIxjTsziaRQVTW9t26tUCh5/QK/8nNYa1NFobG7SpZmox2q0/0EBBssTPToNba4Ch9RRj556F01xGj68/i400xsN+gkl/ozKGGDiPTmaeBMd7AfjXU0o9SBaOeKlVBdURB9e8fX8owaG1/O8jbPXSkVPb9+uM5SPVrvN3RI4dwvh6y3+n673mLsF/9M5qndYVrqgjC8pZiRx4lN5EfFcP2raeRfG1mPBhj/6fLra/f2Vzkt/8gqkcxoaGjvam9va2jo7O7o6urq7uru6ujo6Otrb21taWpqbmxsbGxsaGqCSAcYZqqqqKrH0pHLMIDaoQg3FxcVSqbT3udaWHscN6ll/Sg/U1dV5enqamJisXLnSxsbGw8MjLCyMTqePBgZ/f3+nBw2Pxz8uUODn5+cxyjwx8/X1fdzy5BGjUEhkcqgnfvf1G//PAQMGT89hYCAQsI7OFHDbjQHDcO82JhOXlARygRhM3M2buEWGiJERLiEJPOlPSp4QGYXExBiwOX4iYa4IM3ivnJeXh6UIVt2pKunqpirQLKWyB1V2oGjkkHJFa9sbd2on3S77f8UluJISxN1two/bcDlC0OBZlofweDiJGJRblcpwsAODSIjkCBB+DsLLBqCiAoaUVFD1lckEKUmREUCAQaaA9CoPd8TeUY8QaEcb6cMAoQHqnmGE4ciRI3v37oW08PPPP//00087d+50d3eHYQQIBuO8ZmRkJCcnPyx6Vk2RSCSFhYWqAIuaHFC1/QEeaBoa+qmyal3VndHAYF5RuV5efEPIH3gCOWyLovW72u+m1Q4XVNVi6EBgcKpzfvL9HezoaMjIkJ49x5w3n6KrRxmROkRqa2fq6BRP0avTBwrpx4mkGzDxdNtUUJ61xUC/ZfasLsuf7nhRZ89OM5ia+CAtMA2mxvN47Sg62D/A6Oxa1dqmDVGhuUWzrHwSh6sRywCooOq/xs/RuF36Xlf3caUy/8mP6NVdUkwHAugxDd3sPkdLMh7Y56EB1NUY9VoPMr5CDqC3Fg7zgFKBem9AL36GZhHQhOuotR4qiURba9Fj09AqMdhCbjS6dwIaZgvG+7rQXz9ChbQHtvy6vVEDw+t2xn5rfxVKpWKop+pu82bboE/X47/c4v3ZBvfT9gnS4pohLMWwp6srP09WWJBfUlJceruksry0uqq8vKI8RyonRHB/Phfy1QaXD0xu/XSWWnS7drB/oKenr7u3u7unu6enR8UMra2tMDEJBhlqa2trampGY0MlZmVlZaWlpSWY5eXlVVRUoFir5986CPV8tQce4QF4p9jX1xcREWFiYvL999+bmZlZWVk5ODgGBARQMaNQKL6+vo6YOWHm7Oz8OACAyUsuLi6wnhIEB09PzycpqEohk8nEIGfnuVeu4ewdEFcXxMMD8fJC/PwQAgEJJiKUERlDRASQKMTGIHEM8Ag/KQm0Yshk41JZIBOJzUaysnCJybijxxA3j7+zWKdV6TqqIAM9IiI1NUmWd/Tevc9Q5ToUvYqiF1DUbKD/vaoqLVbG/8vN/VthMa74No5Mxp2/gBMIcHn5iEyGyPIRaR4C+rWJwSAWIQAYMFrI4oMaTWw2kpEBqjYBYEgEwBAbMwwMJDIABk9P5Kb9//n47KeFPaB7Dg0NdXV1PXbsmJWV1f79+1X5SLt37/7pp5927Nhx9OjR5OTkJwky8Pn8uLg4FosFgwwqTlCNQHCqrq5WlVhVM8Mj/j3Uk36fB0TdPeurax6gBZibVFG1jsf2leRggb3xPiO/P//t6nem3Zk+DRZUpWlrkLU0IyezOh68AR1vG/fnKQcHm3Nyip2dk1f8EPrGmypyoGtpp+vo5E/5bZF0g4F+i4F+91S9PIMP3zegTp6arD81/kFmSOZyRf0DK9vaNVWoUFo2KZN9HxViGROZ8RM5XI2Kis96em8oUfAj/iexwV5wxz9GAG038/+z9xXwUZz5++/Su//dERIsAtSuvZ4UWqx2lBYo0JZiCU6CO0VKAsEhQlyIb5zoetzd3RViECUu69kkK/P/vfNuNksSAgUq9PbLfJbZ2ZnZmXc2yTz7fJ/nwaKMx59gdz127R0sSBe7+rYUDGAYJhJiuf5YzyPpyl57Md+DcP7uWizVGc7Q9TCfg5jDt1AC8bgSu/UBxnw8fs+v1XMFYHitLtdzHKxYLEKmIo97mHsu+n+l4xGTWoV7qsL4BRaPW1FVWVVVVVtbW1cHlc2NjY+amppaW1s7O9r6ezv6enoKyxuv2Ud/tN3xvzp2jIQSoXBEODQkGBQIBAI+n49IBnnAgPuswgcZbEBK6NzcXOQIifTQZWVlHA5HgtdznIdiFcUIjB8B2W2iSCSKjo7etWvXypUrV61a9f333+/fv9/IyMjb25tEIrm4uNja2t7Fy87OzsHB4WkAwN/f38XFBQmjnXFt9BR+SuQni0Km+fvZW1m9bWYObG2BowMEDB7uUCjs6wsBA5kMaDRolBQSAiLCxwBDXCJISCIkpxBSUwkZmSArA+TmEOLiCOu+IVy5BpJSDhQU5ZWUlFRUVJaVQ3/V6ur7n32+8sTxf5aWqba1qvA4bwmH/zE4+H51pYa9w/S169/4x/vTfAMItbWgunra/fsE2JWEo4WKClBRAZPaSktBaSkBoYWiQpi9ANULuVLAAI2bUqDfa3y8FDCEhECGAbUkuboCG2tAdN0ja0mi0aQKaD8/v9u3b589e1YeLRw7duzIkSOHDx8+ePCgh4fHc9olpaSkUCiU/Pz8STFDRUVFZSX8rdX7fIYw4z83iueKEXiOESAPMOUFDGPIoal5e139ttiwyPraqXcTyg+b3yk1VFUtUVMiqShRVBbEvt0ixLtZpt54ylc5D2oaSaTMPXvD/vMhZbRhifGkSBrRDvLaaDTfq67WqTbn3JxD/5lDmzc3VgYb1DTiZqtlZGTcEon/xuYo9/Ur1zfMSM9QioySsgpR0dPjE1CuwtIBphuGPZcd05Tn8ft7Mcd3vJIBdSWJJgi4s7ywswCLMX3iHLi9WJIt5nMA89bBbvwdPmIYxBve2phoBHPeBPuUHL7DuuohC2G39oltX8MnCsDwGl60KQ9ZIhHBe3J8ncfdzPIaPPcALpAM8rlV1dVVlRAtyCxTm5ubW1paWluhFVJXT89Afy+fxx0eElTVtZ6/E75wi62xexyPPygcHh4c5PP5PC6XiyQNqCsJSZ87R0vWodTY2BgbG1tYWNjY2Pjo0aOKiorm5ubR45ryBBQvKkZgyhFAnyKxWFxcXHzx0sVVo7Vu3bq9e/fevHnT1tbWxsbGdrTs7e2fBhg8PT3t7OxkgAFhBoQ6yM8sCsPT84qJ6XRzC2BtC71Tke7Zy1tqlBRIguaqsCsJj28LjwBR0SAmBt6XJ+AJbskpMO85PR1kZsKJwYCyh5jYtUWFWSQS5caNW7jlaGlNTe3Spf89fpTw8OGfGh7+rahYyfve33bv/H///Ncbf/872L1zmivxjfx8QtV9nFKohI9SqFAOrVThhNMLJcWgSBbwnAsBQ5a8gCEZFzDE4H5NIYBOh11VPj7AlQisLYGd/XoqlTKKGSBgQLpnAwMDhBZOnDhxDC+EFg4dOqSjo2NoaJibm/s8mCE/Pz8sLCwxMbGyslJGLCBj2aqqqvLy8uzs7ODg4NraWsXvkCl/OBQvvuAI/N8fSIMJAoYxzNDcsi0ve084vayrY4o3MBwwkgoYWufPSpkDAQNd5ZuctYOvLpRA0NHRHhdXeOZM9Gefk0aRA01FJXbmrOI5cxpVoUj6CeSgptY7S613tnrdsg9KN38av27DAXVz9TmxM9VTVGfHbnjPpLH6PSZHqfbhjNTM6RGjDUjRMRAnxMZNzy9Y097hJxJNrtieYihem5fa72PGi5+IcjNeNGLxpaSvefwp9DzELszEmgrGlrO74baeu7DKKKwuHXPVwjx2wlcbC7A7SyClYPEFNiKAcKKQgvkdxWLNx7Z9PecUgOH1vG7PcdSyv6xoRiAQoJyE2trahoYGlMXW0tIii2Pr6enp6+vDRQs8Ho83LOAPi4TxWTVrD7udMmL0MlnCkeFBLo/DhTIGJpOJdM9IydDd3d2Fl4xnaGlpKS8vv3//fmNjY01NTUVFxfAwLgN6jiNXrKIYgeccgfb2dhMTkzVrVn/11VerV8NHTU3NO3fuIMxgg5ezs7O/vz95QlEoFBlgQGZKCDkgQ1USaQpzJDKZTCOTaU5OWkbGBDMzYGUDkO6ZiMsYvL2BLy5jIFHgnXcwDhjCwqWAAUU+JybBLqCUVJCeBpuCMjOhKjoIBr39oyA/0s7OccGCBW+//e7BgweDgkI+//zrI8cI/gGEgwfe+OcHBHU1wrffvmFmPi09fVp93Ru1dTCOrWIUJ0BWoRxvQxqFCohbKMbRAqIXUDNSZiaeC5EKjyQxUap4Dg+H5k40KggIAPe8IW1iZQHs7NZSKaRxgIFEIl2/fv0kXidOnEDqBXnAcOrUqbCwMNSVVFhYWPT0KiwszM7OjouLKywsrMCrurq6qqqqqKgoOjrayspKX18/ODgY5cTLfrM954dEsZpiBJ45Av0i0fGW1h2tbWMgYdQrCS6BJEPd9oSok7HhTxNASzDJnu49CDBoNM1XDp8FBQwhM49WHXvmu7/ACkIOpy8/v8rYOGHdeoqqGhmXOlCUVaJmzsqbg4ukVdV6Zqr1LpjXt/utgfC32M3vcQff5A+oskM+JH+7fffbVy0Wra72Ua5vnZ6RppRIUUoOVEoKnxGXpBSZOD0x8+2K6kP9AykSyR/9r/YgC7P++gnAYLSw+epSVk3O+IvSXY/pzsYeZo8tr4zB9OZg/AHpEvv1mPsOOD/Ew+6uwajnMS9t+DTLG+ofbFZhDVlj276ecwrA8Hpet+c4aolEIhaL0R/X4eFhFMY82obU2Nzc3Nra2tbW1t7ejjIW+vuhxBk1HfF4XBaL1dPTh2EjXT0sXYvIQ9cCWzv6hgVDHA6HxWIhwNCHF8IMKJ+hq6urs7MT+SbV49XQ0FBWVtbd3f0ch6xYRTECzzsCsrvG/v5+Dw+P7du3f/XVV6tWr9LW0bawsLDGCwEGNzc3MpmMPFLJchUQEODi4mJvby/zU0KAAdERUwMGEpkaGOhrbfW5oRHB1AxYWY8BBg8PKGNAXUmBZEDFE9zGpM9RIDYG3pon4KnPySkgLVVKMiQlE4KCQHDIrLR0x7z8wqioqMuXLy9evGTBgreUpit/8V/Ch/8hrFxJuHF9WnQ0oaKcUFvzRnX1tMoKUI5PFThIQFChXEYslEh1C8VFULqQXwCNXHPzoNY5KwuilPQ0aT/SmIAhDLZRUakwTcLbC7gQgaUVsLL5jEz2GXVWlTIMJBLp6tWrJ06cQG6qR/E6fPjwoUOHDh48uH///r1799rb2yNJxjMBA0qCT0tLq6qqKi4uTkxM9Pb2vnDhwpYtW86fP5+ZmakQPT/vz4ZivZ8/AlUCwS6YwCA1VB0PG5qadrZ3Hn5QbZGf1f9kfqLsrfrF/V+0/xcqnlvnq93XUKKqKJGhRZJ5m4VsnV9kRixmVVU3EF1TtbSC3n6brKwSMF3F788qMeqzHh1WZ6bPY3LmMAXKLJ4yi63MZCuzBDN4XcptuX+pzv5Lie+Mql2zmz6Z27FQtetD1Y5P1NrWqD768R99DGvs54fW/SJn90vvdJiPEbeMkz7XXV7WW54y/p27G7DzM7C6tLHlzUXYT8pQxtBWDp2RjBdjJkuxnga4AuUcdhpgKY5wvrMGbmiyFBM8mfAwtqPXZk4BGF6bS/VzD1R2RzU8PFxfX19ZWSlrQ0KiBflEtoGBAQ6Hw+PxoFJBIBAMDg2PDPX193LYbJFoeEQ44kTOPHCZ2tLWJeAPMtn9rAFWP15I+izPMyCSoa2tramp6eHDh5WVlbW1tbgd5M89A8X6ihF46gjIPt4YhgmFwtSUlP37969cuXL//v0WFhZWeCHYgAADeUL5+vra2NggkQOCDQ4ODkQiMTAwcCK6eHJrEplK8/GxNTZ++7YhMDGFt9S2d4GDA3BxAe4eMCDZxwemMQSQAAU3Vw0OAqGhMMQtMgpER0OpQHw8/FIfkQxpqZBkSEyEd+rBQX9OSr6ZX1BUUlxkYmpiaGhoY2PzzrvvvjENzJ8HLl8B6RmE6iqYpVBZAWFDeTm0Pxqb8O4jpG9GxALsRCp8Ai3AZiQcLcjkzomJ8HhioqGbUyjuqUqmAD9/lPQMzK2ApdViMsmbwQim02mIZ2AwGCQS6cqVK8eOHZtILxw4cGDfvn179uy5du1aVlbWM7uSCgsLCwoKUlJSvPAyMDDYsWPH2rVrV69efenSpYcPH8pf7qd+JhQvKEbgRUcghsWeXMDQ2LS9uWV3axuVxeoWiabYfdXIqOK5bf6cPFUkYJgRNjOC+euFBw82tbTRQ7P3bck+O7c1b84AbyZzUHmABQ1SB5gwp5nFVu7uVq6qmZGRq1RzfHaXmmrfLLXeOWq9c9V65uIzs9X6Z6v1qWr0f/E5/9bNkcxMCYczxVm/9i9BwLBVHjCIDRY+0F/WV5E6/tRY7ZjD91iLXACfRIwl2GDWX2LEzVDl3FmDuWzB4vC+o/p0zH077EqCf58EWLA+jJd+/UsBGF7/azjlGQiFwocPH1ZVVTU0NOASZym3gELZurq6ent7+/v72Ww2n88XCAQjIyNCvEZGRoaHhwUCwdDQ0JBAIBYPU2JK9l8mNbR0D3G5fSzYktTf3y9PMqDGJJmMoampqa6urqysjMvlTnmMihcVI/CCIyASidhsdnNzc0NDQ2Rk5IkTx0+ePGlubm5hYWGJl5WVlaurK3my8vLyunv3rp2dnb29PXq0t7d3dXUl4c1I6HGy7WCWA5US7Oamd/PmXw0MwB0TYGEJbGwhYJDmPeNpDH5+wD8QmpMy5EiGiAjoW4rskhISAGxMSgYpKfDL/owMQngYpCaiY/bm5mUVFBRevHjp+vXrlVWVX3+1bt0awqGD0xYsAFpahML8aeVlhLKyaZWV08rLpiGJguyxFGcVZGgBdSLljXILiF7A3w6aI8H4hWSoqUAJDOHhUKJNp0PFs58fdIl1dgbmlsDScskUgOH48eMyrbOMXkCA4eTJkyh1bmqGobi4OCMjw9TUVEdHe/369V9//fUXX3yxZcsWGo3W19f3gh8OxWaKEXjuEQhnsXe1d+xobYNTcwvsQUKPjxq3V1Xuykit7O6cemeeHC+pgKFl/sxYaKiqRFaZFT2neujXcyCVYJ0jmAtvZAVnaBaLPx4qdHUpV1TOSEyaHhYzPcdvBvvdOaw5s3vVVSfqpHvU1ZjqalwN9R51tf7/fsG7fHkoNlbc3j71CLyWrwo4mP238oBhxGBRqe5S9gO51iN0YhIJ1DHLRz6j5SODmPB5YzpeyyGSO2gFYJAbjD/crEgkamxsRGgBZSPIRAudnZ0ILchzC8PDwyMjIyK8hELhCF5DOOPA5wvEwiEXcvrO896Nrd08LndgYAChBQQbent70T47OjqQv+rDhw/Lysra8d8yii8I/3Afrt/shCQSydDQ0MDAwIMHD1JTUxkMhre3t4eHR0BAgJ+fn52dnfloWVhY2NraPi3j2d3d3c7ODmEGO7wcHBzu3bs3af8S+cmikBn29nuvXpl26xYwNgbm5rAryX40jUHWleTnD6XDNEQyBMN78XDcLik6apRkSAJp6SA1FYSGAzMLwjdrwdIlgExenZWTmJObn5OTk5eXV1JaumLFmkMH/19xyZ9Cwwl0Ote3+C8AACAASURBVCgpJJQUE3LzQEjItMwsaH8km0qKCSXFMJqtuAhKnAsLQUGhtA1pzBkJghPYB5Uyql6A/kgxuKFqmFTAEBgIfHyhS6yTIzA1B+YWS8ikewxGkIxhCApiBAYGXr58+ejRo7LsBYQWDhw4sH//fh0dnd27d+vo6AQEBDyTYSgpKUlJSTl69OiqVatWr169cuVKTU3NmJgYmY/qb/ZpU7zx/8YIDIrF9wWCPD4/h8e7/vjxjgcPthXmaaUlacWEaYZQtkcwfCvkvlqeMCbtoo71nd/Ob18ADVVrNGbQoaGqEkXlg/R/dUiegTQm7OxFFojFtfzBGyz2PwcFylyelFKQsQodnTPKypUSEqX5a7HJf0m78dewGf8yUj+eob6cqT7naakOvThyQJEO/Ys/5hw+JCCTRA8fvsgh/j63YXdj5p/Laxi4tz7K0fuc34yTA7/PY/7tjkoBGH67sf9l3lmCF4ZhYrG4ubkZT32qf/ToUTPuhoSIhc7Ozu7u7t7e3gE8xRl1IiG0IBQKxWIxQguQW4DsgmBwcJDL4/J4XMHg4DX7aJ2LAV1dvTwY4Mbq7+/t6+vr7+/v7u6urKy8f/8+AgxNTU3V1dU1NTWKP/m/zHX+39orApwikaizs7O8vDwmJoZEInl6errgRSQSXfHywMve3t7a2trS0tLMzMzGxsbf31/WYkTC2QMymezv7+/s7DxqpCT1YLW3t/fx8ZGtTH5qUUgksoXlt/r6hJs3gaERMDWVkgwyryQU+ezrC0kGEhlihqBRuyScZCAkxIHkBNihZHsX7No17e9/B6pzwRdfgNM/Agr1w4yMkOyc/Nzc3Ly8vKKiotVrvt21e3p+wZ+KSyEYKCgAJWUgLAx8uhze05fh9kelJTCjrbCYgHBCfgHIzwd5eVCxMAYVUCdSOuQWoJVqCqQ44uJwN9VoEB4BIQ0jCJAosJ/K6x4gugI7e2ByB1hYriCT/caJngMDA/X19Y8ePSpPL8jQgvbevbt37965Y4e9vf0zAUNxcXFeXt6FCxe++uqrNWvW6Ovr5+TkCIVCTFGKEfjVRyCyq3N7fKQmPUCTEagZRNIMJm2PDLLOz0a/heQPR4yJ/Xl+3/duWNqxbF4bntfWNl+1VA2iBdxTdXHl0j6JLEpZftNXNi8SFQ0O/sjmvC0YUuZwn4AKTLZye8eMklKl+IRRqBD3/1JS/xIaskR7+cV35tKUNVLMZn+ZOWtG5dy5rWqqvU/Pg+vFIx3YGupsDfX+f/2TqaU16OIiLC2RvO5eJs3FmNFHY4DBaGHr1SW5tzYIuaNS5ld2of4IO1IAhj/CVZQ/BwQYJBJJW1sb0i0gQyQkcYbeqXghtIBUzkNDQ7JOJEQviESikZGRoaGhQeSlyuNxuVxc5ACJhR0XAk7eonb39rOZrL4+GPnc19fX3d3d1tbW0NDQ2tra0tJSV1dXXl6uaEaSvzSK+ZcZAbFYXFZW5uTkZG9v7+zs7IqX24Ryd3d3c3MjEonOzs6IPRjnqUqhUKhUakBAgKOjo7W19UQD1mcCBgqFGhDgedvgw0uXCDduAANDYIJ3JVnbAHt74OSEBzLg0meoZAgYTX2mSzMZwsMAPRjctQcHD4L//JugoU5YvhScPEnw9ZsWHEKgw/4ltbRU35ycnOzs7JycnKKiolWr1m/b/te8vD/l5UO6IC8f8gaZWQQyhZCSBgqK3igsIoSGAxIdLi8uwYOccXGzDC3k4BltKKYtHQcMyXj2Akp3RuqFsDDoAEujwQP284MCBiIR3L0LjI2AldUqCjVgHGDw9/fX09M7cuSIDDAgrfO+ffu0tbX37Nmza9eubdu2GRsb5+fnPxMzlJaWWltbf/XVV2ZmZl1dXS/zUVFsqxiBlxmB+zzezpgwzWCybNoWRvsxPlIgHO/N3zXc9UHpv9QbNOY1Q60zxAw4bFCrVp+dOXdW2twFLW9eHLjYKmx9meOZdFuJRDgiTBoU7Gdz1AVDymzOk1CBpfz4sXJRsVJsvBQqxMX/NSX1b1Tq0jNnrv/7P+GzVJNVNRLUNOKMZ/6Xofw3krIKXWVm/KxZJXPmNKnCJqUnvFnV1eTblvrV1dga6lwN9b533h5Ys5pnaDicnCxhvp7Wq7l+8sFtIoOF5ReXlVrh7kaTjvv/9kIFYPijXX8EGNrb2xG30NjYiCTO7e3tHR0d3d3dPT0947gFhBZkUAGhBSRg4PP5vFG0gEe2cQYHufnljxZr2t5xi+NzeH29fb1w6u3u7kbcQktLy8OHD0tLSzs6oGX1xG9l/mgjrjifX2wERCIRl8ttb2+vrq7Ozc1lMBiOjo4mJia3b9+2tLQkEokIHkxADXCBu7u7q6srkUj09vYODAwkj1ZAQICrq6upqamxsTFSRSPYYGNj4+LiIltTxkWMbjf2P4VC8/SyvHxZXU8PXL0Gbt0e7UqyAnZ3gaMjlD67uQFPD3DvHpQOB+AkA4OBNM0gMpJgYAjenAf+8QHQ3kuwt4N9SilpMJMhNp5AoQEy5W/x8WZZmTnZWdnZ2dl5eXnXrt2+Y/qfvJxpOblSxgDChoJphUWQRsgvgN1HF34iLJhP2LQJOLuCjEy4JC8fhi3AgLZsOCHdAkILKSm4lWoSrnWOAdGRMFoO5rXh/kgBgVC07eEJT8TGFhjcBpZWa6lUshxgoDEYDF9f3/Pnzx85cuTo0aPIHAnRC08ABi2tc2fPpqSkPDPyuaysjE6nu7i49Pb0YHKl+AUiNxiK2V9jBPgSyanMVK1QqgwwaIVQdoRSGwbGcwUsIfuT2M+UGNANaWb87NnZqqrFamrVGhoP581rnT+/DaKI+d0LPmv/vEX0stltsjOXSAZHRkJ4/E1szqxBAdQxI03zABP3QWIpt7TOKChQio2VQoWExOkpqdOzstceOGj83nuRM2elqKrFS/Oe58WTLnoW6/4Uv249RV0DBTtQVWZGz5yZN3tOverczlHk0PskZkD4oU9djakBpQ696mr9y5dxz50dCgkRtb56dCQ791c8IxZh/kfHAIPRwq5rH+de+KTO78YrfqM/yu4UgOEPciXl/6x2dXVVV1fX19c3NkKJc0tLy+PHj5F3KgpbQCpn1ImE0IIYLxEMiYbSheHhYdSMhNACohc4eLHYTIFg0Ckw48MtlqEJJRwOCxqq9vR0d3c9fvy4ubm5sbGxoqKioaFB/pD+IKOsOI1fYAQmfk6EQuHAwEBzc3NJSUlcXFxISAiVSoXZB3jGMCIHDA0Nb9++bWVl5eLigmCD62ghtCCPIjw9Pb28vDw9Pf39/R0dHa9du3bhwoVbt24hMyUrKytra2u0K9JozxJ5skKvUigMouv1Cxdm6F4Al/XB9WvA0ADv27Eg2NrAHh4nJ5h35uEOcwx8fAj4/TdBV5fg5CwlGby9ppmZEShUEBdHgILjeKlpUlwC9DMNDARRkSfTMzKzMrOysrIgzZBbkJ2rnZ0NAYP8lJsL0JSXD+ITgPEd8OWXQFUVfPopweA2iIsnFBYR8vJAdhaEEAgqpOK6heRkaMqUgEc7R0eDiEjY4BQcDOg0qdzZ2xu4u0EBg5UVuH0L2Nntoo4GPNPwCgoKcnd3P3369BG8ULQzMkfS0dHZi/qRdu7U1NQ8fvx4fHx8aWlp0ZRVWFhYWlrKZr/2zoOYol7/EbB6ULUtgiEDDJrB5G0RjLD6molndrr4R6VgXLFAUpkeiHci0VRmMGaqRM5SLVeHgKFjwdLHy9qEeILqxO1/zhKJpHdo2IPLW8nmKPMHoeuRPFQYYCo3NSvn5SvF4FAhKnp6YtL0lFSV+oYdIyPxmVnds2anj0EFjVgcM8TllfBgA7NgiPvoUYO7e/bBQ8Hvvx+grELBp7CZMzNnz66eO7cdb1jqV1ebFDmghiWOhjpTQ63vP/9m79836O0trKrCfuddhfcTsTtjqW0ig4UlukvzdD/pyg39OZflf2hdBWD4o13s3t7e+/fvNzQ0ILQgn7SAeoeYTCabzebxeKgTSSQSIbQgFotlgAGhBUQvcDgcBBjYbA6bzWKzWXwOr6u3e+tZ39VH7lXUNfUPQMjQ2dnd3t7e0tKC4uFGRsazt3+0gVacz8uNwDicIJFIBgcHu7u76+vrc3NzIyMjGQwGlUql4EUdLRoNfrdNp9MDAgKcnJyMjIwMDAxksGEKwsHV1dXFxYVIJLq4uLi5uTk4OFjgZYkXQg5EInFqwECGRaFSGLY2R3888xc93dkGBousrb6xsVxocucvZqYES2tgawfvs91cgbsncHN74949gp//Gx4ehBUrCJf1oaQ4KAiEhhHCI6eFR0C2AZkmxcZKb98pFNjFFBa2LT09NT09KyMjIxPChrzMrD2ZmdOysqAjahbOGEDeIAdOCELk5UNWISMNuLlP26pJePsdwr/+DU6eJgSS4MKMdMhgIJVzEo4WoNA5FkTHSNECVC8wAJUC+RAfH+iPRCTCDiszM3Dr5hsuLqeCgkLpeCHYhtieY8eOHcbr0KFDCC1oa2vv3bNnz+7dO/HaunXr0aNHY2NjnwcwlJeXo2g2TFGKEfhNRyC+p3t7XDguYJA2Jm2PYBhkpoz7rYVhGO0xXYkB9c1jExn6IykFqMwtUpv3eP677X9PFiS/5NmIxU0CgSWb8zF/UJnHnwQqPGqckZOrFB0DWYWo6OlJydMzs9QaG48IhrIxTIJhWERkl8a8JCmxIEULseoacdk542mToe7uttDQwp8uRCxeEjhzFgXPkw5SmZk8a3b53LktarBhaQrkgBqWOBrqfe/9nfndtzxz85GcbIzPf8kRePWb97dijt/L0wutVz7OPL8898pqQfcro4Ne/WH/pntUAIbfdPhf9ZsPDAwgtNDU1IQMkWT6ZuRohDyRuFyuTLcgQwsyrTNqRhocHJRvRsL7kSBeYLFZTPbAII9Djyr493aP8yaM7p5+SC+0dzQ3NyPpgiKQ9VVf2D/s/oRCIYfDaW1tLS8vT01NDQ2Fd6XIpwh9k42QApqn0WhUKhXNo5vXwMBABBtQk5Kzs/MUmMHNzc0VLyKRaG9vb2lpaSFXNjY23t7ezwEYINXh6HjF1HSXm6sRjRYQExMTH0/389tvZv5Xa2vIMFhbE366AFavAdu0pnl4AD8fgq8fcHMn+PkTqKOxDCEo/jkMRERAe6LoaBAXC2JiIMPg5w+CglampcakpWWkp6elp2ekZ2Slp+9OT5+WkSGNhc7MhC1GmZkQP6AUNhxLELJzCPn5hNwcQCGDEyfBB/+YpqFB0LtISEuTZj4kp0BkAlMXYmDydGQUbIgKDYVIhk6DW/n7w+A5dw9oqGpjC+7cATdvKnu432AwQtCYo0cGg3H37t2jR48ewgupF3R0dBBgQHLnbdu3b926RVt7b2ho6NSAoXC0FAmPmKJ+ByPQMjysnRSrFTImY9AKpWqHM7p44y3CWwQtGmELxtACQg54ZJtGw7z5XQtsmLYvc0IicSWff5nFfn9wSJkjZ3+EGpD6B5QbHs7IylaKioZQITpmenLK9Ny8+a2tPwqFZfLvy2C0PydgkG0lZLG6MzLKbt2O//prMh4mTcalDnGzZhXOmfNIVbXrWQ1LLCR1eHPBwIr/cvT1hyIjxb+TCNdBNuZzEDNZhBkuhJPRQt7NRTnnIb3QQMeDFGSjoJiRGwEFYJAbjNd8lsfjoU4kJFqQtSGhpIX+/n6ZJ9Lg4KDMPlXWjIQ6kSaVLkjRApvNQhnPzAEmc6Cru3e3PmXJHq/whKKB/p7Wltb6+vqysjKFYPE1/xz94ocvkUgEAkFvb29TU1Nubm5MTExISAhiEigUijwqkOGEp83Q6XQGg4HYhnFNSlMgB6RtcHR0tLW1tbCwMMfL1tZ2nDya/JSiUChhoWHxcfFp6Rk5uXlFRcWVlQ9KitOIbl+ePUvYsBH8/T0wdw744H3C/gMEVzeCjze45wMCRsUMFCqgM+ANejCOGWCaWyS8d4+Jgd6mIcHA1w+QKf9KSqKnpqanpaVlZmXv33/kzOkFqWnT0tNhZxGCDfKPCDxAs9QMkJkO+YTsLJCdCZUJV68SrG0IsBMpGeKEiAjYARUTC2KiIFCJiCCEhBKCgwGDDigUQAqQ0guurjBWwtIKGBqCmzfn+/jYMIKC5AEDnU43NDSU5TrLmyMhufOOHTu2bdu2ZcvmXbt2hoSElJWVTdmRBF8sLCx8/PjxL/75U7yBYgSeNQJiDLtUWrQtjPZEV1I4PbFxvKPoiGRkU8YWpaCZT2AGksqs1DnzuxdodW/jS17sy3WxSJTL4x1jsuaNsz8awPPX+vqV6+tnpGcoRY5ChZTU6QWFb3d2XhKJHkw8v0BS688FDLKdiAYHWVXVNXb2aZqa1PkLkNSBoqISMXNW9uzZNapzO/BWpad5s/bhDktQJK2u1vfRIu7xYwJ/P+jN+ls1LA3zMLruGFowXAi1znpLMs8vz7+xTtCr+BUku/LjZxSAYfyIvL7P6+vr79+/39zcjCKcu7u7keHpAF7ynUjDw8PIPhUJnRG3gHQL45yR2HghwMDCi8lkDgz0d/X0dDxu8gvK/kiHulOP9OhhS1Pjo4qKiubmZqS6fn2HUXHkr3AE5Bl8oVDI5XI7OztLS0tTU1PDw8PpdDrCCTLe4GnA4JnL6XQ6Yhvu3LmD1Myo9QjBhknBA1JFOzo62tjYmJub29jYPA9gIJFINBotPT29uLi4FK+ioqKYmBhDA4PPPvnHnNmEefPBmm8I585Ns7EBru7AzR2KGVD2MxJAB+Lxz/RR06TQUKgfiIgAEXigW3AwzG4jBb6VkOCTkpKWkpKSkZ512+COhcXSlNRpqakwtCE1FfqiTpyQSgFapqIJXy0jE6SmEZKSCKmpBFe3aWu/IXh7QzYjMhJO4WGE4GACbEaiwmYkfz+cXnCD9MJdW9iPdPMGMDT8mEzypsOYZ2mhmOeLFy8ePHjw0KFDMnMkHR0d2I60a9fOnTu3b9+upaW1adPG3bt3hYaGPidgaG5uFovFr/ATqNiVYgRebARIbW3bY5/sSopkWOZlTdybSa0ZlDHIupLwBAa1Bxr/6fywYqRi4vpTL5FIhkeEMfzB7Sz23HGaZhTV3NunXFs3Iy1dKTIKsgoxsdNT06YXFb/f03NLJGp42s7v+bS8MGAY26dIxG9peeTtnXPoUPD7/0BSB7KySrDKzNTZsyvnzml7joYljoY6S0Ot9x/vs7S0eJaWI+np4oFf0cNUwMEYupjJR1JuwXChxHBhg/7i9HPLci8s78igj52sYm7CCCgAw4QheT0XCASCwsLCpqamtrY2WRtSf38/Ey8Wi8VmszkcDp/Pl6WzybgFJF2QRS7w+XwuXjJigY1zC1J6AYcfrW2tOblZWfnFG05Rlh6kuwemNNQ8qK9rwFMXYMek/J3i6zmiiqN+NSMwPDzMYrGam5vz8/NjY2NDQ0PJeMl6jdBN6DMhwdNWGLcfEonk5eVlbW1tYmJibW3t7Ow8UQONepNkjy4uLpaWllZWVpNGvKGjlT0GBgYGBQUVFRVVVFSUl5dXVVXp6+vPnz//nbff3bxpw4nj7968RbhrC+wcAMxkcCQQiTAqwcsT3oj7+MB2I2S0SsUToBHPEBICMUNYBIiKginL3j7A318tJtopKTk1GVZaekZmWurRpORpSXh4QjLucZSSCjUJ8lPq6NPkVJCcCmMWkpJBUiJISCDEJ0B1dUAAQVdvGpU6LTIahIWDQDKBRieEhk5j4NHOvv6QCfH0BC5EePBW1rAf6coVYGGxkUaj0OljDENQUJCPj8+ZM2cQYJBXL+zes2fnzp07duzQ0tLaunXLDz9sOHBgf2xs7DMBA2pKevjwoSJ+AVPU72AEKvj8HQlRmsEkGcmgFUo9EBnMFAjGHV0Jq1QlaNYYYCCpqMTMmt+5wJXtNm7NqZ+Kxazh4UAe/zsWW4U/+IT9EYIKPb3K9x/MSEmVhwpKZWUf9vWZSyTPaLv39GqeHDBkv2CS+nBfX1toaPGlS5FLlpJmz6HiUgeGikrCrFnFc+Y0qqp241KHKWgH1LDUO39e35LFnCOHB91chaWlkl9U7cDtwyhnMNPRTiS8Gan1ysdpZ5fl6X5S7aknnuCcO/Ul+197VQEY/iBXvL+/v7KyshMv5Jo6MDDAZDJZeCHhMp/Pf5rQWaZbQELncYAB7QTnFgb68erofNxQV//oUf0V68hPj0btuhDwoKZWODKMy6v+IEOqOI0XHgG86Wiwq6uzrq4uIyMjOjo6KCjoVZEJT0MOaDmCHxQKxdvb28bGxsTExMrKysnJydXV1U2uUFcSenR2drawsLh7966/vz/5WUUikfz8/AICAmAGM14ODg63bt2i0xlxcRGOzp8b3yFYmANrK5hgYO8AnJyhY5K7O7wRv3cPsgcyo1UKBdDoUGocjIdAh4ZCnoFGh9DC12dGRJhpYmJqYmJiUlJyakp6ctKxxIQ/JSaBpCQcBuCPCDkk4fhBNg81zUlwSoBQAbonxcVDSiEWtjwR4uMIUVEwo40RBLZpEpYvB7oXIQdCphAgveBFILoDRyd48GbmwMAAXL70Z0fHs6OGqnB0ke4cCRgO4oUAA9Q6Q7XzbtSMpKm5dcuWzd99993Bgwfi4uKeaauKWpLq6uoUfgkv/HOn2PAVjsCgBPuxIGebnLkq9EoKp6e3NI17F66Y+0XiCiX6qO6ZpKJapb6nd69AMh5ajNtQ9lQs7hwaduLyPuPyxtsfIajQ1a1cVT0jOUUpIlLKKqSlK1VWfcSEAonnSpL28Gx6tYBBdvBCPr87La3SzCxhzTfUefORSJqqrBI1c1bu7Nm1qnM71KSpDk9zWOpXV+PgkXA9by4YWPU1VDswGKLmZsnQsOxdXsFMfwt2T0e+EwkzXNh+5eOMs5BbKDLeOsRUxL88Y5gVgOEZA/T7fxl9l9/b29va2opcU5FWAbEKiFjg8Xh8Pl8gEKBmJJkzEuIWJkULo85IULfAYrEGBiBUQMrp3t7ezq7Oh48edT1u8w9O+/xI0OeHaJlF9dCdTSSSQIJBUf/TI1BfXx8REREcDJtYyHi9fNPR1Dhh4qvIZAnBBgMDAxMTExcXF4QQiE8WAgxEIpFCoaCjlX+kUCjoLhm9SqVSTUxMFi9eTKVSc3NzU1JSkpOT4+Pjw8LDAwN9LCw+NjAioOBna2uY4uzogEe5uUIZsafXE5gB9SbRaFLMEBwMeQYKFUILb+8/hQTrxSckJSTEJyQkRUbGhoXrxMa9ER8PMQCaEhOhNSp0Rx2dgfOjr8bjymaU4hwTA2KiIX0RCUUL8F1CQ2HytJEh+PIrwuxZhPc/AHv3EqytgYcXwc0VmiNBesEEXLsGrl19GwoY5PqRECQzNjZGaGH//v379u3T0dHZi6OFnTt2oGakrVu3bNq0cd36dcePH3+eHAYFYMAU9TsbAc+mxu0xoU94JcHI50m6ki5WXpLKGEgqylGzFrYuejAyiQfrxPMTiR8Khkw43IX8QWUu7wn7IwQVOruUKypnJCZBnBAROT02bnp6xvTq+4tZbEcM6564w6ctcfd4GmAY75L0tD08c7lkZITzoKbewyNzz96g9/9Bwo1ZycoqoSoz02DD0typG5ZQWhxTHQY7QBem999jbdnMvXNnOCVF3P/SB9lSgjn9MA4ttFz5OOPcsuyfludf+ZrVUPzME1SsoAAMr/1nQCKRiEQiFMrW19eHWAXURMRms7lcrgwtCAQCmTkSUi+M4DVOt4CgwgTdwoAMLfT09HR1dre2tbS1tSZnFq084vffE/EGTgl4I5JQIpGIxWKJXI0bYlm3kmxNtAJajrYbt4ni6es1Ajk5OTdv3rS2tr537x6VSkU33BPv6X/pJejWlkqlenp62tjYODk5PYkUpM8mBQxIfo0U1ZaWljo6OoaGhggz+Pv7e3p6RkdHR0VFBQcHk0gkb28vRweimektg1vv3b5NMDIGEDNYAGsbYGcHnBygJMDVdYxngL1JfiAAD4EmU6B+AMEGGLSMMwzuHoBC3R0bGxUXF5+UlHz06Oldu+dFRrwBiQJ8iouDCua4ODjJZtBTBBJicZAQHS21QoJyhXApVAgOhmiBAckNAok8zdwCxr0tmA/UVMH6dQT9S9DlycISJlhfvAjuGH9DpZIYtCA6nYbGEz0aGRkdPHjwCa3z7t278Gakbdu2aWpu3bx508aNP6xdt/bKlSt5eXnPTHpWAIbX62f8f+FoS3m8HfGRWkFPdCXtjwzuHRyvY07vS5+BdM8klblFqr5832eOj0hUyh88z2a/MyhQ5nDHEhUGmBA2sNjKHZ3KZeVK8QlSqBAXPz0zS6mmdhmH64xhvc/c/7gV3N0bn8IwvPS9+Lh3wp8OdXe3BgcX6+lFLFkaOGs2oh0YKjMTUJi02jMalnpx/yWWhjpHQ71XQx31LPGJxJGyMskLRLWUhmE2X445qBouFBosqr0EdQtZ55fl6H3RVRA92Ukolo0fAQVgGD8ir9dzdHvN4XC6urqQYkGeWECdSDwebxAvxDAgDQOCCsN4CQQCmYnqRLTAYrFQM1JfnzTRubOzE2W01dfXlZaWbvkpYOXZtI1nKUwOF8PEYogXxKgRGWU7yIYUUR/ygEEGEoRCIa5/gOvKVpBtqJj5NUdAght3I6JIgonFmBheE0wkwh8lmOSZV6iwsPDSpUsXL168devW3bt3fXx8xsGGXwhCTNwtFU9yoFKpgYGBbm5uKIphHGxwdnY2Nzd3cXEh44UONTAw0NHR8dSpU5988omamtqCBQsuXrxIpVJJ5EAqlRIUFBwYGOjp6eHg6GhqanblyrUfT/94+tTqK1f+eusWuG0IM9RMzYCFJXQmtbeDPIOzMwxn8PCABIIX0jP4SfUMJDIgUwANd1xlMKDUwc0NkEjbYqIjYmJik5OTN27c/uWXfw4LW4cggAAAIABJREFUeyMmBhqwwuiGGHyKhtQBIhCkM4hMwHECpBSiYMwCggohIbD3KSgIQgU6DUYukEggIJDgHwB9mWysgc5+wvvvEWbPAR9/DE6fArduAV29Pzk4nGYEBY+2JMEEDDqdHhUVRSaTT548uQ8vKb2wa9covaC5ZcumTZs2fv/992vXrjUzMyvG65kuSQUFBTU1NYqWpF/zh13xXlOMAF+Cnc6fpCspsenRuK2YYubi+KVKVBWVmNn72g+MYFPEEI2MCFN4/IMstvqgQJnNGQ8VmCzl9g7lktLxUKGu/jMu1w3DXlAf7O4+OcOQlfWCGoZxI/C0p6LBwZ60tCozs7ivvqbNm09VViEpq1BVVKJnzcqbM6dBVXXqhqUe3H8J9SyxNNR7FswfWPklR1d3KDgI9iw9M+5pRIAl2mDGH2HGY7oF3q1FZbpLUs9CtJB1fllbSuDTDl6xfNwIKADDuAF5/Z6OjIx0d3cjxQJqQEI4gYsXohcQYBgcHBRMqMHBQZluAWU5y7gFhBNkzUgwnq27u7OzEwW01dXVVVRUsJg9Z80ivj6T+vlhcmFlM4ZhIpFQLBa3traeOHHC2dlZ/s9/cXFxRcUTrhGI3ECwp6WlxcTExMbGZnj4lXYuvn6X9PdyxCJMKowXSkaEmEiMQegAJ4lIPKVapaCgQF9fXw8vGWzw9fWVNSbJZl4tyYAE0OhRvr+IQqFMDRhMTU2dnZ3Rhh4eHrq6uqtWrVJVU1VVVV29erWenp63tzeFQiGRSGQy2c/X38HR0cTU5OqVaz/+eOLA/q17dn+is2/Bj2f+clkfXLsOb7UNjaBi2MwcOpMingFhBiLem+ThCX2T7t2DPIN/ALQnCiQBMhliBhoVeHsBogsICNgZGRkRHR2TmJi4adP2lSv/Ehzyp0i8rQjasEZNNkVLvY8iEE7Au49QAxJEC0GQWKDTYdcTiQzfEUIFXyiZ8PSESIboCg/18BGwaBFBW3ua/mVw8eKbvn52ISGhdDwsD6EFOp0eGxublpZmamqqo6Ojra29dy/UL4w6I0nphR9++GHdunUbN24kkUjPQy8UFRUpAMPv5SdfcRyjI+DR0rQ9Nky+K2lbBMMoO33it1qXqi4rhc38qGjxQ9F469XRnQ2OjIRyuBsHmCrj7I9QqAKTpdz2eEZhkVJs3BirkJun9PDhMj7fE8NeKgTdzW1yhuGXBgyj545JRCJWZWW9h0faVs3g995HnANqWErHw6TbpgyT7lFXQ+BhABc8DKCepa1beMZGI8nJ4p4e2RuNzXQ/xAKOwTYkIzxsAY9c6L72Ud5PS9NwtJB9fllrvPfY+oq5Z42AAjA8a4Re9HV0Ezzx18qL7m/y7SQSCZIWsFgsWSSzDDDI0AKfz0fAgC9XPLwQrpA3RELtTEy8kMS5v7+/By/ELbS0tDQ0NJSXl6OUpZtOCV+dSvzsaKh3cAH+VTQ8dQzDjh8/np+fj2EYajaQSCQtLS19fX0tLS11dXWZmZmDg4NmZmZeXl6IW2hvb2exWGfOnGlvb5/8bBVLf4UREAmxB0VYVT5WVSDmMUWYWCSRCCHPIJI0Vour8kUVWaK+DsQ/PO1wiouLr1y5oqendxEvPT29S5cu3b59++7du76+vhN5gFcCG2Q4QR4qkEcrICBgUobBFU+ANjU1dXFx8fPz27lz51tvvaWurv7ZZ5+dPn0a0Q7I/hXtiUKhOjg4nDp1Slt7yzatz7S03tm5a/qB/YSTp8FP54GeLuHKFXD9Orh1GxjhmMHcHPIMUszgCHkGIhH3WkWSBrw3yd8fticFBELMQCIDTw/g7AT8/DQjIkIjI6Pi4+M2bty2cuVfGEFvhIVDuiA8HLdhjZjkEZIJeOtRWCgICZFOKF6aTodohEIFgWTgT4Laax8fCFo8PaAs29kZSrStbQnWFsDQgHD1Kjh/DhgZfWtsbKCjs9/f3x8FbNPp9JCQkJSUlJycHBqNdvTo0T179kD1wq5dO3D1gqamJlIvfP/996tXrz506FBqaurzKJ4VgOFpP02K5b/hCJTxeTsSop/sSqLsjQh6zOWMO6oiVvH7OR9EcqLGLYfSPnH/8LAnj7eazRmvaZZBhZbWGfkFSjGx08MjIFqIT5ieX6D0qHHpoODFWQX5I3F1/Y0Bg/zB8FtaWmi03MOHQz/8EKU6kJRVGCozE2fNKp07p3nUm/VpDksIPPSpq6GepT51tf6lS1iHDvGJRGF5uZjFxsQirCICs/tGXrQgNJDap6bhuoWsC5+2JfnJH5Vi/pkjoAAMzxyiF18BfT37C2EGtFs+n9/V1SVzTUV3/0i3gNCCHECAZqkykIDIBPmYBfl5GbeA2pB6e3u7u7u7urpQJ1JDQ0NFRUVHRwcaGjPv1BUn4748HX/bORH/Alo6YufOnbt//35wcHBkZKSVlVVwcLCTk1NcXJydnR2RSDQxMUlOTjY2NqbT6TLn9bCwsIsXL8qTEi8++ootX2wEhgTYue8lywnYMiC6ZyqCLWYiEYaJ6yuxze9JlgNs9SxxQcrUn+qmpiYjIyNdXV0EGORhg4mJibu7O5lMflWwQYYTyFOWjGGQ70pycXFxdHS0srKysLDw8PBAgEFHR8fGxoZEItHpdCqVKtsrzi6QqVSqkbHuuvVvfrN2+vcbwFZNsHs3OHQQHD8BfjwNLvwE+/6vXIaY4fZtnGcwwXkGGWbANdAuLsDVDZc0eIF7eHsSdE/CVQ3+AbAfydEReN9bHxpGDw+PjI2N2bhRa+WKv1Bpb8gwQEgIFC5PnGQrBAfjlAJiFWgQKpBxqBBAglnOvr4QLXh5AQ9PKK5wdgIO9sDGBpibE4zvgBs3wUVdcP7cLA93g9u3DbZu3eLn5ycDDPHx8dl4xcfH6+rq7t61aw+uXsC1zppbtmzevHnThg0b1q1bt3r1amNj46KioudkGAoLCx89eiRrTXyxz69iK8UIvMIRGMKwc4V528KoMnNVzWDy9qhgRt14TbMYE9eN1MGOTbkSix8PDdtyuEt5fGUe/wlNM4IKA0zlpuYZuXlSqBAeAaFCYZFSU/PiQYEzhr2yfqGnAYbsF7VVlTvLF58d7u9vj4kt1deP/vQz8uw5FLxhiaYyM2YmbFiqf5bDkox2GOtZmqfB2rhG7H4E9iDJ2pCMFrJuLirRXZp6blnGuWV5F5bn6n/VmRfx4sf9v7qlAjC8gisvgtZAknH24Z2dnb29vYhnmPQ9pr7lmnQT+YVI69zT04OEAQgq8Hg82Yw8VEBNR0+jERCZwJKrgQEoce7v7+/FS74TCaEFaSAr/rvxtnPClycTV55NPX0nTK77HTt79mxJScnZs2e7urpqamp0dXWpVGpKSkpgYGBaWhqSk7q5ueXl5cnOq7y8fP/+/VlZk9hQyNZRzPzSIyCpzpd8oyZZBMRr5ogrciUSiXh4SKK7VbSEIFoIxE5XRTBa64m/i+MOqb293dTUdBxgQLDh0qVL165dMzMzexnYIAMJk5IJ5MkKAQYikejm5ubh4UHEy93dXVtbe/ny5VZWVjJPVUR3PG3PVBrN5M75b9b+efU34NvvwObNYMdOsG8fOHwYnDgBzp4BFy5AzHDtKrhxA9w2AEbGeG+SGbAcxQwODsDJEbi4wBYgaJ2ESxq878E7eF9fOBGJwM4eeHp+GRJMDg0Nj4qK/OEHzRUr/kyhvoG4AgQGUIYDnMenoCA8PRp/ZOCaZjoNaqmpVJy4IEHuIiAAJxZ8IbHg5QndVIlEaOLkYA9sbaFK+44J5EauXAY/ngGXLy+hkH3p9CAaDcqdUVjbrl27zpw5ExISkpeXl5ube+fOnV27dqGktm3btyMr1Y0bN/7www9r167V1NSk0WjPSS+gHIbW1lbZ1wfjPlSKp4oR+E1GAE9we6IrSSuMdjYpbnjKhEGR6IFAcJPD/fdETbMMKjxqmpGdqxQdI2UVEhKnF5fMaGn9SCBwwLDJemxe4vyJT2EYsnNeGSZ5iaPDxENDfTk5D6ytE775hj5fKnUgjTosVeMOS714sMOk3qwIOcDHuapD372LmX0kbUMyWig0XNh8dXHmT8tSf1qedWF5nu7yMmttVn3Ryxzt/+y2CsDwspdeIpE0Nzdra2u7u7vL9iUWi4uKimpra2VLMAxDX/mjJUVFRT/++OOVK1e4XO4LIwc2m93X14d0C/LsAZfLHYcWuFyufKMRnr029tDf348kEEjfjOgF5InUjZesE+nhw4eVlZUtLS24EZIQPxfJj6YRK8+kfH0+U+cKY0Q4jGESgUCQmJh47ty5hoaGmzdvRkZGFhQU+Pj4+Pr6RkVFeXh4REdH+/n5eXl5OTk5RUZGjoyMSCSSzs5OgUBAoVCKihQ/zPIfnF97XoRhEl8L8bJpko+A6ORq8SBbQraXfPKGeCGQHPyvuL9bhElEUMvw1Oro6DAzM7tw4YI8wyCbR9qGq1evmpmZubm5/Sy2AYmYyT+z0N2/j48PorZOnTplZWWFkhkMDAz09PRsbW39/f1JeD0NKqD3pNDo3t7Evdr/+no1Yc1asOEHsG0b2LsXHDgAjhwBJ0+Bc+cgZrh0CVy9MooZkJ4BxwxWuNeqgz3EDMg6yc0dwgYPPNnt3j14K+/iBP1Y3dy+YDD8g4NDw8PDNm3a9sUXfw4IfINGg3plNEGbIzzGAcEDBk2qZoaCZpxPIFOkWgXIKgRCqOCLoIIXLloYRQt2OFqwtAQmJjB44do1yJOcPPlnW9vTdLpM7gwBA5lM3rdv31tvvfXuu+8eOHCARCI5OTlpa2ujZiQtLWkz0g8//LBnz54NG77X19fPz89/fnqhoKCgQ9GO+NSfKsULv80I1AuGdiZEaQWTx0iGEPL2yKDc7skDEITCPP7gCTbnzYmaZgQV+geUHz6akZmlFBUthQqJSUqlZTPa2pYMDTn8LLPU5x+Rp2kYcnJ+EZek5z+wiWuyKysbvLzStm4NfvfvSCQdqKxCV5kZ/yyHpV41td65aj1qqoIN74iNFjENPm6+/HHF+cU5J5fkHfm46ODHteYH+sqSRJho4psqljzPCCgAw/OM0lTroO/DTp8+XVVVJRQKU1JS0tPTRSLR/fv3e3t7a2pqysrKYmJiuFzutWvXfH19hXhVVlb+nzDgzJkziYmJU+396a8NDQ319PSgZiQZq4AYBlkzEkIRCC2w2WwZREDKBMQhyFQKCDOgdRC3AO1Tu7qQyrm5uRmhhcbGRnjKEglXMFj9sH1oaGi7Hmn1Txlf/5S59zJNMAyjaoaGhqysrMLCwsRicVdXl5OTE4PB4HA4Tk5Obm5ujo6Onp6eLi4ubm5u2dnZfn5+w8PDEokkPDycSCQWFxcrvmJ8+mX/NV6RYJiov0u08yPJYoJkyTSRm4Fk10LJIoL4E4IkLkAiEeO0+1QMQ39/v52d3dMAA0IOurq6enp6CDZ4eHjI4g7QF/yyR0QmvBhOIJNhBxHqffL29r5w4cKKFSvmzp07f/7869evubm5uri4uLu7E4lEOzs7Pz8/MplMQo1HcjPkcUWhhoaHGxsdX7P2z6vWgPXrwaZNYPsOsFcbYoajx8CpU+DsWYgZ9C/BmGTIM9wGhoaQZzA1A+YWUAZtg9utQqrBSapqcHWD3/d7esLJwQFY2QBX16/pNH8GPTgsLFRTa9ey5X/y8Z2GRA4kXCFNoQA0UUdnpK/iguZA6IAEZc3+/rDZSQYVPDyggsLVFVIcjo6QyrC2gSoLE1OIFq5fJ+hdBCdPgBvXV4SGUkJCwmVCZxqNFhISkpqaGhIScuHChYULF86dO/fTTz9ds2aNlpbmjh07NDW3btq06Ycffti2bdvhw4c3bdp079690tLSouerwsLC4uLi/pc3XP81fkQU7/E/NAIiDLtWVrItnC4FDEEkzSDStgiGQV7WuF+CQmEWn6/DYs+dqGkewJ1S+/qVGxpmZGTCqObwiOmRUdOTU6aXlSm1d3w+NOz2ChuQJl4eL+/Jk55zc393gEF28LyGhsaAgPTtO4L//h4N71byV1ahqMyMmjkzZ9bs2jlzH89R7Zmr2j0Hn9TU+v71LmvZB8zvljbvW1Z1dvHDr//etujN9n8v6PxgfvfbGr3z5g0s/4Sjs49vbjacnCRubYW2f1OWuLNzJD9vJC9PAttyFYUpAMPLfggkeJ0/f762tvbevXsZGRlWVlZBQUGWlpbp6emWlpYBAQGGhoZpaWl37txJSkoS44Xe1d3dvbGx8QWOQCKRoDhnWS7bOJCAGAYOh9PX1/d/SmJkcIQajRB1gMQJaL63t3cceECiBVknUmtr66NHj6qqqhoaGlCHsQQT8QX8UwYMq3upX5+grruYu+pc2r5rDLEE2snJ7vifkzyRHxMM6qaf8WP8AiOm2OT5RwAa42KYJD1C/JWyeBmQrPir+NM3xIuAxOAIJprCLnDsHZhMpqOj46QtSTKeAc0gYfTVq1fNzc09PT3RzT1CCwgkUPAi//xCCMTX1/fWrVubNm168803NTQ0lixZcuDAARMTE5TJ4IJHuTk6Otra2vr5+U3NLcgOISgohEYLOHb8i9VrCN98A777DmzdCnbuAtraUMxw7Bj0JD13DujqQsxwFe9NuoXrGaReqzhmsLYBd20hNnDEqQaoasA7lNzcga0dvIN3cf2OQg2k0aDIePOWHUsW/9nbexrCAAGBEAygKZAE/Y5kTwMCcJAQAPzw7iOkVYANSLgVkps7FEgQiVLRgp0dRAvmFpBbMDQEN66DS5cIZ34Ep07NDo8g3q9+kJefl52dnZOTk5qaGhERERsbm5GRkZOTk5eXFxcXZ2ZmtmrVKmVl5blz5y5duvT777/ftAk2Ix09evTQoUM3btzIzs4uLi4uLCx8HshQWFhYXl7O4YzXko59qhRzihH4jUYgurcH90oK1IwJ21ZWohUdqhlM2hEZVDmA7rYlQmEmD0KFOYOC8UIFFKrQ26dcVz8jLX0MKqSmKRWXzOju/m5ESMGwX/xj7+fXMmkOQ37+C/q0/pqXgvfwYbOff8bmreEf/JumvoA2f0H4P9+J/fDNvI80Wle9w9/78dCRRSMn/im58cmA8Yoy/eVpep9Uf/OPvllze+eq9qiq9qiq9ahBk6V+dTU2HinNVFfr+/A/zK1beEZGwwkJoubmceBBwuHwDG4zly0dgFupc0+flii+y8AUgOEVferPnz9fXV29f/9+LpdbWlp648YNLy+vnJwcHx+fkpKSgICAkJAQJycnZCqK7qcbGxsLCgrkb6+f51jQzTSTyUT0AofDQTrmSR+5XC6LxRoYGOjp6bl//35TUxPyO+ru7kauR7JHpFXo7e1FS5DEGXUitba2NjY2VldX19bWysuRJZKR8yZR/9Zy+urHyG/187/8MUHPMlpe9Pw8p6NY5/c2ApBekIgkmAjmAdrqipdOg5hhMRBv/7e4uWZqN1XZuXA4HBcXl+cBDPJsw61btxwcHPz9/V8YJJBHi0KhuLu779mz55///OfcuXMXLlyIcIKLi4uHh4ebmxtxtFxdXZ2cnBwdHQMCAka3nup/EolEoVITEpPv+dhu3qr+9Sqwdi3YgKufd+0C+3TAocM4ZjiNY4YL4NJFyDNA36RbMArN+A68OzfDrZNQe5K9nRQ2ODnBW3kiEd7Em5kBF5dNZBKJQqEwGEGHDp349ttZnp4EX5wr8PXD/Vj9YZeRbIIvoVdxQfM93AHJywvGS3t4QIG1mxtwwd2QHB1hNIStDYxzNsd1CwYGEC1c1gfnzoIjRwgODgcqK8tqa+tqah7U4nX//v3S0tKsrKwMvDIzM3NycgoKCtLS0o4ePfruu+8qKyuvWLFiw4YNu3fv/umnn/T09EJDQ59TvYDgBPJUVVgqY4r6/Y1An1h8OCMFdiVFhmyvb9iWm7WVEbg5LNS0qEAoyuTxdrPYs6eACjW1M1LTlCIip4dHTo+Knp6eoVRWPqe3b5tQGINNldjwKgeCRG6dFDAUFLwGgEE6EKx2QaQr21iTe2mZ2OhzzHQ5zGKD00LMZKHAaFH9lSVZPy1LO7cs/aflzR+9BdECbsY66eMYeNBQ7/vPv5mbN3FvXB+KjRU1N4uZTPaxY3wN9YFRL1euhjp7yxbR48ev8pK8hvtSMAwve9G4XC6NRtPT02tvb79+/XpsbGxSUlJISMjdu3djY2Pt7e2Tk5PR7cjdu3djYmKQQrqystLPz6+kpKS2tvbnfqHOZDI7OjqYTKYsyFkeLaD2JPSIdAsoSKEdL0Q1dOGFUEFXV1c3XuOgQkdHB8pbQGjhwYMHQ0NDcoMF7fiNnFO+OB617lL2Ov28z4+Gu9Ny5VZQzL6uIwChAvRRxSQPq8Wr54iXAtEiIPIwlhFH+IlJJBjqTRo9TTlaSCgU+vj4TN2SJE81oJQ3PT09fX19W1vbwMBA8tOLJNc1hOZlS8hkMtqWSqXa2dl9+umnW7ZsMTY29vX1ZTAY/v7+rq6uyCXJ1dV1FDIQEYqQ38nT3xy+QqPR4hLjE5MSr9/Y+83aP61eDdatg2IGLS3omLRvHzh0CGKGUzhmQBroK1egNuDmTdj2Y2QMjE2k7UlW1hAe2NiCu3bA3h6yDY6OUHxsYgIcnX4ICPANDITH5XPPj+i23suT4OUNYxO8vaHUwQcXPCDZg+zR+x7wvgftj1C0AsQJqAFJBhUcgM1dYGkNSQwzcwhgbt+GeEb/EvjpJ3DsKNDV/U96elR9fUONtB7U1NTU1tZWVFTk5ORk4pWVlZWZmZmdnV1XV/d/bYSbN29av349YhhOnz6tra195cqVgoKC5yEWZOsUFBQ8evToyc8Y9vsulHCIjnEiKwpFPvjPhGzm9302iqObcgTutbZsj8Glz5EhG8PCNUOo13IsMtrWM5kzBwcnZxV6epUf1MxITZ0eETU9ImJ6dMz0jEylyqp5/QPHxJIxq48p3/aVvUinP54UMBQW/r4BA7Mda8jCkh0w30OYzdcQHpjgIMF4IZQ149PQ7UXNVz7OxdMVMs4tyzi/LOvM0uZ/LuhXVZ3amFUGJPpxh1a2hjpLQ71v4Yf9K/7bNwFpcDTUWd99K3yhlpBXdhV/6x0pAMPLXoHh4WEnJ6e0tDQMw3p6ery8vKKjowcHB5Ga09XVlUKhuLq6ovSi0NBQkUgkFov9/Pxu3bplYGBQVVUlgX9rxv+9wf/SyN2CjR4mi8VqbGxEoc7jAAMCCbLwNQ6Hw2KxmEwmCmpAXUYIKnSOVkdHB5pFsAEpFpBo4fHjx4hbuI8Xn8+Hf//GDhMem4Nf+mfHo77Tz1l/Kfe/hyiltTC4bZKDHj14xf+//xGQQJJIJMK7ksT3zCSf/km8DIgXA9GRFZJeqZEufpUlYvhPLJLAnD58/bErPzIy8rMAgww86OnpmZmZTQ0YyKMFv+ynUGD6MgnmqVEolM1bthw/fhx1FgUGBvr4+NDwkhc9y3CCbMbFxcXT0xPtZHTfU/1Po9HjE+IzMzOjY0KOn/h81WrCmm+gmOGHjVAAvXuP1DQJYoZRDfTFi+DyZYgZkHWSoREwNoaoAKU0WFrBL/ttbSFmsLeH9ILxnWnOztp+fv7+/v4BAYGkQIqPzwZ3D4KHB6QLPDwhb+DlOX6CIAGPYHP3gHyCqyvkK1xcoFLC0RHyGHfvQnBiaQUxiakphC63b8OkOf3LOFo4Bo4cmxEUZFtXK0MLNQ8eQMDw4MGDwsJCGVrIwgFDcXExj8ej0WibN2/etGnjhg0bdHS0L1++vGXLlt27d6NOpOLi4vLy8rKyMgQMpm5PknqvYa9NQXsA+PMCPSCEeAI6OnRoLIZhQnyCsxK0YOwH5LU5Q8WBjo7Ao+HhXcnxmxh0zRDKjaw76S1r+lkzuey/Mlnjo5pZbOWeXuX792ckp0jz16JjpmdlKVXff4fJ0pNIqkZ3Ofq/WDTh7//oS6/u/5DQ9skAQ2xpGevVvcmr2NPwINZVh1XHYYm2ECTYrobuqAgkGI2lNWNGCyWGC9k3PmrQX4ygQjpumZp1fnme3iep2gvDVGcnzoSpDo2qqt3qan349AyHJRwk9KurMSegBQQtIGZYs1pY94SZzas459dmHwrA8Du4VBL490YM/6bInGckMExXIhy7P8cPs6+vr6ampru7G/kaobA2WVeSLFqBhReCCki+jFMIMEihAy/ENqBHtAQhh/b29sePH7e1tbW2tra0tDQ2Nt6/f7+qqorH440bJvwPopgSWfjZ0fD1+vnrLuV+fpBMiiqCiljFn8Vxg/VaPcVbkuDtjrgyG5qrLgXiJQBihg+BxPGG7AtgSC5IJCIM+k3AeyZ44WWfXmxkZMTPz+/5GYafCxgoFAry+vTw8LC2tkbdRFQq9ezZs9evX0fgASmeyXKFXJJkOEE287MAA4lEotFoyUnJubnZxSVlZIqLptb8VasBxAzfgo2bcMyA8wwHD45poM//hEsa9KXtSTdvQarBEO9QMjWF3/TLxNBWVhBLGBn92cX5jI9PgI/PPV9f34AAso/PRiKR4EqEMMDVFeKBcRNaThwFCc7OUKiAcIK9HYQK1jYQKphbQHLD2AgewM2bEMNcugQj506cBNo6wMJid0V5WV1dnZRdwP9D9EJ2dnZGRoYMM2RnZ6MklqCgoI0bIVrQ0tI8d+4ckUhMTExMTU1F8QsZGRl37tyh0+klJSXl5eVPkzQUFhaWlpa+ZgKGhkpReZaoMkvc1ynGMPg90Oi3JRKxUFRbIq7IFpVkSthM+LMy+tJr9ctAcbBPjMCd6uobRXfTW1YPIKjAnDHAHEMLSKvQ1a1cVT0jKVkKFWJip2fnKD2oeZ/DuS7BGp7YHYZhI4NYaQjmvQeLuD3+pVf9PCq6awJgiFPXiK2seqkA6VdwmCIh1t+CPcrD8vwxui5G3IKZfQoRgpRJkAMJhjilYLhw6Paijqsfl+suyTi3LO1YwiTJAAAgAElEQVTssvRzyzLPw3SFvAufFN1a3xJh+/CeG01NnaysQlZWoarMjJg5K2f27DrVuR3qkHN4TtpBxj/Iz7A11JlfrhBWVPx/9r4DrIlsfT+693fv/7qrW9zmlrt7t+ju2nV3XXet2LAD9i5YsFCCgPQiRVFEkSpdlN57b6EEAknoJaGF3ktoIWVm/vfMgTE0Bbv38j3zJCeTmXPOnJlMzjvv937fCzjwt7CKacDwZpw0MPUCE20EEXT39lU3tvMG+eI9EwqFtbW1eXl59fX1bW1tMLQRDJFE5HWGaRYInCBOLEAaQRwP1OJWV1dXL2YEVOBwOFVVVSUlJbm5ud3d4z6BAKxITAbr5/0Ov8n6/ynrs/rkw41y9pm5leLdni6/jSMAAAC3A7m4FVsyA10xE7uphEr9KFpIQta8j2XEwIdhKIYJhQJRViLqb4swKaiAP2pKFBYWBgXNBBiYTOHJDAPECX5+fq6urvr6+rt37/7qq6+WLl0KgyxBhODj4+M5gT0/YPDAzcfHJyWFwmTmZufkhISEqpD3bdg4c/160gaJIcywV5p04ACYf588RZKVA3PxCxeHZNCAarhK0tAEmgGoaoCJGoxxJyXojKSjTdI3eNfqnoqzk6uzs5Orq5uBgYnCxcV37s68ZwXoAhBbyYZkY/14sbbGV1qBaK1W90iW94bIits4pXDzFg4VroNQSNeMSfqGIDWbtvbfNDT+RlYBGZ3lz5GOHScZGmzLplHKAFooKS0tIzDDKHohNTWVQqEUFBRARZOvr6+kpOSWLVuUlZUfPnyYlpbGZDKheoHBYMTExPzxxx9ffPGFlJTU/fv3qVRqXl7eWG0DTNkmFArFOMw3/aeD2umhq99DfnsHVduPDA4AwIw/KwEv8f6i9R8iv/8dPbgIra8EN3cxOP2mH9h0/8YbAZEos7HvaGf/p73d/+iaACrkF7wXFw+gQkjorMjoWRnUd1ms+X1911AUcO8jrIODpbtg9tKD+ou69ZYghguxIE2s7yUGLIqJaRkXMJSW9o7o2Cv4IBzE2ioxNgVLd8b8rmB20titNcC5yGQYJBj+ghmMXPA1Av2F7VqLylSXZuLeR0CroLCcCnDCiqyra4vsLrVkBPA7h2hwbmlZhYMD5eChwB9+hMmkvWbPCZzzftKHHxbM/ajuk4/bnhU5dH/2adfSpXwK5RUM1ZvWxDRgeCPOiEAkZJTU3XtEOaPvt/W8y5/H7sekMVuAiKABagng3L2mpgbqlWFQo66uLgIzQHqhuxtInKF2GbIKBFRoaGiAIIGDW3V1NYfDqampIZBDXV1dTU1NdXV1VVVVRUVFaWkpk8lsb39CVheE09Bu750eEl+Qyayq4LS2dnL7Bvrfor/8N+Lcv3mdAPTCQzN06UxsIQk5vgrh89AHN9El76ALSeipP5GeLsBCYJjI1QT585/IMhL212zkoTlwyhCzpKQkVVVVAjPAgoqKijJuE4GHcQGDh4cHjI7q7u5uZmZ26NChBQsWzJ0797vvvzt48KCpqekk9coTAQZbW1srKytHR8enRkmCgOE/U+SsrCwGnRESEuzs5GJtbSEn9+fGje+s30gCQZM2k7bvJO2VwjHDEdJJPD/D2bN46KTLwPNH5Qp4qH8V91DSwYOu6huAnNCG18CE3tAAJH3TM/jM2trQ4b6Tg4ODq6vbqVNn//prjpkZDgNwtYOlJci2Nmq5e4d0xwIst28DPgFSCjfMgOOTiQmI62poCFCKnt4/rG3WeXtfdXW5clVzwdmzMw4fJsnJfR0X58NmV+L+R8ANibC8vDxxegEGSmpra4Nn29fXd/PmzVu2brG1tYWhUQlZQg6e4zk+Pt7ExGT16tWfffbZ6tWrTUxM4uPj8/LyRkVcfeKtRuzCemOKaHsDevw35JcZyJ/vopmxgCLGYQHKG0AvbUF/mYGunImGuQ1BhWne9Y05cVPtiFBEH+DJcXs+6e+b1dX17lhWobFpdm7eu7FxQ6xCdMwsaua75RUL+/tvYthImWxPK1YQgfkqY7f+wkwW9uktzlBcnnh5ebPmYuzGIizZZqp9m/z2CYltYwHDZ59HsctHuw9Mvs5JbSkSYJ11WHU24FJib2GPzmHWuzCzP4Cjkcmi8WkEAi3gOEGkv7BTaxFbdQlNaVkKTikkKyxPV1pBI6+gKS8vu3e6Lc1nsK12os4M1Nc3xcZmKypG/rH6Ic45eMye44dndWB+9BHnEyCMnirn0PXZp53zfxwMC5uo0f/W9dOA4TWcWfyfZWheLUKFCRllp7S9Vx62WbzPZukhh5VHnRfLWLv6J7KKi3JyGNAJOD8/v6qqihArw0ConZ2dBGbgcrlduLW1tY1yPSK8jDgcDsQDlZWVFRUVVbhxxKy6urqioqK8HHgwMxiMpqbxE9O8hiGbbvIVjoCoho3s+RFZREJWv4ckBQN3o45mRHadaMkMdBlJ5G2NoKiovQnZ+S0qtQClhGK7f0B2/BvpaBHvY0JCgjhguHLliqqqqqKi4rFjx06dOjWRt9JYwACFCjY2NmfOnFm2bNkHH3zw5Zdfbtu2TVtb283NzRc3z8nZEwDDrVu3rK2tnwoYPD09PTw8fH19ExMTg4ODHYA5Oju72NrcPX36t3UbZmzYgPsm4XqGvXtJ+/eDWKsnToA80FDScOkySVFpKBu0mhrpqgYQHGvrgPzKMC20ri5QO+jp/mBtZWZnZ29nZ2dv7+hw3+mu5ZZbN2fcugVyOIDl9nBB7OOtW6SbN8ECQMKNIe8j42GooK9H0tEmGV371NPjQi4zs6CwmJqZ4+Zmdu7cF0eOzHnoblJSWlZaNkQsQOlCaWlpUVERlUqFzkjE638SMhKJ7T09PSUkJNTV1ZOTk8dSBzk5OUwmMy8vj0aj3b9/f8+ePZ9++un8+fMvX74cFBSUlZWVk5NDo9GKiop4PJDC5e163IBkxqGr/oH8QkIUJbGeLuCYhGGisAfIb39HfyYheifQkSha/AcyXX7zR0Akyh/gXeT2fDo2rwJ0QGpsms3MfTcmdgRUqKpewePdG5F/rasBK47FQnQwi41Dst1rC4X6v+QoL0u4vDzx0vIq9cXg+Xr8nZc3Junp7WMBw9f/iqmqAurEF2YD3VgzG8iU6X5YrDnmeRGz2YXdXIMZDMODITXCSAKBQAiwgOOEQb2FbVqLWGpLspWXURSWJ10G3kdpOE6gk5dV3j7QFXVXUFuATfonJuRyO+j0gmvX4rds9fhoLqQdfOa8H/HBBzkffVSFcw6TETlADyUQbvVfX/MeuL2woXsbKpoGDK/hLBF/ipz61iu3wpfst15yyOG3466rjrv8ji/L9ts5esdXsMuKh5/0lZeXQ7ahqampBQ+KCh2TOnHr6uoiciw0NzdDMqFu2KAggcPhQJxQXl7Owq0CNwgb4GtlZSV0X2YwGNXV1UQ/X8MYTTf5mkYA5Q+i2sfQX2YiP5OQW4pAzwyiIWEILUG04QPkF5Jo65ciVi4S7wNSuVlpgWBKN5WQJSQ0IUBcupeZmamhoQGJhStXrsCCioqKoqIimUzW0tLS0NAYyzMQgAGqmeEE3c/PT0lJ6fPPP1+7du2VK1fs7e0JKbOHB5A7T9ImAgx2dnbm5ub29vaTAQywLQ8PD2dnZwAXHJ0cHJ1cXFzv3jE5dnzJ+g0z1kPMsAXETdq9h7RvH8gDfew4CJ0kJ0c6dw64J13GqQYymXRFFQQz1cBjKGlrk3R0Qd6GKyokfb0VVlZ3rW2sbW1tbGzt7Gxtbt9ec910BhBJ3wB4YMRyA195AzAJpqZgAXyC8RCloG8AlM06uiQtDZKR0Q+BQRbZ2dmFhSWFRUW5eQUplBRbG333hzdKSotYII7qCGOxWMXFxURwpFTc6HR6b+9jN4b4+HgLC4ukpKRRjIE4zwDL0BkpICDg/Pnz33///aJFiwIDAyHGaGkZgTaxt8EAEYehmJU6unQmspiEeN9FUUTUWo/snQ+4OKkFSPVkwxC/DYf7X9hHkahgkG/A41kIRfkoCvAqYSJRYf/AZW7PPN7g7G7uY6FCZxcIiNTNnT0KKkRF/oOa8c+q6l95vPsY1jVUj0iAFcdhPgqY+ToCJww52xj+0qq1iKq0jK22pObqkn7dhZjpIizHl+jACy8wGF2ffhbz6WdRYkvMj/Pj6uqeCTCIhFhPC9ZYDDyL6AFY4j3MXw1zPIjd3Yxd/x33L8LZA0KHMNbLaDyQgBr80quzsEFjUeGVpVScTwA4QWFFhtKKHJUVeSrLqm/s7A68JizPwAYf34KmOlYon99dWlJmeS9JWtpn3hcQOXjPmRP3wYeVeGAlQufwZPzQ8eknnfM+779jgccUnGov3srtpwHDqzttw/NvGIwPic8o3Sh7f5GM7e9HAVT47bjTb8dclh9xXHbQYcFu6zuu0RxOJQt/2F9eXl5XV9eIGwEYWltbIWYQd0+CkZFgsgUIG8RZBQgVSktLi4uLS0pAhHU2mw1hQzluLBarpKSEyWSWlJQQTxBf3QBNt/TiRgBcbGB5LER+et0oBuZA1BhEeoFoxzeo3FqstQHuhcIoq/f1sO3folvnIdYaqPtN7McZqJMxuJrvqWM/kTCPEffN3NxcbW1tIhUDBAyGhoZWVlZ3cbtx44aamhrEDASuuHLliqmpKcyiICMjffPmTTiJd3Jysra2ho5Jo7I0TB4zPAEw3Lp1y8rK6qmAgWgLKiWcnV0cHR0dHB0cHR2dnF0sbhsdPbZ4/XocM0iQNm4ibZMk7dpN2icDwq0eOUw6fnwoS8O58yDo6uVLQG1MJpNUVEiquB766lXwUUmJpK+/2fKu5d279ywtLe/ds75nZXb9xvxrRjOu4TDA2BhEWBq1GBlBtTTwOzLANc0AJwCtAvBxAmyG+iy7++SY6Pj09AwGg55fUFBYWMhgMFJSKJlUWgnOL0C4AOmFsrIyeMfIyMig4AZDqRLEI4IgXV1dRUVFdNyeHAGJCJqUm5ubl5cXExNjZWVFoVBycnJKS0urq6tbW1uHLra3JGwCHh8JQ1rrkNOrsQUkVOontJYtstZEls7EVs5Eoh7hESCmXZGefuN5LVsIBFHcnm/4gtm8wTld3Z93c5f29p0XCtOFwrj+Aflu7heTgwr/jIr8e3raPyhFP/tX6AkxMb0fjwuer8MwoNdGKndx8W7N1cWd2ouGYoNCOe/9fViQFhZ/F8vywAqjgA64mYVxmzHhIJQ4Ps9AFRUPfDIv7oOP49//OO79j+PmfBw/e27S/EXpXcPoZnTliAjj94PWm1lYVRZWFIPRPEF402BtzF0Os94J4heZ/jqEDcSdi2C001F4YNyPOIpADX7p113YormIrbaETl6WrgiYhCQgZV6RSV7BUFlRqLqi8e6BvnAzpDwD63/BQWD7OJxKV9fUY8f8vv232+w5Ye9/UP3Jx5kffVQwd27Vxx834uneoNqhA/dcGuW81I6HVOrT00VHik4xDOMGBLTr6vbGxCDc1y0rH31qn/3zNGB49rGb6p4obhiGDfL7HoVm/HHQatnh+6tOuP5+zGXFEZfFB+7/dthm10U3eUNvzbtBkfHZlRXlZSwwp6+pqamvrycAQ/Nw5gQIGLq7uwcGBgZx4/F4AwMDvb29XC63vb29ubm5vr6+urqajVsZ7m9QWFiYn59fWFhYXFxcWlrKYrHYbDaLxSorK4NKCSaTOTAw8Na5B0z1dPyPbD8oFLZ391RwajNo2ZHRURFhIYH+vg+cnR46Owb4eMfExlOo2YUlrKb2dp4AFyEIBzFeD9rPRfmPH+HgYbwQkYiH9najvd0gfpe7GfYDCXU0AteJpSr2MwnzvCv+oCUvL08cMEBgYGVlFRAQ4Ovr++DBAxcXFyMjIxUVFVVVVTU1NXV1dUVFxUuXLhGAQVJS0sjIyNvb2xMPmQoLns9hTwAMN2/evHPnzuTr9vX1DQ4O9vf3f/TokYuLC3BNcnJwcna5fdsY5xneATzDBlzSsJW0cxeQNOzbRzp0CGR2O3lyyEPp/DnSRRw2KCoAJyUVFZIKGaROU1D4h9G1U7dvW1pYWNy5c8fS0lKZrCAt/aGWFkj9pq+PLwYgyBJc4Bo9fcAk6OoOgQQtLUBWXFUHDMaVKyQlZZKq6sq7lrd9fHxiYmKoVCqTySzAMQOdTk9OTk5PTy8qKhIHDCwWq6qqisViQZckiBlKSkqg1nlwcLCurm5cH6ScSRiTyYSxVul0enNzs6Kiop6eHhGD6634bQJXPVy4g6SEoOs+QJaSUHkJZO0c4MundxwVADWX6C0BP2/FgL/ATiJISU/vD339Q9RBN3d2b9/s/oHZnV1zO7tmD/Bmc3vGsArdsxsaZzNy340eckD6Z3Tk/6Wnz0rI/cUi89LRMJe9MfHpPeB/c8i4TdgdCeBoNN5cGdUHs+TRX8H4oSaLgHO/0ULMcBF2/Tfs1lrMcivmcBDAjxAdLM4CS3PCsr2xvBCsNB5jp4LZfA0Dqy/AmkqxFvbQ0l6NdXCw1gow3QcrWfXZ9P2rnXYvs5NebnXo17snV5mf/8vMSObGQIonRnXFkm1BGNMII4AHfBQx1xOYzW7szibQ+vXfsWuLQX9M8Y5B0sBoIRAhTB4bwEEY3l6k/0svDhIq1JYwlIdAQjJOJqTjZAJTZRlLd22b07nBRDusLh/jT0Fo8Ww/uv76hoaExK7ExLSlSx/MnuM+e44XLnUIe//9hA8/oH74Ye7cuSUfz634+OO6Tz5u/vSTluGcbh2fftL32af9igqoeJZ6BOFISlZ9803lr7+yFyyoP3q0y82NX1kpTsIPXyhv0/s0YHjVZ6u7s93dP3b5gXsrD7utOuH261HnxfttNp91MLYKj0rMKigqqaqoqK5iV1WUs8vYLFZZdXU19C0CCmjcGhsboZihs7NzYGBAIBCIRCIhbnw+n8fj9fX19fb2dnV1QT0DxAwsFquoqKigoICBW15eXkFBQVFREXyaWIxbfn4+jUaD6sNhPuRVj890e88/AkIR0tDaTsvJcb5vrX/plNreP69tn2+9Za6bxDs+EiRvCVLQJlKMJMl3I8l10z/vb/vIfPvXOlK/6Zw/Yn3rWkxsTHVDM08kEu+GCEMREUi2IALuSQhw1HYzxb4noc6mADDcU8d+IWFeVuKAobKyUl9ff5RWwcHBIT4+PjExMSwsLDAw0MHBQUNDQ1lZWU5OTkJC4suvvvr999+JPAzE43zPF2RPAAw3bty4desW1EtMpjUvL6+goKBQ3Hx9fd3cXJ0cnR0cHJ2cnO/duyV75i+JTf8HY61uxFM0SEqSdu0iyUiTDuzHqYZjADbI4sIGyDZcvExSVAScw7lzpMsKn5gYq940N79565a5ufndu3f27JGe9+WMKyp4dCMdAAl0tPEFL2hrgZhLWlogRqqGBnBwUlcHuuorVwACUVbG4yCdf+e07FYtTV07Oxt/f//4+PjMzMzc3NwC3Gg0Wnx8fEpKSmFhIcQMpaWlFRUVHA6HAAwpKSlZWVnd3d2Dg4NNTU1FRUU5OTlPZhWeChxoNFpubm5/fz+8/8B7Tn9/P5vNhs8ssDfYUDz7CB7eDkVMLiBLZqJL8ZzoW+chVUUgOQOIiz0Viu8NPtj/pq6hKL+3bz9vcAQkgFLmbu44Kdi6IFRgvBsdA7QKYeGzYmP/X0rqe2G5q4zS1A/F+e2k5+8tKJROjiOXFA+IU0pNZYjjIcxoEQAGT/XJGYsr4PT6Gj41h0yFMa4EMFkE5u7GcL6+EMzmjZeBtMemv4LJPVxurALC4hurhj6a/oaarBzUXcrXXSLUW4ToL8QMf8auLcCM52Omw3VCoELEM4WQ4BlQgfiBDCME1OAXnh6IcVR7dXHJlaU05WVpOJOARzpaka60IltlJU15eYH67zV3DnJDTEQFURi3cfL6BHh9ckUim44OzVx6c2/PM1+x6WfO+sx5/yGOGdxnz3kwe47b8AI10z5z3g+Y837Y++/HvP9B0gcf0j76qGjuR9WffNyhqYEN/28KGhtL587t9vYWcbl9GRmtBga1W7ZwJCXfdrZhGjA883U1hR3hHyGKYU3NjRHRyVvP2P56yOWP427LDzn8dthW83ZAZnZ+VSXQFpSWFBeXFBaD15KS4pJydnk1bhwOZ1iSUAfFDO3t7TweTygUwtTRIEAHgohEIj6fPzAw0NfXR0RMampqqqurq6ioKCwspNPpNBotKysLBj7Py8srxK2goCA/Pz8rK6uiomIaKkzh1L7WTYHLEfB7AFpR2BFu70B8Yoqptore3hVmf/0zZAup4gCp/yRJKEvCzpIweRJ2YeQiT8LOkRA5Eu8UqeEwibKDZL32Hf1dP18jnw0KCmpu7xj++0MQVCRCBaA52FKMF7pkJmIgBz5qHcF+nYEVUIluYBhWU1NjYGBAJpMJdyMymWxnZ5eZmclgMLKysmJiYmxtbXfv3v3jjz/OmjXrww8/XLZs2aFDh/T19V1dXUf5HXm+CHsCYLh+/bqJicmjJ2aYHtWFwMBACBhCQkKCgoK8vbweuj9ycXF2dnaxsblLJkvt2vPxuvWkIapBAkRP2rmdtGcvSQZ6KB0B+d0AbJAF6RrOniOdv0A6ewZ8JJN/MzUxug7M9ObNmw8fuh8+cnLevJmXLwMYoKYG8ICGOg4McGxwFX9VUwXZmq9cARII6Nd0WQGIJS5eAHFdT58i7d274NixEwb6hg6ODoGBgYmJiVlZWfAOkJ+fn5mZGRUVlZycDJ0VIb0gDhjS0tLYbHZDQwMBFSBayMbtqdhAfAMCZmRnZ5eXl4uG/2XhlZWVlfXjjz9KSUkFBQXBfJFw/Zt3X0IRPNk5cOQrL0D2/SRcPgNZMgN1NQPIesiJBNAM+GYIionwH+s0hIDn87W98vlB3J7RwEA89hEsD2kVGmfTGe9GRwOoEB4xKy5hVirlfVqFlF6V274o/x0BAVJFJTKcWhlOjUxZ6b6crICRLj5IT1uF3fkqjeVtWouE+s8EG8Tn36PKcDo+/utCzHCYAXi8wc8AJ8DF4OfR/Maoyqf6kWjFAKRUE+ov7NUBjkacq0tKVJfQlZdlKC6H2mU8bcKKDGUAEtIVlzOu/lVuebw1+MYgIxRtq8IEz6SpwLAWoVC7selAQ+P+lDjv4mfPk1Bget179pyH4y3uwytBeofZc7w/+DD6u++ydu8p1tRs8fUV1HCIf0BuQEDp3LmCurrHVzmCCIe9Lh+vfNtK04DhVZwx+FfX0tacRkk5r/NwyT773487LT1wf+v5+/4RaWUlRSUlRdBTqGDY8vPzi4uLy8vLKyoqKisrq6qqCORQU1PT2tpKcAsInvMNz+GAAIpcKBwcHBwYGCD0DC0tLfX19VVVVWVlZQwGIy0tDfoiE5ghDzf4kc8fkf/hVYzOdBvPPgIAJ8I5fVVNjYOtzdUDG+5ufI8hReo5SULPkzD5GWA5T5rcMgO7QELlSQOnSOx9JM+t/9DavfTuDYP8oiKhSISjEqBwhp1F+rjIiT+QP99DTc4jq94Tya1F+0Y81OFwOAYGBgTDoKKiQiaTr1+/Di8/KyurHTt2zJs376OPPvrpp59279594cIFVdzU1NRsbW2fKifwnLo9ATCYmJjo6uq6u7tPvt3g4OCwsDCIGUJDQ0NwA05KHo8euLs7OztfN7169OiyTZv+3/p1JJClYSNJYhNp2zaQ3G3vXpLMPtLBAyCG0rFjIIzSqdOkU7IgBuvJkyQVlS1GRsbGJiZGRkY3btxIS0tTVVX/4fvPbt7aZmj0o5rabDL5b0pKwMtISRlgAzJOIygrA/GDogJwarp0kXThAiArIAI5cYJ06DBp5+7/k9y2RFb2tLGxsbOzc0hISHJycnZ2dl5eXlFRUX5+fnp6ekxMDIPBKC8vJ244LBYrMzMzPT0dogv4xEF89v/MZSqVWlhYOPaGMzAwEB0dfejQoW+++WbVqlU2NjY1NTXiaEG8/Ow/nRe3J4KBNDqI4Ulk0Qzkj1kiDks8JwmCCkFGdAwV4jGUECAUmrbXOAID3J4NhDPSWJxAUA1NTe8xc4dYhYjIWfEJsyiUT6qrZQX8dAGGuHT0S6em7A3wlC4skKnmyFRVg9eiwlMsVr1wRIBpfm9X4X1yFnlljvKymquLBXrDnkiGv4j0f+HpgnBJiP5w1CCxafcLntBPFQA8YXuxTqIGoPODegu7tRc14/CApbokjwxSJaQpgjwJuBphOUUR0AhZ5JVZ5JVpyr/SNDeU2sg3hN/ry48TdTdjosHnvCA4fL5KQ+OBpmaZ0lLp8ADFuHDeyLMw+fpbMjIe4QzDKMzggYMEz9lzfL/4MkVGhnXzZk96GtreTrAK4k3Unz1btWoVNkEfkP5+dORTEvF93+TyNGB4FWcHRdG+vj4qNdX5UcSKg7a/HndZuv++zCWXuOSM0sLc3LyCvLy83Nxc6NebixtUHuPRjFjl5eVQnVyJW0NDQ39/P5/PH/VkDmokUBQVCATQMYnL5cK0DE1NTTU1NRUVFQUFBampqQkJCSkpKVQqNTs7m8FgMJlMOp2ekZHR0fESE8e8ioH+n2yjsaXV1tpafcfSAIkZLUcAYwDn/dj5Gdg5gBYAcpjEgoLtSWAvSEScJ/WeJFG2kww2fWWsq1FUVi6exBsoPnPi0VN/IKv+jsitRXKSxb/FMKytre3WrVsQMECVApQr+Pr6njlzZu7cuWvWrLl48aKioqKKioq6urqqqioMpqSqqjrJCKeeU7QnAIbr16+rq6u7uLhMHjAEBgZGRkaGh4cTmCE0NDQsLAwSDr6+vl6e3g73rQz0z587t3b3ns/Xrp+xdh0gHDZtIm3FYcOePaR9uJPSoUMAORw5ioscjr+rpnZc38DQ0NBAS1PTwMCATqfr6en/9NPPsbERFEpISPAdN7fLt8wlNDW/IZPfu3iBJC9POgfRnsMAACAASURBVH8eLOfOkc6dBSBBThb4O506CaIzHT4Mkk8DlCI1R1l525kzp86flzczM3vw4EFoaGhKSkpOTg4UMxD5FgoLC2FENTabXVBQIM4hiJefGSrk4PkZsrOzGxqGVPXED5cAAwiCMJlMDQ2NH374YcGCBVeuXMnNzR11xyP2er0FPC0JiuifRn+Zif7xT0QcMABaAUERPsJMQ7wsUWYqIhK83t7+j7cuEAT29I7jjCSOHDo6QbZmGCw1InJWXPwsSurXHM4VgaCQGL1BDNNhs6RD/aVSEmSqawBgqKqWqaw6UFdv0dI66mYoGuxnexpmKK1IUVhBV142oLdQqL+wUXNxrsrSdMXlVKVlDPKyoitL2WpLOOqLmzQXd2gt4uos7NddKNADoALRB8/sH+MHYr4+tiC+2VTLY2sbdqOCqECEAwOuzqI2rUUNGour1RezVJcUqCzNxtmDVDyNGh7UaDlFYUWqIghtlEVeSSOvpCgsT1f9k24izXqo2xDv2lPBEPZ14Z6txHA+V6GSz79Y33CgoVGmukY6l7HX30M62Ce/pfnZKhX29UWsWesze84jfPGcPcdnzvses+f4/OublCNHKh4+6q+re/J0H+nrq1iyhP3++9Xr17doafUlJ4uG5Q38ioq6Q4eqVq2ql5bucnYeF2w8W7dfzV7TgOFVjDOKogUFBUmJCSc1Hyw/ZL/0kL30JefElAxmLp1OZzAZdPjXS6cPFWDw8hLcSktLy8rKCORQWVkJ3YgFAgHx5yp+DNA3CWKG3t7e7u7ujo4OqH6uqqoqLS2l0+mxsbFxcXGpqalUKpVGo+Xk5KSlpZWVlY1boXjl0+XXOAJiZwfEP8IwrH+Q7+npobLrV99NMzuPk7CLOKVwHscMYOo/lWXU9vAj7r8kOE1K2UG6uuFry9tmLZ0g4ANMaQs4h74erK0BHezDeYch8gEO0cDAgJ2dHZlMVlVVPXv27M6dOxUUFMhkspmZGQxgmpyc7Ovrq6urC/XQ8BWCChsbm8lP3D0nbU8ADObm5ioqKvb29pOUVsNsDJGRkVFRUZBegLAhJCQkLDQU5x0gcggOCg728va2sjLV1DwiK7tCWvqrTZv/ISExY6MESUICEA47d5D27CZJSZGkpMFy/PhXmhpK2jq6ujo68vLyhoaGeXl5enp6C35akJCQlJubz2DmMeiMrMy0+LiAoKA7Do7yN8wkdXSWKSp+ce7cu6dO/e34ccAn7N9PkpYBHlC7dpNk9n2kTN5kY2sQER7s5OR4+vTpCxcu3Lx588GDB2FhYRQKhU6nw0AITCaTQqFQqdT8/HwiYfPzAINx96XT6XFxcUwmEzx5x03s2oYrHr82NTXZ2NisWbPm888/P3PmzASJ5x9v/+pL4IeAYdi10+jimejq/4fVlIkxDIB8QN1uIqv+iS0job/+HXG9geHibrFtXn2X/0dbRDF+T+8OXNw8IWbo5s5uan4vMgpoFeLiZ6VQvq6u1hQIK8cOGXuQfzg5XirEV7q0dIhkwHmG/TW1mX2jRbooitbFu2Vc+SNVaUWp6pIq9cUwsUAKnoMsRWFFisLyDOWVmeSVVOWVVOUVYFFanqm0nKa8jEleVqCytPjK0jLVJeVqSyrVl9SoL667urhRY3GL5uJWrUXtWos6tBZ1aS/i6izq1VnYp7twYHjh6S6EC1/vF74eoALgR7hBv+7CXt2FPToLu3UWdmqDelq1FjVrLm7QWFx7dXGV+mK26pIS1aUFKkuZ5GVZSkCdnKYAPIuAQBkPZASxQboS6DCEB2mKK9KUf8+4up5hdqTMXacmyqE9L5HXXi8afEZfo7EjL76GPQjQwv76BgDYODVS6ZS9fo+kw/xtGTTxzaZUro+LC/xx/sPZc7w++SRk+YrMywqV3t799SNz8E1c40BOTsmcOT1BQd0PH9afPFm5bFn1woXdnp4oj1f1xx+crVsHCwv7EhIqf/utRVsbVIMgvKIiwp1p4opf/zfTgOFVnIP29vakxET/4Og1x21WHnZdfcwuIDwpl07Pysqi4ZY1bMRH6CcA459CGSKEDQS9MFHYU8gziESiwcHBvr6+np4eqH6GJEN5eXlBQUFycnJERERiYiKcHGRkZFCp1DdfZfgqTtXb0QaYZhWWsDQunnaQeLf5GC5OkJ8KPJgSlgCuTaSB06SAzTPVD21KTE4REfO8iYerp6fH2tpaWVlZVVX14KGD8+fPP3v2LKQR7ty5ExAQ4O3tbW5uLo4WCIbhJQEGNzc3Ozs72zEG8zBcunTp7t27kwQMnp6evr6+ERER0dHRkZGR4iTDmDKAD2Fh4cFBIb7ej1ydbczMrhjoy6qqysjLrz1x8pcjR/51/Pg3x098feToF8eOfSUvL6Gurqqpqammprb/wH5LS8vCwkIAGBYsiI+PH45KRAdPGuhMJiM/JyebmpmemhoTHe3l53vHxUXLzl7BxHS/2tU1lxRWaGhK3ra49PDRvajo0Pj4xPj4xLj4WEtLy0uXLhkbG7u5uYWEhCQlJWVlZdHpdAIhEOKEcaf7z7wSVstkMuPj4729vYnwrBNfRI+/6e7uDgkJuXnzJnc4RiGPxyO8MR9v9xpL105ji2Zif/wDqykjPPcA+dBSK9zzHSLzM5rgCxId7vxW1FQJUrwBkDTtnvRKT5hQmN7V/UFX9zhoASoWOjpnNzXPrm+YnZn1bkLSJ5VVanx+xRO66FZXJxMZNIJkqKreX1evUFfPHc/npL0gJUd/B428MkMJuOtQFJZTlVcU3TvTUZjSlpdQl+xZFWJZ6q6TZ3mWZrg74+qGtCt/UpR+pSiuoCoDZx4qeWWG8sp0pZWpSuAR/vCyPFURLGliS7ri8sksxC6pCstT8f5QcKIgBbwCPiRFYQUFbyhNCTAGVBzS0HDeIF1pBUVxZZrKH1SNDdlGUvlW51meBjXRjs20CG5VHr+7DRG8dPfmisHBC3X1+xsahxieao5UXORev0dSwT7nokK4/Gf3dOqtrauNiGjPzRWMwX5PuB7gV60mJuwffkAFQ1wi0tPTFxfHy8/vCQkpfvddfsXQFcUNCWF9842IyxVUV7Pnz2+9caPh1CluSMibLHWYBgxPPfsvYIPCwsLEhDgzG79l++2X7be+aRfEZGRnZmVlZcKXzLEGAUMRbgRsKC8vh5GR+Hz+k0MQIggiEAig+rmnp6ezs7OlpaWhoaG6urqkpIRGo4WFhUVHRyclJaWlpSUnJ//H4/wFHOd0Fa9kBEQI6unlo7plQcaOGSiUMgPXo8lrFaYMLVD5IUxStZ9ksO5jO2urXt6E9+KW5mYPD48DBw5AhADVCwoKClD9rIKblpaWjo4OkdCNgA0vlWHw8PBwcHAYgxds7ezsbt++febMGRMTE8+pWFBQUDRuUVFRQUFBgYGBozyUxMBDWHhYWFhYeHh4VGRkVGREZGhoSGCgn7eX+6OH9x89cnB2unfP8vrt20ZXNVTJZBBq9uTJE1JSez08PIqKinR1dX/66ScxwJCTnQ0W4NuTg4MHRi6Dkctk5jNz83Jz87JptIz05OTkmMzMVCYzl8nMo+XQs7JgtAMalUqNi4uDv/1M/BYkrkIW5zmJ9c8MEsR3hCqphIQEU1PT7OzsJ7AKT/0ZCQSCS5cvBwYGPk8lT21lahvon8QWzMRW/g2rKRUHDEicD7ZkBnJHBWij76oiS0nCWB8QPkmETJMMUxvh5966r19xgDcaLcDISM0ts0vL3ktLfzcqelZi0qyGxsUDvIynNtiJIOepqdKhftLFRY8dk6qqDzQ2ubSP79870MIpcVFPJ/+eoQTyDNB0tw621Y5qCFwcfJ6gt6OvsbyLld3KiK1P9uJE2pf73ih20ci3lqffOJSlJ0nV3JiutjbtyupU5d+gSCBTeSVNBXgBZZEBWZEJyArAV2Qor8hQWp6utCRNcWma4tL0odcl6UpLM5TAt1TlFZk4JqGpgB0BAlFamUpela76V8bV9VStTdnX9jJvnyy0Vyp1160Kvlub8KA5K6yjOL23tpTPbRPxeQRbOOpYXt5HDp8P0ALkFgC3Uy3DLt8bHrjX32NvgKdUiA+tQUxz/PL6MbJmVCis/OOPelnZkavBp/rTp6slJIj13MBA1hdfID093IAA9j//2aik1G5u3nDxYs2ePdDlCRUIkKnDFaL+l1GYBgwvY1RH1Dk4OEhJoSQlxl/Qc/9J2uaAsiuVSs3MzKRmZg4VqFT4jJ+KW0ZGBo1Ggx4CMFtC8bBxOBwul9vX1zc4OAhDqcKY6OLtwX9QEJtjJMkA0zLU1tay2ey8vLzo6OiQkJC4uLjExEQajTaRg5N4zdPl1z0CwPGhb2Dglqnx7fVzWo+TsAtQdTABABjLOcA1xHoIA3C2AUCCJ9IOhBBi4DTJTeLvBmT52vpG8QHp6GiPjIy8eOHCAtwOHz6sra0NQQLEAzhSAC8EThD/lthGVVX1JTEMHh4ejo6OEwEGOTk5XV1dj8mljoabBQQEREZGRkdHx8TEBAQEuLi4eHp6Aq8kMTH0MGYIBrro0JBQsOA2hB/Cw8MjwiMiff38nV1c7ty1VLmipqioIC8vv2vXzn37ZEJCQkpKSnR1dUcyDMQ8HAcNOTnZOTnZ9Gx6Dh0gCKAQyGEwmAxGbg6dAb+EK+FuBJkwFhu8WIRA9BIWmExmQkKCoaGhv78/1DrDm5X4VTRuGcVN/CuRSOTk5JSRkQFrgB6Y4hu8yjKeHxFBbHRQqR+xo8vRxhHuKyKve+gPM1AbLfDrtdEEwYg97ooQYCN8+F5lj/8n20KQtm7uT0SCBUgpdHbN5tS8l53zbkws8EGCUVNjYmd1d3tPcpASOzulI4OlEqKHHnJDMUM153BtXe7AiOzR4hV2FqfnXNudo7+9ozBVfP2kyiiCigQ4ougaaKvrrSvtLqd3FFJacqIa0wPqEh9WR9pXBJizPAxL3DSLHFUKbS/l3ZVLNzjkeXKbn+wmuPjLSvie3uwjuy3L6EDeHdkCm0vFLlfZPqacKIeGVL9WZlxnSQa3Kr+vsXyws0k40IMI+YhI+Oa4zbQJReSGRqBbgAOOO4NJFxXuDfDcGwAAg0yY3/N4JU3qRIy3ESoQdN6/30+ljv2yasOGJhUVYn0TmVz5++8YhtWdPl3911+Qkeiws+Ns2QLHGWywevUbFYl1GjAQp+9lFTra2mOioxMSEqQUnRZJ2dq6h+fkZEFsQLxCwJCBW3p6ek5OTj5uMFUC5BmKiooaGxt7enogYIDhU9vb27tGxnGD/6AQMMBtYB43qH5ubGysrq4uLi6mUCj+/v7R0dHx8fFtbW0gCOBwaM6XNRDT9T73CLR2dhqqXXLd8PdBOfyR/7mZTyQWZuCBkvA4qhdxJfQ50uBpUt8pEGh14CSJL4urHQBsgMGUZqDnSED6PDFyGKrwPClmG0nz2Pay8goerz8jI0NDQ+PXX3/99ttvJSUlHRwcysvLBQKBr6+vkpISRAKTfH3ZDIOjo6ONjY3tSLOzs7OwsJCVlVVQUHjw4IGnp6fHJGCDh4eHt7d3eHh49LCFhIQ4ODjAjNTh4TijAFDB40hKw+BhnHcfHx8HBwdtbW15efkLFy4cPnx4y+Ytx48fi4qKKi4u1tPT++WXXxISEoZdkkZNxd/0jxCiJCQkXL16VU9Pr6Wl5bl/CiMqiIuN3b59u729fXPzMyodR1Q3xQ94qDJE1NOJtjeiHc2oiC+OBFAfK+zHGZiNFqjVRgP7gYR4WQoBXBiljJ1iq9ObT3EEBIIQGE0VQoXW9tksNqAUIiKHcEJ4BAicGh4xKzZuFo9XNsnq/+N3YlCYLx3mL82ki5MM++sbyPUNPcPem2Nr43PbB7te8A9hbCv4GhBzPSW5+csvYr78IuzxMi98/o9RNdU9eBAv8Wt2gmremNUCFL3e3AJiIhFoARcwSNOy9vo/wjEDYBgux0UMCN+gGAMNZ8/W7t8PRxHp6ir74os2MzOUz2d/912nszNc337vXs2ePUCgSKWy589n//STqL0dw7ABKrVeVhbpBdlUCWenV39CpgHDSx/zWk5ddHREVEzCFjl7CVm7FEpaFs4tQLRAQAUqzjNkZGRAwAATq0HAAFMllJSUtLa2crnc3t5emNoZpnNmsViEXy8x74eAQSAQwBCrhPq5paUFkgw0Gs3Pzy80NLSoqIjAGC99LKYbeI4R4Av4KhfOeq+fieCKZDB3H6VUFp/ow0hH8gAhlMiQgjaQbNf9w1xynuneRdf3LTfdt9xUZrnprp9ubfvy3sZ/+m4g0XeDSKwAKlzAoYh4VSPLoF18s5w9pEs7V63fvO3rf329Zs0aExPT/Px8IkomiqIhISGKioqThAqvjGEYBRjs7Ozs7e1v3Lhx8uRJWVlZOzs7z8kZzPIWHBw8jBeiY2NjIyIirK2tDQwMrl275ujoGBoaCiMpTeyqNAQefHx8oLRATk7u5MmTu3ftltgkoaCokJqaymQyQ0NDbW1tMzIyxDmBNx0lDPePjltCQoKWlpa6ujqLxXrhNxw2m62goPDvf/97wYIFOjo6BQUFT/bYfI5f4Ti7IhgIboygKMhviGEIyO/8ePqFet/DfpiB2emCPe20AGDwuCNCEBEmnHZJGmc0X9YqtL//PF8A0i80NM7OL3gvIXGIUiBwAixERM6iZn6NoqPjdz2hX+WDvCMJ0VKRITLlFY/Vz7hjkjM+23vCvq/sq4iolk8/S/jk02ixJeaLr2KqqsSyU7+y3jxfQ7kDA2CcYTRbAjNwaqSS4/b6DQEGQDIE+xS3vRpINqnj4ZeWVixb1nj5cre7O2fz5srly5Hu7n4qtfTjj0ESaNza796tO3wYFQo527e3mZtXrVkj6u1F+fyqNWsalZREnZ1Nioq127bVHTnS/ejRqw+yNA0YJnWmn2ejcjYrCvgtJ6w9YXXZ8BGDnkOlDjkjEZ5IkFuAaCE9PZ3BYOTl5eXn5xfgBgEDi8Vqb2/v7u7u7e3t7++HSKCzs5PNZpeXl4v/RUHYIBKJBAIBJBn6+/uh+rm1tbW+vr6iooLJZAYGBsbExIinRnqew5ze92WPAIJhD9xcrTbN7ZMFzkhDE/eRs3kwlccdjfiypNw9pPub3tXcucRQ4aSHq3NcYgotN7+6pr6+uQ0u7CoOLbcoKY3q6/ngpraK7oG/bLZ8lLqTxD35NNhwDm/lAilXirTxm9mOTi59/eNEwEhNTVVVVSWTyRAMTOb11TMMt2/fvnr1qpyc3MGDB48cOWJmZjalAE1+fn5RUVEQM8TExMTi5unpqaKicuL4cRUVldu3b7u6uvr7+8OgqxEREeHDhtMP4CUiIsLHx0dTU/M4bjLS0tu3b9+8ebOGhgaVSgX6ZvyGMDwDf5veGQwGjUZzd3dXUlKSk5NLS0sj0MKoW9Yz/HzEa0BRtKKi4vbt27///vu333574sSJ2NjYvlfiAQwYBjwpG0AOADg8RgsgAkqsN7p4JmYoC0hcw5PYqn+gOYkosMdZTZ7h2Kd3mdIIoGhPb9+Gpub38vLfi0+YFREJ0MIoqAA/JiTNqqklEyqUSbbiVlsD1M9pKSNmsdWcQ7V1YyMmTbLOF7tZYFDjZ5/Hf/pZlNgS/fm8qPKK0QGdXmy7L6M2Rn8/yJTHGY5mCwUM5RVSEUEEwwBSZIT6eRQ9ewa3l9FzQV1d261bDWfOtOjowLRuLdralatWocMZG9otLBrOn+9yd+fs3i2or6/89VcMRTudnct//lnU0zOQnc3Ztq03IqLb27ti0aJmHR3YSWFz80S0A9LbK5x0fKenHvI0YHjqED3vBmWlpZFREZHRsetP3rNwimDSc4CCATcY2BRSDQThAAFDvhhaKCoCad3YbHZHR0dXVxf0SoKJ2/h8fkdHR0tLi/i/FPgvwjO48fl8iCsgYCC8kiorKwsLC4ODgytxXCu+7/Me7fT+L3EEwETExdHx7ob3B2ShS5KY8AB4E4GJ/qAsKW4byWDLN9fI8kFhUZzGZpB1dhLW1s1NTqOaG+lr7Vzovfmd9qNDrkrjI5NzOL9xkZQjRdI4ubOucRxXkJ6enuTkZFNTUyUlJbJYyueJkIMKboSGwcPDw/O5DVbi5eXl7e3t5eXl7Oxsg5s1bjY2NteuXTt06JC0tPSB/fv37dunpaU1pXYfPXrk7e0NMUMMbjBmMaQa5OXld+MmKyurpqZmaGhoZWXl7u4OD8t32AICAuzt7U+cOHHw4MH9+/fv3LlDUlJy69atZmZmkFKg0+m5ublvE1DIyYEJXiIjI83MzGRlZY8ePWphYZGTk8NisQYHgWL+Jd12Ojs6/Hx9d+7cOW/evA0bNnh6eMDsDURzRGESv4nn3QREServQWX/Qv+Yg2odRla/i5z+E+vtet56p/cfHoHedrfBPqBOJnLQD38z6h1F0drOrigW2yyDKhEX/01UNIiaGhM7KzpmVmTU0JKQOCs7Z60IAU4gU7IuBLmYmQ7UzyCP2+OJ7P66+gt19W3CSd6Dp9Tm1DYOCGgYCxg++zyKXf72AQYhitq2th1obHoMz6o5QMAQ6An9keCrdKifXmriG+7513j5cruZGXEuO+7c4ezeXbV6NS8nR9TUVLV6tYDDYS9YwA0KAhc5ny9saEDx+2eniwv7hx8woRDl8SpXr263smq5erUvKYmoChY67Owqf/tt1Mpn/jgNGJ556Ca7I5vFiogIi4qJ2XHO/mFAHD07OzNrKDgShULJHGlUKhUChgLcCgsLoYChsLCwvLy8o6Ojs7OTAAw8Hk8gEIhEIxlwHC1ASTSfz+fxeAMDAwRgaG9vb2pqqq6uLiwsZDAYYzXTkz2q6e1e+QjAf0QRijnet7OUeH/g9HDWBUgyyAMBdMYuksGmb28b6RWUlAlERNDGEY88J+442AxFser6BhdHB4P9f8bu/b9BSCaM5TFwwAAEDxdIsdtIV+WPd/U+/uMh5mQoipaXl3t5eenq6iorK49VOY8FDzBxm+eLMA8Pj0ePHnl4eLi6ulpYWBgaGJibm98baRYWFnJn5KSkpPbJALt48eKU0rc9evTI1tbWz88vNjY2RsxiY2Pj4+PDw8PNzc0PHDiwbt26v/76a926ddu3bz9w4MDhw4ePHDly8uTJU6dOycnJnT179sSJE3v27JGSktqzZ/f27du3bdu2a9cuBwcHBoNBp9OTEhMD/P2pVOpbgRkgJZKWlvbw4UNVVdVz585pa2s7OzvDyGxRUVGFhYWClxNykbjw+Hx+RkaGnKwsmUwm3OSIK5/YjFjzkgrANRTDRDnxyJm16MaPULk/RVkJb45y9CUd9ausltti3sL+taPmrJBXMvl2BwZqmprDWGyjLNrupORFsXFfRcd8nJj0r8Ki43z+FJyRxFuk9fTIRIdKxYTLVFaJ+9YfaGg0a24Wim/6Osp+fvVjAcOnn0WxWMAt/q0zPoratbXvr60DQ11dLVNTK5WZLk4vgEBJwT6ykcGdvAml52/EUSMIQS9gGNZuaVlNIjWrqWEYNlhaWr1xY6O8fO2RIxiG8fLzq9asqfrjj9rduxsvXarbtat6/XqwPje3dNasGmnpVmXlntBQpL+/w96+1dBwgAYyUdTs3j1uyKZnO/ZpwPBs4zaFvTgcTkRERFxc3N7Ljl5BsYzs7MxMKszAkJ6eDsMaDqdhyMrMzMzIyGAymQW4QWck+ApjqoozDIODgwKBQCjEFXQ4yQ25BSKAEgEYYEIGIuszBAxQ6zyFI5ne9LWOAIIheOB24PFgb3PPdtNsnhzOMOByhdbjJJuN7xmR5Rn5hcPzIZDfDV+e3m+QuRn4SAwZjy/wDw5bPP/7c9/NqDxIQi+IURk4eEDPk1BCQSFPcls3w9ryNszPANHJcB9AhXw+v7i42NHR8cqVKzD381icIL7GxMSEeAzvOXXzwO3Ro0d+fn4REREMBsPe3l5SUnLLli1SUlJaWlrW1tZWVlb37t2ztLS0srIik8l79+6VkpKSlpY+ePDg7du3J5+NwcvLywm3yMhIMbwwVIyLi4uNjfXy8jI0NJSTk9u+ffuaNWv++uuvNWvWrBOzDRs2SGyS2LZt644d26Ft27Zt7969vr6+8FZw586dNWvWxMXFveGiZwIqeHl5GRsba2pqWlhYeHp6hoeHx8XFxcTEREdHR0VFxcTEsNks4XCc8qFr7iW8iUQiIr1MX1+fhYVFSUmJ+JX5EtocUSWe0w1gBqSnG60uRXq6Hv/GRmw4/eEpI4CIOgd7KaOxFspr5xzpKCN1N11DkHG8Ip9SKf61UNTX18fu5jIHeVWT2X6ibcBtuapSJiJQipomTjLIVFXvr28IHBmbZKJKXt56H9/xAUNJSc/La/Sl1ixE0Xtt7SCsakUFoBeAPxIeUDXQSyrYB7A9Yf4W2Rn88RJivNSOPU/l/MrKjps3hY0gAiFAAh9+yP7xR0F1NYaiNbt2cbZsEdTU8MvLub6+7LlzW42NAca4e5c9b56wFgTnRRGk9sCB8u+/b1FTq9mxo8PWlvXvf0N24nl6Rew7DRiIoXhZhba2tsjIyPj4uJ3nbS1cIpg5OVk4w0Cj0aCGISsrC1INNBotE7fc3NwC3AiGobi4mHBJgrpnKGPg8/mQZBANm1AoFOBGoIX+/n4YKKmrq4tgGIqKit7AnKkv6xz819UrRDHbe3esN7zHlwMJnpl7SBrbF3r7+vEGnysohEgkKi4uuXv37qZNm7766qv16zeqa+uSpf+K3zEDBakeRsMGsAanIARyJKN1H8XGxwPcOkE6qs7Ozvj4eDMzMzJuE7ENKioqenp6bm5uk9QSeAx7LkE+AfoIRUZGZmZmVlVV9fSAv0MOh3PhwgU4TT9y5Ii5uTkEDPfu3bOysjI2Nj5w4MDu3bv27t27Z88ebW3tSTbtidujR4+sra0hyQBDrI5CDvHx8XFxseD++wAAIABJREFUcWFhYXZ2dpcuXdy1a9fGjRvWrl27bt269evXb8Rtw4YNGzdu3LJlCwEY9u3bd/v2bX9//8TExPCICHd398zMTEg4EDxDNm7Ex9dSoOPGwC0tLc3Hx8fCwsLU1NTS0jIgIABCJggVoNIDYobY2Ni6urpXOXdva2s7duxYenr6uI2Ou/K/7rbxFh8Qvy+zLv8D/kAehmEiYUt/p0tfxyMM5fEHmC3l29o5x9+EY+tGkEtZ6SBiUmHhY2+Zqup9nJqjNbX5A69TXuztMz5gKCp+WwEDhmE3a2tl0pKkIoL3BnoPowXvQ2EB52MjDKipqfW1womjVL0JF8yT+yBsaqo/cqTT1RVIoQYGWD/80H7vHtyln0IpmTMHBm+t3rSp4fJluL43LKxk1ixebi7Ypaur6q+/yj7/XNTa+uSGJv/tNGCY/Fg945Y8Hi8pKTEuNma/kqO8vjuTTs+igQTP2dnZWVkgviqNRktMTExPT4drEhMTGQxGAW4QMBTjVlZW1tra2tXVBQFDX18fj8fj4wZ5BiFuUOgsrl7o7e3t6enp7u7u6OhobW2F6dtKS0uJZ2/PeGDTu72uEQAcAyZAEJu7t102vxu3haR6cDOzEES7QsBkfZIOSKN7z2KxDhw48P333y9ZskRFRSUlOZnL7cYwrLquTlvxjOfGvwnPjgcYIIq4MKP2IElj/7oGEDFznA7ACRmCIA0NDYGBgVpaWsrKyuPqoZ8BMDzCzcvLKygoKCMjg8PhdHV1jYqTExYWtnnz5rVr127atEldXd3KysoSN+igdOHChZ07d+zevWvXrp1nzpxxdXWdPGbw8vJ68OCBr6/vKJww6iN0UgoLC3NzczM1NT179qyUlJSEhMR63ADJICEhKSkJAcOOHTvOnz+vr69vYGBgZmZmZ2cXGBgYFRWVkpJCo9HgHP1NiJgEe5KdnZ2cnAwDPVlYWDx8+FCcbyFwgnghKiqKQqG0DsdXfTXzdZi+BrYVHR1948YNFosFFQ6jfwzTn9+wEUBRQTN7Q0fNOcEgq5m9rrViY13+nK76K+BBbG9KXf4ciCVee69zenv3xYRJRYfJVFSKOybtr6u/XF/fMqxtffX9nAgwFBZxX31nXkiLHKHwUFoKCIuEEwvQB+lSXERlb0/P6xvnF3JoYytBUbTVyIj9ww9tBgat165V/vYbe/58IGlobCz77LPe2Fi4S92hQxxJSWJ3zvbtnO3bR/NyxNdTL0wDhqmP2dT3YDAY0VHhioYPVx+9F5eYlpNDo9Fo8HEgDJVIoVBSU1Ph88KoqCgajQahAgEYSnFrbm6GLkm9vb19fX0wuOrg4CCEDcTrIG48Hq+/vx86I3G5XEgvQMBQWVlZXl4+/U859TP5ZuyBAuckDMOECGp501jp3PHKmlrcpwhk9Jl8F1EUbWtrg8/gMQxjs9lqamr+fn7ivmpw7t/d02egqeay8e8imFh6FNUAyQd5UthmkuXN69Ax6QndGBwcZDKZ9vb2V69eHStsmCRgIPgEDw+P4OBgCoVSUlLS0dEhnOCvoq2tTU1Nbc2aNWvXrj18+LA4yWBjY6Orq7tnz+4dO3bs3LkdPtqfPGDwwCkOPz8/8VnyKLQAmQeoh05MTITyBnd3dzMzM0VFhYMHD27HVc4QLUhKSh48eFBbW9vAwEBfX19PT09fX98IN3Nzc2dn58DAwLi4uLS0tBxcW/yK8QP0O2IwGNnZ2RQKJSwszMnJycbGxsnJydvbOyIiAso5xOHBuOXIyMjU1NTOzs4nXCov76vAwMCFCxd+991358+fT09Phzrsl9fcdM3PPwI8bmR94bx2ztEBbhSITN8dWpv3rmCgAEORlvKN7ZzTz9/EC6nBhVMlHR6AR0x6rH6Wqao+0NBo1NQ8+JqeeU/kklT01gIGj9ZWmbgIAi2AsEhh/vdzc17ISXwzK+lLSem0tOyJjOx68KD0k094DEZPUBDriy+EOIeA8PkAUdy6BTuPDg6WL1jQZmHxAo9lGjC8wMGcsKrm5uaoyHDDO54LZBxNrEJyGTnZ2fAxIQOGTaTRaBQKBfobxMbGUiiU4mErKSmBaKGsrKyurg4yDOK6Zx6PBxEC8crDbWBgoK+vDzojdXd3d3Z2tra2trS01NfXs9nsRtxJbsIeT3/xloyAQCjiC54EEqhUqqen51jdJ9BUDQ6eOHHCxsZm1MP4cQ99YJBvpK3pIfEOzAIxjnuSPMgEp73+S0ZewbCgZkKqAcOw3t5eCoVibGysjBvhoQQBw4MHDyaaskM+wdPTMyAgANJxra2tk0G/+fn5x48fX7t27caNGy9evHj37l1IL1hbW5ubmx85cmTr1q07gIxA8qq6uuewQTww/GnCdy8vr5CQkFE4YaKPkG2AyCEoKMjBwUFTQwMKGCQlJXfu3KmgoHDt2jUD3AwNDdXV1aF0WE9PT1dX18DAAPr8uLq6+vn5hYeHJycnwwzxcDbPZDKhjxDEEuKuSk/O5Sz+LcFgEAiByWTS6XQqlZqUlBQQEODs7HwLN0dHx+DgYBhVdlxsMO7KqKioyKiotLS0zo4O7HVYR0fHgwcPNm/ePG/evB07dvj4+LS/MVHzX8d4vLlt9rY58fvpGCZsZv3ZWLpwuKNIM3tDO+cUDh4i6vLf5/NyRYIaDOUPb/B63nkYpkGnyYT5S+cyR4kZDjQ0urZ3jHNbfPk99Z1Aw/CWAoY+DLtYmC8V4jMiLFKIb07TM2rWX/4ZeJEtoAJBT3Bwf0ZG6/Xr9ceOwapFXG7Zl19y/fzgxz4KpWT2bF5BwQtseBowvMDBnLAqkUiUlZVh4+yz6oT7pjNukXHJdDoNj64OntJlZ2fDf2ImbmlpabGxscXFxQROKCsrY+FWVVXV3g5SMfTgBkmGgYEBnpjBmEiQWyDQAkEvNDU11dTU/OdZ8quJUD7hiEx/8UpGoKysbPHixcrKyvC5O4/Hy87ODg8PJ56nUigUNpsNnTSe0CM81gvW3duvo3w+bsfM8QOtghQQM9K2k25oqkxm+o5HZEKrq6v9/PwMDQ3JuF25cmUiwAApBQ8Pj6CgoNjY2Pz8/Pr6+sHBwad2njguBEGioqJ27NixZs2anTt3mpiY2NjYQMxw7949FRUVye2SW7du27Zt26lTp5ydnSdCLJ7jmYeHh4+PT3h4+EQggVgPH8BHREQEBgZ6eHg4OTnZ29uTycoEvXD48GEdHR1DQ0PokmRkZHTkyJEvvvhCWVkZrtTDTRc3fX19Q0PDmzdvWllZOTk5eXl5hYaGQkCSnp5Oo9EIDEBgCQIAQFABn1MQZQg2CF8jGo1GpVIhJeLt7e3k5HTr1i3YDSsrKy8vr7CwMHho46KCp66MiIig0Wi81xfJhMcbSEpKOn369L/+9a9ff/31Pyn8qqqqJn9REVfXdOF5RwAVdNYp83riiXqEvNKuBvWuhqsNRd+2VYNZUX+Xf13+B8LBCrjNQHdEfcEHwkE2igpbK3a2lH3SUr4ZEb0wj22iJ1MtVPN4J5NipSKCZMrKxMUMMtWcfTW10dzX4AXk7z9OWNVPP4t6SwFDen+/NCVRPCySdIjPhZiwvpcTfm2qF8Ar2x7p6xMRJC2KcrZvr925U1BVxS8v52zYULF0KYzB+qL6Mw0YXtRITlgP/O9pbWkJDgndfcl15emQs7ru2VlZ9Jzs7Gwag8HIyMhgMBi5ublMJhO+pqen5+fns8SMzWazWKzy8vKGhoaurq7u7m4ul9vT0wN9k/r7+wmcACXOvbhxuVzILXR0dLS1tTU3N9fX11dVVdXV1U3mofKEhzT9xdswAh0dHVBE29bWVlFRYWlpKSkp+fHHHx87dmyU3v2p0yME5KMCPEZNU6vK/k2l+2eMjZuEcw4z0DMkg/XzGLn5wyTD00dKKBSWlZXZ2NioqqoqKyuTyWQ9PT2CYSBcj3x8fCIjI2k0WmNjI4/He2qfx224v7/f3Nx8/fr1a9askZeXJwCDtbX1zZs3Dxw4AJXHkpKS+vr6UwIMnrj5+vpCn5zYCSwqKiooKOjhw4fOzs62trbW1tY2NjbXr1/fv3//tm3bJCUld+zYoaioaIibAW7GxsaHDh2aO3eukpISnKnrDxskHHRw09bWhgUDPM+0sbGxhYXF/fv3XVxc3N3diZhRUVFRCQkJycnJKWMsOTk5ISEhMjIyLCzMy8vLzc3NxcXF3t7+1q1bOjo6V69ehZoTY2Pj+/fv+/v7R0dHQ/ADdcxPxQZjN4iPj6dQKFlZWd3drzo1wdjrp6yszMDA4McffySTyZNEvONeY9Mrn20EUFTY02Ih4BXC3QUDuQ3FP/S0XOf1xLdVStflzxIMlqEov7F0SVej1nATwuay39trzgGJp7CVxw0R8kGsmDfBEtrbZCKDpRJixJUMMlXV+2pqj9bWMl+5ADowcHzAUPwWip4RDDOuqZGJChH3R5IJD/AoApr4/2UTcDh1R47Ubd1ad+IE++uvGxUUXuxoTAOGFzue49QG/5lQFM3LpauaPFx9NuL30/56FgF0Rk4mlZoNoqxmZmVlwgd7EDPk5eXl5uaWlpayxay8vLyiogKSDF24QcwAYQMOEMAL/AhVzl1dXR0dHe3t7RAtNDY21tbWVlRUQLnz2L/McXo/vertHAEejycnJ/f5558bGBgcO3bsu+++W7x4sZKSYlJSUndX17OeesClUzIyDbf8axBEdB0dNwmVB2kZ0neSzHTUBYhoSsx7R0dHcnKyhYXFf/gQHR0dV1dXCBW8vb3Dw8Ozs7M5HE5PT8/zA92mpiZNTc01a9Zs3bpVXV2dYBisrKyUlJQ2bdokISEhsUni+PFj8IG951TMy8srODg4IuJxOufw8PCIiIjIyMjQ0FBfX193d3c7OztrMbOxsVFSUhpLLxgYGBgaGhoYGBgZGR09enTu3LkKCgpjAQNkG+Ar5BwI/KAlZpqamjo6OpCOMDExMTU1vXHjxnXcTIfNxMTE0NBQW1tbS0tLTU1NBTcymaykpASBnLGx8aNHjwipxlgA8IQ14oGSYmNjk5KSUlNTMzMzc3Bjs9mvkWQgfuIoijY2NhLhm7hcbkZGBiHyITabLrzAERAJ6gQDhSgCcvmBWA68IuEgCG/aVa/WXL4JrkSRvhbWyo7aCxiG9XW41Rd+LhI2wa8GugLaqmTQZ42pCit5Ga8ohtlWsEGU1YzUUY5J++vqz9XVV+Hpt15G0+PWGRj03wMYKgTCA1kZ4s5IUkFeh8P8a/AoHeMe/v/USmF7O8rjdT182J+W9mIPfBowvNjxfFJtra0t/sHRG+UebCBnrDrldsM2MJNKTUvPyKLRkpMp2dk5jBx6NoNOZzAZuAdyQUEBW8wgYCgvL6+rq4MZ3CDPQLAN0E+JO2xdXV2dnZ0QLbS0tDQ1NUF6YdpP90kn6b/iO5FIZGhoOHPmzHfeeeebb745ffp0WFjYC1KXAhRgbX7Td9NMBKSKm/k4G8N5EgAM50n9p0kam78rr+ZMEjAQ6AVF0aamJi8vL1NT0wcPHoSGhqam/n/23gOurfve+yd97n3+dzxNb9okTpr2dt3btL1dcZukbWI7cRzHbuwaAbaTeMQxS9jsaWw2BswwYPbeAgQSQ+y9p9ggiam9EZIQGyHpf3V+5kQGjJkGzPm+sHx09vn8juC89V31Y2Njk5OT2+cEzYH9X/fdl19++Ze//EVbW/v+/fuhoaHBwcGhoaF+fv5Xr14BpU5PfXrKyclpC06GjIyMpKSkkJCQIMhA5dbQ0NDw8PCQkBDQAgLmhfDw8AcPHly8ePEMZJ9//jlwLwBacHFxuXfvnqur62pggPOhVwMDwAYnJydADvfu3QMM4OjoeAcyBwcHe8jsILNdNhsbG2sNA2+tILO0tLx9+7anpyfoqLAOGKxeBNpgg/nl5eUg3UIzUIpIVDtbh4eH4Ug5zcHak2lwT/b09Bw/fhy0boDvUvh8eDwekgkGq7GFifnpejHzC/7g2zzyj4QjJ5YWGCqVapx6UUS/rFKppDxX3uC7quUazTMSLKvvP+QLNKVimkP+bynXBRxRHS2p3FYt6S2c+QY3mVGq7nZ36BTgUF0dK5mBzbHgcIRPqdCwwf1varXcPO6ajdtIpD2Ij9rUma9eOUYg1Kks1oxH0iFk32+qXb0mMmdnFUCAYWf1XG9v8/Pz7a3Ntt4Z7+njPrFu/eu1dBuftPLy0np1eEBNZVVlc3NLU0tzZ3trG1EdedzR0UEikYaGhoaHh0c0bHR0lMvlAmYArgbpKgO0ACKRBAIBCEai0WgCgWD1X771ThpZdgAV6CASjx49+k//9E9aWlp//OMfbW1tk5KSNpKr8MxrVUIlU9l8oQ3qQ84XajxQN3uGiyaBVm4mWpkfaWESYjb1lA/flnNzc4ODgxQKRSKR7HgncnAUhUJRXl6ura39wQcfXPnqqwe+vqGhoY8ePQoNDXVxcTl79uwJtR2/cuWrjbeDwGhYSkoKCDTy0LD79+/7+voGBgaCAwFmCA8Pd3R0PHfuHAAGOHvBDTIXFxcbyHR1db///e+j0WhnZ+d79+6BukmaqKDpYVgTGGBmAF4HBweHO3fuOEAG4MHe3t7Ozg6wAzgoYAcrKysLCwszMzNjY2MjIyMfH5/8/HxNd8FqQlg9p7S0tKqqqr6+vqmpCUYFooa1Q0aj0dbMzn/mbblLKywsLDCZTPiU2tvbm5ub4Wil4ODg48ePs9nsXTr6i7pbpXJhXlYlpl8YH/3dJM95cW5gcbabR/kVSF+en6pk9nxvcY6yMENkdv/7/EwL0GFpgc7s1pJwHNT1EkQxEwwjdRHpfW/shQWj+mpUIV6HRNKhryya5MTlTT6vtmJ5+S8IMAgUiq+7OrRz0r/1MODTdfOz2rjIJ3HXPw8IMOy6xPABlErlQH9PUXHFP4zjj9+q+cS26d0beN3b8SHxGcVlpRUVJeWlZd6hWFxhVXNTQ2NjIyh7QiaThyEDyADzA5vNFovFMDaInzQ4EkkgEHC5XA6Hw2AwBALBpp7h4DNHJg6WArOzszQarbW1NTEx0dLS8rPPTv/yl7+8c+cOeNaBH823c1FpqWnJp/5FARq3wcAAJoy1eF9oOenrymbmtnOI3d62tLQUMAMajQ4ODn60bPr6+hAwnPj000//N6gds2xpUPnU5Xfr/Z+enp6amhoZGekL2YMHD3x8fEAgkLe3N8AG4G0ICwszNTU9e/YsKI5kZmYGiiO5uLi4u7uHh4eHhYUFBARcv379lVde0dfXd3BwsLa2trOzc3BwcHR0hNkA+BPgt7BvAbgX7i4boIU7ywa7GgAtrAYGS0tLCwsLU1NTNBptBJmxsXFgYCDI7V4NBppzgGOhrKyssrKyoaGhtbVVAxDWnuzo6IDTq3bkLlXtqDk6Or711lsoFCo/P392dpbJZB49etTAwAAQxT484R29+h3b2YwYx+3/PzL+vSW5AN6pTBjCJf1SqZhTKhX84RMihoFKpRKMnBKMnAZOhtnJEv7QBxzST5cWWCo1KhwAWgBX1yWTXS4rVHdmGB5+IgGaSrvI5XkLBDPPpdAqoYC3pofhwHV6zpiY0Kkp08xeQOVhratKFp6LjPAdezgnEGB4ruM+OjpaV1MZn5b34fWE4+b1p2xaPrxd+uH1xK+soh5GYvDZWd/YRX5uFJaNL6isLK+sLK+GGroNDAyA71wHIaNANjQ0RKfTRSLRxLKBafgVFFHl8XgsyCRbj1x/rhIhB9txBRYWFkDLvx3cs0Asu6t7THRNw70AY4OxluIbrfuf/XRgcGh/PkWBs1IqlWVlZefPnz916tSdO3dgJ8P9+/d1dHROnDhx7NixS5cugUyGNMgwGzCwJmAGEHHk6+sbEBDg7+8PXn0g8/f3DwoKCg4ONjAwAOnOwL0AEp1dXFx8fX2zsrLy8/MLCwudnZ2PHDni4eERExMTHBx8//59F6g/g5OT0507d2xtbQE/3Fk2GBhgxwJYAnkU1C/Aq2BnZ2dvbw/ewrRga2trY2NjaWlpZ2fn5ubm5+fn4eEBgMHY2NjQ0NDU1DQ+Pr68vFwz1mg1KpSXl9fW1jY3N28EFYhEYkdHR3t7e2dn57hQuD9vm8nJycLCwsuXL7/x5hvvv/9+dHR0UlLSG2++ERsbq1Ao9uc5q/afLcm57P43ZYIgjVNTiujXxqm6j9lASmD1viJfoC3O9nH6X51gXJLx3QSjny3MdHDIv5oaj9XY8GBMFvB5OsW52pUlOmPU1cwQJBQuKDcYv7n16y0ofBGAQaJUGvT1audiv3Uv4DCo/Kxy2uPCWVsXCNlyAwogwLABkXZuFR6PV1FRUV1d9TAy66/XEk+Y13xqS/zEquVvJuV/+TrttGH0xzdC37uC+Qc6Ji4xnZCTlZuTn5eXV1ZW1tHRMTAw0NfX19vbq/lKoVAYDAaMDcDnMD4+DrdcYLFYAoFgPyQU7pyKyJ42qsCuPsTEhgUXnvk/SuO12j+baCWd0MrNwSl2/w/hRrXQWA+WRalUlpaW/uMf/zh9+vRdR8cQyEJDQ+3s7E6dOnX8+ImTJz92cnLKyMjAbNjSNBwRqampUVFRAQEBPj4+fn5+gYGBQUFBAQEBXl5e3t7ewPNw9eqV06dPf/7530FOMwAGNze3mJiYPMiKi4vv37//xptvhoaGFhYW5uXl4XA4LBabmZkJmkxHRUXFxMRER0eD9tVBQUHh4eERyxYZGRkcHOwGGdg5zCQuLi6Ojo42NjZWVlZmZma3NczExMTOzi4pKSkrK8vd3R12LxgZGRkaGtpBfSrKyso0OQGehlFhzeijtZ0L0Fx12bi2tp6enh3Kt1Hthi3J5T09PfYO9u+888ff//53r7766ltvvdXe3q6EbDeO+OLtU8p15fT/SLGkjp6fn26coJ1n9ryyOEcGV6pULvCH3hMzzaGlLROMq1K25eLckEql4g8fk/K9DpwgSpUqnkbVKcSj6qp0aDT1D3X5h0a/yOWFjY8v7jIyFBXxXwAPQ5ZEqlNXqZm9oH0oq6nu1UcAAYbnqvzk5GR1dXVlRUV1VWlwdNZHX8d8iC47ZdvyqU3LKZuWkxbNx2+Vn7So/Su66PTNcG+/kIjw0Kio6Li4uNTU1IqKCpAd2Nra2tzc3ARZfX19XV1dc3NzT0/P8PAwjUZjMBg0Gm1kZAS4IGQyGfx49FwvFTnYi65AVz/F78yPl/TXAga0Vts5rSAXW4VCXYx1P9+BSqWyvLz88uXLZ86cuXv3bmhoKMhXNjAwOHny5PHjx1EoVHBw8KaYAaNh6VB9Un9/f09PT29vbz/I3CG7f/++s7Oznp7uZ5+d/uKLy6D3AqiD5Ovri8Vi8/PV3xcUFxd7eXn94NVXgx89IhAIeXl5+fn5BAIBvIIJeA7YBJAGeM3Pz8fj8RpnpJ5Mgyw9PT0qKsrc3NzAwEBfX/8mZPr6+sbGxiYmJiA+6uHDh7dv3zYyMjLWMENDQ3t7+4yMDM1khtLS0srKSuBV2CwqwBTRDtk+Z4aZmZmhoaGkpKSTJ0++9NJLWlpaH374IYdzKDpGqXbClhbZrL7XxSxTEe0Sb/B3YpYBj/LrCdrfoe5s6gPMiDNYfT+Qz9PV9VKXZAsz7YqlCZkwgkP+5eLcwE6cwvPex6JK5UseUDNDc8MKJ4MOjX6Rw4sYFy3u5tcrxcVrAwOFMvW8tdjq8SYUCv2BflQe9gIuDfYw6BRk44ceo+ZWd4xst1EFEGDYqFI7st7CwkJDQ0NFRUV5eVlVZVlSeq6eafyfb2BPWNR+atN6yqb5lHXzp9Ytp2xbPjQp/ehL/68NjE3QaHNzc2tr63v37oWEhODx+MrKyoqKipKSkoKCgtzcXCwWm5aWlpCQEB0dHRUVhcFgSktLBwYGkBikHRkyZCdPU2BqdsHlxj94X72kWu1kMNbif6HldO3vk1Aaw34GBsAzvb29dnZ2f//73+/cuRMC1TLy8/O7cuXK8ePHjx07ZmxsnJKSkp6ejtmkpUHeBsAMoaGhIJPB09MTZC27u7tbW1ufP38e9F6AsxdcXV0TExOLiorAEz+BQIiIiLh8+XJ8fDwABk0e2OB0/iojQJaXl3f//n0DA4ObN2/qQ2ZgYIBGo83MzMzNzU1MTNBotAYpPJ4EfgYnJycsFlsOWXV1NegTBz/6b3kCMMPAwMDkXvS3etrdrjnf3Nz8rbfeevvtt995551jx47p6emZmpp2d3fv8/tc8xL2fFrMdhANakm5zkuL6gKp8gXaBOMKs+e7Eo6VfIGlVExPi2IVchHUxbmIR/kZj/Lr8THUwkzbnp/5lk9gWql07unUKcSh2ltXFE3SodH1ONyI8fHdi00qLRWs6WE4QMCQPDGhW1d1ISsVpgXt3MybxXni+X2dLLflG2YfbogAw3MdFIVC0dHRUV5eDjFDRWVlaWFJkYt/6lnj6L9cz/nApPykZf0p6+bTNk3HzGrev+h39pzexx+d+CtkJ0+ePHv2rLa2tpGRkYuLC4iB9vf3f/DggYeHh7Ozs5ubW2lpKZ1O3z/VCZ+ruMjBnrsCwX4+VZ9/Z+2oJH0tl09/PkZj7bKnfceumc/j+fj4XLp06e7du8HBwZGRkR4eHp9//vmxY8fOnDnj6+u7BWDALBvYNiEhISwszMvLCyQfOzs7m5iYnDnzGVwcydXV1dnZOSQkpK6urqmpqbq6uqKioqioqKBA3cwhLy8vF7INQsJGVsvPz09PT7e1tf0GMgMDAyMjIzQaDYojwakLa2KDoaGhm5tbenp6TU3Nll0KT+OKtrY2CoUCOsbs2Bjv0I66uroqKyt7enqYTKZUKoXLKO3Q7g/FbuTzo6ze/5gRZ2he7YwEN0H/dEZC0JypUqmW5IKlRZYKKtG2YtHBejsul1u2t+gU4FGdxBVFk9R+Bi4vRDg+vzt+hrLypwGD7EBoyJCAFFTNAAAgAElEQVTLr6izFzI10511CrIzyY87/R2IqzjoJ4kAw/MeQQqFUlFRUVVVVVlVWVlVWV1RVVNTmV9Q7BmcesUy6uMbMceMiz40rTz2VdA3+ugrl788DwVYgxTMDz744G+Qffjhh8ePH//ss8/OnTuno6Nz6dKlK1eumJqa0mi0fR4B8rzlRo63OwqAP2qlZeXRH/2zwlhLZQj9wHnP0ETgR99taVXHdh+Ue1Imk+Xm5qLRaDs7u7CwsLi4OFdXV8AMX3zxRXh4+JYDkzCQAWyIi4vz8vJycXFxcnK6evXq559/bmpqCrsX3NzccnJyuiDr7OwkEonNzc2gZlplZWUpZAUFBSAwCUYIABLw6zqoAK8DJvLy8ggEQkhICIAENze3gICAmJiYxMRET0/PFZFIwL8AzwSJDTY2NmlpaSBr+WlP/1uYDwhkbGwMeRzfnU/w3u91gmnMo/xOqZh98lQOyjcMT571ht8xFhaMG2pQBThUd9eazPBQOD61CwV/yisOMDAoVSp/vkCdvaDhXkDlZhqU5IvnVtw/Gx4JZMXNK4AAw+Y12+oW4MmJxWJBec/VNZDV1tbU1tbWVFeXlRbn5mS7+0f/6VL8ia9DzO2dbawsjIyMrl+/fvnyZW1t7c8///z06dMgrvrYsWMnTpw4efLk6dOnz507p6ure+XKFWNj48bGxq2eHbIdosCmFFD/Xe8fHHX76HUFaL8AXmFmMNZKOfXPefn5BytOQ6FQDAwMBAYGWlpaenh4REVFOTk5nT175sMPP7xx48bW2jJgnrTExESQJuHt7f3F5ctXrlxxdnZ2hQxUU42Pj8/PzwfJAB0dHZ2QgfwlkBZcX19fo27eUltdXV1cXFz4pBEIhBVUoPmWQCBorl5SUlJTU1NaWorBYIqLixsaGlpaWmpra3Nzc52cnAwNDVfHI8FzwFJzc3MXF5eioiKAN1tgg6dtApiBwWAsPcf+VirEnpcCi3MkRvf/nZ5Ie14H3C/HGZ2bN6ivUjNDz2pmUNda9eDzRTt9z1dVj68ZkjQ4eAByGBpnZvX6+7Tx6ZruBRQhizAyuF8G9XCcBwIMz3ucJyYmqqqqapetsaFR/Q1iQ2NFRQUhL/dhZOKZb0JdvB4+8Lp/756zhZW5kZHRtWvXLl68qK2tff78+TNnzny2bGfPnj1//ryuru5XX3118+ZNExOT7Ozs5309yPEOqQJqYGDwhC6od2ZvQGkMGsCghLCBcForJV5dcfLAKSSVSvPz8x0dHS0tLb29ve3t7c+cOfPJJ5/cu3cPg8FsJzYpLS0tKioK9G578ODB1atXLSws3NzcADCAV2dnZ3d3dz8/v5CQEAwGU1hYmJycrKenl5uTCzkeujo7OzsgIxKJLausvr4epDlVPGlgZn19/YotgHMA7Lm7u5tIJJaXl4eGht66dQt2JsCQAE+AyCVzc3MQvOTh4VFSUtLR0fG0p/+tzQf9KxkMhnynn58O3D35Qp6wTBg4J6t8IS9t/YsanJ29WVcJMUP3Sj8DlabH5TlweZzFnexgXVe3BjAceaN4aHi/A4NUobjNZKGqSp9wL+RjLatKZ5FfC+vfZzu9FAGGnVb0WfubnZ1taGioqampr68H/Yza29tbW1tramoIhXnJKdkhkTFR4RG+/j6eHp5379yxsLQwNja+cePG1atXL1++rKuri4JMW1tbV1f30qVLX3755Y0bNwwNDc3MzMLDw5G/rM8aAWT5FhRQQgHEIBBJqVIpoJbPSrFs2u36p4Ira+c9V5/Rig72O4jAAATicrlYLNbCwsLe3t7MzOws1F7Nw8NjO8CQkpISHh4eHBwcGhrq5uZmZGQEuxcALQB4cHZ2Br3YADyAMq/e3t7p6ekFBQWVlZVwhjFwQXR1dXV3dwNfBMAJMK35BA+m4XXARFeXGj/AJiD8qby8PCkpyd7efn33gpGRkampKSjJCpjB29u7srJS84hbgwTNrdqXjcPhHNwbaQufN2STF16B/pnpr2sr1vYzUGl6HO5tNocyt2PpvE1NE6s9DEfeKB4e2e/AECaa0OvrgXwLj4sjaeMxuvlZrUhr5+f+IUGA4flJDmIzlpaWiERiTU1NQ0NDU1MTkUgE3+rV19cXFRVBBRDTYmJigoKDHjx44Obm5ujoaGVlZWpqamxsrK+vf/369atXr16B7OrVq9evX79586axsbGZmZmtra2Pjw8oYX6w4kCe3xggR9qiAmsHFs8tKl0NdGmXX1Kinyiuqk6DRms1nNWKCvR+Wk+rA3GLKhWKnp6ewMBAMzOzr7766uTHH589e9bd3X3Lfob4+HjQUTokJMTV1dXW1hZwwopXFw0DJZVcXV3v3bvn5OTk7u7u5eX18OHDiIiIxMTEtLS0zMxMAoEAuh/UQ9bU1AQqL4NWaMAdATqjrajIXFVVVVRUlJmZmZSUFBcX5+fn5+DgYG5uDrKc18x1BjNv3bplaWlpvWxWVlaWlpYPHz6srq7ejdikzs5OPp8P7pkDceds8XOGbHaYFOiZnv6mFvIzdHWu4Wdgc75hcxqnpndEktZW8ZrAMDK6M/vfkZNcvZO6mZmLVKp2AV6z94IOIduntWHtv0mrd4HM2TkFEGDYOS03vCcymVxTU9MMWXt7e29vb1dXV1NTU1lZWW5ubkZGRmJiYiTUbsnHxwcwg62trYWFxe3bt0GhQ1DWEAQM3L5929zc3NbW9t69e+7u7iMjIwclx3TDgiEr7rECExOittaW5ta2PjJlhEofHqOO0ll0Lp9CY1t8+bkaGJ6srAqAof6sVqC32wvwxfD8/HxbW9uDBw90dXX/+te/njt3ztfXF7N5S0tLi46ODl42Ly8vN8hcXV1dXFwAM2iQwreTgBmA2wF4Hu5Cdu/ePdDL2dnZ2cPDA7SUDggICAoKCoSaxIFuxImQJSUlxcbGBgYG+kPm4+Pj7e0Nfr1YW1tbQHb79m1DyMDvljVDkkCus7m5uY2NjbWGAWaIjo5ua2vbWT8DkUhsa2vr7e2VSqV7/ElADo8osKMKUGZmDeuqUAXZqI62lbVWqTRdJusyk4WVSLYf1kkkil8/Uvb6kWKNn5I3f1gyuo+Bgb6wcJPF1mmsvZCV8m0p1ZyMrwpw9EnkV8GO3ogb2xkCDBvTaUfXYjAYNTU1jY2Nra2twMMAgKGioqKgoCArKys1NTU2NjYsLCwgIAD8Ub937569vT34u25mZmZqago4wRIyW1vbu3fvuri4uLm51dfXI8Cwo8OF7EzV19d//OjvbI7+s/ex7/oce9npb99zPPHm/bM/tz/1s7t//a7kay11K4YVOQxorV6UlvHf33/gZOvnfi/i4YOwgAeRjwIxSQnpibFpqWn7uZvvmkM+PT3d3Nzs4OBw/Pjx8+fPP3jwIB0yDNQHDbMBg+ORgJMhODg4MDAQ9IH28PBwc3MDiOAM2be44OLi7Ozs6Ojo5OQEFjlp2D3IAD/cWTYHyOzt7eEJe8hsbW01n/KtnjRLS8vbt2/DWQpPmzAyMjIxMbGystKAhceTVlZW9vb2KSkpu8EM7e3tAwMDU1P7PYJChRiiwGYUGJmdM2msRRGyUa1NK3u6UWm6dIYemxMkHJcsqZtgbtm6uqSvHynRoIXi14+U/PCt/QsMUwqFI1+gO9APocK3ndpQ+Vm5w5Qt64BsuB0FEGDYjnpb3HZ8fByEJAFgAAHEzc3NIDwAj8eDTk/AyeDv7w+YwcnJycHBwdbWFvyVB/EAdnZ2Dg4OTk5Orq6uHh4enp6eWVlZiMt+iwODbPZ0BQoJBS6f/FD0lTr0SGmotaSvpdTXUuirp1VwZaQnJxYNtcYuafVoaxH/odVxQavlnFb5Ga2qM1rWv9L68pKeaGL86Ufbv0v4fH5ycvLXX1/X1dXx8PBIS0vbSEpDWloaBoNJTk4OCwsLDg4GwAC/AnIAPVW8vLzc3d0BPAA8cHV1NTMz+81vfoNGo11dXTVgwQnQAvwKsMFx2e4smyY22EEGyMEGMmtra+AcMDMzWzMGSZMcAC1oBiOtwAZLS0s7O7u0tDSQr0zcUWtvbyeTSdPT+zqIYv/eu8iZ7VcF6PMLFm1NOgXZ2o11OlTaSmyA2rpZszmD8/NbvoKeHumTtKAGhrd+VDI2th8/TXKlMnB8/CKNdoGAWxGM5NlUu3gAC2lseeD21YYIMOzBcMzMzDQ2NtbX17e2tsINj1paWmpqakpKSvLy8rBYbEpKSlxcHMiP9PPzA48R4ItGOzs7W8gcHBzu3r0L0iK9vLwePHjg6+sbEREhkx2MVix7ID1yyC0pIFeqXeKZOJzrsVcnr2up0C+pDF96zAlPBiN9OxP4HKBkBhUa2sTkJRVaK+8TrbvGV3l8wZZOZF9spFQqWSxWbGysgYGBq6trSkoK5lkGgCE2NjYYMhgVVk9ouh3u37/v6enp4eFhbGz8r//6rzdu3FgNDHfv3gWBSStowdHR8c6yOTg4QA4Ge5gWYFeDpaWlGWQg4lGTDdacNjIyMjMzWwEJK95aWFg4OzsTCIQdD0wiEont7e2Dg4NzO5cMqkIMUWAfKCCUL93rIuoU4LSry3RGRlcyA5Wmx+Z8zWKXyGQKUHtik+fc2zv5xptqSNDAhpIf/biUSp3Z5J6ex+pJE2I9Flu7vEizMpJ2HtaghMCb2Y+E8zxE2QfHQIBhDwZBLpcTiURQ3xAwQ3t7OwCGsrKygoICHA6HwWCSkpJiYmLAV5IBAQG+vr4+Pj6enp5ubm7Ozs737t0DqODp6enj4wPCl4OCgh4+fDg6OroHV4Uc8sVVQKFORFAzQ1Jy4v0Tr0zfgGKQjLS+xYYnfQvq8CQAEoYvqQwhL4ThS0uGWqkfv+RsYTw+PqGOmjvgfVsVCgWZTH7w4IGXl1diYiJmXUtLS0tNTY2KiloNDKvnAIoA80E2grW19csvv2xoaOjp6ekOGej7pulbAH6FOxqm6Viwt39MC+DrBsi1YGNnZ+fi4uLn55eamlpWVubv778mJGjOvH37NvBwWq9rVlZWPj4+FRUVO54ADTwWo6OjoKEb4k1VIfaiKDClUPiRB1AFOO3SAhSFsjoNWpfB1GWyAoXj45svJ9rfP/nWWyVrAcO+e/7OFkv0WGyUOnUh9dvUBXz6xfysDgHvRRntA3kdCDDszbD19/eDxkxtbW2tkLW0tNTV1VVVVZWUlIBOrnl5eWlpaaGhodHR0eHh4aGQPXr0CFCB37IFBAQEBgYGBweHhIQAj0RVVdXeXBVy1EOgQFRkxMPj/75wU0uJfklhtDLdeVV4EuSIMNZSGGrFnviO2x1bsfSFcn+JxeLm5ub09PT4+HgMZOnp6cCfAN7Cr0lJSSEhIavjkVY7GVbMCQsLc3R0/N73vmdhYQE+/kFBQX5+ft7e3l4a5u7uDqKVNCkCpEQvxyg5urm5eULmAZmXl1dpaalQKFQoFMPDw46OjuuXUjU2NrawsFiXFL5daGlpGRAQUFdXtxvM0NHRwWKxlpajuhFsUCH2QiiwqFIl0amoArx2AR7V2706DVqHRr/I5d1ic5o3GZhHIsl+9OPSFcDw4/8spdH2l4cBL5VeZHNQHW3qOqrZGqkLhOzMQdILMcgH+CIQYNibwRsbG4Pzntva2lpaWlpbWxsbG+vq6kAD1/LycgKBgMVi09LSUlJSkpOTEyCLj4+Pi4uLiYmJgiw6OjomJiZ62WJiYsLDw1NTUxGX/d6M6yE4qkKlCgkKfPTBvyzqQ/kMK3wLT75VGn1HZfydRQOt0A//ydv97tTMjpUV31dKi8ViIpGYmZmZmpq6Ogc6DbK4uLiQkJAVMLCRtzAwWFpahoaGPm0T8D3CQw0LWGWPHj0KCQkBXz2EQNba2qpUKmdnZ+Pi4vT19TWdCSumjYyMbt269S0QPGsK5EUEBgbW1dXteGwSSJDg8b79uhFhhn31iUBOZjsKFPN5X5YVqtOgW5rWSGmg0vRY7IsMZoRINLFhVwOFIvvJT8tWAMNPflq2f4BBoVRmiCUX2Vyd3u4LTzZ11inIfkhskSOpC9u5q3ZiWwQYdkLFze+Dw+HU1tY2NqrbPLe2tgJgaGpqqq2traqqKi8vh/0MOTk5OBwuKysrMzMzHTLwRJKamgoeRDAYDCCK5OTkxMTEuLi4yMjI4eHhzZ8UsgWiwLMUgMJn5xeX/Lw8Yj76/5aenvEMJzPM3dQKOPavQf4PZucX1JFIL2j1bIlEAoqbAWbAPGmpqamghsHTHvfXmR8WFubs7Pzyyy+bmZmFhoYCH4WmpwJmAEAC8GtYWFhoaGiYhmmuCeilpaVFpVL19fVZWFgYGBisgATNtysaLzyLF9SJ1IAZIiMjm5ubd4MZOjs7BYLHzRmedeMiyxEFDpICpOlp85YGdRp0VZnO0NDTXA0mLHa1TLaR8kmDQ1M/+/lKYPjpz8ro9H3hYVhQKmPGRbpsLmqg/0IuVjPRGZWfdaeucnpH+14fpFthP50rAgx7Mxoikai2trahoQEwAwCG1tZW0FYJOBkqICsvLy8tLS0sLCwoKMjPz8/Ly8vNzcXj8TgcDrzicDgsFpuRkQGqtaSkpMTGxhIIBLlcroRsb64QOeoLrcDM/KLHvTuJJ76jULsU1IkKmsFISuPvqGeitea+0fL627+Hh4YsytUpEC+kge+26XR6TExMSEhIVFRUQkLCikzopKSksLCwdahgnUVhYWEuLi4vv/yyqanpCh8F8BLAhLCpCRgYlErlhGgiNDQU9jCsaL9gBNnqxgvPZAZra2tQNCkzM3M3ApPa2to6OzuFQuELeV8hF3XIFRAvLfmR+lCEbO2iXFQPFJ5Eo6sdDho/ekyWLoPpIxSOQl/HrKPYyOj0L/6r/PUjpZpJzz/7eTmDMbvOVs9nkUgu9xYKL/L4alrIz3qCFvKwRuWFXCTR+fmMxLOOggDDsxTaneVTU1N1dXX1kDU1NWmWS2pvbwc+h5aWFsAPDQ0NwPNQWVlZUVFRVlZWCllJSUlRURFIeMBDlp2djcViU1NT4+LiyGQy9IXuC/qN7u6MC7LXjSswOT3jYn0b+wlgBi2lRjDSkpoWXpJc03L78D8S42IW5Rv5CmzjR95fawJgIJFIYWFh8BN8ZGRkfHw8jA1bjkd69OgRDAzAwwDCiuADbQoSNFfWBAaVSsVkMn18fEAOw2pgALnOmj0cNkILYB1LS0snJ6fc3NwddzIQIevr60Mauu2vjwRyNjukgEKlKuHzrpYXqbGhsVZndGwtVwPtIod7jclKFktET49QGhub/uXbFSuA4ee/KGcw9tjD0D83Z8Hhqmmhr1dNCxqJzqg87PWi3CGxukgGYvtBAQQY9mYU5ufnGxoa6uvr6+rqmpubiURiJ2RdqwzMBxTR3Nzc2NgINqyFrKamprKyErggCJDl5+fjcLiUlBQ8Hg8KiezNFSJHfcEVUIOoUCK9c+sbwmffUdOCZn1VtJbgitbd42+kp2fI1ZmpyoNeE2mdwQTAMDAwEBoaCp7j4af5iIiI2NhYUO5sHR/C+osAMHz3u981NzfXZBLNp/8tTK8ABpVKRaVS3d3dV+Q9GxkZodHodRovrEMOMGBYWFg4OTkVFBTsIDN0QFZZWZmTk8PlctcZIGQRosCBVoA6O+fU0aZmhtICVF+vmhlWuRp06YyLXJ4Bk5UnlU6tFetPo828/auVwPCL/ypnMvfMwzCvVOZIpFeYbD0OF9XddSEPq5nlrJ2b+VUBrpv/bZ7SgR7EF+PkEWDYm3GUy+Wtra319fVNTU1tbW1dXV29vb19fX39/f0DAwPglUKhDA4OUiAbGBjo6+vr7e3t6uoCzv02yJqbm+vr6wE2lJWVlUBWWFiIx+OTk5OZTObeXB5y1BdeATUBqJmBPy62u3mx4vRLamYwhH6MtbhfatkdezMnL39rJcMPonhDQ0MRERGaX//D8BAGGXhAX58N1lwaGhrq7u7+k5/8xMbGBgYGeOeaE5vCBlCyqaW5GQAP0JxIJFpbW4NkBhCJZGRkZGpqug4VPHMR6AltaWl5//79kpKSrcUmwf1qwHcrXV1ddXV1SUlJLi4uOBwOqfFwED8yyDlvXIE5pTKHxbhSVqBTgEM11OgMj6wuuqpDpekymHocjhmbUzwpW4ENTObs//y26kkPQ+l//Xc5k7k3Hobh+QVXvkCPw9VlsFDE1gs5GZq0gMrDXinEd/A4G5cIWfM5KIAAw3MQeY1DKJXKjo6OhoYGIpHY0dHR3d3d399PhmxwcHBoaGhkZIROpzOZTDqdzmAwaDTa2NjY6Ojo8PDw4OAgiUTq7+/v7e3t7OxsbW1tamqCsaGioqK8vLyoqCgjI4NIJK5xbGQWosCOKsDgsK2u/L3hPMQMaHV3Z+uPf1pQVqL5JLqjB9yPO2MwGNHR0aC6MXiIXw0PmnPWZIPVM8GugoKC3NzcAgMDNfFgm9OPHj0KDg6GgQEM1uLiYm5uLhqNhmOTbt26ZWVlBfsKnokH66wACq02NDRsgRkAMHR1dXV0dNTW1qakpLi4uBgaGkZFRSHxSPvx84Cc0y4oMDY749LRisrPRhXnoTqJUAElhmZKA5hW11Bic8zY7MLJScly6WEOZ+4Pf6x+/UiZRg5D6X//soLFet4eBvHSUrJYfI3F1uPwdMao2g21F3BPVlDNzzIozR8QIblJu3APbW+XCDBsT79tbN3T01NfX9/V1dXT09Pb20sikYBLYWRkhEql0ul0NpvN4/H4y8blcjkcDpvNZjKZgB+Gh4cHBga6u7uJRGJLSwuIVoKrsubm5lZXVx+qh7ZtjAay6VYVgMoejTBYlrqfdF54aURHy/rUrytr66AbT3F4EmhWA8M2H+hXbA46sayYuZ23K4ABzneamZnJzMw0MTEBzGBubm5trS55tA4JPHMR2By8RkdHb7ZoUkdHR2dnZ0dHR1VVVXJysouLCxqNvnnzpr29PYVCAb/ikF90W/0AI9sdJAUWlMoiNtOgsliHkK1dUYLq71OHJ62KUNKBSq/qcbjGTFa6RMJaXBSNL/z5zzUrgOGXb1ew2c+v1PWMQlk0OWnCYl3k8nSZLJ1BinZFMZS0sNxvAY/RIWQ71FYwZNKDNCqH5lwRYNiDoQZ/2/r7+wEw9PX1DQwMkMlk4FigUqk0Go3JZHK5XIFAMD4+LhKJJiYmRCLROGR8Pp/L5bJYLBqNNjQ0RCKR+vr6urq6QJ5DU1NTQ0NDTU1NUVFRWVnZIlKMbA9G+HAdElDBwOCI6YUPb332TkuburQ/yFs4PMDA5/MTExM1PQzbeZpfvW1ERMSKHInV62xqDgCGpqam1Y/a09PTcXFxcDDSNmlBEyesIIuJiWlpaYEyltd4AXkOcAwScEdUVVXFxsbevXvXxMTEyMjIwMDAwsKisbFx9ckfrs8ecrWHUgHh4kLIQI9OHlbdq6GmEkUmPw0bdJnqp/PrbJb7IOf371W//lqJpofh7V9VsNnPw8MwuaQolcks2Rw9NkePxdGh0VGdxAuEbM0UZ+3cDO3czIhuIlJBdd/e1Agw7NnQkEikurq6rq6uvr4+Eok0ODg4PDw8OjpKo9EYDAaLxeJyuXw+f3x8fGJiQgyZRCKZmJgYHx8XCARcLpfBYFCp1OHhYQqF0t/f393d3dnZ2d7e3tzc3NDQUFpaWlVVhQDDng3w4TswZXiUPDh8OJ/hxGIx6MsevnkDDROeloEQFhb28OHDq1evurm5AT8DvOamCGH1yoGBgSUlJfK1KquwWazMzExXV1dLS8sdBAZQaNXe3h6DwYDma2sQAzQLeBWIRGJ5eXlMTMzdu3eNjY3hhGwDA4P4+PjpTTa7PXyfSOSKX2QFiKJxm4YqVB5WOz9bu75ah0JZMx9a7W1gMrQpzJ++X/XaD4qeAIa3Kzi77GHgyeU4ieQWi61GBTZHnXpBoWhXl1/AYTSTFnQI2TfLCNVM+os8YAf/2hBg2LMxBMDQ3d0NgAG4F8bGxkDqAgwMQqEQeBgkkE1ABoCBxWLR6XQqlToyMjI4OAgSo4GroaWlpaqqqqamBgGGPRvgQ3ZgtU9BfckvckGkdYYUAENYWFjEVu1poBEZGenn5/f2229bWVlFRkbCq0VERIB06s2+wp6KoKAgPB4/O7v2V4xyuby6utrOzs7CwmKnmMEKMgsLC2dn54KCgjWTGTo6Orq6utra2ioqKqKiou7cuWNsbAyKNRlDpq+v//DhQ6Qy0jp3I7LokCgwp1AUMmgGFYU6+RA21FWhSCTI2/BkbgOddoE09rMPngCG135Q8ps/VDH4a3/8tyngjELRNTsbOj7+jdrFwdVjsdWoMDqq3doM1U5NUQMD9KOdm6mdj31AbOZMT23zoMjmu60AAgy7rfBT90+hUOrq6gAwkMlk2L3AZDLZbDaXy+XxeAKBQCgUgkgkgAoiyIRCoUAg4PF4HA6HxWIBVwPABhKJ1NPT09bWVlNTU1tbiwDDUwcAWbDDCihVSuBdODyBSN8qCAMDeKCPjIyM2DkLDw/39/cPCQnZyC7Dw8PBavDE6q3AST569Ki0tHSdXxHz8/O5ubk2NjY7BQxweJKlpaWnp2dZWRlgBhCABLwKLS0tBAIhKCjI3t4eRgU0Gm1iYgKysc3NzTs7O8GtpkIMUeDQKyCan4sf6P2yEK/2NuRmaleVqRu9jVG/dTjQaKhR6i8+rnn1+489DK++UvTTv1SdxpNvsVjBQmGpTEaem5MuZ0hvTdE5hWJ0fr5KJgsXiW4xWTo0mjpXgcFSo8LYGKqz/UJRnjoGKftxxoI2Pl2nIBtdVVrNZrywfT23JuV+3QoBhj0bmcHBQQAMoD4SAAYqlcpkMlksFofD4fP5QshAGoN42UA+g1Ao5PP5gBlAMaWxsbGRkREKhdLb20skEmshW+dpYM+uHDkwosALpwAckhSxOxYVFbVBCIE5ITo6OqyqSBkAACAASURBVCoq6mmnEx4eHhYW1tLSsv6T98zMDBabuf28ZxgVwISVlZWFhYWXl1dFRQVIaAYF3wgEQmBgoKWlpYGBARyAhF42kFZRVFSE/Fp74T5AyAVtVwHqpDSwo1UvF6uTl6mNT9cuIaDaWx4XYKXRdZn0/z5b++orRa+/XvzqK4W/vtRwvmNEj8PQY7IucrgXubzLNLo+g3mHww0bH8dJpfXT072zs9SFBdHS0oxCsaRSKaDfFErodUmpXFAqRUtLwwsLjVPT6RKJJ4+PZjK/UreDUO9NjwlxAp2hMzKK6mi/UJyv5oTs1MdeBXy6DiH7y6LcBApJjKRZbnfkn9/2CDA8P63hI4G/0IODg/X19XBB1RUeBuBegDOeNdMYxGIxyGQAzMDlckHpJCqVOjo6CvIZOjs76+rqEA8DrDkygSiwqwqIxWIMBhMWFgaihiJ22mJiYtZ5+oePFhkZGRUVFR8fn5SUhMFg4uLi4EUrJgAwtLaCDHXVOiYSjYeHh1tYWKx46N/mW8AMAQEBtbW17e3tBALh4cOHlpaWhoaGoNX0MiZ8+/8333wTHh4+OTkJ13Ra57SRRYgCh1AB0sS4T2uDTk6GTh5WXa6UgNNurEUNDOix6b+52vTqfxS99oOi3xs3o4aouky6ZklWXTpDl8nSY3PU/MDj67G5elTaFTrjGwbTkMG8xWJbszk2bI4Dh2vH5pix2MZM1g064wsaXV3FVb0+R4/J0qVDTeXUrzQUmazdVH+hMGc1KugV4AJ6OsamZIdwgA70JSPAsAfDtxoYKBTKyMjI2NgYjUYD7gUejwdnLwBakEAmlUolEolYLAZFk0BgEpvNBvkMY2NjQ0NDAwMDXV1dDQ0NdXV1yFdxezDAyCEPnwJSiSQ3Nzc1NTUpKSniSYuE7Ml5m3gXGRkZHBx86dIlZ2fnqKio8OWIo4iICM3pqKgo0FUag8Fgsdjs7Oz09PTo6OinHWnjwKBSqSgUiqenJyixuk1O0NwcRDoFQWZlZQVQ4Vs+eHLKyMjI1dV1ZGQE/P5UIYYogCjwFAX6hQKflvqLuZk6+VnaeMyFnAxUdeEfblb84LuFv/umUYdG12Wt0S5akx/U0zS6miIYTDVIMFl6LLbmjy6TpV4ECEFd2pXxuJfc4CCqvVW7ouRCbuYTAUg56gAkvQKcf08HeRKpmvqUkdvfsxFg2IPxAX/whoaGYA/D4OAgAAY6nQ6nO8PuBRCLJF02QA4gpQGumMRms0F/t+HhYRKJ1N3d3djYWF9fjwDDHgwwcsjDp8CkVFpcXJyVlYXFYlNSUpKSkuLi4qKjo9eJI1pnUYSGRUVF+fn5vfbaazdv3gQAEA4ZcCbExMQkJiYmJyfDnJANWVZWVkJCgsZuVk5uEBjgp/Pe3l53d3dLS0vNJ/7tT1tZWd2+fdsQMmNjY5CoAAoigURnQA2GhoYmJib19fXgfOCzUiGGKIAo8BQFRiQT0T3Eb4pytXMy9Iozf4vK+f7Lxe/blugRq1GdRHV69OiYDpWqTpKmQ4/73z7901bCAxWaA3o+wHgA1h+j6gwPo3p7tFsatcsKL+Ri1aFHcK4CPh2Vh9UpwF0rJUSQekcQr8JTButAzEaAYc+GaXR0FPRh6O/vBx4GKpXKYDDgjGfYwyAWiyUSySRkUsiAkwH4GUDFJBCVRKfTR0ZGyGRyd3d3U1MT4mHYs9FFDnzIFJBKpUVFRVgsFgdZdnZ2ZmZmRkZGWlpaAmRxcXHA1QBewzUcBRHrWnR0tJ+/32uvv3ZT/2ZsbGxcXFxCQkJ8fHxycnJ6enpmZibwJwBOyM7OzsrKAkePiYlZ5ygbBAbNYWxpaXF2dt5ZZjAzMwNIAF6NjY1v375tamp69erVa9euwfyARqNTU1OnppA6KpoDgkwjCjxbAfHcbDWD6tVU9/axwle/X/LDXxR+5JGByk/VzstQByyVFWrXVKAaa1GtzaiuTtRAv87goM7IqJolwM/Y8sToKGpkBDU4iBoYQPV0ozraUc0N2jUV2qWEC3lZF/DpECQs5zTj07VzM3UKcDr5WbZNtTm0McH8/LPPFVljfyuAAMOejc/Y2BjowwCAAe7AAIABdGAQiUSAFmBggLEBMINIJBIKhZrlkkAaQ29vb3NzM+Jh2LPRRQ58yBSYnJwsLi5e8ewOf9mflZWVmZmJwWDSli0pKSk2NjbmWRYbG5uamhoZGfnmm2+amJhkZmZmaRgMJzAtwBNZWVnx8fERkK3pygDA8MykZ81hXFxcyM7ONjc336miSebm5jASQBVTjUH1JC8vLysrKwMDA0ARoI6qWCzWPBlkGlEAUWDjCnA4s7/7Q9XrR0pfe7X4jR8VHf0Gfy4Vg8rDaKv7IaReyEpRP+7j1MFL6lCivKwLBfgLhTnaRbkXivMuFOaqUxEIuAt5WPXSnAw1HmSnPd4qWwMScjKAP0EnN/NWVUkiub9fIl5UHsa6eRsfmgO0JgIMezZYVCoVVEkaGBgYHBwEwMBkMjkcjmYCg1gslkImgwwAw+TkpAQykP0MZzIwmUwqlTo0NNTX19fS0tLQ0LB+SJKmZ1+pYSqVCn4HBILfam6yZ9ohB0YU2GcKrAMM8EN8dnY2eMTH4XAAITKfZVgsFo/HJyUl/ed//qeJiUlubq7m3tafTklJAaiwDjBsJOkZKA0++CKRKDY21hKybcYjgWAkNGSgfKqXlxcWi83NzU1ISHB0dESj0SA2ycbGpqOjA/nNs89ueeR0DpIC/f2TP3yr9PUj6k7Pr71W/Or3i392tEDnUf6VoiztnAydAhwqH6udk6GNT4cKGaVBmcprveKgoqh4jLoQU06Gdk4GKjdTh5CtU4i7gE+/UoBzrK/OGBzonxDNba9I60ES99CcKwIMezDU4C8fjUbTzGEAGc9MJhN0YBAKhSBLQSKRSKVSmBNkMtlqJwMABtCTAfR+7u/vb21tfSYwzM/Pl5aW4nA4kUgEQ4JUKsVisUvQpx1wgkqlUigUcrlcoVCXVhMKhX19fXsgHHJIRIH9qsAGgWH9R/w1lwJg+MlPfoJGozcFDBtJet6UhwFoz+fzQ0NDrSDbMjNYWVmZmpqaQIZGo42MjAIDAxsbG4lEYllZmY+PD/A8GBoampubV1dXI7SwX2985LwOhgJtbeIjb5RrtHkufv3VyuvXOwVzU3UMWlJfl3N91Y1C/KXczAs4DCo/S7cQr1uI0ynE6ap/8LqFeJ1CvJorCNnaeWq0uJSHvUrAfVOcZ15V+rC9CUvub+ayJ+ZmlxB/wsG4I7ZylggwbEW1bW6zAhhWeBjWBAbgXpDJZFNTU7CrQQpVTJqYmNCsr0qj0UZGRgYGBlpbWxsbG9fxMCiVSoVCERoaamhoOD8/L5fLJRKJSqWSy+VCoVAB2fT0tFKppNFoOTk5CwsLCoW6v0pDQwMajd6mCMjmiAIvkgKbBQaQabAmIayYCQODsbHxpoAhOzs7MTEx4ikWDiVRbK0DGolEcnFxAUWTbGxsNogNmmuam5sDBwJ49fX1bWtrGxoaam9vDwkJgRfp6+tHREQgqQsv0icFuZY9UaC8XHDkjQpNYDjyRgUK1QafjEKpnFpYGBVPtHBYZdSRnCFyJrk/g9yXSe7HDZLyhgcJYyNFtLFyFqOOx+0SjVNlk5ML83Ny+QL0VADvB5l4gRVAgGHXBxd6Ll9aUiwp1V/Pq5RQtI9KpaLT6cDDMDAwMDQ0NDY2RqfT1/EwAFqYWrbJyUmpVAr3ZODz+aAhAwAGEokEgEEul6+6wsduA6VS/fQPOiXNzMwkJCQEBwdnZWWNjY35+/sPDg46OjrGxMRER0dnZ2dfv36dx+OBXbHZbAcHh1W7RWYgChxeBTYLDCuoYJ232wGG9PT0FZVYI5YtPDw8Kirqfx/9t/DlvUKhqK2tdXBw2FQCNAwMmsFIRkZGlpaWBQUFAwMDFRUVQUFBt27dMjY2RqPRBgYGnp6eLBbr8N5VyJUjCuyQAnn53FXAUP7Zmaa5uaUdOsKu70ahVHrU1lZRqeBInVyuU3V1wdDQrh8YOcCyAggwLCuxa/8rlAqlSqFUyRWqJYVCrlwCDRNVDAajvr6+p6dnNTCAHs8gJAmOR1omhcf/awKDSCSCayUxGIzR0VEymdzW1tbU1LQWMKggdFEDjEqlys3NffToESghv7i4+NVXX/X19d29e5fFYpmamk5MTKDR6Obm5pCQEFghJpNpb28Pv0UmEAUQBfYnMGCx2NjY2Ii1bDvAAPyQlZWVDg4Om0qAtrGxsbKygisjgdQFe3v75OTk+Ph4BwcH0N0ZjUYbGhqamprW1NRsgWeQuxFRAFFghQLZ2ZwVwPD6G+WnTjVOT6/+SnHFpvvlrUKpNCooeDssbGphoWBo6A9RUZ+mpf2bl1dSd/d+OcUX/TwQYNj1EZ6bneYIxFzhJEcwMTU7o8YFqGaAJjAMDw+DmqqaTRhA+4XJycmpqakZyDSZASQzSCSSiYkJAAygVhKTyQTA0N7e3tzcvCYwLMoXZTNzSqVcJpPhcPiwsLC0tLTAwECVSmVlZUUikdzc3AQCwZ07d6amptBodENDQ1BQEEhyUKlUHA7nzp07uy4ccgBEgYOjwNz8fHV1dUZGBvZJ06hptPbkOr4FsAiPxycmJv74xz82MjLabEhSdnZ2cnJyxFq2ZWCA85rm5+dxOJy1tfWmmAGmBfSy3bp1y8zMDFRJAsFI+vr6JiYmeDx+YWHh4NwCyJkiCuxfBdIwzJXAcKT845ONU1MHBhhUKtXE7Oyvw8ONCgoMCYQePl+lUvk1Nb3y4MEglIS5f9V/Uc4MAYZdH8mWHup7l0P/di3qzxdD8ip7oeOpiQEGBhKJNDQ0pAkMAoFgfHwcAAOLxQIriMXi6WUD5CCTyUBUElxclcvljoyMVFdXd3V1dXZ2Njc3a+YwQDFIytnZeZeQYlNPvHxpKTQkzNLKsre3d2Jiws7OLi0traCgoKur6+LFi4WFhZcuXaqsrDx37lxNTY2FhQWHwwFiFRUV6ejoCIVCKNpK7aZADFHgkCuwtLTU19dXXFxctGyFhYX5+fl5eXk5Txoej4drJeFwuOzsbIAYOTk5+GUDK8BVlTIzM93c3CIjI/F4fFZWFtgKTICCSzCLrMaPzMxM0JABJC1ELBsABjJ5KyFJ8FgLhcLQ0FALCwuADWuSAxyJZG1tbWlpeevWrWVS+PZ/QAvGxsYGBgYmJib3798vLy+fnp6GD4RMIAogCmxHgaRkxmpgOPFRg0x2kIBBpVJVUanfcXe/npsL1FhcWjqdmnoqJUWOpFJs5/7Y2LYIMGxMp22sRRrl/vly+J++iv+NdmhIah20pyeAgUwmPw0YxsfHQZVVKpU6Ojq6zAvTMDBMTk6CNAaQ98zj8SgUSkdHB4lE6urqamlp0QAG9UFprHF9l6z3b+b+6WtcFLZ5dm5GKFBjukqlmpqaotPpKpVKIpEARwebzeZwOAwGQyaTiUQikPS8tLTE5/NZLNbMzIwCMrA58ooocMgVWFxclMlkEqisGShIwOfzQQt2uoaBz/LospHJ5M7Ozrq6uuDg4IcPH4aFhcXHxycmJiYlJaWkpKRBhsFgQBnWjIyM9PR0DAaTmZkJswFMFyCRWjOdGmwFWj6HPmmPHj2KjIwcGRnZZswPlUr18/OzsLBYp24SAAnN1AW0hpmYmMCo4OnpWVJSwoe+O4Rdmof8vkIuH1Fg+wrEJ9APNDDQJBIchdLO4SwpFNZlZa/7+/OXv1AYFIm+7+sb1NKyfZWQPayvAAIM6+uzA0tn5+d0LJKOXop753LMVbuMxeUsZNjDoAkMbDabx+PBHgbQZkEikUxPT8tksjWBAUQljY+Pg+KqTCZzeHiYRCJ1dna2trbCwKBQKitaKGeM4t7VL/jEtu1j87p3LkVWtg2qS6Yql5TLkVKgcOqKywaPFGAR/HgB3Avw2xWbIG8RBRAFNq4Ai8W6devWl19+efv27bt373p6evr5+T169CgiIiI6OjouLi4JsoSEhNjY2KioqOjo6JiYmJCQkLS0NBqNRqfTBwcHSSRSX19fP2R9kAFPI5FIbG5urqmpIRAIsHMD9IRmMBjb/wj39va6u7uDoknrVExaHYwEApBAgzYvLy+ACts/n43LjqyJKHBIFIiLXwMYPvr4YHgYUnt7fxsR8Zf4+J8FB7ez2dOLi78NDzckEOCxi+zoeN3fnzE5Cc9BJnZDAQQYdkPVFftUphd0/QYV9uer8e/ohhRWP45KYjCZ9fUNvb29ZDJ5eHiYRqMxGAwYGEQikQT6qnJychLUR5qdnYUSGWYANoAcBvBFplgshoGBxWLRaLTBwUEYGNTVU5VKOkfkHVv14bWo47dKP7ZqOn4z9c7Dstjsxpm5RaVS7ZRc/++0ErKVFwayMVbMRd4iCiAKbFIBLpdrYmLyxeXLpqamTk5Ovr6+ISEhMTExSUlJaWlpGRkZUVFRqamp2dnZqampwGMQEBDg7OyclJQ0Pz8PHw18Th9/WjXfQDWU5+fnp6amwC+QqakpiUQCf6EA72ELEwqForS01NbWFi6atDo2ycLCwsTEBK1hwKuARqO9vLxKS0thr8IWTgDZBFEAUWB9BWJiaas9DB+fbNifOQxzy9+rqlSqPoHg12FhzUymOkRCIlmEQo/q6PR/8/IqGB4GVz0nl1fTaLOLi+uLgCzdpgIIMGxTwA1tPjs3Z+CU9VtU1NGvYk4bxI0xhSqVksFk1tXV9fT0kMnkkZGVwAByGECJJBCABGhhZuZbYIDznlcDw9DQUFdXV1tbm3xxUaFcWlDIVaqlsoaBD/UzT9o0v3MtIxJTr1KXeFXIl5bUsUqIIQogCuydAmw228TE5Nq1azY2Nu7u7g8fPoyMjExISEhLS8vKykpPTz9+/LilpWVhYWFubi4Oh0tNTY2MjHR3d09OTtYEhr26gunpaSwWq5kADTMDCFUyNTUFxVJBAJKhoSEajfb09CwuLhYIBOt/W7FXF4UcF1HghVEgOmYNYDj5yf5KehbNzICnEdvycjyZ3MJiSebmEru7jycmrhgIpUrlUFHx48BAJuJVWCHNbr5FgGE31YX2Df4WsrgTupYJv9OJ+OOlKJRFEo0zwaRTcbjsuvq6rs4uMpkMkp65XC7o8SwWi0F9pOnp6VnI5ufn5yADb4HDAYQqgUwGkPrM5XIZDMbw8DCR2N7Y2CRflKtUUL+F6v5j+piPLJr//HW2Y3Dh/MIC8DxAgUZI4vKu3wbIARAF1lFAExjc3Nw0gQELmZ+fX2xsLIFAyMvLw+FwGRkZiYmJ/v7+GAxmbm5unT2vs2hnH9Onp6cxGAxgBs1EZ2tra9CmDQQgGRoampiYeHh4FBcX8/l8zXN47BhZ54yRRYgCiAJbUiAqer8DA0kofDcmhgZ1jw1ubf2uv//p1FTu1FQdnf4DP79+gQC+breamoTu7tnFxfPp6dVQ4iW8CJnYVQUQYNhVeZ/YOYMnvn4n4ze6Eb+7GHHOJCklp7qoKL+woJBAyM/Pyy/ILyosIJSVldXX1zc0NDQ2Nra3t3d0dPT29pJIJDJkIyMjo6OjI5ANDw+TyeT+/v6+vr7u7m7QdaEOspKSEjwen5mRRSgsVCrl03Pz4Rm1H9xM/9utiveuYe9Hls/NzatUSyAS6YlTRN4gCiAK7IUCABiuX7++2sOAxWLxeHxBQQGouaQJDEFBQZmZmfvBwwA0EwgEcNEkmBlAZSQQgGRsbOzq6kogELhc7l7IjBwTUeCQKhAVRV0dkrSvPAwzi4t1dPrCkrqRnG15+Q8CAw2gLIV5ufyvcXFnMZil5TpIn6WmOlZWqrvBLM85pIP63C8bAYbnKrlsesYvrvr9L6N+pRN+9GKEqXtmbX07dWyYRh2lUsfGqDQajQaKqIwMD4+MjAwNDcGQAKZHRkaGIRsaGhpcNgqFQoKMQqEMDlKA9ff3tjXXNxAHb7jg/3wN96cb+efM0vOre6Fe0yqlOs0ZcSw819FHDoYo8DQF1gcGHA6Xn58P3AuawBAYGLivgEGlUvX397u7u4NCq3AdVeBVcHNzA6igWP4zr+leeJoyyHxEAUSB7SuwJjDskxwGqkTCWo4s8q6vf9jcPLO4WEOj/T9v71wKRaVSETmcl3189AmEAaEwtbf3l6Ghncg3Dtu/Jza/BwQYNq/ZVreAgn/U9NwxQLvtgX/3UtjbF0I+uB7pGpxf29rHYjKEXK6AzxMKBaIJ0YRYLJNNyqZkU9PTIHthYX5hYV79T/3f/Nzs7Mz0zPT0jLp6klQyKZqQjItE40KhgM/ncVijo2PZeXU3HNPevZb2+y8zPjZM9Y4u541LoZpIalSAvP9bvRJkO0QBRIEdVeCZwBAfH5+WlgacDHBI0j4EBpVK1dbe7uzsbGlpaWVldevWLUNDQ+BVYLPZMCHAEzuqIrIzRAFEgbUVeAow7HEOQzeP94/MzJ+FhLwZGOhWW6tQKhO7u//Ny6sL4gHHysr/DAriTU2pVKpGBuO9uLj/iYz8KCmpkkpd+yKRubusAAIMuyzwU3avVCryqztv2Ce8fyX8v84F/+nyIxPXjPSClj7y2LhQKJOKZFLxlGxycmpyanpqdmYGQIM6b2FO/TM9MzszNaNuxyCbmlKDhVQ2JZFJxQKBoK1rMDS58rJt+h8uxf/hctLntzGPUhtpLOFTTgSZjSiAKLD3CqwDDFlZWVgs9r333gPtFEHSM8hhCAwMzMjI2HIOww5etmb6wZJcXlJSYmVlZWJiYmtrm5qaSqfTNQkBrKw5ZwfPBNkVogCiwGoF1gSGPSyrurC05NvY+PPQ0DtVVd08XhSR+C/e3tkkkkql+hKPfy82dk4un15cfCcq6lpOzpxczpqcnF9aokuloErS6gtE5jwHBRBgeA4irziEUqGuWaRisWhNTbWVNe2PEkqMnDOOX4v+rXbAe5cffWmb4hNbWlLfPzjC4gtF4yKRUMiXSSVLiwuKpUWFHPpZnJcvzi7Oz8mkMjZf2ENh4EqJzqHFupYpf7r06DeosBPXo6y9s6NTSomd3dDhl5aQAKQV44C8RRTYNwqw2exbt25dv37d1tZ2dZUkLBZ79OhRbW3toqIiGBgSEhIePnyYkZGxf3IY4OrMEokkIyMjPj6eRCItLizsG5mRE0EUOKQK7CtgoEokZzCYv8bFtbBY8HhczMpCZWaqH40mJ9+CHA5qdyWb/X1f31+Fh1/PzUUyFmCt9moCAYY9UB5KHlAMj4w2NjaR+3tGR4ao1NF+8nBRZadvTJmJezbKNP7Tm1EfXQ8/ZxJ71TbZxCXd9gE2IKE8Ja8ttbA9MbctNKXufmS5nX/BdYf0M0axH92I+uRmNMoswcY3Lz67AZdfU1paQWdQKaT+DmL74uKCOgJJgVRP3YOxRg6JKLARBTYCDCgUav8DA3yxMzMz+8H1AZ8PMoEocJgVWBMYTny0N43byOPj/+bt7VJTozkiOpmZxgUFYE42ifTv3t4AJ6pptKC2NuHMjObKyPSeKIAAw57Irj7o0NBQQ0PDch+GESaTwedxxoU8qVjtUqDS2Z0kRnlDf255R0ZRe1x2S2RGY2RGY1RmY2RGQxS2MRbfnF5EzKvsqW8dJA8x6Wy+YFwslUgkYhGPyx4eVqdE9/f3NzQ0jI+Pw9/87dnVIgdGFEAUeLoC6wNDVlbW0aNHDxYwPP1akSWIAogCz1uBNYHh+ImGycm9aXYW1dHxL/fvAyRYWFpyqa7++aNHFOhZBUhzLTf395GRU4h/8nnfKesdDwGG9dTZpWUgeJdCoWgCA51OZzKZHA5HwOcLx0ViiXhmanJuVp2pMD8/q5AvKJcWlQq5CvpRLi0uyRcX5+fm52ZmZmSTU5PSSbFELFL3exYKeTwei8UaGxsjk8ktLS39/f1qB8NyZZJduihkt4gCiAJbVuBpwIDBYLIgO3r0qI6OTlFREVwlCYQkpaenI1/kb1l2ZENEgUOiwFrAUPa3v9VJJHsTMbikUPwdg3k3JqaFzT6dmvpBQkIPn685FuzJyUetrZotnzWXItN7ogACDHsgOwAGEomkCQw0Gu0xMAgEwnGhaEIkEUulUunUlGxmWjY9Oz0zNzs7Pzc7Nzc7PzczOzszMzM1PS2TTUknJyelateCSCwWqZFByOfzuVwunU4H/Z5ra2v50EcRyTXcg8FGDokosAEFNgIMKBSqsLBQExgCAgIQYNiAusgqiAKHXYE1geHP79aIRHsDDCqViiqRvBEQ8Iq/v2tNzezi3jg6DvttscnrR4Bhk4Lt3OoDAwONjY2gL9vw8DCNRmMwGGoPg0AgFApFIpFEIpFKpTKZbBqqrDo7Ows6Pc/NzWm2eZZK1cigjkUSiycmJtROBoEAdjIMDAy0trY2NjbKZDIkMGnnRg/ZE6LATiqwPjBkZ2e/++6758+fX+FhQIBhJ8cA2ReiwIurQEzM6k7Ppf/z2yqhcH4PLzqhu/uf3N01U5/38GSQQz9TAQQYninRbq3Q19cHgIFCoewIMEgkkomJCZG6qpJQIBBwOBwGgzE0NNTX19fS0tLa2ioWixFm2K3hRPaLKLANBdYEhsTERBCShMfj33///TNnz8DAkJ6enpCQ4O/vj8FgkJCkbQiPbIoocCgUCAoaXdXpueSHb5VVVu5lyXWlSoXKzPxtRIR0fi+55VDcATtxkQgw7ISKW9pHd3d3U1NTb2/vCmDg8/maHobJycmpqamZmZnZ2Vl1w7Zlm52dnZqagsKRpFKpVAwZAAZ1WNJyJgOVSh0cHOzt7W1ra2tpadHsnbSls0Y2QhRAFNh5BTYCDJ999hkCDDsvPbJHRIEXWoGRkWk7u4H/+u/y14+UvH6k+Mmfkp/9vPzGN92treovE/fEGFLp9319bcvK9uToyEE3pQACDJuSa8dWViqVHR0dTU1NPT09mhXPdAAAIABJREFUJBIJeBjgHIbx8fGJiQmJRDI5OSmTyWBgWICaPC9ANjc3Nz09LZVKJyATLds4ZMDJwOVyGQzG2NjY4OAgiUTq6elpbW0dHh6Wy9WNIJDGSTs2nMiOEAW2p8D6wIDD4a5evWpigoZzGNLT0+Pj4wMCAoqKisDHeXvHR7ZGFEAUeAEVIJNlv/t9zZE3ql4/UvokKsDkUHrkjeqf/6KivV2yV9dfODRUODy8V0dHjrtxBRBg2LhWO7mmQqFob29vbm7u7e0lk8kwMHC5XIFAoAkMmsywsLCwuLgol8sXFhbm5uampqYkEglIWtDkBCFkAoGAz+eDwKSxsbGRkZGhoSEymUwkEvv6+mZnZ3fyepB9IQogCmxDAQAMX3/9ta2trYeHR2BgYGRkZEJCAhySVABZfn4+SHqGgaG0tHRpaWkbR0Y2RRRAFHhhFejpnXzrR2Vr+RZgYCiGlpZVVe1lbNILOwAv1oUhwLA34ymXy9va2gAwUCiUkZERGo3GYrE4HI5QKATAIIVMAplUKp2enp6dnV1YWADAMD09DWiBx+NxuVweZIJlAx4GUC6JzWYzGAw6nU6j0cbGxoaGhjohAykNe3P9yFERBRAFNBR4JjDk5eURCAQEGDQ0QyYRBRAFnq2AnV0/5GHQJIQnpo+8UXHpcvv/3965B8Vx3fme2tp/tmpvtvhnpcr9I1shyW7uPm5cQfZ6s5tNyo6psh3bd5OYe+2K7fUbyVZsx481iW7FduwY+SbaxLKNLCWKZVnoiQALPyUEGiEe4jFCQiBgYIAZYGZ63owk9Dp3Z37op6Pu082gmWEY5tulQme6T5/H55zf6fPt8+gzZ/DeYW6Yee4DgiE7FeDcuXPNzc0tLS09PT0nT54cHBx0Op0sGDRNoylJtPGR3+8PBALhcJg0A22RFA6HvV7vxMSEy+VyJg66fXJycipx0AgDaYbxxDE2NjY6Ojo8PDwwMNDT09PW1jY+Po6JSdmpAYgVBCQCcwqG6urqvXv3GgXDxx9/jBEGCSScIAACVxEYGYl9/X/s/8tln5pNSfrSX33W2RW66h78AAEVAQgGFZXMnzt9+nRTU1N7e7uFYKA9Uml+EUkImp5ESxd8Pt/ExMTY6NjIyMjg4ACtnB4eHh4dHWXlQOKBBh9IWoyPj4+NjTmdTvqsW1tb2+Dg4DlsgZz5EkcMIGBBwCgYKisreUpSdXX1ww8//OijjxrXMHz00UdYw2ABFpdAAATWvjGwbPl+pWBYtnz/088cByIQSIYABEMylNLvJxqNNjY2trW10S5JPCWJ1zBo9A02kguJSUper3doaOjEiRMDAwNTnqkpj9flcrvd7vHxceeo89SpgROJg1ZE0HejXS4XDzjQzCW32+1yuWioYWRkhKYnHT9+PBaL0TLoixcvYswh/eWNEEHAksCcgmHlypVPPPHEvn379u7du2vXLl7DgBEGS664CAIgIHy+meIVjcuWGwcZPv3q1z4fHJwGIxBIhgAEQzKU0u8nEAgcPHiwvb3dKBhII/AiZvpJ32KjtQrxFcxDDo/HMzYysrfu497ewenpM6FwfF9Vj8fjcrkSYw6DDodjeHh4fHycNAPNUNLJBqfTOTQ0ZLfbOzs7eUnDpcSR/jwjRBAAARMCsmB46aWXjIuea2tr6+rqampqqqurSTBs2rTpjTfewAiDCVGcBgEQuEJg0++dxkGGZcv3v/xy/xVPcIGAJQEIBks8Gbvo9XobGhpowyJ50fPExAQLBlq4TMuYvV4vbXlEeyhNTU35fJ5tH7ateqPp7fcPhULBSOKIJg768LOmaR6PZyJxkGbgldAsG8bHx2lVQ29vb0dHx8TEBIYXMlbmCBgETAnIgoF2SZKnJO3Zs8coGDZu3Lh27Vpsq2rKFBdAAAQuE5ievvDd7x5etvwzaWLSp//zGw3uiTOXveB/EJiDAATDHIAydHliYqKhoaGjo+P48eMkGGjRMwsGHljweDzU0ee/iW2UtNGR0Rd/f/TZOvHG1hGvxxcJxz/iFolEwpcPkhD0ZTf5g26kQGhh9OTk5OykJqdzYGCgq6vL4XDwVxogHjJU+ggWBHQEZMFAIwyVlZXyl553J47a2lrdCMOnn36KRc86mPgJAiBgJFBd7V4uC4bl+3/7uyGjN5wBATMCEAxmZDJ7fmRkpKGhobOzM3nBQJOU6G8gEDje0/vEm/bVu8T6nVPeKXc4HCKlQDpBVg6XFUT8/1AoFAwGST/IAw4kGxwOB02R4iUNmaWA0EEABBIErAVDTU3Ngw8+eO+999bW1vIaho0bN1ZUVDQ0NFy8eBEUQQAEQMCawMzMxbv+V+uy5Z//5bKPli3/7MYbmwKBc9a34CoIyAQgGGQaC+ceGBhobGxMXjDQxxl8Ph990DkSDjUf6Xp8/cknd17aVO3WvBOhUCQUimsGo2DQnY9Gozz4QOLB5/NNTU3RaMPo6Gh/f//x48fD4fDC4UBMIJDfBGTBwFOSeIThww8/vOmmm2644QYWDB988MGmTZvWrl178OBBCIb8rjvIPQgkS+DgQd8X/3v8O27Lln+2ZctYsrfBHwgkCEAwZKci9PT0NDU1dXV10QjD0NDQ6Oioy+WiXZJoGhLNSiKpoBMM05HwR/tbyt51PVF1/oP64YDmC4WCOmGgG2RgIaFzkH4IhUJ+v9/r9ZJscDgcvb29Z87Mzm7E3KTs1BLEmjcErAVDXV3dzTfffOONN9bV1dEIAwkGjDDkTQVBRkEgDQQuXLj07w92Llve9L1bmmMxfKktDUjzKggIhiwU96VLlzo6Og4fPtzd3X3ixIn+/n5ZMExNTcmCgWci0dgCzSaKhoNbq5se2+xbveVs7efDgYAvlDjkEQYSDCwbdDqBf9I6af4bCoUCgYDX6x1IHJAKWagfiDL/CEAw5F+ZI8cgkAUC3d2hv/rygZqaiSzEjShznAAEQxYKkD7z3NzcTILh1KlTDoeDRhj4swm8VxJPQyLB4PP5/H5/NBx864OGx7eGn9o8/XnTcDDgC8YHGK6akkRDB9FofDE0Cwl2KAXD9OUjEol4vV6emATZkIVagijzicCcguF73/uecoTBZrPBPPOppiCvIJASgUuXxI4dLgwvpAQxX2+GYMhCydNnnltbW+12e29v78DAAH2hmaYk0f5FNMhAwwskFehjz5qmBQL+UMBfseGTx98OPPOW91BbfzgQSOiF2XXPRlVA8iAcDvO+qywneGwhGo1e1guz/4+Ojo6Nxac5okeShVqCKPOJwJyC4dZbb73++uv3Jo5du3Z98MEHtOi5tbUV5plPNQV5BQEQAIHsEIBgyAL3QCDQ0NDQ1tY2L8Hgv3wEgwGfZ2rN242rtsw89/ZUx7H+SDAcCsV3QKLBBDPB4PV6I5HIzMzM2bNnT58+LUsFcrNgiEajsVgsEAj09/efP38ePZIs1BJEmU8EXC7XqlWrHrj//ueee8646Lmuru72229fsWJFdXU1r2F49913X3/99ZaWFphnPtUU5BUEQAAEskMAgiEL3N1u94EDB9rb23WCYXx8fGJiQjnCcFksxP8PBYNul+u5d9qf3CN+vsnb2zccCQZDwUAwGEwIhlAkEo7FpmkyEo0tRKNRv197770/njh+4qP6jzb/YXN3V/eZ02d0moEFw/T0dCwWm56eHhoaikajWWCEKEEgnwi43e5Vq1bdbykYjCMMEAz5VEeQVxAAARDIJgEIhizQ7+/vP3jw4NGjR48dO3by5EmakjQ2NuZyuSYnJz0eD6975slINB8pEAj4/f5wOHKqf3D1O91P7hG/2Dg+MDgSjs9ICpJgIJ0wNDTkdrvPnIlLgkgkQn87Oo/+3zVr7vz+HXd8/7YH7ru/8WDTzMwMaQNZObBsiMViY2NjHo8nC4wQJQjkE4E5BcOtt976zW9+UzfC8Ktf/QojDPlUTZBXEAABEMgagWQFwyWTI2sJz3DEJtlN6fTFixfp/s7OzkOHDnV0dPT09Jw8eXJwcHBkZGR8fNztdpNg4BXP8sBCIBAfQwgGA5FIxN7TV/Zu3xNV4vXNztGx4fgAw+URhoBf+7C25rFHHn7llZcPHz5M+iEYDJ49e/b997c+9uhjP/9ZeXn586ufXPX4o4/6fL7JyUmn06lp2vT0dCQSYbVAQsLj8bhcLlrGQInPMHgEDwL5SMBaMNTW1t5zzz0lJSW6NQwQDPlYV5BnEAABEMgGgWQFg9xlpI4j/81GspOKk1O4SBwkGM6dO2ez2Q4fPkwfYaA9VXWCwefzeb1eGlXw+/0B6fAH/NPRyKGW44+/5/zJNrFuy9CEeywYDF8efAi3tx35/LP6D2trH/z3B7/xjW+89tprjsTxX2GuXLlq1aqVLzz/0x/94M7777v3jju+39PT43a7h4aGpqampqfjs5jkpc+xWEzTtJGRkXQBTKrY4AkE8o+AtWCorq7esmXLe++9V1tbW11dvWvXrq1bt7777ruvvfZac3Mz1jDkX31BjvOdwPnzIhAQPl/8H2YN53ttWKj8Lx3BkK5OLYVzMenDLF5jAOQzHA43NDS0tLR0dXXxRxicTieNMExNTem+0SYLhmAw4A/4Y9HwR009ZTu0J7dcqNze59d8sdiZcDji88W/xrBr17Z1v3n9/61dd/2Kf1yxYsXNN9/c3d3tThwPPPDAypVlP1n9xD13/+D+++696647T5w44fF4zEYYpqenQ6HQ8PDwuXPnzLI5r/MLVasRDwjkGAFrwbBnz57a2tq6ujr+0jMEQ44VMJILAmkl0N4u/uZvxD/8g7juOvH3fy++9S3x61+LSCStcSAwELiaQHYEw7x6mal7Nvbd03jmwnyOS5cuTUxM7N+/v7W1lQTDqVOnhoaGrAVDUDr8wUAsGtxW1/7I294n14ff/GNTzd7qHTt2NTQ0+v3+YDC4veqDnz6z+qVfvHTD9f943XXXPfTQQ319fZOTk+FwuLy8/JGHH1r9xMrHHnpg1crHVq1aqWkafdlNuYZheno6HA4PDw/HYrFLly7poKVeLtcWwtUVGL9AYCkQmFMw1NXV1dfXQzAshcJGHkAgZQKNjeLP/kx8/rno7xfHjonf/1589auipESEw1cFPTNz1c8kf5w7l6RHeMsvAmkQDNfW7UvXXbpe7Lx+JtPVP3+txznDMTMzc+HChb6+PnmLJPpqm9Pp5I8w0AgDz0ciGRBMHKFQKBgKTkfDG/cefXxz5Knfhd7a9PFXir78hf9WuHXr9kgk7Pf7Dx1qWrny8Wefffruu3/07W9/e+fOnS6XKxgMxmKx5ubmRx55+P/879K77rzjvvt+XF+/j1ZF87oF3ZQkEgxOpzMajTJYhkZnuBzZg4WDPWfCkV+Gi9wuLQLWgmHv3r2//e1vX3vtNV7DsHXr1g0bNrz66quYkrS0KgJyAwJJEWhqEn/xFyLxnaRZ/6OjYtkysXZt/OeFC+Ktt+LDDv/yL+KWW+K6Qgjx5JNi9+5Zz2vWiEceESQM2trEQw+JkyfFj38sPvxQ/PCH8VGLu+4So6OznvEfCBCBXBUMFr3S5C9x39foMJMJBhVw5cTMzMy5c+foKwdnpePM5ePs2bNnzpxpbW1tbGykLZLoq230mWe32017qsqCIZA4EmKBvsyW+NxCIPCbvd1l1RdXrQ983HDiUNOB/fsPeD2++IQlv39sbGzHjqqf//xnL7zwwsaNG9va2oaHhzVNC4fDsVisqanpmWeeef755/fV12/e/Mfx8fHTp09bCIZIJDI6OhoIBC5evGiklDxqpc9MyAZlmLB2EFjkBMwEw7Zt23bu3FlXV/foY4/edtttu3fvrqmp4TUMr7766uHDh7GGYZEXLpIHAmknQIJhePiqgFevFv/0T/Ezf/hDXDx88olwOOJTlZYtE8PD4qmn4mJAiPjMpeuvF8uXi5GR+M9f/ELcdltcHvz5n4sf/EC0t8eHLL7+dbF69VWB4wcI5Ixg0PU4ufOqO2/xk28xOszkwfnz568IgoRrxnCwNLisC86cThyxWIwdscuH3+9vaGiw2Wy8RRJ/5pm2SOI1DLyPajB4WSokPswWioQ0n/Zybe/KavHkm95Dzaei8a8uxEKhEG2j5PF4jh8/vm/fvm3btn3yySd2u93pdPp8vmAwGIlEOJGxWOz48eOBQIDVgnHRM+2bNDY25vV6Ey8t4uRkVkaSFy5cSL4IjD6V3f20nISpg8BiJmAtGPbs2bNjx47t27fX1NTs3bt3586dtIYBgmExlynSBgKZI6AUDOvWia98RczMiO9+V/zkJ7ORX7wY7/2vXy8OHhR/93fxqy0tcYXwwx+KnTvjfm65JT4c4fGIwkJRXT17109/Gj+PAwRkAjkjGKjXaOxiJnNG2a/V9X2pH8zywKALZk/o5AFJAp0w4J43ff2AFgmEQqFoNOpwOPbv39/S0tLZ2Sl/hGF0dJT3VPUlDk3TSADw95spnEg4Mjk5WV4z+PjuC0+96enoOBUM+ONTlRK6IhgMer3eoaGhzs7OI0eOdHZ29vf3U48/FArRBxk4eTQfiT/UICsHdkej0YmJiZMnT7rd7nA4TLOqWEcRNB3e8+fP685Yq4hkSpA2mGLlYLFnF/uZl0M2CbhBYOEJzCkYaNGzLBg2bNjwy1/+8tChQxhhWPjyQowgkF0CSsGwZk18GXQoJP76r8Wbb15J4He+E9cPwWBcOfT1xS8995zYtEmsXBkfbfja1+ILIVyu+JhDe/vsXS++CMFwBSBcRGApCwZjt5XOyO/IdW6jTmCFQA4zhcC9cOp/h8Ph0OUj/umExAfXgsFgV1fXgQMHaMWz/BGGsbExmpLk8XhIMPgTu6mSDEh8vzk8Kxgi4dHxiRfq3WXbLz37u5HeEwPBYOByVPFBBp/P53K5hoaG+vv7BwYGnE7nf/VFaAMlEgzyQgVe68wKQc4IuaemppqbmxsbG0l+uN3u6elpmnlFuEhl6Ujq9BiRT1IbzOltXmJgvp7JMPAXBBaSAAmGBxJfen7llVfWrVtXWVm5efNmmpJUXV1dV1e378N9NTU11dXVNMJQWVm5du3a7u5uCIaFLCnEBQKLgYBRMFy8KG64YXZlwt/+bXwmEh/f+pZ49tn4r9tui89W+tGPxL59oqND3H672LMnvs5hZiY+JUkWDP/xHxAMzA+OWQJLWTBwv1OnHHRdW35fTisQqBPMOoHm8LBOuDy3KEY9bPpqgTySQFKBRALpBE3TSAO43W6aj9TW1tbd3X3ixIm+vr6hoaGRkRGdYKD5SKwWpJGBSHQ60ucYf/rTwKr3xM/e7HMMOeILoS8fgUBA07SpqSm32z2aOGgttc/no/EKThj59F4+PB7PxMSEy+UaHx8fGxtzOp0jIyMOh2NoaOjkyZMtLS2tra29vb19fX12u72jo4M+I82Uzp49S9xoIce5c+eMkHWlIP/kkrJwsH/yM18ZkLx/gQMEFpyAtWCoqan59a9/XV5evnv3bp6S9M4776xfv97pdEIwLHhxIUIQyDIBEgyJr6rGU3L6tFizRnzhC6KzM/7z7rvFv/3bbAo1TXzxi2LbtvjPN98UN90k/vVf46sXYjHxz/8cX7Tw9NPxSyMjEAyzxPCfGYHFLhgsepDJXOJeJjuMHVkSDNzf5VXLSqmg0wk8nsBDCjye4Pf7SSp4vd6pxNHb20vzkY4ePWq323t7e/mrbWNjYxMTE/SZZ5/PR/ORaDLS1NTUwMDA4OCg1+uNRKPTsWhv/+CDv6y9b03Tz35Vc2qg35U4xsbGRkZGhoeHBwcH+/r6ent7e3p6jh07ZpeOrq6uzs7Ojo6O9vb2tra21tbWI0eONDc3H04cNputNXEcOnTIlvi03JEjR1pbW9va2jo6Oo4dO0YKZ2BgoK+vr6urKxqNso46c+YMiQemRzxp8EGnH2jwgUskXcupk5cE1j7NTAXnQSBzBKwFQ21t7TPPPHPnnXdu375dFgxvvfXW6OgoBEPmygUhg8DiJHDwoPiTPxH33BOfVvTAA2LFCvHlL19ZgdDWJr70pbgS2LBB3HprfElDKBTPR0+P+NM/jf88fz7+8+67RUGB2L8/7h4ejq9haG2dze6zz4rvfGfWjf9AgAgsdsGg69slIxJkP9wrZce1CYZY7MqQAk3jiSQOWScEEwe9uWep4PF4JicnJxJHW1vb1q1bDxw40NnZ2dvbe+rUqcHBwZGRkdHRUfmrbbzcmQKnbVW9Xq/T6UzEGZryabt31a59460NG7Y0Jw6bzdbU1HTkyBEasujr62tra9u/f/+BAwcOHjzY2Nhos9mo93/06NHOzs7u7u6enh4aMRgcHORRDhIttPCaF1GEw/HZUNFolCBEo1G/39/a2ur3+3ki0+nTp3mRtyweeOcoVg468aBc8MCFpdQSdFUuZZ1bV2fm+xNNAwgsPAGjYNiwYYM8JWnPnj20RRJNSXr//ffffvvt9evXQzAsfGEhRhDIOoHxcbFunXj5ZfHSS+KVV8TWrWJq6qpE9fSIF1+Mb6X6n/8ZX71Ax9mzorJSfPzx7M+WFvGb38x+KDoUEu+8IyYnZy8dOnRlD9bZU/gv7wnMQzDkPSsAAAEQAAEQAAEQAAEQAIG8IwDBkHdFjgyDAAiAAAiAAAiAAAiAQPIEIBiSZwWfIAACIAACIAACIAACIJB3BCAY8q7IkWEQAAEQAAEQAAEQAAEQSJ4ABEPyrOATBEAABEAABEAABEAABPKOAARD3hU5MgwCIAACIAACIAACIAACyROAYEieFXyCAAiAAAiAAAiAAAiAQN4RgGDIuyJHhkEABEAABEAABEAABEAgeQIQDMmzgk8QAAEQAAEQAAEQAAEQyDsCEAx5V+TIMAiAAAiAAAiAAAiAAAgkTwCCIXlW8AkCIAACIAACIAACIAACeUcAgiHvihwZBgEQAAEQAAEQAAEQAIHkCUAwJM8KPkEABEAABEAABEAABEAg7whAMORdkSPDIAACIAACIAACIAACIJA8AQiG5FnlgE9N00pLS0tKSux2eyrJraqqKikpKS8vTyUQ3AsCIJC7BNCY5G7ZIeUgsAgJ2O32kpKS0tJSTdNSSV55eXlJSUlVVVUqgeDeayAAwSBKS0sLEsc14LO+paKiIl0h2+12TmdBgWmpcYwVFRXWabO+Ssm2iMj6dlwFgTQSsNlsZWVlhYWFVC2Li4srKipSfOTokseGozt/DT81TZNTa7PZzAKh7KRoqmaBCyFKSkoKCgpKSkp0fpKMl5mkmEI0Jjr++JlFAg6Ho7y8vKioiBuT8vLyJdCYCCGS7CRkET61SAUFBRat4pzJs9lsVHbGlm3Oe+EhRQKmXc8Uw82J2+nRnrnnGT9xU6ThcDi4t0SpNQvQ4XAUFRUVFhamYpBCCEp5aWmpWURJnrfb7TabLcXhjiTjgrclSaCqqootVHYUFhamsV6ly1SFEMXFxXI6LSyRvKXYHbco9BQFAxoTC7a4lIsE6uvrdU9SssEl0Jgk30nIYsER/6KiohQVGrVslZWVKebFljgcDkeK4eTP7fkrGGw2m+7RnvZST1cvpLKyktq1NPaQ0p5ZZYBmXRalZ5wEAR2B+vp6qvnFxcX19fW2xFFVVUVP/cLCwhQfPBxdukzVbrdTgpN5mJHPRSsYGM4icaAxWSQFkaPJYNssLCysqqpyOBx2u33JNCa520nIYnXKdAucxaxlKOp8FAyVlZUsFQoLC3l0Mu2I09ULKS8vLygoKC4uTnsKMx0gnvGZJry0wyfbLC4u1gkDfvYn0y9PBlG6TJUVji7ByjRk+nFlZn2ZjleZ2dRPmmUn9ZARQj4QoPpTWFioe6PMjUm6pHtWGpPc7SRkse7laEuYTWJZjDtbUVPDQbN7HQ4H/0x7etLVcOTukzJ3U572yoAA50uAH+TKxW3prVrpMtV5hZPpx5UZokzHO9+CTtK/WXaSvB3e8pkAT3xXNiZlZWVpfCU3r0bAolDmFQ6sw4Kk2aUcbQnNsrMA5/NxhKE8cfD0HrK0VBb4appWUVHBoxbFxcX04tPa4GknIqqypF6MbRlf1TnMagY3i8qZ0w6Ho6ysjEdUioqKysrKdK9bKGRl6yNnxyzLnDBdgvmnMmF8FxwgwASsK7OyivK9Zg6zeivXbeO9yZgqNyNc1clhUeHJA73XtNlsvKuBhWEKIWjVJrc21HQoR1rMEMnxGjPLZ6z5ozFhUHAsfgL0Ar6wsDCNSV0kjYmuweGflFPZimnRJnUAZA7J2zKHWVpayqtBkt+FwrqZpTYwmWCVLZjc3OlyVFJSIrfDzIRZsUPGAreRQD4KBh0FftLrzif5U9M0+eHNNa+kpMTMPMxuoZcc8mQGDk3nMEsbW4JsHuTZbPFoQUGBUajItsdxcXbsdjurDjlh8pCufF52GxPG4cMBAskToBo4r51/zewuLabKzYhc2633AyGfFRUV9IJTd6NyIaa8arOoqKikpER+vuroKa1YCMHx6vzrfqIx0QHBz9wlYGYL15yjxdOY6NoN/klZYyuur6+XOyqccbOOgdnWKcrGirouypePHBFvpqJ8OWsWrLIZVLZgXMS8LoVRkIP7HsxE50GZMDn9cEMwzG4+eM11hTsKvNUjvXgoKCjgZ7munpHd0tIruqRpGtdy41oFtgRdOMafbAlsG+SHp1aXlpayVcu7sNXX18uhKWNkwVCYOGjdGO3mxi0RB06hKcORI4IbBK6BAD/kdPXcOqgFMFW2EeuU0FV6XJHykQ2Tm4KioiI5HE3TqEkpLi6WM87rHWXFnvq2qmhMZPhw5zQBsjWdgaSSo8XWmJg9atmKqekoKyuj3SMo7zztU25/HA4HjXYa13vITQ2/2eT9Y4xdFx1hs+aRxn8KCgq4EyWEYIVjTIayNIkAt5D19fWUQn7JomtOk391ostFPv+EYEhJMLA1Glsi7tPopAjE+M85AAALjUlEQVSflx/5VAU5NN0EA7O2wFhxOQRd4NQpUe5bzGYmh6aMka3duAiV49UNVijDkSOCGwTmRcBut3M91JmJdThcRTNqqpw268TQVX6/ZcwItxKyQfFJnSwXYvZjMroHtpn1KR+3xgQzMTQmRjg4k1sE5DpfVVUlz6i5hk+AsWksnsbEzNg5qcoRAxIGunaDSpb6DGVlZXJBU3fcOK7rcDiIsNxeyTeSW9k88r1GmPyKRLfDu1yaHAsRKCgo0KVZHtnQtZzKcDhAOIwEIBhSEgw8jsZqW0bMV+WT9DJeaaL8UlB31awtkIMlN7cO8jOehxd0wwi6W+SryhjZ2pWZVdqeMhxjsnEGBOYkwM8DGv6Wq+uc9woh2BiVtZevykFdg6myjcjhmLnJZJQyXgjBIw98OwWuaxzoqjJeM+tTmirHwg40JowCjlwnQHWevhBMbt1fYy/TIsvcXCyexsTM2NmKjV15TdMIgrItpcEEedUHdySUuSbtoevZ6xgqmykeXlAGy7fIVynZOoFBBJTNI0OQ+0UYYdCVTjI/IRhSEgxmVkroua7LJaGs6+yBb5HNwzoWvlcIoTQMDlP2KbspSfJrA2WM1uEo86UMR44abhBIkgDVJapmpBl0rb91ONZVUVm3lVWaY+FbZFPlk+zNwmEdPnVK5Ae2RVDKeM2ybB0vx4LGhFHAkesEuN2gTQLq6+vtdrumaTzvhebDJJlNM8ui25XGaG10fMs1NyZmSVJaMaWTL8mRMgG+ym/lKZHKHjm/xTdO++EA2Y9uzoVZynWJlFWNEqZFOJwX3SNDGY6cYLh1BCAYUhIMVOHkrrbMl1sBPslTBo1yn/woa7aFJXDIFreT9LewZGP4xjNm1s4JUNqeMhy+BQ4QuAYC9fX1VK+U6/XNAlwYUzWavFl65ny/ZRYU9XIqKipKS0tLEgevIErySaw0VWM6lW0RGhMjKJxZ/ASoziun5WiaRqN5SYpzttxMP/fNWgAlbbNHrdKKKQQOn5oR3V9uVbiTTVEUFhbqfNJPYqhrgnRJ5Rjl87y4Qj4pu43tlfEMz85QDtiaQVCGI0cNt44ABMMcgoGqlO4vm5B1hTOah1nF5VJRejBrC/gudlzb7cbwjWcgGBgyHIuBAPVc5Y8964yUfi6wqRpN3oLVfFsPskF6vtK9xcXFFk9rpRVzd0c3oG9MJxoTIxOcyVEC1rbGa3l1zQXdxX91V80syNgIKE1JJqn0YAxHvkXnNjN2Zch0L4fPGVQ6ONcUhdKPfFKXMPknxyifpHvNYCrbK+UtZgTMZl4oQ5YTBreRAARDGgSD2fRHo3mw9S7kCAMZEkYYjLUfZ3KXAJsSP8/khxa7dVczbapGk7cgrHzssX9jUDzZt7S0lPNF/o2eLV65WcfLCTAS5jDRmDAlOHKCAL0vNzN/Y1XnBkR2sNHRSbPQjMbI4c/ruW8MxwK1WXeZo+bEcyAsk/iMtYPe0Shf4VvfyFeVObKGqezW0y06jWFGAIKB+afugGCYQzDYVAfP+bOoo2av5JV1nQuSLYqj4Id0MoaqbB04TI5F56AkyQOsynxZh6PMlzIcXez4CQJKAtaVx1jVbaqD7cg6NGXdVlZpTirfwlGYmTzfonNYh2+c/EP+lcsKOTFyFGZZto6XQzASTiaDFDgaE8YIx2IgQCuCzISusarbVAdbupllUU6VxmhtdHwLR5GMrclgzZJkzBrfxZf4I7Z8SemgRCY/ccsYCGdTvmSWcvLDicQaBhlattwQDHMIBuuCsd4tgd8IyoFYb71i7CWkLhh4cwPZ5DhJymUVShtWWjuHo2wQU38nweHDkW8ElLbAELhWG9+csR/ZsTCmam0jcnr4zZnZe0qaesTygB+cyvwqmxqlFXO8uvdzurSZvZZj7GhMjMRwZtES4HprbT68wNc6I4uwMTEzdot2g3dJMm7rTJ9XoumOLCeYIZ+RKVVVVZF/+aTOrWweue2SxRLfyMMgctEoOxtmBMyaMiEEtbFztoScGDggGFISDGxCxjrHm6brlgHxeWPLxbatM2ALS9DVYA5BFzgtSFKOUVDgutcGyhiV1s4JUNow3aILnG+BAwQsCLCl6MyBbqEqWlBQoHzMGINdGFO1thFdqshklEu3+THJcxiUwp4CtNlsvLBBjkJpxSkKBt7vFY2JjBruxU+AHoJFRUW6FoM3+1dWaWW+FmFjYmbsZl0CyhcPvOiYmI1vEEN+iyHDMUuA7EfZPKb3OwzKQjSDQGlWZkdONtxMAIIhJcEghODNBPgjhXa7nUSz8inOt+Tul5659sgOpWDgPh/3e+Rb4AYBawJsXGVlZTabTdM0h8Mh75IkT32xDortTv6eaNpNVflENEsYmQy/5aJXaA6HgwPR7WDIPR5+2caelU0Nv7rTvT4gqkVFRRyOMoVmT1nuLclfh73mz8Yro0ZjosSCk6kQ4PpcVFRUVVXlSBz19fVkVgUFBTozsY6Lm6bMPfe5HbBOCV01669zrpW5czgc1HSYfTxe18AqbV/TNB5yUQ48cvrNcsQtFcO85i89z0swULILCwuVYyacbDiYAARDqoLBbrfz05pfGdJmz2wGjJscmqZxcyPfQrvLG7W+WVugC9Zi6E0IwX13XYzKF5zKGM2snZKhfMbzjnUcqbLZMmYEZ0BACGFhKcovelpDWwBTtbYRXfLIKMrLy5WtQVFRke4xxk9rtiYOga1bjoJf3ZE3Nj32TOflW2S3RVdDF4KcHuOrATQmMlW4s0igqqpK+bBWPgSt07nYGhOllVl3CSiDZhmhPoyxN2Jh+zp1YQRo0Tyy5JAbk4KCAmVvnvzopnWYEbCAwMO2HKkxzTgjE4BgSFUwULemrKyMX1Twp+bZtGTi7KY5f1xT+S72wA4LS2A/5LB4xgshHA6HnM6ioqKysjLlW0ZljBbWbjHPweFwlJaWcjPNvRZdyvETBMwIVFVVyVWI6u21VSR6GZY5U7W2EV0G5cdeRUUFy4bi4mL5TZt8l81mo6Ud9Cjl7ZLY8GXP9KRk/zKxqqoqjk53C//kMOUb+SoaE0YBRw4RcDgc5eXl3AJYPATnzNSiakyUj2yLvrKcO03T5PansLCwtLTUqPz5FqXtK1sJvoUc1s0jNW7cVbBoBuWWk6MwI2ANwWaz0Y0UJocGh5IABIMSS66etH7G52qukG4QAIEFJ4DGZMGRI0IQWMoErAXDUs75UskbBMNSKclEPvCMX1LFicyAQPYIoDHJHnvEDAJLkAAEQ64XKgRDrpfglfRrmsbTD4xTD6/4gwsEQAAELAmgMbHEg4sgAALzI+BwOGgmmG4vh/mFAt9ZJQDBkFX86Yuc10IUFBTMufYofdEiJBAAgaVGAI3JUitR5AcEskeAxyqpYbHeSSl7yUTMcxOAYJibUU74IFMsKirSbR2QE4lHIkEABBYPATQmi6cskBIQyHUCLBiKi4uhFnK6NCEYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBP4/eU86PD5hLswAAAAASUVORK5CYII=" - } - }, "cell_type": "markdown", "id": "d11a3c68-fccb-4e09-916e-ca3a9a048861", "metadata": {}, @@ -195,18 +148,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "19902729", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[0, 1, 2, 3, 4, 5, 6, 7, 8]\n" - ] - } - ], + "outputs": [], "source": [ "jnt_names = [\n", " \"joint1\",\n", @@ -250,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "6151d602-0c5d-4e20-a41b-1b01864f0151", "metadata": {}, "outputs": [], @@ -289,18 +234,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "c89dd04d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████████████████████████████████████████████████████████| 150/150 [00:04<00:00, 34.14it/s]\n" - ] - } - ], + "outputs": [], "source": [ "import logging\n", "\n", @@ -311,7 +248,7 @@ "\n", "# Camera recording\n", "rgb, depth, segmentation, normal = cam.render(rgb=True, depth=True, segmentation=True, normal=True)\n", - "cam.start_recording()\n", + "cam.start_recording(save_to_filename=\"Videos/video_02.mp4\", fps=60)\n", "\n", "# Hard reset\n", "for i in tqdm(range(150), ncols=100):\n", @@ -337,18 +274,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "bc94198b-ffb3-4e5c-822f-4f42dbff1eb5", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████| 1250/1250 [00:29<00:00, 42.31it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# PD control\n", "for i in tqdm(range(1250), ncols=100):\n", @@ -386,7 +315,7 @@ " cam.render()\n", " scene.step()\n", "\n", - "cam.stop_recording(save_to_filename=\"Videos/video_02.mp4\", fps=60)" + "cam.stop_recording()" ] }, { @@ -401,26 +330,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "b1a79fdc-c447-41ed-8319-0a5f19608b58", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<video src=\"Videos/video_02.mp4\" controls >\n", - " Your browser does not support the <code>video</code> element.\n", - " </video>" - ], - "text/plain": [ - "<IPython.core.display.Video object>" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from IPython.display import Video\n", "\n", diff --git a/projects/PhySim/PhySim03_motion_planning.ipynb b/projects/PhySim/PhySim03_motion_planning.ipynb index cbe7d6a0..037bc2bb 100644 --- a/projects/PhySim/PhySim03_motion_planning.ipynb +++ b/projects/PhySim/PhySim03_motion_planning.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0b774e89-ff67-4285-a5da-f47170945e31", "metadata": {}, "outputs": [], @@ -48,21 +48,7 @@ "execution_count": null, "id": "b7a9c1c3-7661-40d7-b895-79de1d0337e0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [08:58:51] [INFO] \u001b[38;5;23m╭───────────────────────────────────────────────╮\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:51] [INFO] \u001b[38;5;23m│┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈\u001b[0m\u001b[38;5;17m \u001b[38;5;23m\u001b[1m\u001b[3mGenesis\u001b[0m\u001b[38;5;17m \u001b[38;5;23m┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈│\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:51] [INFO] \u001b[38;5;23m╰───────────────────────────────────────────────╯\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:52] [INFO] Consider setting 'performance_mode=True' in production to maximise runtime speed, if significantly increasing compilation time is not a concern.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:52] [INFO] Running on \u001b[38;5;23m\u001b[4m[AMD RYZEN AI MAX+ 395 w/ Radeon 8060S]\u001b[0m\u001b[38;5;17m with backend \u001b[38;5;23m\u001b[4mgs.cpu\u001b[0m\u001b[38;5;17m. Device memory: \u001b[38;5;23m\u001b[4m121.50\u001b[0m\u001b[38;5;17m GB.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:52] [INFO] 🚀 Genesis initialized. 🔖 version: \u001b[38;5;23m\u001b[4m0.3.3\u001b[0m\u001b[38;5;17m, 🌱 seed: \u001b[38;5;23m\u001b[4mNone\u001b[0m\u001b[38;5;17m, 📏 precision: '\u001b[38;5;23m\u001b[4m32\u001b[0m\u001b[38;5;17m', 🐛 debug: \u001b[38;5;23m\u001b[4mFalse\u001b[0m\u001b[38;5;17m, 🎨 theme: '\u001b[38;5;23m\u001b[4mlight\u001b[0m\u001b[38;5;17m'.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:52] [INFO] Scene \u001b[38;5;23m\u001b[3m<0078298>\u001b[0m\u001b[38;5;17m created.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "import genesis as gs\n", "import numpy as np\n", @@ -97,55 +83,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "85fca6cf-23ec-4925-b691-8208ae7489af", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [08:58:55] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m0\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<0d66be6>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.Plane>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:58] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m0\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:59] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m1\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<3bb69d6>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.Box>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:59] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m1\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:59] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m2\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<279e873>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.MJCF(file='/opt/conda/envs/py_3.12/lib/python3.12/site-packages/genesis/assets/xml/franka_emika_panda/panda.xml')>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:58:59] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint1`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:58:59] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint1`.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:58:59] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint2`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:58:59] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint2`.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:59] [INFO] Applying offset to base link's pose with user provided value in morph.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:58:59] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m2\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:00] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m3\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:00] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m4\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:01] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m5\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:01] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m6\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:01] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m7\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:01] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m8\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:02] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m9\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:02] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m10\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:02] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m11\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:02] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m12\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:02] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m13\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:03] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m14\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:03] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m15\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:03] [INFO] Preprocessing geom idx \u001b[38;5;23m\u001b[4m17\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:03] [INFO] Building scene \u001b[38;5;23m\u001b[3m<0078298>\u001b[0m\u001b[38;5;17m...\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:59:04] [WARNING] Reference robot position exceeds joint limits.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [08:59:04] [WARNING] Constraint solver time constant should be greater than 2*substep_dt. timeconst is changed from `0.005` to `0.02`). Decrease simulation timestep or increase timeconst to avoid altering the original value.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:08] [INFO] Compiling simulation kernels...\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [08:59:27] [INFO] Building visualizer...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "amdgpu: os_same_file_description couldn't determine if two DRM fds reference the same file description.\n", - "If they do, bad things may happen!\n" - ] - } - ], + "outputs": [], "source": [ "########################## entities ##########################\n", "plane = scene.add_entity(\n", @@ -183,13 +124,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "cac12a53-5b5b-4533-b1e8-a5b901444dff", "metadata": {}, "outputs": [], "source": [ "rgb, depth, segmentation, normal = cam.render(rgb=True, depth=True, segmentation=True, normal=True)\n", - "cam.start_recording()\n", + "cam.start_recording(save_to_filename=\"Videos/video_03.mp4\", fps=60)\n", "\n", "motors_dof = np.arange(7)\n", "fingers_dof = np.arange(7, 9)\n", @@ -229,7 +170,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "361bd79d-3710-4fb8-9c82-967c536d7dc2", "metadata": {}, "outputs": [], @@ -249,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "bedf4dfe-d51c-484b-baaf-b351bc1c8406", "metadata": {}, "outputs": [], @@ -274,19 +215,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "b3ef0fd2-1b2e-408c-adb3-d16862a09112", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Executing motion path: 100%|██████████████████████████████████████| 200/200 [00:04<00:00, 46.65it/s]\n", - "Reach the last waypoint: 100%|████████████████████████████████████| 100/100 [00:02<00:00, 47.82it/s]\n" - ] - } - ], + "outputs": [], "source": [ "import logging\n", "\n", @@ -329,18 +261,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "3c9b2e21-6f85-4220-8d59-d05990823ca9", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Lower the gripper: 100%|██████████████████████████████████████████| 100/100 [00:02<00:00, 36.97it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# reach\n", "qpos = franka.inverse_kinematics(\n", @@ -367,18 +291,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "583befbf-0a92-4540-840a-64204dbeedc7", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Close the finger: 100%|███████████████████████████████████████████| 100/100 [00:02<00:00, 45.12it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# grasp\n", "franka.control_dofs_position(qpos[:-2], motors_dof)\n", @@ -401,18 +317,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "45581280-45c5-44a7-afcb-33cbf44a8ec3", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Lift the cube: 100%|██████████████████████████████████████████████| 100/100 [00:02<00:00, 46.23it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# lift\n", "qpos = franka.inverse_kinematics(\n", @@ -426,7 +334,7 @@ " cam.render()\n", " scene.step()\n", "\n", - "cam.stop_recording(save_to_filename=\"Videos/video_03.mp4\", fps=60)" + "cam.stop_recording()" ] }, { @@ -441,26 +349,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "eda4709b-4663-41ac-850e-6be12eb52311", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<video src=\"Videos/video_03.mp4\" controls >\n", - " Your browser does not support the <code>video</code> element.\n", - " </video>" - ], - "text/plain": [ - "<IPython.core.display.Video object>" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from IPython.display import Video\n", "\n", diff --git a/projects/PhySim/PhySim04_parallel_simulation.ipynb b/projects/PhySim/PhySim04_parallel_simulation.ipynb index b23e7c95..aab133b1 100644 --- a/projects/PhySim/PhySim04_parallel_simulation.ipynb +++ b/projects/PhySim/PhySim04_parallel_simulation.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "3caab361-5625-4df3-bbac-024295980c0b", "metadata": {}, "outputs": [], @@ -44,21 +44,7 @@ "execution_count": null, "id": "bc4e3ed3", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] \u001b[38;5;23m╭───────────────────────────────────────────────╮\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] \u001b[38;5;23m│┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈\u001b[0m\u001b[38;5;17m \u001b[38;5;23m\u001b[1m\u001b[3mGenesis\u001b[0m\u001b[38;5;17m \u001b[38;5;23m┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈┉┈│\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] \u001b[38;5;23m╰───────────────────────────────────────────────╯\u001b[0m\u001b[38;5;17m\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] Consider setting 'performance_mode=True' in production to maximise runtime speed, if significantly increasing compilation time is not a concern.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] Running on \u001b[38;5;23m\u001b[4m[AMD Radeon Graphics]\u001b[0m\u001b[38;5;17m with backend \u001b[38;5;23m\u001b[4mgs.vulkan\u001b[0m\u001b[38;5;17m. Device memory: \u001b[38;5;23m\u001b[4m60.75\u001b[0m\u001b[38;5;17m GB.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] 🚀 Genesis initialized. 🔖 version: \u001b[38;5;23m\u001b[4m0.3.3\u001b[0m\u001b[38;5;17m, 🌱 seed: \u001b[38;5;23m\u001b[4mNone\u001b[0m\u001b[38;5;17m, 📏 precision: '\u001b[38;5;23m\u001b[4m32\u001b[0m\u001b[38;5;17m', 🐛 debug: \u001b[38;5;23m\u001b[4mFalse\u001b[0m\u001b[38;5;17m, 🎨 theme: '\u001b[38;5;23m\u001b[4mlight\u001b[0m\u001b[38;5;17m'.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:34:49] [INFO] Scene \u001b[38;5;23m\u001b[3m<397158a>\u001b[0m\u001b[38;5;17m created.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "import genesis as gs\n", "import numpy as np\n", @@ -93,24 +79,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "67347a5e", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [10:35:17] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m0\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<1c869ef>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.Plane>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:35:17] [INFO] Adding \u001b[38;5;23m<gs.RigidEntity>\u001b[0m\u001b[38;5;17m. idx: \u001b[38;5;23m1\u001b[0m\u001b[38;5;17m, uid: \u001b[38;5;23m\u001b[3m<c3b0391>\u001b[0m\u001b[38;5;17m, morph: \u001b[38;5;23m<gs.morphs.MJCF(file='/opt/conda/envs/py_3.12/lib/python3.12/site-packages/genesis/assets/xml/franka_emika_panda/panda.xml')>\u001b[0m\u001b[38;5;17m, material: \u001b[38;5;23m<gs.materials.Rigid>\u001b[0m\u001b[38;5;17m.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:18] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint1`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:18] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint1`.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:18] [WARNING] (MJCF) Approximating tendon by joint actuator for `finger_joint2`\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:18] [WARNING] (MJCF) Actuator control gain and bias parameters cannot be reduced to a unique PD control position gain. Using max between gain and bias for joint `finger_joint2`.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:35:18] [INFO] Applying offset to base link's pose with user provided value in morph.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "########################## entities ##########################\n", "plane = scene.add_entity(\n", @@ -140,30 +112,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "a9f5096b-1131-47ec-aec0-8ff95eab9855", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[38;5;17m[Genesis] [10:35:38] [INFO] Building scene \u001b[38;5;23m\u001b[3m<397158a>\u001b[0m\u001b[38;5;17m...\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:38] [WARNING] Reference robot position exceeds joint limits.\u001b[0m\n", - "\u001b[38;5;3m[Genesis] [10:35:38] [WARNING] Constraint solver time constant should be greater than 2*substep_dt. timeconst is changed from `0.005` to `0.02`). Decrease simulation timestep or increase timeconst to avoid altering the original value.\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:35:39] [INFO] Compiling simulation kernels...\u001b[0m\n", - "\u001b[38;5;17m[Genesis] [10:35:43] [INFO] Building visualizer...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "amdgpu: os_same_file_description couldn't determine if two DRM fds reference the same file description.\n", - "If they do, bad things may happen!\n" - ] - } - ], + "outputs": [], "source": [ "########################## build ##########################\n", "n_envs = 9\n", @@ -186,15 +138,7 @@ "execution_count": null, "id": "d6c14a54", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████| 1000/1000 [00:46<00:00, 21.58it/s]\n" - ] - } - ], + "outputs": [], "source": [ "import logging\n", "\n", @@ -204,7 +148,7 @@ "gs.logger._logger.setLevel(logging.WARNING)\n", "\n", "rgb, depth, segmentation, normal = cam.render(rgb=True, depth=True, segmentation=True, normal=True)\n", - "cam.start_recording()\n", + "cam.start_recording(save_to_filename=\"Videos/video_04.mp4\", fps=60)\n", "\n", "target_quat = np.tile(np.array([0, 1, 0, 0]), [n_envs, 1]) # pointing downwards\n", "center = np.tile(np.array([0.4, -0.2, 0.25]), [n_envs, 1])\n", @@ -231,7 +175,7 @@ " scene.step()\n", " cam.render()\n", "\n", - "cam.stop_recording(save_to_filename=\"Videos/video_04.mp4\", fps=60)" + "cam.stop_recording()" ] }, { @@ -246,26 +190,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "a236c16e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<video src=\"Videos/video_04.mp4\" controls >\n", - " Your browser does not support the <code>video</code> element.\n", - " </video>" - ], - "text/plain": [ - "<IPython.core.display.Video object>" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from IPython.display import Video\n", "\n", From 21f90f9416a3f123b2b1c97c73dd5522ffef1d35 Mon Sep 17 00:00:00 2001 From: Sonya <195730002+sonyyang-tw@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:06:03 +0800 Subject: [PATCH 114/180] fix(physim): restore notebook image attachment Restore the embedded joints and degrees-of-freedom illustration that was removed while cleaning notebook outputs. Co-authored-by: Cursor <cursoragent@cursor.com> --- projects/PhySim/PhySim02_control_your_robot.ipynb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/PhySim/PhySim02_control_your_robot.ipynb b/projects/PhySim/PhySim02_control_your_robot.ipynb index d7c5cf9a..059b4ae6 100644 --- a/projects/PhySim/PhySim02_control_your_robot.ipynb +++ b/projects/PhySim/PhySim02_control_your_robot.ipynb @@ -144,7 +144,12 @@ "![image.png](attachment:267b2468-7d14-45fd-a20d-3eb514f2c857.png)\n", "\n", "Take Franka Panda arm for example, it has 7 revolute joints in the arm and 2 prismatic joints in its gripper. Since each joint has only 1 DOF, the robot ends up with 9 DOFs in total." - ] + ], + "attachments": { + "267b2468-7d14-45fd-a20d-3eb514f2c857.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABBAAAAF1CAIAAADeBo7pAAAAAXNSR0IArs4c6QAAIABJREFUeAHsvQdcVGe+/+/+9v/733tXo0lUirvZ3dzdze7du5tsysYUG1Y6AvaK0vtQpYOxa+woKApE6U1RqqAgIL0qRYr0mQGmMMP0cs75ZebRk5EAUWwDfh/nNZ459Xne5zBzPufbZhDQgAAQAAJAAAgAASAABIAAEAAC4xCYMc58mA0EgAAQAAJAAAgAASAABIAAECBAMMBFAASAABAAAkAACAABIAAEgMC4BEAwjIsGFgABIAAEgAAQAAJAAAgAASAAggGuASAABIAAEAACQAAIAAEgAATGJQCCYVw0sAAIAAEgAASAABAAAkAACAABEAxwDQABIAAEgAAQAAJAAAgAASAwLgEQDOOigQVAAAgAASAABIAAEAACQAAIgGCAawAIAAEgAASAABAAAkAACACBcQmAYBgXDSwAAkAACAABIAAEgAAQAAJAAAQDXANAAAgAASAABIAAEAACQAAIjEsABMO4aGABEAACQAAIAAEgAASAABAAAiAY4BoAAkAACAABIAAEgAAQAAJAYFwCIBjGRQMLgAAQAAJAAAgAASAABIAAEADBANcAEAACQAAIAAEgAASAABAAAuMSAMEwLhpYAASAABAAAkAACAABIAAEgAAIBrgGgAAQAAJAAAgAASAABIAAEBiXAAiGcdHAAiAABIAAEAACQAAIAAEgAARAMMA1AASAABAAAkAACAABIAAEgMC4BEAwjIsGFgABIAAEgAAQAAJAAAgAASAAggGuASAABIAAEAACQAAIAAEgAATGJQCCYVw0sAAIAAEgAASAABAAAkAACAABEAxwDQABIAAEgAAQAAJAYDoQWLeO2Lz5qYFUVxNz5hAPHjw1Ez4AgeclAILheYnB+kAACAABIAAEgAAQUEcCEwsGiUTRZ6mUYDIJmUwd+w99UlsCIBjU9tRAx4AAEAAC04fA0NBQaWlpeXl5dXV1TU1NnUqrqamprKwsLS2l0+nTZ8AwEiDwJgiMJxiamwmhkNi4kbh0idi0ifjiC2L5cqK29k10EY45NQmAYJia5w16DQSAABCYOgTkcvnly5e3bt1qZ2dHoVA8PT19fHz8/Px8fX337Nnj7u5uo2w1NTU4jk+dYUFPgYDaEZhAMPD5xAcfEJ99RpSVEd3dhJkZsXAhIRKp3RCgQ+pJAASDep4X6BUQAAJAYPoQePjwoYuLi42NjYeHh6+vb2BgYHBwcEhISFBQkJ+fn6enp6Wl5eHDh3k83vQZM4wECLwJAuMJhqYmQiAg/vhH4uDBx90qKiLeeYfo6HgTvYRjTkECIBim4EmDLgMBIAAEpg4BqVR66dIlCwsLFxcXb29vf3//oKCgYGULCAjw8fFxdna2srK6c+cOmBemzlmFnqopgXXriC1bnupbTY0i6BkJhg8/JJKTHy998EAxv6rqqZXhAxAYjwAIhvHIwHwgAASAABB4CQSam5sdHR2tra3d3d19fHyQWhhlXjh48CCLxXoJB4NdAIG3m4CFBaGr+xSC/HzivfeI3l6Czyc+/JCIiXm8tK6OmD2baGh4amX4AATGIwCCYTwyMB8IAAEgAARelACGYZcvX96+fbuzszNpXggJCQkODg4ICNizZ4+Li4u1tXVBQQGYF16UNWwPBAgiNpZ4912iru4xC7GYMDZWxDfLZASXq3BJcnB4vCghgdDUJAYHgRoQeCYCIBieCROsBASAABAAApMg0N/f7+PjY2lp6e7u7ufnFxAQEKJsQUFBvr6+np6eNjY2pHkBNMMkCMMmQECVgFhMWFsrLAkWFgpt8PnnxD//Sdy/r1iFwyE++ohYupTw8yOOHiV+/3vCx0d1U5gGAhMRAMEwER1YBgSAABB4BQRwjMBwghgzHxCaP+aiV9CTV7tLHMdTU1N37tzp5OTk7e0dEBCAQheCgoL8/f2RecHGxgbMC6/2NMDe3z4C+fnEvn2Evz8RHa0ouYAah0P86U9EQoLi5eensEWgsgxvHx4Y8WQIgGCYDDXYBggAASDwIgQwqRhPPIuf9CDOeI964ae98e/d5DejcLn0RQ6hDtv29fV5eHgg8wKZHCk4OBglR/Ly8rKysjp8+PDw8LA69Bb6AASmNwEOR+GSlJ4+vUcJo3tVBEAwvCqysF8gAASAwJgEcILARDx8w6fEf/+K+MuMn14fzSD+PgP/3xn4n2dgLsa4eMonSE9KStqxYwcyL6DkSCh6ITAw0MfHx9XVFaIXxrxCYCYQeBUEOBxFHYZr117FvmGf058ACIbpf45hhEAACKgbAUzCww87EFZLCZvlj1+2K3D7VfLl72H/moH9bQZGMcHFQnXr9nP1p6enx8PDY9euXSg5kmrtBX9/fy8vLxsbm0OHDkFypOeiCisDgUkTkMmI8nKCwZj0DmDDt5oACIa3+vTD4IEAEHgjBHBcjon4mHCEEPIULxEfl0vkqeHYN7Owf8zAlmtgd69P9QjgrKys7du3Ozo6enl5keaFkJAQZF5wcXGxs7OD6IU3cvnBQYEAEAACz0sABMPzEoP1gQAQAAIvSkBOYBiOYcq458dRzrnxmM572D9nYMvexwtSMRwn8Ckc+TwyMnL06NFdu3a5ubn5+Pgg8wKKXiDNCxC98KKXEWwPBIAAEHhdBEAwvC7ScBwgAASAwBMCuCJDkuKFEQrZgDVWY8vm4v+cQXzxayIvGZ/qxgWCKCoqsrGxcXBw8PLy8vPzIwVDYGCgr6+vq6urra1tYWHh1B/okzMK/wMBIAAEpjUBEAzT+vTC4IAAEFBvAooA6N52bPdS/JNfEZ/MwA45Y3KZend5ot4hAcDhcA4ePEiaF0ZlU/X29raxsTly5AibzZ5oX7AMCAABIAAE1IYACAa1ORXQESAABN4+AnjfI3z3t8Q/fkV8PAM/YKfInjT1IRQVFVlZWdnb23t5ealmU1U1L+Tl5YF5YeqfahjB1CMwMjKFH0lMPdzTqMcgGKbRyYShAAEgMDUIKOwKCockwQjurIf981fyj2fIgy3kQi6m6P9jyaBYQXFPjaFoBhyXT4nBcTicw4cP79y5k0Kh+Pj4IPNCSEgIKtbm5eVlZ2fn5+dHp9OnxHCgk0BgOhGgUoWmppXNzSPTaVAwltdDAATD6+EMRwECQAAIPCaAwhcwTI6FBeJf/Fr2vzOIDf+U97fLCRzDFPIAradQCwqVgMtlYjkmU0ZIKwWFeoOsrq62tra2t7f39PT08/MLCgoKUTaUHIlCoVhbW1+/fh1TxG5AAwJA4LUScPd4oKVdvHVrlUQCf4Cvlfw0OBgIhmlwEmEIQAAITDECuFypFj7/v/J/zMA3/BNrrZMrtQGuCIN+3BTTIr488ZzcfiXuZY6X5qq/D49EIjl//ryFhQWFQtmzZ88o84K3t7ednZ2/v39/f/+TUcL/QAAIvCYCd+4M/fZ3tzQ0szU0b8XE9L6mo8JhpgsBEAzT5UzCOIAAEJgiBBQWhqvfE5/9Gv94htzkz1htASYV4sIRXMDDBVxcOEIIRnDhCEYQ8vTL+Mf/BzP+M/7te/iaD/CuJtJhST3HWldXZ29vb2tri6IXkHkhODh4lHlB/ZWPeuKFXgGBSRPgcmXLV5RoauVpaGZpaN769NM7ff1TuzTkpFHAhpMjAIJhctxgKyAABIDACxAI3E787//B/zUDM/4zYb9abq1DWC8jbBQv3HoZbrlU5rAK72nFHHSxRe9gXc1Ychj2918RsaeUWVhf4LivZlMkACQSSWho6I4dO1xdXVXNC8HBwQEBAd7e3vb29n5+fn19fYpADWV7Nd2BvQIBIDCawIGDDzW18pVqIUtDM0tT67ab+/3RK8FnIDA+ARAM47OBJUAACACBV0TAfxvxl/+D/2MG9j8zsI9myP86A/9oBv4XxYv4q3Lik/8Py0vAl8/Ftn0uE45g9cX4wv/AnXQJuTqGPiPBUF9fb2tjY2tr6+HhgaIXgpUtMDDQz8/Pzc1NNXoBjAyv6MqC3QKBnxOoreX895/yNTRzSMGgoZn92w9uFRYyfr4yzAECYxIAwTAmFpgJBIAAEHiVBPKTiVNexFlfItQPC/VTTvgq3pUv/KwvcWk/XnEb+3Ymvu3fmGBEXl+Mf/UfuOUSQiZ9ld2a/L5lMlno2bM7tm93cXb29vYmoxeCg4P9/f2RecHf359Go6FjgGCYPGvYEgg8DwGhUG5mVqFqXkCyQVMrb82aewKBOj6DeJ7xwbqviQAIhtcEGg4DBIAAEHguAnjHA/zr3xDbPseEPHn9PfnC/8Cs1VcwdHd3u7u7W1tbe3h4kLUXUDZVX19fCoViY2OTnp4OOuG5rgFYGQi8OIGIiO4noQsKZyTVl6ZW3sWL3S9+CNjD20AABMPbcJZhjEAACExBAo8e4N/MxLd+hgu48voibOH/T1BM1NMlCcOw5ORkS0tL5/HNC0FBQVQqdQqeBugyEJjCBDra+f/z93wNzVxVnfDTtFbuJ5/c6e8TTeERQtdfFwEQDK+LNBwHCAABIPA8BHABF9v1Nfblf2KNZVjkIfyvM4jUCyppV59nX69sXWQx6O3t9fT0VDUvhISEoORIqtELYF54ZecBdgwExiAglxO7d9f83BnpJ8GgiH7O9/FpGmNjmAUEniYAguFpHvAJCAABIKAeBHCCkN9OlS78T3zxu/Iv/q9s3d8waqd6dG10L5KSknbu3EmaF8hsqig5koODQ0BAADIvgGYYzQ4+A4FXRiA1laqlnassvPCUJ5KqYNDQzPn9H25VVbFfWS9gx9OEAAiGaXIiYRhAAAhMMwIKwSAXY4XXsL27seNu2MNq9bzb7u3t9fLysrS0dHd3HxW9gMwLtra2EL0wzS5OGI76E6DTxP/+slBTaxxnJJVgBk2t29t31Mhlj2vMq//QoIdvhAAIhqewo9TgcrkcTajnz/NTPYYPQAAITFMCOIEpX4Si0JviheE/lYFWozEnJiZaWFg4OTl5e3v7+/sHBQWhbKoBAQF79uxxcHAICgqi0+lq1GPoChB4CwgcPdauqXX7aWPCeHaG7AW/zb17F1KsvgWXxQsMEQTDU/AwZSMIAsMwJBueWgwfgAAQAAJAQIUAm80OCQnZvXu3u7u7j48PyqYaHBwcFBTk5+fn7u6+a9eu6OhomUymshFMAgEg8MoJnDrVMXH0gqqW0NTKD7/Q9cr7BAeYygTedsEgUjah8HGBdCQYWlpahoeHMQybwMIgEAjkallBaSpfjdB3IAAEphiBvLy8Xbt2OTo6kuYFFO6MzAuOjo5OTk7379+f4Lt0ig0YugsEpgiBhw95f/ww9+libeNZGHL+8MfclpaRKTIy6OabIfC2C4aCggJ9ff27d++q4i8tLR0aGlKdQ6VSpVIp8lPCMCwmJubkyZMsFgt+BVUpwTQQAAJvFQEWixUcHIzMC76+vmSxNmRe8PDwsLS0DA8PF4kgaeNbdV3AYNWFwDGFV1K+hmbe+HHPOZpaeVrahf7+zTiEMKjLeVPTfrzVggHHcTabvXv3bqlUSqfTk5KSSktLMQy7d+8eg8EoKCgoLCzMzMykUqkbN24sLi5G5/Dy5ctHjx5F0xiGqemJhW4BASAwLQi0tbWNen7xZoel+pQkNzfXwsLCwcFhPPOCi4vLgwcPVDd5s52HowOBt4qAVIpFRHQvWlysqZU9pmb4y1/zjE0qzod1joyA0+BbdWlMZrBvtWAgCILFYtnb2yM33B9/mB0cHGpra11dXe/fv29lZVVRUWFjY/Pw4UMXFxcUtCcQCAwNDSMjI/fv38/hcOCHcDIXHWwDBIDAsxGQSCQGBgY/FjRAHpJq9YXDZDKDg4MtLCzc3Nz27NkTGBgYHBwcEhISGBjo5+fn4eGxe/fuiIgIiUTybGOFtYAAEHglBAQCeXx83x8/vPW0Zsj+3Qc5JSXMV3JI2Ol0JDDlBQPpJkSeHTSH/DjxBI/Hs7e3b2xs3L17N0EQYWFhsbGxBw4caGtrCw4O5vF4FAqlrq7Ox8eHz+cTBEGj0Xbu3EkQREhISGpq6sQ7h6VAAAgAgRchEBsX9+tf/3rBggW1tbXP9c32Igd9xm0LCwt37dpFmhdQZqTg4OCAgAAfHx8HBwcXFxeIXnhGmLAaEHilBPr7RX/9W97T8QzZv/9Dbl/f4wDOV3p02Pn0IDDlBcOLnIaWlpZz584FBQVxuVwLC4v79++fOXOmubnZ2dm5tLTUzs6us7Nz+/btpaWlFAqlqakJwzCxWOzt7V1fX3/69OmampoXOTpsCwSAABCYgMDQ0NDnn3/+q1/9asaMGdu2bROL1SgSQCwWh4aGWlpaUigUHx8f0ryAohc8PT0tLS0jIiLEYvEEA4RFQAAIvB4Czc0jv//DqApu2R98kNPZKXg9HYCjTAMCU1swiGUiJovR0dFRW1tXXFx879692tranp4eqVT6LOdmaGjowoULPT09BEF0dHRcuXKlvr5eIBCEhYWlp6eHhobm5eWFhoaWl5cXFBTU1NQgfwAajRYTE1NWVqZW7gHPMl5YBwgAgSlE4MCBA3/5y1/+9a9/LVq0SFtbOyUlRX2+c+pqa+3t7R0cHDw9Pf38/FBpZ+SPtGfPHicnJ4hemEJXGnR12hNoaOBoLxgVxpD9299lP3qkcJ2ABgSehYC6C4ZRVngcx2k0WmFh4dWYHw4fOer/nZ+Hj5uPj8/hA0cuXLhw7fq14uLitrY2Ho8Hab+f5fTDOkAACKgngaampg8//PDMmTNGRkYBAQEODg5fLVxIpVLVobdisfjcuXO7du1SNS+g2gu+vr4eHh5WVlaXLl2C5EjqcLKgD0CAIIjaWs7TAQxZGprZ2guyOzpAMMAF8qwE1FowkI/TRkZGioqKwsLC9u/f7+Li4u3tffr0qbS067XVdb29vQKBgMxWhOO4UChkMplDQ0MQbPesVwGsBwSAgJoROHbsmL6+PoPBMDY29vX17enpWbhwYUZGBvmt+Ab7W1dX5+DgYGdnp2peIKMXnJycKBRKY2OjOnT1DVKCQwMB9SFQVTWsqXVLtVKbhma2plZWOwgG9TlJat8TNRUM5C+NQCC8kX4T1RCNiooqKSnp6+sTjeXLi+O4XC4XiUQjIyMMBqO/v5/BYChrqymsFMoToSjEppzGCQK9lElRIfew2l+m0EEg8LYR6Ovro/b3EwRhbm7u4uKCYRiVSuVyuW+cg0gkCg0NtbCwoFAoe/bsGbP2wqVLlyB64Y2fKegAECAJlJezNbXyfiYYslvbeOQ6MAEEJiag1oKhvr7B29t37979tbX1v/jzg+O4TCZDgoHJZPb391OpVA6HgynqMcs6+4YGWByCkOGYFMMxnMBwHCNwDMMxOVQrmfgagaVAAAi8RgJPHnA8PiQSDOpTV76x8YGdnZ2trS0yLwQGBqLSzoGBgb6+vsi8gGovjBrIa0QIhwICQOApAiUlrJ8LBg3NrIcPQTA8BQo+TEBAfQVDUVGRu7t7RkamUKjIDfLEODDuWEjBwOPxmEwmlUrt7+9nsViNDxo5HOaxyMJ953MIQi6TYVyeUCASyeRSnJAqTQ3j7hMWAAEgAATeLAEzMzMXFxe5XP5m77/lcrlYLB4eHo6Ojt61a5erqytpXkDRC/7+/p6entbW1pcvXxYKFbka32yH3+xZg6MDAbUicOfO0JiCoanpzRst1QoUdGYCAmoqGDgczu3bt7u6ugQCgVAolEqlqG7RBCNRFQxsNptKpfb29rLZ7JamptaOR5u9khduj3I6lL5jT7IZJW69R+Jm70S7falnYovvlLeyOGRmsZ/8k5C9guzAqEMjAQO/iKOwwEcgAAReLgFTU1MkGF7ubifeG47jKIs0h8NBHp5tbW11dXWJiYm2yubh4UEmRyKjF5ydnR0cHOrq6uCLcWK8sBQIvGYCeXmDYwqGB40gGF7zqZjCh1M7wYB+aTgcztDQEIPBGB4eRoLhF3+BxhQMQwyGWMivbOxYZBG31LXwa7ucJc6FS1xKlrgWLXYp/Nbx1leWGQst4vWcYo9dKmhq65fJJLTe3txbORcuXAgJCfHy8nJzc/Pw8PD39w8NDc3MzOzs7EQB1iAYpvBVD10HAlOHQGJSYm5u7i9+Ab7ggDAMk0gkfD5/eHh4YIDe3d3d3t7e1NRUU1NTpWwVFRV5eXmBgYHbt293cXFRNS8EBwf7+fkh88KJEyc4HM4LdgY2BwJA4OUSyMoa+JlgyNLQzKpvgL/Wl0t6Ou9N7QQDQRAymayvr6+/v5/JZLJYLIFAIJPJfvH3khQM6DePRqN1d3fT6QNiEe9cfMmXu9JXepSt9Li3yrNspWf5cvdSHfdiHfei5e73VniWL3MtXmiZscQy1tI3coe1/cbNmzZu3Lh58+YtW7Zs2rRp8+bN69evNzU1Xbt27ZYtW0JCQkpKSlCpBzI703S+RmBsQAAIvCECv/i9N7l+oW9LpBCQA2dXV1dTU1N9fX1tbW1VVVVlZWWFsqHp6urqysrKK1euWFpa2traenh4+Pr6omJtZDZVFxcXBweHe/fu/aL76OT6DFsBASAwaQLpN+hjCoa6ehAMk4b61m2oXoIB/TryeLyuri46nf7gwYMbN260tbU9S4JU9BMoFouRYKDT6T09PXQ6ncVmbfWK/deOVB2Xuys8K5a55H9jnbxkV9xKx8TVdgk61nFLrZO/tb251CV/BaVsoXXWlxvP6m9w3rZ9y7atW7dv27Z169YtW7Zs3Lhx/fr15ubmJiYmenp6+vr6Pj4+Dx48mFrXC3IzwJSNtJCQv+6qtybkUuQJRn5UXWdqjR16CwSmHAHyb/Ol9BzHcZFINDw8PDQ01N3d/fDhw6ampoaGhurq6oqKikplQ/IAmRTQe3V1dVVVVXV1dUFBQWBg4I4dO5B5wd/fP1jZgoKCAgICvL29bWxsTpw4MTIy8lJ6CzsBAkDgJRJISaWCYHiJPN/OXamjYGCxWJ2dnTQa7f79+9nZ2Y8ePXoWCwOGYVKpVCQS8Xg8NptNp9N7e3tpNBqDycgqfBB4MsvENfFri9hPNsessbv8oK17iDkyyOL2D7DCIpNWrLP7Zl3AV1vCv7G+ruNcvHB3ss6GPVu27ti+bfuWLVs2b968cePGdevWmZqaGhkZ6evr6+npLV++3MjIKDIyks9X1D2ZEnfSqolWSCWAuGEYNiqqEt2sqM5EeuPt/DuBUQOBN0VAoGwveHQcx6uqqvLy8mpqapC5ABkQKioqkEggBcMozYAEQ01NzbVr12yUDUUvjEqO5OrqamdnV1paOiW+CV8QJmwOBKYcgZQUEAxT7qSpXYfVSzCgO28qldrV1UWlUplMJofDEQqFzyIY5HK5RCIRCoUcDofJZNJotJ6ent7eXsbQEE/IlUqEDDanvKHzdGzJVq+Y6wUPBDx2V3d7eHjoOnNzMyMjMzMTk/WbV6+z/3bj4a+trn1jeX3J+kMbt23fvk1hYTA3N0dSQVdXd82aNatXr165cuXy5cuXLVvm4uLS3t6udid2rA5hGHb37t0TJ07U1taSyzEMu3Llyii3Y/LRJoZhPB4vNDQ0Li4OhZ6TG8IEEAACr4HAnj17/P39VdX+JA7KGBo6fvx4YGDgzZs3kcWA1Amq9gQ0XVlZqTqzurq6vLz89OnTu3fvVjUvhISEBAcHo+RINjY2x48fJ79GQDZM4hzBJkDg1REAwfDq2L49e1Y7wYBhWE9PT3d3N41GGxoaekbBgKq2IX8kJBioVGqPstFpVCabxRsRiBSZ/uQEQfT09eUVlN4tKPT23rNGd42+voGRsbGp6VpzM7ON6802bNpusMFl4Zaz/7ZIWrb+iPmmjSZGJps2bXJ2dnZzc7OzszM3N1+5cuUyZdPR0Vm0aJGZmdmdO3fQRfPGfymRHQB/XJlu9JXc0tKiq6vL5XKlUmlPT49MJiMIgslkSqVSPp/PZDK5XC6DwThz5gyDwUBjSU1NTUpKam5uVrU2jN4vfAYCQODVEMjPzy8oKJjcFwu51d27d93c3FxdXb/77ruMjAxkZ1BVBeNNV1ZW1tTUZGRkODs7W1tbj0qOFBgY6OPj4+LiYmNjk5aWhsytU8Xi+mpOF+wVCKgjARAM6nhWplqf1Esw4DgulUo7OztVBcOzBD2TEc88Ho8UDN3d3V1dXSh4mscbEYlEfD6/ubm5rra2oqLcy8tLT09PV1dXX1/f0NBw7dq1ZmZm69ev27Jly45tWzdts1y6cd/fzM9/YxqSkpZG7e/nKhuDwWhubk5ISLCzs1uxYsVSZVuyZMmKFSvCwsIEgsfpWcnf6dd/PeAKUYThmLKI9c8OT6PRbGxsJBJJeHh4TEzM3r17fzTjUCiUH1PQuri4xMfHe3h4VFZWrl+/vqurC2mPw4cPb9mypaKigiAICPL+GVGYAQReFYEX/xpBexAIBBERERQKxd3dnUKhIM2AfI3G0wnk/GplO336tIWFhZOTE0qOFBQURJoXvLy87OzsgoODi4qKWlpa1KEW9as6H7BfIDBlCSQng0vSlD15atNx9RIMBEGIRKKOjo6enh7SwvCMgoEMYCAFQ2dn56NHj3p6ehgMBo/HGxkZuX//fm1tbU1Njb+/v7Gx8cqVK1evXq2np2dgYGBiYmJubr5hw/qtWzfv3Llzt8VOc/P19n6hn6w7eyq2RHm+cLky8yASHj09PVFRUebm5qRm+Oabb1xdXVtbW9HJffEf+8ldJGK5jM5gK4/+U00Jcld0Ot3W1rasrOzHDCcEQdjZ2ZWUlLi7uw8ODnp6enK5XBcXl6qqqh+jGdFjQjSK5uZmBwcHkUj0pgZF9h8mgAAQeF4CdXV1/v7+bm5u7sqmqhl+UTbU1tbm5eVRKBRLS0t3d3fV5EiotDOFQrGzs4uNja2pqamsrGxububxoHbs854iWB8IvFoCIBheLd+3Y+9qJxgEAkF7e/vjeGUGg8PhCAQCqVQ68a2bQwhZAAAgAElEQVQqitwVCoWqFoaOjo62trauri5Uz6G1tbWurq6+vv7EiRMbNmxAcQhIMBgaGiLBsHHjxu3bt+3atct07doLF8Jlcll8Tu3npidzyx4SBCHHMblMJpZIBALByMjI8PBwaWmpk5PT0qVLlyjbN998Y2xsHB8fr2pqIFMMvdIrCsMxgsCEEmnA2cy4jCqCGMPCwGaz6+vrnZ2da2pqdu/eTRBESEhIfX29u7s7m8328fHhcrnOzs4VFRU+Pj4oYkGubARBoOqtE5+FVzpA2DkQeDsJMBiMoaGhSY9dKBRGR0e7uroiteDu7q7qm/SLgqGiouL48eM7d+50cnLy8vJCyZGQeQElR7KzswsKCiouLkYVGyoqKpqbmyFX0qTPF2wIBF4FgRcXDFJCJsAFXJzLJtgsgsUhOFycy5FzuHIuX86XEQr3ZmjTm4DaCYaRkZFJCwZ0H09aGDo6OlpaWjo6OgYHB1taWsrKylCuj+3btxsbG+vo6KxYsWLVqlW6urpIMKxbt27Tps0WFhYbNmzw8fERiUTKcy+/mlGzcld4Z/8ggWMyuVQikaBgCS6XOzw8/OjRo4MHD65cuXLJkiXLli1bsmTJt99+a21tnZubS3r0vh6nXvoQ23Ffxl/XXoi7UaPs+WgLQ3l5ubW1dX5+vlwuP3ny5Llz59LS0qhU6oYNG9LT07dt25aVlbVhw4bc3NyAgIDa2locx5lM5sGDB7Oysrq7u8ElaXp/F8Do1JNAYGCgs7PzpIOe29vbg4KCkD+SqmagUCj79u3LyckhS7ORbkjkRG1tbX5+vqOj4+7du93d3X18fFDthZCQEFXzQlxcHBIeKNUS0gxgZ1DPywl69XYSSErqHzOtam3duHUYuDi3QdoQL0g4xD3kMOxoSjdb1LP4Hw//+ae6P/+h7MMP7/7pT/kf/Tnnr3/N+ftnt75YU6jnUOF4sPFQWt+1XkGfFJO+nZyn96jVTjAMDw9PTjBInjz4VxUMTU1NSDbcuXOnuLi4tLTUz8/P3Nx81apVpGBYs2YNcklav379li1btm/f/mOZtoaGBmRSUAYDYIcu5jvuS8EwTKbMxSQSiQQCAY/HQ5phYGAgOjra2NgYuSctXbr022+/XbJkiYWFxZUrV1pbW5+ljsSkrzOFBYMgGtto5u6xC60zl9rnGbhcib5WxeYq8r0qtMrjN8X/QkXkN4HCl1FKE6lUOvKk8Xg8Pp8vEokkEolMJkOpV7lcLp/PR2HlEMOAkMI7EHhtBGxtbdevXz85wSCXyxMTE3+uFtyUjUKhnDhxorCwcDw7Q01NTVFRUUhIiL29vbe3t5+fX3BwsGr0gr29fUhISFFREUrVipQGkg1NTU2gGV7bRQIHAgITE0hKGjuGYZRg4GLccnH5Ke5pC5bF5/TPf9/7B81OrfnNGu9XzJuT9/471+bMSpozM2H2zDjlK372zPjZio8Js2cmzZ6ZMmdW6pyZ8bO1U36rV2B4vPnEQ67CNQPatCGgdoKByWQiwUCn08m0qhKJZOJbVZRTlbQwsFgsGo3W0dHR2NjY3NxcVFSUl5dXUlKSlpa2detWMzMzHR2d5cuXkxYGfX19ExOTdevWoaLOP+YxROmDlLmGFOdaJJHt8o/LuNtMEJhIIhKJxAIBH8VFcDgcNpvNZDLz8vIsLS11dHRQAiXS2qCrq+vg4BAWFnbv3r2BgYFRlw6ZwHTU/Gf8iBMYjmN9NIae1eWv7HJWelas8Li31PnulxYpZq7xJdXtBIFjmByTyzBMkSEKGhAAAlOLgKurq6mp6eQEw8DAwMGDB11cXEjbwqgJNze3U6dOIc0wKpsquvuvrq6+du2an5+ft7e3amlnPz8/CoXi6OgYFxc3po0C4hmm1mUGvZ3eBBITx7YwPKhXPENkY+wsYbY72+Mb+re/7f+d1sACzS6t9yvmz8l5b1bynJmxs2fGKF+xsxXTSC2M944kRPLsmcmzNZK1NxdvzaXekuNw+zEdri+1EwwMBuPFBQOTyaRSqR0dHQ8ePKiqqsrOzs7Pzy8uLj5+/LiZouCCyc8Fg7GxsZmZ2aZNm1AQgmrILzrP5XWda52iGKwRmUQoEAr5SgvDyMgIR9nYytbU1LR//35DQ0O0f6RJli9fvmTJksWLFy9btszU1NTLyys6Orquro7MWU5eR5OJEMBxOSbHMHl4culnOxNWuZet8ihZ5Vm2wrN8keOdryyuXIi/J5VLCBwJhtFOSuShYQIIAAH1JODp6WlkZPRcggF9k+A4np+f7+XlRYY7j1ILKJ4BaYaCgoIx7/urqqoqKytTUlIOHz4cpGyo9oK3t7etra23t3dhYeGYG5J2huHh4cl8s6nnyYBeAYGpSWBMwaClkRtdVRjI8/837UstqrY2Y4Fml/bcqvmzs95TGA1URcITzfCbq7N/c1VFPKjMf2p90gSRMntW/Jy1hWaFA4VTkxz0+icCaicYBgcH29raent7J21hGB4eJgVDfX397du3MzIy8vLyCgoK3N3dzc3NDQwMxhQMpqamPzojmZqa1tfXjxIMuByXyCUWPolHom5jckwg4JOCgcvlcjic4eFhNpvNYrGoVGp2dra7u7uhoeHKlStXKdtKZUOF3hYvXvztt98uX75806ZNgYGBqampE+QV+eUfWpyQY3JlcIUs+FzOQotUHUrRUufbq9zLV7qX6biVfLEj4WjUXQyXEiDxf7rsYQoITBkCkxAMaGwMBuP7779XDXf+uWBAmoFCoZw5c6akpGRM36RqZbtz584PP/xw5MiR4OBgX19fV1dXKyurEydOlJeXj7kV6Z5UX1/PZrOnDG7oKBCYjgQSfx7DMC9H4+/x2jX/o83S0qIt0Hio+V7h3FkpT+wJyJiANEPC7HdS58zJfO/d/PffK5r7ftm8uZXz59bMn1c7f17N/LnV8+dWzn//3rx378ydk/WewispYc5P4iFO6baUPPu9pHn2FY49/N7pSPdtGZPaCQY6na4qGLhcrkAgeC6XpOHhYQaD0d/f39HRUV1dffPmzczMzPz8/OzsbEtLS3Nzcz09PRTAgG7o16xZo6+vb2xsvFbZNmzYgAJ8yUtAUY5ALicIPOXWg39vONXaRZdJBHz+Y5ekUYIBpTTp7u6+desWhULR0dFBNaHJkImVK1euULZly5YtUrbVq1fv3Lnz8OHDmZmZjx49ehJsrTg+qoTwy7JB2VeeUOh8OP3jTT/o213+1ipxkV3uKvd7K9xLvtgeFxpzF3+SN+kZ96bcJbwBASDwhgkoBIOh4XNZGFCPi4qKfsyVPIF5gdQPbsoWGhpKJjtCt/uq7yhr6u3bt69du3blypXz589fuXLl7t27E6gFtHllZWVDQwNohjd8GcHh324CCT93SXo3T3NnkBZdQ6NZ49289x/f5ZM6IVYhEt7Ne39u1fz5zZpaPdratAXaAwu0hxZoM5SvoQXag8o59AXa1AVafdpavdpaPdrKmAfNucidKemJckCyIWX23zP+kdid9Hafiik8erUTDDQabXKCQSwWCwQCFIWMBEN7e3tZWVlqampmZubt27fT09O3bt1qbm6ur6+PohdWrVq1evVqVLvNyMjIRNk2b97c399PnlJ0yy6XS3FcTh1kf7XxzHfh2XKplMdXFHZAagFZGIaVjcViMZnMoaEhBoNRW1u7bt06HR2d9evXOzo6GhkZrVixAvkprVa2lcqGCsAtWrRo8eLF+vr61tbWJ06cuH37NpVKnThyg+ykQloo9AA2wOYaOkRdSi2taey2+S7tC4ukZW5lOu6ln22Lzi5pQgoEBIMqN5gGAmpOwMPD43ldkgiC4I2MhIWFTRC9QKoFNOHm5kahUEJDQ4uKisZ0MUJ3/zU1NbW1tSjE+dnLRVdUVNTV1YFmUPMrDbo3jQmkJAxqauZraGb99Jqfoxm7+t17s2fFq7gYxcyelThnzq3359VrKETC4AJt5oLf9X3wT+rHS+hLTQfMbBg2FCbFmeliw7AxH1y3hLb04/5Pftv7O4WQGFigRVXKhl5tpB80WjXfL5s3O/3dx1EQcbNnJs6elTDHtdptRDoyjWlP16GpnWCgUqmqMQzPbmEgU52y2eyhoaG+vr6Ojo67d+8mJiZmZGTk5+enpqZu2rTJzMzMwMAAlWxbs2aNrq6unp4eKvZsrGzr1q1rb28n760xDFOUIpBJpHIZJhPZhiR/szWsu38IBViT5gWkFpByQL5JQ0NDTCbz/PnzixYtQukLExMTv//++40bN27atGmU2WHFihXI8rBs2TLks7R48WJjY2MKhRIWFnb79u1Hjx5xOByRSCQUCpFxQyAQKIKvxeLHGY3kmAyTEATR1E4tKGtRBGpLxReTShfvil/mcnepc6GefUz/AEs5rjFKNEzX6xvGBQSmOgFfX19dXd3nzbRWW1u7Z8+eUfmRRomEUR9JzTCmnQHFJCDZQL6POZNcSk5UVlZWVFSAnWGqX4rQ/ylKQEpID16/pamRrzE/+7FgeO+W5vLz79/Vnhn3juJuXhmKMCt5znsl8zQfaWkzFvyu/4OF1K9smDY/8H4oFZdS5VQ5MUbgshyX0+S0ElHJ5ZHInYyd/6J+qk3VViiHfoW1QWFz6NN+HBeR/q7CTylW6aGUMmf1bb32EcWNFrQpREB9BcPAwACLxSIFw8QWeblcTgoGFos1ODjY29vb1tZ2+/ZtJBhu3bqVkpKyefNmU1NTQ0PD1atXI7WABIOBgYGhoaGRkZGxsbGBgUFxcTHKPYrKlslkMlR7gcBl4Umlf17zfeyNSplYyOUqLAxIKqAYBtVpJpPJYrFaWlrWrVvn4eERHBycnZ198+bNlJSU2NjYiIiIwMBABweHDRs2IPFAviPlgAIeFi1ahDK0Ghoa2traHj9+/ObNmw0NDd3d3b29vT3K1tfXR6PRBgYGhgYVEkVRUY7DEQhFEqmEIGQ5Ja1LdkcvpRQvtLzhezJLESOtKPEGDQgAgalBwM/Pb/ny5aqeir/Yb5lUGhcX91xqgbQzuLm5kXaGMfMmkTJg4olR2yJpAZrhF88drAAEXi6BOnHdFuYW7a7faXo6KwTDe7c05uVqaObMO7Vm1vXfPJYKSXPeK52r2aP1P/S/b2BsDOWeKxeXczHu8/ZkQD54jX/NgrHrr9S/aQ8+kQ09StnQqfX+vXmzEpVOSnGKHEr/zPjk3lDZ8x4C1n+DBKa8YMCVTSaTicViVOaZxWINDAz09PS0tLTk5eUlJiZmZmbeunUrPT19586da9euNTExIW0LyLxACgYjI6M1a9acP3+eIAipVCqTyaRSKVILIpEIk0vyypr/bnTWKjiFz+eT5gWUIonNZiPBwFE2UkKkpKRs2rRp9erVW7du9fX1PX/+fFpaWn5+fklJSU1Nzblz53x9fX18fFauXLljxw4DA4Ply5eTIRajxMPixYt1dHTWrVvn4+Pzww8/lJWV9fT00Gg0Op0+ODjIQI3JGGIxhphDDCZnmMUS8Ueu59d9uytGx+3e1xZxja0/eVu9wcsODg0EgMAzEoiKivLx8XkuC0N7e3tISMgkBAOKgUaaYUw7w8QiQXXpKONDdXU1cmFqaGgYGhp6dmfLZ6QEqwEBIDCKgBgXn+Kc/ivtb4pH/n1aWlRNrWgjTZ0wjT+lzKVsm3X9v2bGzpqZMPvdwvd/9/AD0yHTKF5Uu/TlPPVvk7YdGD7wcf8nCtnQp7Q29Clkw/xmzdkZSlODUjP8Pu3DW7S8Ud2Gj2pLQO0EQ39/f3t7e09PD51OZ7FYHA5HIBCIxWKpVDrmbwxyGZJIJEKhUPFwXZkiaWBgoLu7u6mpKTc3Nz4+/ubNmzk5OdnZ2Q4ODkZGRmvXrkWGBT1lM1A2ZF4wMjIyMDCwsrJiMpnIsIAEA/IFkkrE9x/2frnx3OIdF9u7qPwRhXmBzVbkR0KNtDCoxjYwGIwHDx5kZmaePHnSyclp7dq1qOBDYGBgREREWFhYfn5+XFychYVFQUFBaGior6+vra2tmZnZSmVbvnw5kg0o/gGVd1i0aNHSpUtNTEy8vLyysrJ6e3v7+/vpdLoidoLJZLOHORwuhzPMHeGOjIxIhIKwuLtfWaR8bZvlcyJHIBRIpVKIZFDbv0noGBAgCeA4jr7c0JMRcv4EEzKZLDk5mUKhPEu48yivpFF2BvRQQ1UGPPt0tbKhgIeqqqqysrL8/Pxr166hAvOogiR8C01wHmEREHgRAh3Sjs2MLUqp8Ng7SLNDU6Nz7rw6zbkR384L2jj38jdzct7/S8NHnmzPGmk19iQtyoscdNS2vbLewOGgP/X/eX6TxvxGDYWHUr+2ZpfWe4VzH/tBJc7WSv1tDjV31IbwUT0JqJ1g6OvrU41hYLPZPB5PKBSKxWJUexiVH0axyIrSy0p/IZFIRJoXhoaGqFTqo0eP6uvrMzMzY2Jibt68mZ2dnZubGxwcrKenZ2xsrK/SSPMCimEwMTHR1dVNTU1VhAEogwTEYjESDGKRqKd/aMnOi5+Yns0qqhfyRpQ6YQzBwOFwuFzFzTqXy0Vl3ZhM5uDgYHd3d21tbVpa2rFjx+zt7VHlh507d3p4eNjb22dkZMTFxVVVVRUXF6empl66dMna2nrbtm26uroooRNyW1qubDo6OkuWLFm6dKmBgcGJEyfa2tq6u7tpNBoyNbBYLDabzeFwRkZG+Hw+XzDitO/6QpucZZZXG1oecRS2B4W7l0gkmtjXSz2vWugVEHhLCKjeUj+jZujv79+3b98vZlMdUyqQM5HeOHfu3L1798aMgUZ6YJR+QGaEGmWrrKy8e/duRkZGVFTUuXPnDh8+7OXl5ejo6ObmVl9fj8alOrq35ITCMIHAayCQK8j9jPa5Qi2gQIJe7ffL572TMmdmym/mXP6f+YvOabx7W+N3GZqrz+a2Nrzq/lRKKg37jd4tev/dvPc1mjW1laaGueXzFVWiYxVh0L9N+33hwN1X3Q3Y/4sTUDvBQGZJotFoKG6Yy+Xy+XyhUIiie2VPN4lEUXgZ5UdisVhDQ0N0Oh35I5WXl6elpYWHh6enp6PabREREcbGxkbKhiQDUgtkAANKrmpoaLhly5aOjg6ZTCZSNoGi8gJfKBD00wZXWEb879rQ01cLhPwRtvK+HJkX0D06ckxCgoGnbGScA0ulDQ0NdXV1VVVVpaamHjlyxNbWdv369du2bbO2tvb09ERuS7W1tQEBAaWlpZaWlkuXLkUZWpVWB8Ubkg3I4KCjo3P06NGOjo7e3l6kGVAEBZvN5vN5fJ5AKuE3tXUvs4z53CLth+sVcrmUx+NxuVyU00m5Gh+Uw4v/OcEegMDLJfDct9Q4npmZ6aZs5N3/5CbQTsLDw8vKysbLnVpdXV37pFVXV9+7dy83NzcpKSk6OvrEiROBgYFubm52dnbWT9ru3bsjIyPJeIznHt3LhQt7AwLTjgCGY+e55/9I+29FqlOlWtDs0nq3QPlEP37mnKsfanx1UeNdZbokDUU8Q0MN/zUwGMFHgphB79+ePzNp9nuFczXbtbSpC+bVaSjKwyk1w4fX/1zPfuXS5TWMdHof4kUFA3ro9RK/95lMZmtrq+qNL5fLRUYGibLJZDJF1qInTSwWC4VCUjAMDg5SqdSurq779+8XFBQkJCQcOXIkNjY2Kyvr1q1b165d27Vrl66urpGRkeGThvQDckkyNjY2MTFBPksUCoVGo0mlUjIxEZ/P6+sfWG4Z8Q/z83uOZyoMDEoNgAQDepzP4ynSrY6MjCC1gDIaIWsDmYYViQqUTAlFXHR0dJSXlycmJh44cMDKysrMzGz9+vU7d+48cuTIlStXAgICli5dGhIScu7cuR07dqCUrKtWrUKyQUdHZ+nSpStWrEhISOjr6+vv7x8YGGAwGEieDA8P8/k8gUCE49Jjl/M/3pJkG5Iu4POECmwKbsiPi8FgDA0NcTgcsVj8Es/m9P7jgdEBgddAoKioKCoqCsUw/OLfJofDOXHixLNnU51YSyCnpvDwcFU7AzIjIF+j8vLy27dvZ2ZmJiYmnjt37sCBA56eno6Ojvb29jbKZmtra/ek2djYeHl5NTc3/+IoXgNVOAQQmH4EhLjQj+33uCqCUi1otGvNyX5PmZ7onXeS39UwOaCwLagkV62t5bw2Dkn85A+Kfj8zfvY7KXPmVs3X7lswr15jVoKyVFzy7G9yFg2Khl5bZ+BAkyDwooKBdBB6Wb8BQqGwtbW1q6sLOeUzGAwOh8Pj8VAkw+MUoijVqVzRkL8Qn8/ncDhMJnNgYKC3t7ejo6OmpiY3NzcmJubEiRMXLlzIzs6+detWbm7u8ePHUQDDE72gSI6EGumSpCzgtlZPT8/JyamxsRH5EPN4itIL7Z30b3dc+Hh9uHVQMo+viBEYHh5GvkbNzc11dXW1tbVtbW10Ol0RPKBsKH0T0g9IS6hGOJCygcViMRgMOp3+6NGj0tLSmJiYoKCgHTt26OnpmZiYHDhwoL29fWBgoKGh4eTJk8bGxmvWrFmlbCjCYcmSJZaWlq2trX19fSiYgakIZmA33G8YGBhQRmxLe+iM5TYxyyzjWjv7OFxFhANytRIIBMihi/mkgsTIyIhUKp3E9QSbAAEg8HIJpKamBgUFCQSCCXZLfv2Wl5f7+PhMLtx5lHhAaoGibOHh4aWlpbW1tVVVVSUlJXfu3Ll27VpUVNT333/v4+Pj5ubm5OSErAg/1wlIL9ja2lpbW/8YwC0WiycYCCwCAkBgcgSGMfZuhqUiyFhZQE2rT1ujRVNRAwElM037r/d9zTXm5qmqBQ3NrNcpGAiCuCMs+HPxR8iwMCf3fc12rbnV82fGKe0MKbM3FW8RY/D9MLnz/zq2mqRgIH+fcBwXCARUKvVldVYul3d0dLS2tnZ3d6OH5Ww2Gznio9BnuVyOPWlyuRy5JPF4vOHh4aGhIRqN1tPT8/Dhw4qKiuzs7KtXr547dy4qKiozMzM3NxcZGaytrVetWqXqjPRzwYB8k/T09NavX3/mzJm6ujoWiyUWCSsbOr7YdP7TjRHrXKMKi+5e/eHK/v37nJyctm/fbm5ujgwXZmZmO3bs8PT0PHfu3K1bt1pbW4eHh1HnRSKRIqJApUo0V9lQAQeUWAnd6DOZTBqN1traiiIF29rakBFgcHCQRqNdvXrVwMBgjbKtXr161apVK5SVHLKysqhUKunNhUQUi8USiURSiZQgpIFnbn287lJBdYdUIiElDSkbkA0EeXahtLZCoZA81y/rFMN+gAAQeHYC6Ntu4vXRHymPxzt//vwLRi+Mkg0ob5K7u/u5c+cSEhJOnz4dEhLi5+fn4uJia2trY2NjbW1tY2Njq2xPbAlj/I+cLZuamuD7ZOJTCUuBwCQI0OQ00wFThVroeZySaN59DUXQQoyiKNs7N9+Zn/UPjb8naMzLebOCgSCIMlHZX+7+dWbi7JlXZ89Smhrm5CptIMq8ScebT0xi+LDJ6yHwfILhx9QWwcHBTk5OTCaT7B+dTs/JySE/KkuD4RzOYztXb2+vq6urs7NzUFAQaY5QXVl1Gv2WdHd3379//9GjR+hhOQrP5fF4IpFIIpGMKRhGRhTxx0gwdHV1NTU1lZSUZGRkxMTEREREIJekrKwslCvp4sWLZmZmq1evRppBVS0glyRU8hm9GxkZrV69eq2Jsasr5cDB7xy9D3++Iezf26K+NN+ra6C3RlHPYZWenp7qrgwNDQ0MDHR1dVEZaXNzc0dHx+PHj9+4caOpqYnJZKIAblSaGukHdKeOxINqtlYmk4l0AuliNDQ0NDg42NXVZWdnt2rVKl1dXVI2LFu27PTp06qCYXh4WGEYUUaAiCVigpAX13T8wzQ0IlWR/Jj07JJKpWRgN5/PR50ZHh5GFo/BwUEejzdmiirVcwfTQAAIvDoC6I8UGS3R1yCubKpHrK2tRc/7f37T/4Jz3NzcnJ2dHRwcbGxsrKysSJFgZ2dnb28/sVpAXklWVlaRkZFgXlA9XzD99hBoHGpkCRWFU19F65H16A7o/aQWerXn1cx/XPEgZvacW+8t6f/aJ+6GxvtPV3pWOibV1Ay/ii5NvM9iQfHvcv/wOIAhbvaspDkKI0Pc7JkJs+cna5YxoDjDxPze2NLnEwwEQfzwww/Hjh0jCKKhoSE/P18kEjEYjM7OzqGhocbGxjt37gwMDNy4cWPPnj0sluLPA6XuaWxsvHTpEtISEzxhQov6+voaGhrIXEksFmt4eJjL5aJcSSi/KvqxRBYG5FFDCobOzs7Gxsbi4uL09PSYmJirV6+Gh4enpaXdvHkzMzMzJycnMzPz6NGjRkZGurq6yDGJDGD4uWBAssHA2HCNrq6u7vKFeq7/3hz15baohaYBRsYGSi8mI+TLNN67kZGRvr4+qvxgZmZmY2Nz8ODBlJSU+vr6wcFBoVCIbgXQKEifJVI2DCsbSnk0PKzIbsRgMGg0mre3NxIMin4pZYOOjo6vry+qzDA4OMhkMoeHh1E0hdKhSySXyVgcno7FBZ/jCoGHyZ+KBlGVDaSTEpvNZjAYVCq1v7+fy+WCbHhjf6lw4LeGAEqlyufz+/v7GxoaUMBxXl5eTk4OMpPm5+eXlZXV1tY2NDQ8fPhwcHAQVaG5cuWKi4vLpLOpjicqPDw83NzckDCwm1Sztrb28PCA6IW35hKGgT5FQI7J9WP1V15ZOSIZeWrBy/jwSNa5cmCVqlqYWz7vcQKi2NnvFs1dT9tAJ6gpsUOaWqP9kTQ0s96IYCAIIo2dNvemxmOdEKtUC0gzJM3+NmfxiOzlg3oZsN/2fTy3YIiPj7948WJpaWlkZGRiYuKhQ4dKSkpCQkJKSkooFEpiYuKRI0dyc3O/++47hd889riocF5eXm1tLRIMEyBHgqG/v7+urq6trQ3d+yIXHSQYpFKpXC7HnzQMw6RSqUgkQgl/kIXh0btpUp0AACAASURBVKNH9+/fLyoqSk9Pj42NjY+P37t3b3h4+PXr12/cuIE0Q3p6+sGDB83MzHR1dQ0MDCYQDEgGGJkYmRgZ6RubfWl28N/bor/cHPXNWh9jYz1jYxNj44kEg4mJCdqDiYkJclhC4mHlypVGRka7du0KCQmJi4urrKyk0WgCgQAViUNh1iMjIyjbEnrkjz6yleXhenp6HBwckAhBgkGhZpYvp1Aojx49IhMlsdlsFG/N5/PFIpFULpFKxFb+sVYBqTguw+WK0hbKSBC5aok6sbKRIdEoyxOKJqfT6fCMcIILGBYBgckRkMlkPB6PRqU2Njbm5OTExMRcvnz55MmTR44csbGxMTY23rdvX2hoaHh4+MWLFyMjI3/44YcrV65cvXo1NjY2PT39zp07ycnJvr6+LyV6QVU5IPnh4uIyKaWg2AhFL0RHR5PJkSaHCLYCAlOOAI7jvdxeDMfaWG0fnf1IP1afJ+G9xFG0ydqWDehokZ5Ivdrv35unuAuPUdyCv1c6z4HpMIIrbr5jYnvVSjAQBHG6/8ystCe2BaQW0HvK7H0P9r9ESrCrl0XgWQUD6UeLBMP+/fuLioqkUunWrVurqqqOHDnS0tJy+vTprq6uPXv2FBcXX7x4UfEMW3k/iuN4VFQUj/fLfyekhQGFDvf29vb19TEYDPSkXCQSyWQyJEKeSAYcxT2TLklUKrWzs/PBgwfIwhAfH5+SkhIREXHy5MkLFy4kJSWlp6dnZmZmZ2dfu3bt+++/37p1K7rhNjQ0VDUvqJoLkM+SiZH+SmPLf20MXbgt8otNlxYZOxnq6q5es3rVqpWrFY5Ja1DyIuSGhPK0ks5OyExB7h+5Oa1SNlSOzcDAYNu2bX5+ftHR0cXFxd3d3aMCDJCnEErBJBQKa2pqNm3apFp+Tk9Pb9WqVRQKpbW1tb+/H1VjYD8pxSAQCEQioVgilYpFJy/d2ugRK5dJMBxXFQykZkCiBaWf4nA4qMYFMm6glLWo6NLLugRhP0Dg7SSAYxifz29tbb19+3ZycvKFCxd+fM6yZ88eCoXiomxubm7e3t46OjqamppOTk5BQUF79+7dt2/f6dOnL1y4EBkZGR8fn5SUlJiYmJCQcOzYsZduW3B3d/fw8KBQKA4ODpMWDNbW1r6+vu3t7RPYlt/OCwBGPe0JsIXsT8M/3Vu4lyCIdlb7n07/ySTehC99OZlMm6XNiwcWaw8ogxaUOZHeK1amT42ZPSth9tzKeQHDATJChiCroWDACdyq0WZm6s80Q7yimlv9MGRZVbu/j+cQDDiO5+fnf//996mpqVevXj127Fhvb++hQ4eqq6v9/Pxqamr27t374MEDJyenoqKiAwcOIAsDjuPd3d2RkZHPPnSUFLWzs7OtrS0tLa2urg5lLBWLxci8oLorDMMkEgkKeka+Oo8ePWpsbCwpKblx40ZiYmJaWlpycnJkZOTBgwdPnTqVmJiYmpqKZsbGxp47d87FxcXY2HjFihUo3Sp5c09qBsV9v4mRsZHxV2bBn22NWLjth8/Nw5Yb7dhtYXHw4MGrV6+mpqZmZWVlZGRER0fv27fP0tLS2NgY6Qc9ZXgDuSu0c/IQyFtJV1cXiQcUjWBoaLhp0yYKhXLixImUlJR79+61trbSaDRU54HJZLa0tISEhBgaGurp6ek/aUgw+Pn5PXz4sLe3F2VWfVowiKRSmVDIzbxdbeYaLRGLcRXzAspSi8pak8mdhEIhCphGgRAoASsqlIGSPKqeCJgGAkBgYgJSqRR5TnZ3d7e0tNy7dy8+Pv706dP79+/39/f38PDw9PT08vJyd3dHuYnc3Ny8vLx0dHQWLFjg4OCA4o/RTG9v76CgoGPHjkVERCQnJ8fExAQEBCDB4OHhoWoiePFpZ2fnSasFZF5ITU1FD3om5gNLgcA0I4ATeHJT8juH3zlScoQgiIfMh3889cd1SeuEMuELjvSh5OG39G8fl2ZTqoX3i+cpahrEzJ6ZOGdejcYR7hGcwMmjxMT2jWNheH1pVcnOkBMD8oF/Fy5UBECrWhgU0c9z1haaSTAJuSZMqAOBZxUM6OFQSkpKeHg4ij+OjIy8evUqk8nMz8/fv39/UlLSoUOHUlNTAwIC2trawsPDGQwG2qq1tbWlpeUZRyuXy1tbW5ubm7u6urq7uysrKzs6OlDtNolE8vNfHRzHpVIpn89HXkl0Or27u7upqamsrCwzMzM5Ofn69espKSlXr169cOHC4cNHzp49e+XKlaioqEjULl8+e/ZsSEiInZ2dmZkZumUnEygZGBjo6euuWb1GX2/NMmOHzzZe+Gpr5GebI1fsCistr+Zyx44W4vF4LS0t6enpqCIbcnxavXo1Eg+k2YE0OKxduxaVk0NuS8bGxoaGhigsYfXq1fr6+qamptu3b7e3t3d0dLS1tV23bt0TmfDU/2vWrDl27Fhra2tPT89YgkEskYp5I9zy6rZ1blf4QgEulyn+KZv0SSOjKpGdoaenh06ns1gsshjF8PBwZ2dnX1/fM55QWA0IvJ0EkMMkh8Pp6+trbm4m0zBERkZevHjx/PnzYeGKdunSpaioqIiIiLAn7ejRo/v27du/f7+fnx8SDNra2g4ODqQSQCXV0Lu3t/eBAweOHDlCpjN6cYWguocXNy8EBAT09PS8ndcAjBoIEASR1JQ089DME2WK/D+Ng40fnPhgU8omkUw0aTgd0o7FA0ueUgv3ntgWkuZo1GudGTkzaufqKRgIgijmFs+/oaUIuhilGRLnpPVdGzUK+PhmCTyrYHhtveTz+c3NzahoMZ1OR85IyLl/PIs2juMymUwgEKAsol1dXa2trZWVlbm5uSkpKRcvXvwxQVNgYMDpM2ezsm9V19Tl37lzIeLi9yeOH//++PdHjh4+eOjQoUMHDhwIDg52d3e3sbHZtm3bunXrNmzYsGvXbk9Pr+PfH/EKPvjFulNfbrv81ZbL/zANCwnNfUYgPB7v4cOHN27cOHr0qL29PdIkKFiZjJ1QjaAgbRHkBBIY+vr6yHVqzZo1SHgYPGkobltfX9/IyOjq1asoIy3KrErGi/P5fEVmVamUzWI2tXZb+CeP8IXYY7GgUAxP9IJUVTBIpdLh4eFHjx6x2WxUThsVehsYGGhsbOTxeLiyPSMKWA0ITHsCOI7zeDwqlVpTU5OTk5OcnBwaGrp///6QkJCAgAA/P7/AwMCgoKCQkJC9e/eeOnXqgrJFR0fHxcUlPGnxT9rly5dDQ0MNDQ0XLFhgZWXl7Ozs6uqKjA+q4sHBwcHR0fGlRy8g2fDi5oWUlBSZ7LFfxLS/AGCAQGBMAnH3435z8DdnKhT38Q8GHyw4vmB72vbJ+Sb1yft06MufUgtlj20Ls1LmaD7QPsc79/M+XI0ZL4bhTVoYUD/9Wv1n/dwxKUlRyk0gf1FTzM9RwJxJE1A7wcBgMFD1saGhIWRYEIlEYrEYPfAWKUsGoDoGyE9G6Z2vmM3j8ZCRoa+vr6ur68GDB4WFhdevX7979+6tW7ciIiKCg0P27PG5HBlVVl5ZVVN77Xr66dNn9u3bHxQUHBgYFBAQ5O8f4O/vHxAQsH///sjISGXtBTaByx90DBg7/vDZpoiF2y4t3BL1702hD1onU3diZGSkra0tIyPj5MmTzs7O69evJ7MnIfGAtAEpFSaeQDqBfNfX19+xY0dBQUFLS0tXVxcpGFDYNFkqm0aj9vQP2oYkc0Z40nF0Agp6Jt/RtqhWA7IzsFgs5Pj0i1Hsk74uYUMgMLUIjIyMtLa2ZmZkhIWF/eiQuWfPHtcnDd3io/vvUcYBX2Xbv3//6dOno6Ki4uPjk5OT09LSUpXt2rVrGRkZtra2H374YVBQkLe3t4ODg729vYODAxIPbm5urq6udnZ2NjY2r0IzuLm5vWD0ApgXptZlDL19uQQKugrOV52vp9cTBHGl4cp/HfivsKowgiBqaDVax7UOlRx63sMx5AyjAeOfciL1ac+tfBzlPCt5jkaj5umR02PuU50FA1PO/Oz2F2M5Js2O6FBk14SmJgTURzA8drbr7e0tLy9H1cpQqpCoqKgffvghIiLiwoULqampxcXFtbW19+/fb2xsrK2tzc7JuXL1ytmzZw8cOLB///5Tp05dvHjx0qVL0dHRV65cjYmJYTIel4wQCQWPOtozMjIuRlyKjr5642ZmRmZ2Suq1y5HRp06dPXz46JkzobGxscXFxVQqVS6Xo6DtlLwHOrvCP1l/aeHWyK+2Rv/d9HxIaA6h4ho4wYmc4AG8QCDo6OjIzc09efKkg4ODqakpimDW19cnLQ8TCwa0FGkMFAtx4MCBmpqalpaWzs5OKpU6ODjIYrE4yoZu+sUScX9fb+uj/s1ecdwRoewpc4LCtECKBIUCe9JQ9DP6JFQ2DofT1dXV2NiIKE1AABYBgWlJgLR24jjOYDAKCwvDwsICAgIoFAqSCc/yvJ8UDyi2GD2qOH/+/NWrV9PS0tKVLSsry8nJ6aOPPoqJiUlISDhx4oSXlxdKPeSgbPZPyiDY2to6ODi83LjnF0yOZGNjA9EL0/L6h0H9IgEZJnPPdV8YsfDry1/POTxnf5Ei7U9kXeR/7PuPizWKlDANAw2tzNZf3I/qClyMu2Voq6pamFevMStBUZ1tVvKc+U0ah0bGVSDjCYbq6rE9q1WP+xqm04fSZ6e99zOvpNmfZHzKkb55G8hrIDAlDvEmBQNfKGjtond0D3XSGHK5DMcxHMevX7+O0n5TKBRPT889e/YEBATs3bv3yJEjFy9evH2ngEYfJMmKRKLG5qaU1JSjR4+6ublZWVk5Ojp+9913hw8fPnDgwMGDh0+fDh0cGED39yNC4eCwIjuBSCKi0+n379+vqKgoLy9vbGzs7e1lsVhSqZTcM5oQS2UW/gn/WBv25bbLX2yL+Nf6S0aOl6lDHAL/KZZo1CaT+Mjlcqurq8+dO2dra7t+/XoTExMkG1CqpYllg5GRESoSt2PHjps3b9bV1ZGCYWBgAJViQIYaoUAkEI70dnVVPegycr7M54tkUoXpRrU90Qhj/I+kAsq1yuVy+/r66urqBALBJMYLmwCB6UGAyWQWFhYeP37c3d0dOQuh+/Vnv2snPYtQBAL60vP39z9+/HhMTMzNmzfz8vJcXFw++uij+Pj43NzcrKys5OTkU6dOeXl52dvb2ykbqpuG3h0cHJ5FqyBzx8TvLxi9YGVlBeaF6XGdwygmQeBk+UmDWAOWkCWRSy7VXPqvQ/+FciVdqL7wn/v/82brzefdpwyXuTBdtYd+quU8v0lzVvIcRb3k5DnzmzV9h/3kuOJB55jt6tWxXZLURDBgBGZaYT4zeXQkw6zkOZc7nyNlzphjh5kvi8CbFAylDZ0fm574dP15PduLXB6fwBVFG27fLti797v9+/d/9913Icr23XffHTp06NSpU5cvX87Lu93bS5VKZDKZXCqVslis+oaGlNSUU6dOonSEgYGBZ8+ejYiICA+/EB5+MTz8orIoNdYzxN7pG3smtkjhRUM8rg4xJsRRZoH7D3u+2RL27y2XP9lwacn28OqmHoLAcTn2jEaGMQ9BzlQ9Fo7jXV1d0dHRKJTCwsLC3NwcmQ5I5UBWdUBB0qik9OrVq83NzS9dulRaWlpfX//w4cPOzs7+/n46nY4EA4fDUVgYBEIOh03t60nMaTB0uiwUCiRiMakMkGxAfkfkTNUJgUrjcrk0Gq2+vh7V5iOHAxNA4C0hIJFIqqqqjh8/7uHh4erq+uwKYeJ7dCQbkOXB19cX5UlDgiEhISFX2VAFt9TU1JMnT7q5uamqBTTt5OT0i0d5lhWcnZ1JTYKUybO/29raWllZ/ZgDA4yQb8lfBAxTlQCO459e+FRVFUTVRf3mwG8q+isUNcta0tpYbarrP8v04eEj2vQFWr3KJKq92hqtmu9ce1ehFhLnzG/WcB/2kOKjn3iq7lbNBQNBEKXcsrnX54+Ofk6avTD7a74cHk2qnsw3Nv0mBUN71+DCzeGfb7r09dbznb1MdAt+505hcPDeoKCgffv2nTp1KiEhoaSkpKWlpa+vr6+3t7mlpa+fimGPH/CTFobjx497eno6OzsjwXD58uXIyMiIS5ePHz9Fo/aWN3QaOcd9uiN9V9A1hSlDxaEI3bKr3rirngrkeHAmvujPeifXWF+obOhVeu0rChi8RMGgevTBwcGLFy9GRERERUWFhYUdPHjQ3d3dwsLC7P+x9x5gTWR7/zjP+/zbT1fUdVdl3b2v9969e/febbp9LWBDRRGQKh2kC0gXUURUkCYgvfcaktB7L9JLIPQaSkIoAQIkAVL/mzkyhiJrwxU3nyfPcGbmzJlzvmdCzme+TVr64sWLFyCIQwBu0GfPnlVSUoqMjCwvL6+rq2ttbe3s7Ozv7x8eHh4dHQUpLMhk8tzcHJVKHSMSx4iEu3558pZxtAXK4sIzwgArEJ5XgPkCCEhFJBKbm5snJyd5xcUv8yXwV5DA2NhYTEzMrVu3jI2N39Tr/LUreFNTUzMzMzs7Ozk5uS+//BKJRBYWFuZDKCgoKCkpycrKun37to6ODi9n0NXV1dfXf30O88rqBT09PX19fW1t7YcPHxIIr+Lr9Vd4hPhjfP8kgCFiEtsScTM4MLSj4UdNckx4h3ku5pxtsS3vkRcvR8/HHCB8un/kKVvYh9svmLWbyxYSBD9q36szrUNj/4Fz8LtPGDgcjmaT1lolw/aknYnDiBeXFb/m5kngzyQMi3SGonX8Ifngr6V9I5NruG/u2eza2tqEhITy8vKBgQEQh4d38HQGY2JqksFigvU6lbrQ2t6ORqM93D2sLC2vX79+7949f3//iIiI6KjoqPBwN3d3B5/kU9pxIoalZ8yrRHTih/ETvA1uXObqItjs6VmKY3BBz9A4Z0PVxMZNveDZ+fn56Ohob2/vqKio1NTUnJyc7OzslJSUyMhIT0/PW7duXbt2TV1dXUlJSUNDQ1dX18bGBoVC1dTU1NbWNjU1tba2dnV1AcJAIBAmJibgQEk02sIgbpA4RpQ0irzumLJEX6BRaTANWMUT4ONrCxQKhUwmA5suvobhBaeVX21LSwD2WKDTlxobG318fEwhrF3lv8EjwFrJ1NRUR0dHU1MzMDAwOzu7CEJhYWFJSUloaKihoaGuri4vYQBlAwOD69evv05ngC+1/itBV1fXyMiotLQUltuWnn1+5/kS2FgCdBbdPM9cyF3o08efCnkI5fVxgygmtiX+3w/+78S2RPha2SRZr5r1PZLhOusWKhcq/0X4Yj/+WYK2XcV7tscIbo8T/Kjl40sTElOsqXUv5D0YHb2uSVLOO2KSBLpaN1u3J2WNkgG582KJ+Mb6E96R8subJ4E/kzBwOJyskvZDMl6HFELPawf14EY5HA6DwaBQKLAOAXqj/8xhgMVmT81M05e1BCsIg5WViYmJg4NDUFBQVFRUTEyUb1CotMHjn9VQJ80qzlhWilpW/qSRlFr8UukD2Uwmiw05QHP5DGsjW6Y3MkksFis5Odnb2/vx48exsbGZmZlFRUVPnjypq6traGiora2tqKgoLCzMyckpLi4G28bGxqampubmZiwW297e3tPTMzAwMDw8DPI9g3Rvs7Oz8/OUgb6equaBryU8XcMK2Ew6jUqlLGMVMVg+vM5fkCOPQCDwTZLeyIzzG9kqEqAvLeXn5d28edPExGSTkh6su8S3gGBlZeXs7JycnFxSUlJaWlpQUAByRK5LGHR1dQ0MDEA/123zeQdhn4pXjqaqp6enpaXl5uY2M/NOOFNulaeL388tKoFF5qJqsuovIb9gx7EzCzPKycpf+X01vzTP4XBuFNz4P07/x7HCsX+6363STSRChER7GoLlxQc7xBj6dfQ3rjHS0FP1wh4oiOq2WMEP6z76lfgbjvFUp7Fxm1uCMLA4LMmqy6uVDAmCOxEf1k3XbTxA/tm3IIE/lTBwfQEY9j6Fh+T8v5ELNLyHTk5Nd3R8YHL9+p27dzDNmLXjZ7PZU9PTdG5Uby6LoFBo2LZWFArF1TBYWZmamjo6OoaFhcTGxjg/Dj2j+fhXrbSTJsVnzCtETMuPGBYeUk+2fJS5sQ/DipuyOUw2C3LGZnPNoN6or/OKG/HsYDAYNzc3T0/PoKCgpKSk9PT0wsLCysrKxsbG9vb2rq6uvr6+fgg9PT2dnZ3t7e1tbW3tELq6unp7e4EPw+joKPB7BrGSxscncAM99wMK/33RI7kAw1ykUSmU+WWsYgbLh5/+5T0Lx66Fws7+8YuNp0Zc0IRBImTxWITxDJtf5EvgHZYAg8HIy8uztLR8fWuf5y3Wn3ccZIA2MzMzNTW9c+dOcHBwQUFBXl6eh4fHtWvX1iUMsJ7hFTgDcOB+JdUC9yJdXd1r167x1Qvv8LPM79obkwCNTlNGK5+IPDFBeWq50DDacMDjAH4Oz/3h47ADGwK/9PvyoPdBaYR0/3T/y96YyqbKjcs/S7kwIvQxlhsWaVuM4O7yPf8d/W/t0osuo6OihtbL9PxuaRg4HE7GROYO9K7V4ZJQO00azV5Wevz6b1wCfyZhYEE+Oh3d3WeUnQ5fCT0sG/D9pRui5yUkL4lJSV4yNro+M7NOOC1uOKOlpwnDKRRaSysWJgwmJiZOTg8jI8L8gqPUTPwuaPv8pOh7RDPxN72c0/pxRg5ZBg8y7H3yFxY38g164yJ+2QapVKq/v7+7u7u3t3dMTAwajc7KyiosLHzy5ElDQ0Nra2t3d/cABBwO19fX17WMnp4ewCWGhoZqa2srKyvHx8cnJydJJNLc3NxAf19LR6eoVuhPV/xbuoYXqdT5+fm5FwNvTTKZPDk5OTg42Nzc/IJRkljA6YPD4LCZLCabb6jwso8Ev/6fKwEajZaVlXXjxg2w/oZfwz9vif9mj+vp6SkpKYFbA5cJT0/P6OjohISER48ebcwZ9PX1X8o2CXhvGxkZvTJh0NLSevTo0dTUS79J/XOnmH93vgReVgI0Ok0uSe7H4B/nlubga++V3vst9DfeLM40Om2c+iy0I1zzRQr3px88C6I6LLSvb/+OlF3bonfszN39v/iDGbSXCLW0VQgDlUX9oeDn1TkZEgX/kfqvySW+z+SLPDWbWOetEgaQkhmYy7O5Bj6sWQrV7vZt8YsSJ6QMv5f3OXwl+HtZd1EpXVl5eVUlOUwLZmlpCdQHNjPz85SxsTFu/FPoZT9Xw9DK1TB4eniAl39Ozg9jomIQCQg331Dpaz6/XkX8pIHSvIvqHSSyWHQ2i0FnMlibb1n0ajPGXUpD46qvr3/06NHjx4/Dw8MRCERaWlpOTk5JSUl1dXVTU1N7e3tvb29/fz8OhxsYGOjr6+vt7QVUYWBgYGhoqL6+3s/PLz8/fwICiUTiplrrbAtCVP5X0kfWLGZ6emZunjo3Oze7DEAclvc2+jszMzM+Pt7f39/a2voiCVzZHA4wMOvBjYGItC+h4Xk1OfKv4kvgzUmATqdnZ2dbWlqCxTowEHqzlGCD1qysrCQkJA4ePKivrw97NZiamt6+fTswMDAhIcHZ2Rl4MjxP1fCynMHU1NTA4GnA1pelDXp6eoaGhmVlZfyXAm/uAeS39I5KgEqnKqIV97nvqx6pBl2MxcbucdtzwP3A0fCjehl6oU2hzWPNTK7L5asglZr2GeFvT8MiQfZIO/M/3Ba9Y0fyzn0D+/3nuAngXhxRUev7MNTXv3Omg869LmsTP3+QtDNh+JlDyIsPnF/zDUrgbRAGsA6mUCiTk5N4PB6Hw83MzADC0NPfIysjI37poqTEpXMSKr9dtj8sH/C9nP9vsnYXFIyaGjGQJQuLyaQzGEtQYuJF0hSJzqBznZE5bAqV1tLaikKjPD09blhZWJqbuDk7e/qE6t4OOaISdlgl4bhKaGBiBfOZE8T6ogM9fBd+5EBPFhcX4+PjPTw8QkJC4uLi0Gh0RkZGfn5+RUVFXV1dS0tLR0dHd3d3f38/UDXACofBwUEcDldRUVFZWdnf308kEsfHx6anSKMjo0XldWe1w/8j7e0eVkKjzM6QZ8kzG4EMYW2NqampsbGxrq6unh5uYLgNhMadYujD4bB7hycu6YU8aeJaW7K4VJG7gcy81p8R/lG+BN4FCdDp9NzcvBs3rDcvGtIGbAGcMjQ01NTUXNUBMzOz27dvgyxvDx8+BLZAenp6q2gD2AX+DC8Y+/WV1QvAewFSL7yQpeK7ML/8PvAl8DoSoDFo8kj5j1w/qsXXxmBjDj4+mNWbVT1S7V3jLY+UP/j4oEaqBo3+B/GL1u3AAH3gO8IhIcKy68KI0J7qj7fFCm5PEPy4e6/JtCmT83I8JDJyfZOkd5AwdFI796ceWB1fFSmoWKXM2vzAM+tOB/8gkMBbIgxkMrmvr6+jo6OtrQ2LxRKJxIWFBQqFUldbJyUlJSl5SYoLSSmpy2KXtY/J2x+W9fpa2lP8WoRbZGkNdpA8R4UjFFEoc2w2A/SeTl/q6GpLSUF5uLsbmtnIad85q+nxnbzvd3KBZ7VCtW762tjdT4yNysvLLSkpATnaRkeJFAplaWlpcnKyp6cHh8NRKNxsbtw17LukeRgaGgoICAgKCgK2B2g0OjMzs7i4uLKysr6+HovFdnR09PX1DQwM4HC4wWUMQejp6RkaGiIQCBBhGJ8ikdqxTUYOCd/IBB5RCmhs75+bm53mwdQaAKXEqsPT09NTU1OTk5MEAgGDwfT19YFZWHfL5nCYXErAjWeFI5BkzBK+kY8MQJRSaAscDp3NZjLYrA2yzKzbJv8gXwJvUwL0paW8vDyQ4OUPl/WbWmFdIygzM7MbN244OTmFhoba2dmtogp6K/GHcZPMzMwsLCxeR72gq6vLVS/wgyO9zWeUf68/WwKUJYp0orSgq+C/vP9VNVLF2x3KEmWJ+dR8mvf4H5YX2UuK40orXBfa9nEz1DmKbwAAIABJREFUOscIflj/kcSE5Cx79g8bWVVhCxEGDocjWy23PWllErdEQSHUpyO0kVXj4u++TQm8JcKAw+FaWlqwWCx4Mw0H0Oju7paWlpaSkrx8+bKMjKyM7GVlBRlNzavyqjoSKpbX7iWJ6oQcVfY9qxOieRvpGFgUkVKTkFVTUtdb2Ywrru6Nzai745miaBosrOz5g4z7zzIeZ9S89G2CH/nFRkREhwUG6GhrffnlV19++eU333zz3XffHT58+Pjx4xISEjIyMqdPnz506NCPP/54/vz5zMzMtyn0P7wXeG3f1tbm4+Pj5+cXFRWVkJCARqOzs7OLi4urqqoaGxvhlAs4HG6IB9yEFSMjeDweJgzE0RHvsNQf5P2/UwiWNw4YHOybmZnhJQMkHkxCAAdW1ZmamiKRSGNjY4ODg8XFxY2NjRsPhMVmcdiskQmyknXCr3r5J03KjmjESpvF+8ZWEEkv/f9u43vxz/Il8GYlwGazS0tLQSiFTSUDf9i4hYWFlZXVqmpAXQC2jo6OwcHBICfDBrTBwMBglZpibZuvo17Q1NR8/PgxmbyO49mbnRp+a3wJvFMSmF+al0iQ2OOyBzuOfSMd8531exYWaUiIm3UhYzfXdaHgw+8Ih7rona9wl61FGJAE5Hb0zlWuzx+gdoYMhL7C2PmXvCkJvA3CwGKxhoeHYbP7wcFBEL+fzWbPzs6am5uLiYlJSUlJS0tLX5aWk1dQVlGUviz50PEBi81YZDCHiXPVLSPIXKx3XLmlK9raLf2Bf6FDQOGDgKK7PgXmrkkm90ONbj7SM75tZGzl6uKcEBeblpIaExPr5+8vJyv/9Vf/PXTo0PcQDh069PXXX/8bwldfffXtt99+8803n3/++c8//2xhYdHU1ARsbDYws3lTcn/Bdnp7exEIhI+PT0BAQFRUFAKBSE9PB4FW6+vrgQM0cGYAOobh4WEowR03piqBMDpKJMxMT6Tklh1X9v1BMeyQrE9IQi5pYhws/XlowtMiYAswZ+AlDKA8MTExMjLS0dGRkpLS0dHxRzoZFm2Rrmef9KM66rRVtahF5RmLGuHr5b+oocSN41B5jXQG1/scikL1VB7sZXBtlniwdl7ASVCNtQxg5/aCsuVX40tgXQmArz8eP+Lk5GRsbLxqVf3iuyBhs7m5+eu4PVhYWOjp6cnJyT2vJ2ZmZiYmJvfv3w8KCrp586a2tva6nAEkdDMyMgIcY12VhYmJycbeC2vP6kLQ09MzMDC4ceNGfX39u/PPc93J5R/kS2AzJDC3OHch7sInHp80E5tfs33MEuZz/L+e5mgbEto/IrS7/KNtMTt2JO/6pO9A+sJLODrz9iRi65gkcTiccfr45xn/3p64UsmAFFSoUuRbJfFO61suvw3CwGazx8bGQKxPAoEwNDTU2dk5PDxMJpMXFhZ6e3uTkpLi4uLi4+NjYmJioqMjIyPj4uLweG5gMiaTsbS0QJocb8U2V1aWVVWWj47iQUxVDoeztEjr7GzLSE/18/GyvWVzw8ry0SP3+AREalpGXHyCj6/PlStXfvnp56NHjx6HcOLEiXPnzklLS8vIyJw6deoHCMePH/fw8HBzcztx4gQCwU0o+E795jGZTBwOl5GRERAQ4OvrGx4ejkKhgA90bW0tBoPp7OyEbZOGhoaGIYyM4Al4/CxporS2/ZSG3yHFoG/lA3XuJU2OTU5PkXiJweTkJPCNhrfwWV5GAc6Ojo7icLji4mIkEpmdnT039yw6xNoHl+vAwGJml7ee0I4SNnpyxqIKyoZRddayVvh62Y+q8dYemeR5GofHFhOWPJvNZjKZgA/Q6XQymQxTCPhGgDMwmUzgew3YAtwCXI1f4EvgZSUwPT0dGBgIr/hfnCTw1lzXZ2DdlTrvVavKVlZWly5d2rVrl7a29vOuhTlDYGCgtbW1trb2SnOkFXuGhobrdszc3PwP1Qt6etwszjBJuHbtGnC/fvDggaenp6+vb3t7O/8L+LIPG7/++yGBucU50RjRv3n+bWT21c1mqCyqxJjkM/XCiNBH2L3gRftHHfscZh1eWVZbizBwsz43rMn6nCh4IOVvhEV+/vhXfgpe98K3QRg4HA6VSm1vby8qKoqLi3NwcJCSkhIXF5eRkVFSUtLU1ASxNUxNTW/cuHHnzp0bN25cv34dhAtUU1OzsrJ65O4eGhaWnZNTW9fQ1NJWVlFV14jp6O5taeuormsoLinJy8/LzSvIzs3PyMxBoVMys3LQySkxsTHOTs6+3j4JCQnJycmpqakZGRklJSW1EDIyMnx9ff39/VNTUzEYTFdXl5+fn4qKysam+a8r75e8nvfXd3JysrKyMjw83NvbGwRIyczMBMNpaWnh5mfo7cPhBgFlII7ix8fHUooaT2sGHVYI+lEp7JhyYG1z18w04AfjMD2YmJgYXwn41Pj4s2rj4+Ojo6NDQ0ONjY3JyckoFOru3bvd3d28PVw5ODYDWvJzOJwnTX1ndCKPGRQc1UQd0UoVNi4VtagSNa/5ST1FzSZxmPjMRZLJZEZGRpqbm/f3PwtZPT8/n5CQsEqbQaPReJUMZWVldXV14MjKbvD3+BJ4OQmw2ezc3FxTU9PnLaxXremftwvW96ampiYmJsBDAGgbLC0tYZ3DH97ixo0bkpKSe/bs2YAwgCxypqamQM9gbW2to6Ozrp4BUAcjI6O1fd5YvQCTBGDXZGNj4+Tk5OHh4e3t7eHhce/ePSsrq9u3b3d0dDz/H8LLzQK/Nl8CW04CE5QJ71pvkLXt1TrvSX78jC0MCe0b2L8jjRtHdXflnssT0lQ29dWa5XA4W44wpI6mrZOQAb3Tf/jlwkO9ssQ2uJC+0EUaVKMvtNMXmsnEe2zuS+aFDeq/N6feBmEAPyFEIhGJRDo7Oz948CAgICA8PDwoKMjPz8/Hx8fb29vLywveurq62tvb29nZ2dra6unpffPNN599+tnFCxcfP/bKzMopr6hsa+8cGsYPDuHrG5rQKWmBQaHe3gFIdGpp+ZMkVLKxidm33x4+evS4rKysmZmZj48POjk5Ozs7Pz+/oKAgPz8/Ozs7E0J6enpycjICgYiKigoPD4+KigoMDCwsLHyXZ3dhYaGrqystLS0gIMDHxyciIgKNRBXk5VdVVbc0N3d3teMGevHDOExbj71Pyk8K3t9fCf1JJfSby/6PIwtJk2OEUa4n9NgaAMoADsP0gXeXSCSOjo52dHSEhYUFBQWFhIS4ubk1NjZuvD5YDk/FamofOW8Qe0wlyCmsUMEq/mc1hLBxiahl7U+6KSo3EKSZOUivww37UFpaqqenx+FwiERiQ0PDEgQ8Hr+wsDAyMtLe3j4+Pv67NdSdO3fGx58Gt56dndXV1UUike/yxPH7tlUk0Nvbe//+/Y3N/dcuuOGlPyADpqamhoaG165ds7S0dHd3DwwMtLGxMTY2NjQ0BBoAAwMDYGVkAmEtPwENWllZSUlJ7dmzR0tLy8LCAr7L2g6AhGv3798PDAy8ceMGvMRfoV9Y3oFtk+B2VqV2Xq7I/auvr29sbHzjxg0HBwcnJydPCI6OjlZWVteuXdPQ0FBSUlJWVkahUNyA13zwJcCXwCtJoG2p7Qv8v58ZIw0L7S7fsy1G8IOUnV8NfNPF6HqlVp9eFB4xuG7itncwShLo8djS2D/TvlidkCFB8Lvqw1j6m/EVeWV5jnX/Njlwhc2iMpYG8K0HJvrF5ycDX7m1LXThWyIMTCZzfn4erEQXFhbYbPb8/PzU1BR4vT0xMQHyiwEbmImJCSKRSCAQQHYwNzc3cXFxeXl5Ozu7gMCAwuIC0jQ3fweLw+zt70lMSrh7766JmYmnp3syGpWUhLhjd/fHn34SFT2roKBga2ubkpICwoxWVlaWl5cXFxcXFhbm5eVlZmampaWh0WgkEpmQkBAfHx8bGxseHp6SkvIi6QX+rDmGF+hkMrm9vT07KzsiKiowODgmKgyJRmZk5ScmF9k+Tj6lFfKtTPCPqqG/qkR8LeNvdD9xeBCyUsJzQeD6N4wSIIxCIEIYHR0FxAAchLdjY2MjIyO5ubkGBgY//PDD4cOHv/3224sXL5aUlJBIpK6urtLS0sTExPDw8NDQ0Li4uIKCgra2NhKJBHrLgmIldQ2MX7GKGiKSFhYXk7KbJIxjfr2aetqq5hfNdCvnNOrSAsitUVNTc/v27dHRUU9PTzQafffu3d7eXiMjo+7ubm1tbSQSeevWrbq6Oi0tLdA+k8ksKSkJDg5OTU39syaFf9+tKwE6nT4/Pw+vdBcXF+Pi4q5fv77x0hxeZ/MWLC0tzc3NDAwM1NXVJCUlZWVl7e3tY2Nji4qKQkJCrhsb6+nqysnJnYYgKnrGycmpoqIiKSnJy8vrzp07ZmZm169fB8zBDIK5ubmVlZWsrOzu3buvXr36PJMkuA8gG/T9+/e9vb2trKw2UDIApS7Qe/AGRwKBWWGHB0tLy7t37zo6OrpBePDggbW1tYGBgYaGhoqKijIEFRUVRUXF390ngBHp1n0S+D3nS+BPlACDw1gRGWlY6GNuZCTB7bGCH9XvTaAmvGbfnkcY6uqmX7PlzbtctPjc6lhJsYKCubv/S/jKmewyxnzFdHiv3+GJvotTQ9qLlKoZgtVE186xnhNMxjNDiddv/51t4W0QBjqdPjExQSAQ4BUkk8kEYVXnePC7A/Tc3NwsD2ZmZggEAgqFAu/qAgIC4hMSiopL+wcGJyZJo2PjTc0tSFSyu8djB0en8Mjo9MxsFDrl7j17JWXlBw8e+Pj4IBCIqqqq1tbWdgitra0YDKahoaGmpubJkydlZWVFRUV5eXkZGRkwbUhJSYFXD+/gtMF2OHDfFhepTe09knqeohqPhVV8f5Dz+VYu8HulsJ9Uw35WDv9WNkD2euiT6vruro6uzk7get7b29vT0wPSOAwODg5BgHwfuD7TIyMj4Ah8ClAFYWHhS5cumZube3p6IhCI2NhY4BWqqqoKlg6qEJSVlZWUlFRUVAwMDJydnfPycmFVwDB+nDg5BZwWJmfmb3tm/agSf9qi6nvVpABEFXBNqa2tvXPnTkJCQmgoNx6CkpJSU1PTjRs3xsfHbWxs5ubmrl27hsViHRyeWnOWlZVlZGT4+fkFBAS8y0wPni9+4Z2SwODgYH5+fmVlJQ6Hm5+fr6+vv3Xr1suqF0BaN21tbTk5OVFR0XPnzuno6Pj5+aWlpWVnZ/v5+RkbG6upqUlKSpw6dUpEROTYsWM2NjbDw8McDodOp8/OznZ0dOTm5np7e9va2ppCADSAlzBYWlq+CI0xNTV98OCBt7e3mZmZjo4Or66At6yvrw/0DGZmZnC6aGCtZGdn5+Li4g7h/v37lpaW+vr6mpqagCSoqKioqqqCLSgoKyunpqbC7zLeqfnld4YvgS0hgbj5uP0Eof1QgjbudlBoZ9bubTE7BPN2G4wZvGzWhbVDDo9YPw/DO0sY2By2aOW51RqGOMEPEDv39e8XmvjkF+KvhbQ/wR6EzV6awd8ca/uf8b7Ti3OFlKlYfOt+xtLAWpm/f0c2kTCApS2TyRwfHweBPmk0bgYTFovFZDJnZ2eBVmFmZoZMJsM0AZTJZPI0BBKJVFJS4vjw4X2HB+ER4SFhoUUlJROkSSaTuUSn9/T1IpKSHB0dbWxuBQaFpKZlJKemP3BwvHnzVlRUNAKBKCgoaGxs7Ozs7O7u7uzsbG9vb21tbW5ubmxsrKurq6qqKi8vLy0tLSoqysnJQSKRCAQiMzPzXSYMqx5BYPPTPzx1WM73sGLoD8qhP6uE/qIc+otK2A9Xgr+W8Ze55vvIw9vf3zc4JDg+Ph6FQiUlJcXHx+fm5qZAyMzMzIaQn59fCCEvL6+wsLCoqKisrCwlJUVfX19ERERJScnV1RWJRJaVldXX10dEROjo6EhLS8vJyV25ckVZWRmwBRUVlStXrsjKyl6+fPnChQuioqJnzpy+cuVKaGjoMm3gPhdgFEwm0z+h8hf1WBGTCuGrUd2DY7293UlJiS4urgUFBZaWlgsLC9bW1r29vSYmJkQi0dLScmpqSktLC4vFWltbA1VVcXGxn5+fmpqagYEBlfrqJp6rBMvffY8lAD+Bs7OzxcXFiYmJSUlJqampaWlpDx8+fJFFOVjNg5pmZmZ6enry8vKnT58+duzYpUuXXF1dk5OT8/LyciH4+/tramqeP39OREREWFj46NGjt27dIhBWu+6xWKzp6en29va4uDg7OzsTExNTU1MrKysZWZndu3cDkyRYmbBxwczMzMnJycXFBSSB5uUJq8pGRkbgLnfv3nVycvL29vb09HR1dQUd0NLSgtUIMEMA33R4q6ioaGNjMzKy2tETFvJ7/CDxh8aXwBuRAJE59iP+p2eEYUTow+qPtscKbk/a+UPrTwTW6v8Vr3DTsPD1TZLeWcIQQYnYXy/EFUI8zycOSl3Xunf/sJDQ+Ce/EY7MsTeKvPIKgvrDS9gsyvSICbHj79OEm1CMHDqx64dpvPkfXvgeVNh0wjA1NYWDMouNj4+DnxA2m00mk0G4pMHBwdHR0bGxMWCVNDExMTY2BvxrYcuZurq6sPDw6LiY2Pi4uPj40rLSwaEh0tTU2Ph4M7YFiUI9evTowYMHIaHhGZnZmVk58QmIx4+909Mz8/LygHoBvE3v6+vr6enp7OwEyeOAqqG2tvbJkyelpaXFxcWZmZlIJDInJ2fLEYa+ockfFXx/Ugn7WSXsF+XwH5WDv5XxP6oaFIyqmZ6Zwg30FRQU+Pn5GRoaysnJ3bt3r7S0tLu7u62traSkJCUlBYFAJCUlIRAIJBKZmJiYkJCQmJiYnJzs4+MjKioqIyPj5OSEQCDy8/Nramqqq6sfPnwoA0FeXl4BgoqKipqaGnjdqKioKC8vLy0tLSEhceHChbNnz548efLo0aOysrJpaWkrfZdZHA47EFH9k2rcr9p5lm5ZGVmZt2/dwuEG6XS6v79/QEAAFottaWnR19dPTk42MjLKyMjQ0dFpbm52dnbmdbkGVmeAjr4HX0v+EN6CBFgsVmNjIxICCoVCIpEeHh4vrluwsLCwtLS8fv26goLC2bOiIiIiJ06cUFFV8fPzy8vLy87ORqFQ4eHhXl5eJqYmYmJioIKIiIilpSXQLTxvjHQ6vaurKz4+3s7OzsLCQlpaGpgkQSZPGzOFp2eBbZKzs7Orq+vGnEFXV9fS0tLLy8vNzc3Ozs7MzExLS0tNTQ22OIKJwboFwCJSU1PB95rFZC4uLpLJ5N/pPUiI+bwx8o/zJcCXACwBu2k7ofHlpM7DQnt79n+A2rktVnBP0d4M6ivGUYUbB4WtRRhSqan/O3pwb/d+bmTVuJWEIV7w47Z9+4eF9uOFvsZ/M8GcWDXSzdhl0sfIRIeJvvMzIwZM+iCHw6FMxRBa9zPpXC43PxWDx+4F5c24+7vT5iYSBg6Hw2AwhoeHcRDgEJxUKhWPx+NwuO7ubvC+v7m5uaWlpbm5GYPBNDU1NTY2NkBoamrCYrENDQ1FRUXxCQl37toZXDNQUlKSkpK6cPHCebHzFy+Ji4uLX7x4UVJS8soVJS1tHQtLKw9Pr/j4xLKy8qqqqubm5q6uroGBgUEIAwMDvb29QNXQ0tKCwWB4VQ1FRUVoNDovL2/LEYbewYnvZHy+Uwj6VibgOxl/YbWgO95ZnQNj0HO27HjM4czNzQGDH39//4yMjMbGxoGBgZ6entra2rKyspycHDc3t6ysrPJyruiSkpKUlZUtLCzCwsJAdKmampqqqiobGxtJSUkZGRlZWVk5CGpqalpaWlevXtXQ0FBXV1dQUJCQkOBODIRz586dOnXqxIkTx44dO3r0qJOT0/z8PPgCsNksrr6JzXYKzvtJI/lX9fiqRhAciRuRFQqqy3WDfroWgfJwM5ncI8uXs6Grn6kswO7yef5fvgQ2ksDAwEB6enpSUhIKhUKj0bGxscCR4IWW5FAlQ0NDWRmZEydOiIiInD592tjYODw8PCIiwtXV1draWl1dTVxc/OzZsyIQTp48eezYMQ0NDV6iu0H/lpaWurq6EhISlJSUdu3apamp+Yc+DLw9NzExsbS09Pf3d3NzMzAw2MCfQReCurq6MoTfI8XBygS48Dy2oKioaG1tPTAwMD8/PzY2hsPhOjo6sFhsc3Pz0NDQyrcDG4yVf4ovgb+uBDBLmH+M/JPX13lX8YfbYwR3pO4ywl1nc579gr+OjLYQYSCxSD8TftlP/GT/sNDusj1cJQP8iRPcVfjh/kGu7ZYQ8RPxMYm3kJaBSR8e6z46NXyNRk6dGpQZ7fiaScezWbTR9v/MjN7lKhlYVGLHV9N4C/pCO5M+9DrT9I5fu7mEgUql4nC4/v7+wcHBxcVFSH3DnpycHBkZ6e/v7+7uLi8vj4uLi4EQDSEmJiY2NhbKxxANynFcvQK3TkBAgLOz8927d21sbIDRsKGhoYGBgR6UNsjQ0NDExMTe/m5YWFhycnJ6enp2dnZhYWFZWVllZWVNTU1tbS14QV5VVVVTU4OFADhDfX19VVVVRUVFenp6SUnJllKmc5fLg4RxedMIzVsIO+/c+BzMIIH0vMcOiUQqKSnl5eXdvHnT398/Ly+vvb19YGCgq6urqKjI3t4+JyenpaWlrKzMyMjIzs4uKioqPT29uLgYyPDu3bvi4uLSy5CRkVFXVwcLDm1tbTU1tcuXL2tpaT1+/Dg7O7u6urqysjIjI8Pd3V1DQ+P06dMiIiK//PLL76bYs7MrMj3TFhdVb6EPq6BNnNJYbMab+hf5PCHwj/MlQKFQ8vPzEQgECgLQp/EuuNctm0EApzQ0NC5cuHACgoiIiKSkBOTurC4pKXny5Mnjx48LCwuLiIicPHny9OnTZ86cOX369KVLl17W1p/FYvn7++/Zs0dDQ2PdLm1w0NTU1NraOigoyMXFBeRP0FsJfX19PT09DQ0NXpNCXm4ACANQHoLjsOZBUVFRTU1NV1c3LCysu7sbi8U2NTVhlgHe9czMzPCfNL4E+BLYQAJMDlN1QlVo7Jl64WPsvu0JO7cnCn5bfojIIm5w7UudCg3bMiZJE8zJQ/hDTw20BoU+rP1oZ+6HO7N37yr88CPMx4AtcAnD+CePyY9fSggbVGaz2YylQSZ9HUdqMsF6ckAGXLtExQw3/z8zBCsOhzM36Y9v+5TF4Mbgoc4kEVp3THR/tTCbv8FdtvqpzSUMU1NTfX19vb29w8PD4N0wnU4fGxsjEAjAszYlJSUsLCwyMjJiGeEQwsLCwsPDw8LCQpcREhISFhYGaq2tDyqHh4eHhIQEBgYGBAQEQghYA2DoEhAQkJeX19bWBvQMTU1N9fX11dXVOTk5nZ2vknf9z3oO2GwWg8lkMZhLzCU2m8mNHcX9MLnpk9f0iclkKigo3Lx58+HDh6dOnXJ0dPTx8cnKysJisX0Qmpqa2traOjo6Hj58eOfOndjY2LS0tPz8/LKystra2oCAAHFxcSkpqcvLUFFR0YGgp6enqqqqoKAQFRVFIj2jK9wvIYOxsLAwNDQUEREhLS0NOIO9vf1KNQ4b24s/rhH7m1p0c89qY+g14+Af4EvgtSRAp9Pr6+sBVQDqBQQC4eDg8ILeC2bm5hqammfPnhUWFj5x4sRJCKdPc9VogCScOnUKkARRUdGzZ8+eO3fu7NmzoqKiXl5eCwsvEa4bvLno7+9/9OhRTEzM7du3QUoH8xeGiYmJjY1NWFiYi4vLWj2Dvr4+4Pm8JGHdMswTVFVVdXR0TE1NHz586OzsnJaWVltb27weQHKblxrva00q/2K+BLagBPJoeZ/gD3ANbIC786DQzszd22IFd6TsSpzkppF9U9hChIHD4dyfuf/MRmtE6Kl8Rrh5r58KakToHyP/bFtqe1PyYbOXxnvPTg0brG1wckBuatiATmubwV8b6/5plmgHrI/YrFlC+z9mx5zAJUu0Fsbie+76vLmEYXx8HBAGIpEIfvwoFMr4+DhIBdDf3x8bGxsRERG5jGXWwP0LlPuAP4SEhISGhoZBAAwiZBlBEIIhgHJAQEBQUBA4EhwcDA6CLcwi/P39UShUGwSgQMdgMHV1dRUVFdPT726UsbWPMofNYbIZHK65DhtQBTabyeKw1lWSTE1NCQsLHzx48MCBAwcPHrSysvL29g4ODs7Pz8disT09PTgcbmhoKD093draOiIiIi0tLTc3t7Cw8MmTJ5mZmVeuXBEXF5eEICEhoaCgoKWlBVJEqaqqqqmpZWdnU6lUGo3GaynEYrEYDMbi4iKVSn3y5MnVq1eFhYV/++23pKQkeDgsrgES+3Fs6ZfSoY+jy+Hj/AJfApshgcHBweTkZCQSCTgDGo0OCgqysrJ6QcKgoaEhKsp1WgBUAd7ykoTz58+LQbgIQUxMzMTEpLe3l6sQXHb6f6mhMRiM6upqJycnU1PTF+wnoBUgaBICgXj48KHeSujq6mpqaqo+B7A9kpqamoaGhoGBwc2bN52cnEJDQ9FodFVVVWpqakFBQWtrKzAobV4JDAbT3Nw8NgYMI19qrPzKW1ICDDyescaVf0uO5G11msaiXSBefJapbURoT/3HXPMblKBsndzrR0biHcfWIgxkFll8XJzLGYYhkgATquUoUkLET+TG5d+sPdL8ZPBIsyBj6VnSWCDAuXE3Yvu2sZ5js8R7zCXuC80lah1jkftmeW780eSADJvN4BX1e1zeRMLAZrPxeHxvb29fX9/UFDdILXB3HocA4oGEhIQAYgC0B4AeAHUBoAfgeFlZGRaLBTZFVVVVT548KS4uLigoAM6FWRAyMzNBISsrC6RgA7SBlzDAtMHf3z8iIqKxsbGrq6ujo6O1tRWLxWIwmKqqqvf4F25qaur48eOffvrpgQMH9u/fr6am5unp6efnFxERUVJSgsVicTgcBoNxcHDw9/dPTk7OycnJz88Cev9IAAAgAElEQVQvKSmpqKiwtbU9e/as+DIkJSXV1NQ0NTWvXr2qrq6uqqra3NxMo9FAJFwikchrvsxms+l0OpVKnZ+fb2xsVFFROXbsmJSUFBxZhcVmcthswgTphFbEFYuExaWl9/grxx/anyUBsFKfmZnJz89PSkqCCUNSUpKzszNwd153LQ4ftLCw0NLSAmzhFKRGOH2Ga24ENAmAJFy4IHbhwgVx8YuXLolLSFySkpKUkLgkKyubn5//alSB9yocDhcUFPRSnAF03tvbOzEx8f79+8AMCRCH56kXlJWVVVRUNJZhZWUVEBAQGxubnp4OgqcB97CampqsrKy6ujqYKQCSAHaBdVJXVxcIjvdnTTr/vm9NAgQ9Pdz+/cNiYpMuLottb+zV71vr/9u/Udx8vNAotCaG1sH7+vfvSN21PV5QKO0Adv4N5yYLCV3fJKm29h19Q9pL7/0Z/+vHHXs/xu7d17ufq1jgoQ1Co5/Ez79ubopVM85mzRPaP5/GW6w6zlwawGN3zI49jeTOZlFG2/9Dm02H1rSLLOYK++pV175nu5tIGJhM5tDQELB1AR7PDAaDRCKBgEggkkZ2djYaAohSgkAgQJSeuLg42JMhNzcXBGAF7hADEHp7e7u6ukB2hcbGRiyWG0sHaAz6+vpqa2uDgoICeODv7++3EsHBwdXV1SBuUkdHB4i4WldXNzT03vqsLC0tSUtLf/LJJwcOHPjss88uX77s7Ozs4eEBFhPFxcUhISHa2trOzs5JSUkgNzZwAklJSZGSkjp37pwYhAsXLsjJyalB0NTUlJGRSU5O5nA4TChGColEamtrgz2boS8V1zBpcXGRQqHMz89nZWWJiYkdOXIkIICb453NZrPYTAbkzewVX/a1tGdH3zp2hO/ZF48/nD9FAkwmE0RGAv9wUCgUGo0G1j6mpqZ/6FV8/fp14KIASIKoKNfcCPCEixcviotzPxISlyQlLwHLPWkIEhIStra2vKZ6LzV2BoNBoVBgBk4ikeLi4kC2NZjJbGyjZGpqamlpGRwcjEQi7ezsQHIGXV1dDQ0NENkMWBwBlYKGhoaRkZGtre3vydSBFZO1tXV6enpdXV1NTU1jYyPMClpbW8vLy8vKylqgkBUwbQAF+ODw8DA/QcpLzfhWrMyam+s+eHDc2nomNHRIWrrn4EGiiQmbH+f6+XM5x5o7ThDmEgbw1hwOpYreeavt9vOve8UzW44wcDicuoX6z0u+2J4k+AFy587s3XvboeBIQ0L7CUKH8IcnWVzngTeLuQmvEeweJh2/qtnZcc/hlp3zE49oM0kTfRdIg+psNtcp96+GTSQMdDodRCXq6+sDlqyLi4uALUxPT8Np2mZmZqYhAC4Bcj/D+YYJBMLExASZTJ6CAJJAj0JZioeHh4eGhgCL6O/v7+vrA1scFMW1urq6DEJpaWlJSQlI8Jyfn5+Xl5eVlZWRkVFUVNTV1dXT09Pd3d3V1QVCJzU1NfX19b3HD4Gent7+/fs/++yzgwcPSkhIODg4uLq6enp6+vj4GBkZXbp0yc7OLj4+Pj09HagXioqKSkpKnJ2dT548CRtkX7hwAeReAFkXfo/jDhIgQNYWXG4AUufCb0ZB5o3FxUUajTY7O/t7gF07O7tjx44pKyuTyWSIM7C4hlUcTv/w2Hcy3ogczHs8Bfyh/YkS6OvrS0lJgXULgDB4enr+4cobLNAVFRXPQxATE7twQezixWeaBElJCSkpSSkpKUASZGRk5EAYMVlZVVXV0tJS+OvwssNvbW39fSnf398PtzA7O4tEIm1sbIBWZGO2AM6amJjcuXMHgUDExMRYWVnp6OhoaWmpqKgoKSmpqqpevXpVV1dXC4KhoaG1tbWxsbG6urqaqqqBgcH169cDAgLq6+thDtAMoaWlpbGxMScnp6qqatUpEPIOVMNisXzv55ed9C1Xn1pe3iEoCCsWaDU1Hbt2zYSEbLmBvLUOh86FPTNGGhba28sNpbo9UfCLjC+Ji2/M1xkezlYkDBwOp3i2+EDu37jBVaMFd+bsBuRKaPyTm9M28NDeYIHFmMK3fjpLtF/bJmUqfAp3YWpQjkIK/+vYIK2SwyYShoWFhf7+/t7eXhwOtwQZmVAolImJCRKJNDs7Oz8/PwcBvHWeX8aqXVCNTCYDXjE1NUUikSYnJ0HGBiKRSFgGfhkgSRyRSBwbG4Osn8YnIADfidHRUeBvPTg4ODAwAFwseiB0dXU1Nzd3dnbCP8yrhPUe7N67d09ISOif//znwYMH5eXlHzx44ODg4OLi4uvr6+bm5uTkFBMTA2JMZWVl5eXlFRQU5ObmamlpCQsLnz7Ntb44c+aMuLi4AgRFRUUpKSmgXgBsYa3o2Gw2k8mE3Rjm5ubIZHJxcTGIMFNbW8srVRabqX47yd6ngPcgv8yXwBuRAJlMLiws5DVGQqPRCATi3r175ubmphA2YA46OjoSEpcucUM5A3MjCQkJCUlJSRACQFpaWmY51rC8vLyCgsKVK1cUFRVlZWUfPnwIB5V+hYHg8fjIyMjJyRWv0xYWFp5UPLG/a29iYgIowR9uzczMXF1ds7KyIiIizM3NNTU19fX1LS0tXVxcQkNDHRwcrl69CrIuSkpKcLUlFy9euXLF0NDQ2NjYysoqOTm5BQKgATAZyM/PR6FQjY2NazkDqNPU1DQw0M9XMrzC1G+hS8bv3On9xz+Y0AsgbreZzJ5//nPiwYMtNIS32dUp5tRvhCNChGfqhd1le7ZB3guPu702oyfBIbh9+wv27ste+cl5Z02SYCFkTWd9kvUplzMkCu7t2rcfL/TZ8N9qFlesHODKr18gEx3wWCEmFPuIw+HQFzun8caLlEqo5bWhZF7/hluphU0kDBQKBQ6RBPTpILszmUyen59fSwyWKQP3L+ASc3NzvBmgyRBg5jAJYWwZxGWMjo6CIkwYxsfHx8bGRiEQCAQ8Hj80NDQ4OAi0E0A1AbhNW1tbe3s7b7D/rTSZL9DXoKCgAwcO/Pvf//7iiy+0tLTuQXj48KG3t3dgYGBYWBgKhUpNTc3IyMjOzs7NzQWhJy9evHj8+PGTJ0+egiAhISELAaR5xuFwG9wZOD2vIgy/J+zT1dX97bffoqOjgVUS3EJ8ZoOGTeI6MZ7gGvwCXwIvLwEGg1FbWwuyLqCWgUajQ0NDXVxcQMjmW7du/R7z1wyCOQ8sLCyMjIyuXLmyHCJM6vLly9IQnpIEWVkuSZCXv7LME5QhKCkpaWhoFBYWriXSfzQCNhTGgPf3ibfMvZrFYtXWVt+7d8/MzNTc3Az68HR6TdHMzOzGjRthYWGZmZkRERHe3t7R0dHp6en5+fnl5eVeXl5SUlKAC128eOHCBTEpKSldXd2bN28+evQoMjIyPz8fUALYJKm5uRkoGdLT0ysqKsBZXjoBl7FY7BYLJvFH08M/v0ICTCbu5MneXbtwx46NGhlNBwaOGhv3/Otfi93d4Eml1dVRS0sZE28jx9aKjr2rO4GzQbyhVPd27/8gaed2hOChrO9nljYlGPHWJQwcDidjMkMo89Pt8YJ7aj8SmvhEelyGwY0J+Tyw5uerCQS/Efzjqel4BqOLw6E8r+ra4ywGcaTl49kxtyUahjSoTmg7SBpUoS9AT/La2n+xI5tIGObm5nogEAgE8PoZ6Afm5uZgwvA82gATBlCYnZ0lk8lgC7QNMzMzsLYBMIdxHsAqhfHxZ+oFEM4VEAbYnGlgYKC/v38AQl9fX1dXV2trK9CHvJdPgru7+9///vcvvvji119/NTY2trOzs7e3d3R09PLyCgsLS0hISE5OzszMzM7OzsvLy4cQFBR06tQpOCbMqVOn4LeqkpKS165d2yClK5vNBoRhaWlpYWGBQqHMzc3NzMxMTEw4OjoeOXLEyelpSLJlaTM6cWPi14LJ81wDQdYbylmz3Dj/719UAtxcJYODaWlpqwgDEon09fUNCQlJSUlJSkqKjo729/d3cnK6desWr5OAmZmZtrY2MDECxEAOgrycHCAJileuKEEA7sIg8bmampqioqK9vT0I+cBkMmk0GolEwuPxg9DbCvDCAphQjoyMkEgkMpm8sLDAYDJWEQw2940tm7awNDtHmZuljo2RBoeH8XjC0NBwcnLy7du3N1CMwMQBjMjW1jY5ObmoqKi8vLyioqK2traxsbG5ubmiouLGjRtiYucvXRKXlZXV0dF2c3NDo9FFRUV1dXVYLBZEoAZsoaWlBYvFtra2tre3t7W1VVRUFBQU8BIJmCqAAgaD6ezs5Hs/v69fv6WBgc7du8nx8dTy8jFr674vv+w/fJi+bNw7ZmnZfeDAsLh4/6FDEw4OHMZzQ8qwmczF9nYWjfa+CgqMi8wiHyEc4fVe2F0KpSdDCob1hW/S2INDtpjT8yo5pE2k7884sKt0jxDxkyTKsxCLq6oxmZOtrYp5+bsKCrcVFm3LzdtWUvpxY9PXAwNqU1MeS/QyDmcSiie56roVuzMEW0LL/4x2/AuKptq64txfe2cTCQOZTO7p6ent7QVxh5hM5tTUFKxeoGwIWNsA2MXz9AzT09OQa8NTOyXAHMAWWC6BMuAPY2NjwDUCj8fDVkk4CIAwgLTH7e3t7+sPG5vNtra2/s9//vPvf/9bSkrKysrKzs7uwYMHbm5uAQEBUVFRSCQSJLzLzc0FhCEvL8/T0xNElwfhI0+fPi0hISEFQVxc/ObNm3Q6fdX6Bv5OAe8FOp0OHBjm5+dnZ2cB2QsMDDx+/LitrS1cGSow5mlLSjfi+ka4yRy42Z75+MtIgM1mg/SOb3zEU1NT2dnZgC3wOjAgkci4uDiYRaAhIJHIqKgof3//+/fvW1tbm5ubGxoaqigrA78dQAkUFRWvKCgoQQBHVCGoqampL0NVVVVdXT0+Pn50dLSzs7OhoaGioiI3NzczMzMjIyMjPT0jHUZGdnZuVVUNFtva0dHR29PV3orpaK/p7i7DYDJKS6MyMgPzcgMzMh6jUR4paB8EwgeRGJGUFItCJaFQKUFBYbdv33kRzmBubm5mZubp6fnkyZOWlpZmCMDQqLW1NTk5+c6dO76+vggEori4uKmpqRUCXBOQhLa2ttbW1qamptLSUhQKFRgYmJWVVVxczNsmaBluH7g04PH45/2jeOMzzm/wbUpgJiKia98+xnIGHub09JCExOTDh6APlNLSRSw35s9ia2vX3r1zKSngOHN+fqm7m8lja0fv7+/+7DPaSjvVtzmQt3Ov8LmIFd4L3fuAeuGXnN+oTOom9SEoeKuaJMECySBl/qP1nydGT8yz5+GDKwvsvj7DwuJtGZnb0tK3paZxtxmZ27Kyt+UXbCsq3l5SKljf+PcBnPQM2YfDeW4wRubS8Ny4A2ORr1VYKV0OZxMJw/T0NCAMIDwInU6fmZkBugUKhUKlUuHt87gDL22AVQ2APJAhzCwDZg5A7UDiAcwZgGESL2cAeoahoafmSYOQV0NHR8frGByvFvC7tL+0tGRgYPDdd98dPnxYW1vb1tbW0dHx0aNHfn5+kZGRiYmJqampwHUhPz+/EEJ+fr6rq+vx48dFREROQXEkz5w5w7XdhnDhwgVbW1s6nc5kMtcuBdiQ9wKdTl9aWqLRaFQqFdiYTU1NTU9Px8TEHD9+/NatW7wXsrkJGTgmD1OLG7jfVX7K53fp8dn0vvz+TdTQ0Ghvb2dDAPcDjwfvQ/Li/QBX0en0uro6OKnzsjnSs79IJBI4NsBcIhlCYmJieHi4p6enpaXl1atXVSCoq6uD4EIwTwAkQQOCpqamhoaGpqamgYHB3bt3w8PDCwsLc3NzU1NTU1JSUrlIS0tLT0vPSE9LT0tLy8jIzMjISktLR6NikxA+iCTbuHjjiAglL69f3R79y8VVyNX1Q1Oz7f/977brpttcXLe7Ou9wcdnl6rbf0/M/3t5H/P0lQ0L1IyMcXJxvcY2SzC0szMzMzS1grcLagpmZmYWFBRKJbIHAu7JvampqaGgAegPe7ApYLBbYara2ttbV1eXm5vr6+trY2ChDJMrd3R1kvUQgEBt4P2MwmPb2dir1JWwDXnyi+TX/XAkMS0kNiYnx9mHC3r7v0CGuL8PkJDkqaioggPrkCWt6uu+rryY9PDgcDq2ysv/773GHD/d99x3X1QFSO0yHhfX84x+suTnept6z8jx7Xpgg8ky9MCy0q+SpeiFqIGbzBhsYtOUJA4fDqVmqqV6sfp6UaLTO/II9GZnbSkq3Y5q3t3d8UFu3PTNrW3oGlzaAT2bWtqLibTW137PZm8XNnte99+D4JhKGqakp3iQMi4uLs7OzgCRQqVQaD6hrsIpCwF4NsKoBNk8iLwNwBrAF77Bh1rCWMwDDJKBqGBkZ4WUOnZ2ds7PvZ2Dd+fl5TU3NH3744ejRo6ampk5OTh4eHn5+fmFhYfHx8SkpKYAtAKpQXFxcUlJSUFDg6urKyxbOnj17aRliYmKWlpY0Go0JuTUzIbCWwYBAp9MXFhZoNBqwRwIBr6anp8PDw48fP25vvyIcAZurVGBbuGSklnDfSLFYG9gpvgffPv4QVkhgdHTUxcVlcHAQLPQXFhbgdOCvQBjgS7q7u9FoNEwGnhEFFCqJB8hloFAouDIajUahUDExMQEBAcA3GsQX0tTUVIegCeHqMkBM0kePHqWnp1dXV7e0tNTU1OTk5KQ/RUYaV7GQmpmRmZ6Zk5qWjkSFRUXfDQhUcHM79NBx/737/9e9+wIODgJOzgKuLgJubgKeXgJm5gIffyxgai7g7SPg9fjpx/OxgIengLu7gLv7/3h67nFz+dzmloSZuYWZhbm5meXG/gympqb29vYg51ozD3gpBEwS2trampqaSkpK4uLiXFxcDA0N5eXlRUREjhw5IiYmFhwcjMFgALsAykme9lYXMRgMHj8CXgqsmHj+zhaXwMDPP/f9618EXd1ZJJI+OLjQ1NTz978T9PTYFMrAkSODv/46Zm09oqAwcOxYx7ZttJoa5sxMzz//OWZpyaJSF1pbe/72NxBPaVhBgaCqusWF8QfdR1AQz3ydh4X2du37IElwO0Lwh+yf5xmbSKcDA98HwrCxcCcnM+vqtvf2fUCa2kGe3TE7x912d3+Qlb2CM+Tlbxse9t+4Kf7ZdSWwiYSBRCJ1d3f39fUBdzc6nT43N8dLFagvAMAcYMIA+zbAzIHMg5mZGTKZPL0MWNsACiC2Eqxn4OUMw8sYGhrq6up6XwnD7Oysurr6zz//fO7cudu3b3t6evr7+4eFhcXGxiYlJaWnp+dBYZGKi4tLS0tLSkrKysqKioq8vLxOnTp15swZOKyquLg4oAwXL178/WXq+Pg4g8Gg0+mAIcBbOgTgvUCj0cAkAsJAIpHc3d2FhYU9PT15n0s2h8nhsK3csqMzGrjHuRms+fhrSQBe6Ht4eCgoKOTk5PDm9HhZWZBIJJCmjZcngDISiUQgEEkrkZiYiEAg0BAAg0ChUEDhgEQiY2Njvby87O3tTU1NtbW1NTQ0rl69CvKda2tr6+vr29vbo1CohoYGLI/Rf01NTW5ubjqEjIzMzPTMFHRUYoJzUNAVD4/P3R5tc/f4Hy8vAV8/Af9A7tbHR8DbW8DHl1sIDBC4dUtg3z4BC0uBoGCBwCDoEyAQGCgQECjgHyDg6y/gC9X08Nhxx/aimbmpGaRrWKtbgI+YmZldv37dz88PhDaCLY4AYWiD0NLSAvK7BwQE3Lp1S0ND4+zZsydOnDh27Njx48eFhYXFxMT09PTQaDRgC3CIVUCTmtcDBoNpa2vjh1h92Wf43a/Pmp2lFBYSb97ECQv3fv5579/+NigmxiAS5/PzO3bupA8OgiGQ3N27Dx5kLy6So6M7P/oI9oEmeXpOPnzIXljoOXiQHMN9y85eWJgJDaWWlr77Y3+pHi6xly4QL+4nCj3NvTAs9DQ4ElIwuHdzQ9D6+/dv0ShJLy5hNnuRQlWk0nZMzzz9zJB3zFF2NGG2A9ukjMxtObnbyisOMxib4ln+4l3dojU3kTBMT0/DhIHFYrHZbGCXQqVSFyDw6BhWF2EqARMGYJ40twZrmQMvbQAeDqsIA+AMIJkDfiUGBwe7u7tBcoAtOqMbdHt+fv7q1atHjx69fPkySOccGhoaExOTlJSERqOzs7MLCgoATyiHHCLLy8tLSkoCAgLOnz8vKip6DspRdf78eZgwXLp0SVJSsqGhgcViLUEAJAGmCouLi0C9ANsjzczMkEgkAoFgYmJy6tSphIQVyRpZXJMkltF9tFdsGfdnA7JQ2mBE/FPvsQQKCvIlJC59/PHHIiIiwcHBwBUKHi8bwvN2wXEGg1FdXb3WGAkwgaSkJAQEUADEISAg4MGDB76+vrGxsWg0Ojk5GSgckEgk2E1OTkYikZGRkR4eHr+nYzMyMtLT09PR0bl582ZUVFRlZSVYdmO4aIY+mObmpprqmpycvIyMHDQ6OiLc2MPzkNujD909/sfXRyAoUCAsTCAqSiA6RiAuTiAxQSAhQSAmViAqWiAkRCAsVMDeXkBISMDCQiAinFszNIy75X5CuZ/QUG614GCB4CABb6+P79hegYiBGUwP1i2YmpreunUrOzsbEBtghtTe3t7Y2FhWVhYXF/fo0SM9PT2Qpe7o0aPAKPEEBBERkYsXL+rq6pqbm9vb2xcVFWGxWAwGg8ViKyoqiouLYQayljVgMJienh5YcQRPH7/wnkiAxWIQifThYTAcSlFRx44d1PJy1uwsra6u+x//wGtocDicEWXlQXHxFUNmsylFRZ379zPGxiilpbjjx4dOnqSWl6+os/V38mh5BwifPs1YPCy0r3f/B0hucKRvMw+R6dyURJsHP/+B954wMBhVc/On5+a5bGGGzFUvEMd2tLV/UFL6VMOQmcUlDITR+M2T8/vd8iYSBgqF0tvbOzQ0NDExsbCwACzawbJy1XZxGYBIgCUmTCp4yQMcXgk4Q/NqHniZA2ywNANhenoaNlLi1TOMQiBAwOPxBAJhaGiop6dng7A/W/ppWFhYMDAwOHbsmIyMjJubW0hISHR0NAKBSE5OTktLy87OTk5OLisrq6ioePLkSWVl5ZMnT0pLS3+3OpCUlDx37tyFZYiLiy97MUiIiYn5+fmBqKnL07gI5peXLVAoFDApwIHhd1NpGRmZ39urq6tbIVIuQ2Cr30Q4BufzCcMKyfwld0AsVENDw88///w///nP3bt329raQNRjWBGxSjB0Op1EIvX19TU3N9fV1aWnp8P2RSgeIBCIhIQEQBiAngEciYqKunfvnqKiooqqyh27O35+fiB0GBqN5vVzACoI4OTg4OBgZ2eXnZ0NFsoQVXi6aeKiGdOMxTRjC/PTw8Ovu7t/5+T8/z3y4GoPgoIEwsMFoiIFYmIE4uIFEpMEkCgBNFogNU0gPYP7QScLoJACTg8FhD4RuGnNrRYVtfoTGSUQEcFtJzxCIDDwfywtT5qYmFps6MYAKISpqamzs3NlZSUWi62pqcnMzAwLC7OwsFBVVT137pyIiIiwsPDx48dPnDhxchkiIiLHjh27evVqYGAgULOYmpoGBQVxBwmhqakpLS2tpKRkA87Q3Nw8wQ+vueqpfU932UtLEw4O/T/+OHL5MtHConPvXpK7O4fDwZ09SzQzWzXocVvbvv/+d+z69b5vv53y82NDuZtW1XnuLoPBJhIp9fWTEeGzJSXvpmqaxWEpTSg/i6Y6IrT7yUfbYwU/QAl6dfk8d2hv6ISv3/usYWAyGyhUbfLsXgqVSxVmyDtG8DsaGrfn5XH9nmEfhrz8bdU1Z9ns57o7vyFhv7fNbCJhYLPZExMTOByOSCROT0+Dn3kWi8WEwIYCbrIggCPAlIUJWcMzGAy4AMxdwGIUZhRwAdZNUNYAphMgqzRsrcSrcIATS4MccIODg0NDQyBrxPs35ywW6+bNm8eOHbt8+bKLi0tkZGRCQgIwRsrOzs7JyUGj0eXl5dXV1TUQqqqqysrK4uPjlZWVz507d3EZvIRBXFxcWVl5YGCAwWDQaDR4XnjZApiI2dnZ6elpEok0MzPj5+d37tw5NTW1VemoOBzWwuLSJf1IBz5heP+ev9cYUX9/v6ur66FDh/73f/9XXV29trZ2XcJAIpHq6+sB9U2CsC5bQCKRiYmJCQkJiTxISEiIi4tLSEjw9/e/cuXKkSNHTp48eeHCBUNDw0ePHkVERMCEASgogM4BiUTGxMSkpKQ0NDQ0Nzc/JQo8f1paWjCYxrR0Vx/fo46O/+8jV665UUCQQEioQEQkV5OQkCCAQAggkQLJyRBVSBfIzBTIyhbIzhXIyxMoLhbw8xc4cEDAxoarf4iJXecTHcPVTsTECji57FFTldQ3MNzYhwEQBjMzM0tLS3d3dwcHh2vXrklJSZ05cwZYHIlAWKYJT/+eOHHi6NGjV69ezcrKampqQiAQt2/fvn79uq2tbX5+PhaLBWkZioqK0Gh0Q0PD8zgDBoPp7u5+j0NXv8Zj/n5eypqboxMIHA5nwt5+UFSUOTU1Zms7cPQoe5EbO3sWgZjPzuZwOAPHjvUICXXt2TNuZ/cigmCTyYzOzoXExHkry2kpqYovvkgS3JmwQxC5c1eH/T0Oe3Xekhdpc1PrNC41HsT//Zl6oW//B8nc1M5fpv93cnFFWsbN6IaP73tIGNhsFp3xhErVJM/upS3s4OoWyDuGhj+oqeX6LfBShYzMbZB6YffU9JPNEO9fpM1NJAzcCAlM5sjISGtra1dXV3d3d08PN8pqf38/DocbGhoCL/XHxsYmIJBIJJCMeXx8fGJiYnJykgQB6AfIy5ibmwPZG2D/aV6/CLBg5WUXYOUKkgDMQwCxeshk8szMzDQEoH8gEomDg4Pvq3oBPNB+fn7Hjx8XExO7efNmdHR0YmJiZGRkZmZmfn5+Xl5eenp6WVlZbW1t3TIqKyszMzONjY3PnUz8ED0AACAASURBVDsnvoxLly7BGgYJCa6SwcXFBeiFYP4GWyIBX2dYvUAmk+vr6xUUFM6fP+/o6AhoJM+XjT4+NfvLFT/n8CK+hoFHLH/F4lpKQCaT4+Pjz549GxQUBKwceeVCIpFKSkqWgxGlAn9l1HpAIBDx8fGrCEN8fHwchKCgoLt378rISANjfWFhYVFR0du3b/MSBlBOSkoCVyUmJubl5TU2Nj5lCk3NmKYmDKalpaWjrCw5Okrr/2fvPOCautf/f+L93f+9VwQEBLXt7bi3Uzu0ra2tC8VRQBy4cFWtLW5luEcduGWvsAlkEAJhhrAhkIQEskiYQaYMZe9Ncv73e74QEZWqtY42z+u8wslZOec5h+S8z/N8nufKFb0rVxAnZ8QDD9KHAgJBxhGJjFCpIKoAUCEKiYlG4hgIk4nExyOJSUhyCpKSgmRkgoXffgd38QKOFoZQQsAqIG2JCkgjJARMoVDACCHobwcPf7Vl64979+2xtrJ6ZCaSChWwgkq2+/btW7lypUqWoOq1snjxYkNDQ9VbOD5//vydO3cyGAyJRCISiQQCAZVKhcxw69YtGKmQSCRisTghIYHFYkGEyH2USaXSe/fuPXx+R59N9fif0gNDTU3o0NBgY2OVqWnVwoV3TEyK9fXbiMSBysoiPb1uLrclKKhs9mxFb++jD39wYLC4uC84qH2PZcv8+Q1vvNFqoN851SBLRydIU4ugqRWIvZJ19Vokkkdv4eVNPdp8bHr9cGvn6dXTdfkgvKARrn01//oL2ClXtz8VMCiVvf39zK6uda1tOsOo0KJZUTEpiwfKIo1GhZhYgAqxDFBZNb/g2Atw9Z/4I/5YYACNtQcGqqqqYHori8XKxIzNZnO53LCwsLNnzzo5OQUFBVEoFCKR6OPj4+7u7uHh4YaZh4cHHjNPT088Hu+JGR6P98LMx8cHzvX29vZ50PwxCwwMJBAIgYEEP/9AKjUsJQX0KuJwODA1n4MZFzMej5ebm3vnzp329vY/989Yamrq4sWLjYyMDh065OfnFxkZSaFQGAxGenp6CmZsNluImUgkEgqF2dnZLBbLzc1t1apVK1euNDMzW4kZBIbVmMFZISEhqoJIUKwCuU6FZy0tLe3t7eXl5UeOHDE2NjYzM8vIgEKF0Y+ClDl5FR+vdPCgZqmB4U/8vfN7Dg1CKdxCWVmZi4sLVNKnpqbCJuWxsbFRUVGPIoVwWC4pJCQEAkPIiMH7fgqFQiAQ3N3d3dzc7Ozsfvzxx6VLly5cuHD1qlVubm7ho6oqwSADlUqFKxKJRAqFkpCQIBQKh5khV5qTzWUwXBydZ128OOHmTcTVBfH0BHqDwEBMroAFFsLDkchIJDoaiY1F4uKQBCyqkJQMaCE1FUlPR9hcwBXvvYe7dRNJTUMiIpEwGkLDhjAaGAcDFqBwdp6ybeuan376+eCBg1jv53GQAczas2fPypUrFy5cODrpSBVVgHIFQ0PDhSO2e/duSAtCoVAgEMBvCSKRCHtjh4SEwABLbm4un89PSkqCimqg5HjIYGGlh7XsT/jdq8Ts91xC6nVfugeUAwOdCQntAQG9YjGKos0eHiXvvafs6xtsapK/805XcvLDezhQX1+2fl3z2/9uM9DvmGrQaqDfaKDfZKB/e4oeEeMEpvZksZ4uV1cnUktbYmf38BZe4pQ7g3dmVM+cVjMid66YrhUzWYOi9W/6u1VdVS9gx1xc/yTAoER7BgbCOjqXtrZp9vSCUkjNLZq3Syex2Rqw/YKqgiocYXNAHVVm/MT09M/7+tS9xn/XtfYHAoPq2181MmZPc3NzDx48ePbsWSKRGBISQiAQeDwebAwM05NgMlJ/f78qSgCrc6ruRKGCtqmpCWYW3bt3r66uDpZJraysrKioKC8vFwpFZ8+dP3HytDtWP9TDwyMwMLCsrKyqqgp2Wi0pKSkrK2toaFA1jXrcDo/Z/9fxbW1t7aZNmxYtWrR7925XV1d4axUVFZWZmclisbhcruqOB/78C4VCDocTHR1taWkJ7/JheGH16tVr1qxZvXo1JAczzIhEYmdnZ39/f3d3tyoNqW3E2tvbi4uLjx07ZmJiYmZmdujQIRjMGeNtv/Ds9364GcIUAfeqqyS9jhfZH7PPY64T+CFZWVnbt28vLCxMT0+n0+mwsmdsbOzjwgswGYkyYiGYwZt+MplMJBK9vb1dMHNzc3NwcNi/f//y5ctPnjypUkirCiuFhoaSMSONGIVCSUxMFAtEUlmBUJgV4L/r1/OTLl9CHB2wekdeQKBMICAkIggRhNEwuUIUEhuDBRbiAS0kJyMpqQAM0tMRFgvJyEC4PIAH77yD3LiJZAvB9JgYJDwcDHQ6NoQjdOztmbMfb9u2Zc+efYcPHbaxAc0WHkcMtra2e/bsMTU1hajwMDAsXLgQS0paZGRktGXLluPHjzs4OCQkJEgkEsEoE4lEXC7X29vbysrq8uXLXC43NzcX6hlSU1OTkpIeIoX7EyQSSXl5+Z07VbW1tR0dHYOP7/6LPTgAzxSUSmV+fj6JRIKds/+Yq0y91ZfjgV6RqJ1Oh59ds2PHHQuLh3OKBtva0r76qlp/SrOBfsPI0GignzpZx3+SFktHp95AvxlDiFr9KSWmpsquP7BK6dO6ybXNTRVemFY9fYpEX4MCwgtWQpun3dSzLe/sUvq6i56Vyq6+flJHp2F7h2Z3D0CFpmbNktuTMjI1YmKHe7TBOkgJiRPj4kFUAaqc45gTmfHa9fUg801tv8cDfyAwqHbrcQ+EsrKy9uzZe/Xq1WAikUql+vv78/l81c+DavXfOdLY2Hj12o1Ldle8fXyDgoK9vb1DQ0MfqVJ45B3J7/z0V2p1qBu5du2aoaGhmZnZ9evXIyMjExMTo6OjYewlIyNDJBKpftUlEolQKOTz+enp6Tdv3oSxBRhkMDIyMjY2XrNmzapVq2CcwczMzMTE5Ny5c2KxuLOzEzJeN2ZdXV11dXXR0dE7d+40NjY2NzdfuXJlbGys6lyrqiEplIqfz9I/Wu2cllMCeEFdJemVuoBe6s6M+feEbxUKRXt7e1ZWFpVKjYiI2L9//+HDh4OCgmCjNFWQQaVkUCURwQjDCDiAZCQSieTv7+/p6enq6gqZwdXV1dnZ+cqVKwEBAaOBIRQzqHkgkUjkURYSEpKUlJSUHOLltebXX/9x+TLi4IBzcwe1UAEtBGF1kKj3FQuMGIQZhyTEI0lJgBZS05C0dCQdQ4VMNggv8HlIdBRuyRIE74UTCHB8PpKZicQygB46IhLkMkHlQ4Cf5i8/L9/1088HDx60sjpia3v0ccBga2trabnH1NQEZBwBQTPgBRU5wDSk5cuXr1mz5tChQ76+vgkJCdnZ2WLMcnJyBAIBfFWBA5fL9fT0tLW19fLygkKO3NxcHo9HpVJhnpLq+2TMiEgkYjAYUVFRycnJQqGwoaFBiT0gGHOiVRfd4ODg3r17v//+e7lc/rhlVAurR15fD/RKJHX79ike1Q2pgkaLnKxTqKfXhIFBo4F+pf4UsqZWuLZ2LRZwgCAB5k6f1rrUqPPkif74+KGRqk0vyyfdim7DOsNpdcP5SNOqpmsn6GiQtaaET5O2Sl/MXjk5v8bAoFS29vX7dXTO6+waRoXGJs1i+aS0dCBoVmmaExJB3pFI/F1RsSk36+OUVI2ExImJSaAy0u3Smy/Gz3/uT3kRwPA4D5aUlHrivckUKplMCQ8Pj46OLioqUt1EPm6tp53e1Nzs7unh5u7p4+sfFAyynmDyzNNu50+wPMSk/6VMrF692tDQ0Nramk6nJyUlpaWlsdlsPp/PYrGys7OheBFqFsVicU5OTmZmJpVK3blzp6mpKSSEOXPmvP/++6amppAWVK/GxsZr166FJSaTkpJYLFZsbKy7u/u+fftMTU1Xrlxpbm6+atWqAwcOjG52oVAqsH1TFpXXzbXw+GYrvqSqHkWVildPuPYnuAz+ZIdQW1sLMSA0NHTjxo1T9PTefffdzZs3u7u70+n0qKj7YgYYXhhzi08eMRheCAwM9Pb2dnZ2dnFxcXZ29vDwIJPJNBoNCh5geAEKpkfWe+AviUSmUkO9vG3Pnde8cgUB1ZBcQc8EP38stkBGQqmg8FFEBJaGhCkWEhMALaSkAn0zKx1EFTIzEQ4b4XKRrCyEx0O4WUgKC4QahDkTcgSIQISwOQgjDsgewBCNRMcg12++u23b1j179h4+fBj2craxsbG2th4dZLDFzNLS0sTE1HDREkPDJYaLFhsuWowlJRkuWbLExMRk7dq127Zt27Nnz7Fjx1xdXTMzM3Nzc2EOkooQxozAOIObm9upU6cYDAYm8gZpWVAT9ciUJEgOEomEw+EkJCQkJiampqZmZWXV1NTA7yjVMyaRSPS/TtsqXVlFRUVNTc1DwidUbX9CDzzqy185NMTdfyBUUyt5sk6enl6Dgb5AV9d/khZbR2d02AFiQ9tULHNpqkHTjE/aLTb1eHoMSiTKl9FGOr474Y3aN1VyZ/0CAw2qlkaYlgV76ws7cY5OryUwKJXNfX2e7R1fdXVrdnWDSqkNjZqFRZNS00BUAaICIw5QQXKKVmHhqo4OBooCAYxCca+5JaS07Ke8vE13qgmqh5IvzOF/yg96mcDQ2dWdK82T5MqYzPi4OAass676qXhe7u7t7ZXl52VkciKjYuh0kLLPymA9MsLwvD7xld0O9K1CoXBxcTEyMlqxYoWbm1t8fHxqKlB38Hg8Lpebnp4Of+NlmMEkBA6HEx8ff+3aNZh69L/eCytXrpwxY8bXX38NE5NgbtLoUMMPP/xgbGxsYmJibGz8ww8/mJqarlmzZu3atebm5qtXr05+MEVVqUSVStDU+YpP8oer3FYfCurq7cdyEF5ZX6p37JXwwMDAQFpaWkBAQFBQEIlECgkJcXNzs7CwePfdd/X09JYvX3758uWwsDCIDTCJiEgkkkaMPMoCMYMyBjc3N2dnZ1dXV19fX5XagTrKRq338CiFSAzE47c6OExyhbQAM5FIgBbCaCO0MCJaGJ2GlJGBsNkIhwsgIYuH8PkILxvJyUHEIkQkxImEiEgIxsViwBIJCSA9KToG6B9u2X+wa9fPBw4ctMLkzra2trA725EjR1TUYGtru3fvXlNTk4ULDBfMX7pw4WLDxYtWLJtntnr55s2bd+3adejQISvMDh8+DAsohYeH83i88YEhJydHJBJxOBwHBwdXV9ecnByo4sjOzo6MjISNKcbEFuBbiBbZ2dkcDicrK4vP50skkqampu7ublUNpZSUlAMHDoyppaYOL6B/YRvo6Eg0MaFg+uYcXZ1obe1ATS2Brm7LSMxBla2kGmk20G+fatAx1aBh+rTWBfM7rI70RUUOlZW9MC/ubNh5Px/pznSddD0NkpZWqE7yvZQXtg+Ojq8ZMCiUd3v7nNvbZ3X3jKBCg2ZB4aSU1PuaZkYcCCmkphnI5Tu7u7kvzJl/2Q96mcCAomhjY3NRcUkWjx8VHc3j8cevsP7MJ0mJKuUlZZlsblRUNIPBgJ2nn3lrr++KKhhrbm62srJasGDB+vXrg4ODk5KS0tPTORwOj8djsVhsNlv1iy6VSsViMY/HS05ODgwM/Omnn1RBhqVLly5evFgVW1AxgypJSTWyZsTWrl27evXqh9ULWCrCUN291sU7fD5Z53HKPg44GVDEaD306+t49Z7/UR5oamoikUh4PN7Pzy8oKIhGo0Vi5u/vf+jQoS+++EJfX3/27Nk2NjawiDARsxFeuJ9NFBwcDJsYUigUEonk5eXl7Ozs6ekZFBQEdQ7wFfaBhslI5McZ2GoImRTs77fDy0sXqpyJRCSEAtQIoM1C1LDEOT4eSU4C4mZVGhIbCyxwsxAeHwzZOUiOAEBCrgwnycWJxYhEhEjEiESCSHMBTiQkItGxoAwrhax95jTQBVlZDUcVrKysDxw4+PMvuw8e3APyk2xtf/llr7HxygULDI2WLFht9vXmzZ8dPvT26bOap07POnHC6tix47a2R62srA8dOrR//34rK6tbt265u7sHBwdzOJzHMYNqOuz45unpGRsbC2slicViNpudkpICv0zGeRWLxVKplMfjEYnE/fv3z5s3LykpEf7vKxSKMV3eVF9if9Qlpd7uK++B7pqauAULQzS1iJpawZpaQZpakVraPF1dmZ5elf4UqIRuHBE5qLChActZasUE0+1TDZo/+rB1zZoeZ+cBQY6ys/OPO2j5oPz96g+mVWNy5zvTp5ZO0wwHzdoWJC7qV7y4hgAODq8NMCgUNb19N9o7PunpHUaF+gbNvPxJySn3USGOCVAhLX16Scm+nh7ZA6dvqB+tKxjITemTsoea6x6YpX7z+zzwcoFBOTQ0VFFRKcsrSExKiYlhVFWBcgHP8TYRbqruXl2OQBzHjA8LDysqBFlPaisvL7e0tPz22283b94cFBSUnJyclpaWkZHB4XCSk5PZbDZ8TCiVSkUiEZvNjo+PJxAIp06dMjMzMzU1haEG+KoCAxU8qPTQo6esXbt2zZo1pqamN2+OzSbEwgtDzsEZM9d5fGbuHpteoD5Bag88iQeKi4vd3d1dXFzweDyRSFSJFiIjI2NiYv7XR/z8+fMLFy40MDCwsjpCJpODg4NJJBKsyQZDDWTMAgICAgMD4bgqyODj40Mmk1U6BxU5wMV+6xV0dQgmWBICdIKIoPIpLRTQQmQU6MgWxwDxgeFMpNRRmUhYbIHHR/hYYEEoAHiQmYE7eQIXRkOkMlyuBCeVIlIpkpuLyGRgscQkrG9DHEKhTL10aY21ta2N9XFra9uDh47s3r3P0tLs8tXvThzfb/nzvjVrl60z//KXXz45d266u/u/vL0neHsj7p7IjVt/O3vGyMbWxsra5siRI4cOHTpw4MDBgwcvXLjg4uICe1BkZWWp2GBMSpLqrUgkYrFYoaGhGRkZUPMgEomSk5N5PB7MbxzDDHl5eQUFBTKZLDo6+vjx43PnzjUwMJgxY8bevXsLCwuf468AqrY/nQc6ysvjvp8XggEDZAZYUJWqpR2vPVmkq1sxZUoDFnNoehQ5NGDyaFXYoXn+vC4b677Y2KGa6ufuKvs2h/vhherpejn6GmQtjTBtfInXc/+scTZo73D71Rc9KxRVvX127R0f9fRqdnYNt2qW5U1KSh6LCumst2+X2vb2ye8fcl8nWpGDpjn1uW+vtl5dfWjDnYOb7pzc3Zn1iIpb99dSjz2NB14yMKAo2tXVXVxckiMQxcQw0tLSVaWKnuYoHr0s/Mnp7euTyvLT0jPoEREsFmvgqfpHPnrDr/1UmJFVWVlpZWX11VdfmZmZwYpJTCYTKDaTkxkMBpvNzs7OhklKTCYzPDycQCBcvnzZwsICphvBKqsqZhjdmUFVPWkMMKxdu9bIyCgiIuJhLJSVVM/dgp+1yWfZbv/GlrbX3sXqA3ghHuByuVeuXLlx44a3tzeNRlMBAxyh0+nR0dFhYWGurq4+Pj7BwcFEItHZ2fnixYtBQUFEIpGMWVBQkL+/v+otGRNAw6LMqnKrKmyAIQi44vivJMAnBwhBWsEkrN/CiHSBEQuaLSQmDtdESmcB0QLIROKANKTRtCASIdLcCWlpiPlaxNcfyc/DyaQAGGSy+0NODpKcgmMycMw4JDxM0939uwsXth09Zml15Jf9+zdfufIxNUTj2vVvzp35zNlFJ4z2T0YMjskEOxBKQwL8/+ntPRmPf8/dzczR6Ya9vYOjo4Ozs7O9vf2NGzdgipG7u/u1a9f8/Px4PB5sv6AihIdHIDOkpqaKQRsKUFA1MzMzLCwsOzt7DDOIxeL4+PirV68aGRm99dZbH3300aZNm7y9vdlsdl5eXn19Pao2tQfG9UBHeXn84sXUEWYIHok2kDS1yJpaIVraDO3JfF3dkil69/RB9dWmUaroR4YdWqfqt3z+WdumjT0+3oN5ec+lRl832r2wbtH00XLnOB0Nita7kf+923tv3ON7zjNv2b/SwKBQlPb2nu3ofF+FCnV3J0lyNRKSxqICK+O9isqzff0VwEED/WhRKsoLRsOPoS7G6MUZinNzqw6uL95jUbRnY/GejaX7LMoOWvQVC5+zN/+qm3vpwAByTu7VNxQUFiWnpNFoYRKJBLZ5/j2h59FPp6qra7J4/MioqOiY6IYGdRXeB6701tZWPB4/b968WbNmHT58ODAwkE6nR0ZG0un0iIiIhISEuLi46Ojo8PDwkJCQwMDAGzduYKJJkx8wMzExgaWTRtPCmHHIDDC8sGzZsqVLl+bn54/shFKBKlAUbe3s2WJD/mIj/vO1ntcD07C56mSkESep/z7eA2lpaWfOnLl27VpQUNAYWoBvYT9mOp1OIpEIBAKZTP7ll1/mzp3r5+dHwgocwd6FqvACecQgGMBl4DQVM4wsMv5fSgiFQKZsIFOmUkL+Fjo6GQkLL0DpQlo6woK6BQ7QLdynBUyugGUf4SRioGQQixHpCCfk5SH5+Qh8zc8HcYb4eKw5dDwSH4ejhWsHEd8ODHwzmGBAj/h7VDRCC/sbPQKJZ0J1BI7FmpSc/HEMwzQs7ASNdjks3DsiMjwikh4eHhYZGRUREQHV4cHBwf7+/rDO7Pnz5319fcfRMwiFQlg9CQYiRCIRDDIIhUI6nZ6YmCiVSvMxy83NlclkGRkZ33///VtvvWlsbGxvb5+SnFxQUJCfnw9zIAsKCnp6elC1qT0wrge66+pSTFeGamrD3CSSphZVU0u4Y0fmli20t/4NyYGMJSxl6oDaSrX6U8YhhwYD/ZYRtUPTe++2Gv/QbW8/kJ2N/o5LMbkn+QG5c/5UjRAtjXCtI0LrcY/s+c+8eesVBYahoaLevhMdne/29g1HFWrrJonEkxIS76MCMx4kIGVkflRReaW/f1QUKMUJvfopenkmajcTvTRj8Oys0n1riyw3FWO0AF8r9m++53wSHXpx2V/P/+S9Mlt8ucAw7AaFQpHOYvn5BxKJRCsrK0tLy6NHjz5zK1BIGhUVFdHR0X5+fufPnz937lxQUJBM9mCi2ytzDl7WjqiwKi8v7+jRo7NmzZo7d+6hQ4c8PDyCgoL8/PxgPgZ84ArvHhwcHGxsbMzNzU1MADOswAz2VRgdaoCQADs2rFq1as2aNbBF1Pfff29hYdHWNhxAwHZA0dvff8I+6vN1nl9v9ft+m09pddPDFbhflovUn/uKeyAhIcHa2trd3V1VOPWR2ADbvBAww+PxDg4OEB6IROKZM2du3LhBoVBUfRVgVEGlc1Axg6pdA3lcUy1Po4XFxFBjo72io/dHRn8VHf0mI2ZiXBzCjAfag+HWbFhNJCBdwFTOvGygWxBiugUobgZyBSlSkIfkyRBZHi4/72/5+YAW4FBQgBQUgmhDZiZgBjAwwZCQgMQnIPFxSFwsyICKjkaiov4WGTmZGT8nNXV3ctJlZlwwIzY6Li6JRosMCgL/6FRqSFgYLSYmJgWzxMREBoNBp9PJZLK/v7+9vf2JEycCAwOzs7MhEuTk5IyfpCQUCmHraz6fD1vakSnkQEKgBOuALRQK8Xh8bGws6IZdWJiXlwcbSKs2XlenTj5+xf/5Xond629o4G3dFjpFn6SpFfPRx/mXLsGfj+6KikoymbP9x4j3P4DkQNLUCtPSTp6sk6unW6X/GwlLTQb6bVMNOqcaNLwxvXne912nTvYlJSmamp72mPc3Hhidj6TDAnLnyWF6WU2gM+mLtBs3S161lKTBQWlPz8H2jrd6+zQ7OjVb2zRraicJhBrM+LGowGZ/WlXl2N9/d6zH6MfQy5+i52eA4cInd62XFVpaQE4o2rOpZN86+Z4NRZYWlYfNFfVYRGLs+ur3T+eBVwIYUBSNT0x0dfUgEAg//vjjvHnzVqxYAfUMT3c02NLwPtjFxWXWrFkLFiyYN2/ehg0bfHx8KisrH86EeYbt/5lWUSgU0F0oihYVFdnb269evXrhwoXbtm3zwNpse3l5eXt7+/n5BQQE+Pj4ODk5nT59evv27aampj/88MPy5cuXLVu2fPlyWAdp5cqVqvCCSgNtZmZmZGS0du1aExOTL774wt7efrQDm9s7j92M+Xwtfs72gJnmHuc9EoCGRd2vbbSP1OOP9wCHw7lw4QKRSHxcp7bw8PCwsDAikQiLIBEIhKCgoGDMKBSKr6/vF198MW3aNFNT08uXLxOJRFihlUQiYeroR9dTIj+B0Wjg5jsujhnPTI5PSExIpCUlBSelXExM3piUPDslRTc19f/S05HMDISdOZyMlMXDVM45QOUMYguYuDk3FwQW8gsm5BUgefm4ggIchISCQoAKBYVIYRFSUISIc5HUFAAMMN0IvDKAtoHB+FtsrF4s4zsGc1d8vGtCfFhsDNPPN+jKlWtnTp87fPjIjh0/mpubr1u3buPGDRYWFtu3b7eysrazs3N0dCSRSDC6SCKRfHx8Lly4cOLECSKRyOfzx0cF2AcagIFMJhAIxGJxeno6m80+fPjwzp07ITDk5ubm5+fLZDKVsAEWZMvJyeHz+VwuVy6Xq8unPv7CV88Z5YGhoc7bt9uLinrvPnRDiaK9dXW1TGbOgQPRn31OxLKVQCBidMLSiNRhHJ1051QD0BJu9qxOy196abQnbOzQqmidU/PNtNrh9gtTy6ZNogO589K05f3KF/3A+/qNVwgYBgdFPT172zumq1DhTvWkHMHDqKDB4X5eXe06MNA46nyPGs2moJdmYsDwyeC5z0v3mRftGQ4vNNkaDpz7ouf01zWHjcv3rxsoyx21mnr0GT3wqgBDdGys3eWrQUHBO3funD9/vqWlZcez1kuGd8AEAmHBggUrVqxYvHjxli1bvL29y8vLn9FJf97VVLSgOsT+/v7i4uLw8HAPDw9nZ2cnJydXV1cPDw8fHx88Hu/k5PTrr7/u2bNn9erVxsbGy5YtW7x4ASXc/AAAIABJREFU8cKFC5csWbJs2bIVK1bAUqqmpqYmWEHV5cuXGxoaHjlypKCgwM7OzsRkZUVFBah/hJkgv3KTNeEzc89vtvrMsvBdvjuwtqEdQJ1qb9Qjag+M64HKysqQkJBHRhVUE2E2HZQ1EwgEKE4gYflIJBLJ0dHxxx9//M9//qOrq/vtt98ePXrUz8+PSqWqhNGPK8NKHmvDNZcwzXTQtavXPDw8IiOjkpISklOSUlLTU1Mz09LZLBYrgxWTmeGWyT6VyV7N5sxgcyZzuDgepl7IFiBCrHCqRIJIcpFcKS5PCjKOrGwQMmVCcREuvwAHIaGwCKACHIqKkPxCJFsARBHDcYY4JIGpwWR+FhdvyYy/mZQUk5ySRqcz/PwCjx49aWa2dsFCwwULDOfNW/T9vIULFixcAG3h/AUL5i9cANo8Yz2eN1+8eDEwMJBKpQYFBTk7O5/AjEajQSQY08ENShpgyaPsnOyoqKjjx46tX78+JSUlJycnKioqLi6Ox+OpCGHMiEQiycnJ4XK5bDbwUnZ2dnd397gnXz1T7YGn8EB/Y2N9Rqb45MmERYuIk3XIWJElkqYWXUs7XUdHpqd3Z6TC0iN10o1YwlLHVIO2qfrNMz5p37Sxx9d3qLhonHg4p5fzZs1bw+0XqqdPEUO5s5ZHmedT7PdzWvTqNfkjIwx8fstz+oQn2szQEK+nd3d7hwFEhZZWzao7k/jZGnHMsVGFLN4XNTWug4PN4223pQa9Phe9AMIL3Se/gclIRZabGmwWoxc+Qc9/gr3O6Dkzd0gQOd521POezAOvCjDQIyLsLl8NDibu2rVr/vz527Zta24e90J5/OHBm+D//fAYGRnB0p8WFhbe3t7YrerjV1PPGeWB6upqZ2fnG5g5Ojq6urri8XgPDw9HR8dLly5ZW1tv3rx55cqVxsbGixcvhjcb8+bNMzQ0NDIyWozZokWL5s2bN3/+/K1bt6anp9vY2q5ZvZrLzURRtKevV1hQecqF8e1m1y82en27LXCOhd/sDW4JmeoCVqPOgXr0tzygVCgKCgpgzr0KD8aMhIWFkUikAMxgkCEgIIBAIEBggDBApVIDAwNtbGzmzJkzefLkDz74YOfOnS4uLjAQoQo1kMc1bIOAGSgUyuVLF4xXLF++zGT/voOenvjIiIiUlBQWi5Wenp6RAVqesDMFXG4OLyuTnxXP47ll84/kZK/Ozv5IINASiYBoQSJBxBIkVwpSj9gc5IP3ETs7nPw2UoiFFFSoMDwCJuIKiwBaJCThklPezuJuY7HsExOoaamsjAx+YmK6k6Pbjh27TE1+MDX5brXZnDVr5qxZ882aNd8aG39vaGi4cMHiRQuWYN3csNbPhoaLFhkuXLgQYsO5c+dg4dqrV6/a2NhcuHAhLi4OCqBVoQbYIV4oFEIp84oVK6ZPn/7OO+9YWFgkJydLJJL4+Hg6nQ6V0PB1DDDADnGZmZmpqamwlRusfw2/z3/rWlDPV3vgST0w1N3dmpdXcP1G6g/GFH0DMhZ2CNbUCtXSZk6eLNDVLZ2id2/csIOqsUPz+/9tNTHusrcfEAiUD9VTudZ67X4+0p3p2kmgu/P0iLeqekA1yBdsV64Wv1xgGBxkdXdvbe+YokKFispJWTwNRtxYVODxZtXWug8NPQHJKJVo6BEgYLjwSevx+UWWm4r2bCo/sHro188ALcBUpfMz0EufoK4/oDXqpPTfe9G9KsAQn5jo7eP3v2afP+/e/d133/3www81NTXPdnCwBJCHh8ecOXOWLFmycOHC7du3x8bGqpNin8Sf8OdZIBDY29tfx8zBwcHFxcUdMycnp6tXr546dernn39eu3btypUrTUxMli9fvmTJknnz5i1YsGDx4sXzMJs/f/7XX389f/78xYsXL1269NixY+VlZQWl947djF13hPDlRvdP1+G/2hLwzfaAOVv8vljrFhAuAKEFdeOFJzlJ6mUwDzQ2NjIYjIeLI41mBhqNRiAQYE6d/4gFYWn7ME8JFkcKCQmhUqkkEunixYsrVqwwMDB49713L9ldIpFIwVhtJajkIY9nFDKZRCZTg4Jdr1z/co/lm6tXz1pq9J2JsbGFxdaLF+zCwmgwM4fNBp3KuNwsHi8nmyfMyZEIhRKhkCcURYrFDhLxvlzJMnHue7m5/5LJQDyBw0E+/hi5cgUBwDASVSgsQooKkaIipLgIkWMjhcWa+Xlzs3N+lkrD5MWFEml+Fk8kEIjpdNqpUz8eODTr+Mk3r1/X8/HRIhI1KCGTyBQNIknDy1frypVp1lYfbLb4avmyBYaLlixatNjQEED/EswWL168fPnybdu23rhxw9XV9fTp00eOHLl8+XJ8fLyqaFJOTk5qaqqHh4e5ufmHH344/Y03li9bdvXqVbiMSCSCkmgmk8nhcEanIY1mBkgRWVlZTCYzOho0zBnTr0191as98Hw9oOzv7yovL/Px4fy4I+K/7xM1tShYqSWyllaUtnamjk6+nl4NFnZofkyFpeYRqUPTv99qWbSw69dz/fFMldTB5J7J9LtYPtKd6QYl0ybRtDVoWhb8LQoUdCl9wXb5yssChqHBwaTung1t7ToQFZpbNMvKJ3G4D6OCRnb27Lt3PRWK1qdwTh4TvTgTvfBxo60hAAbLTU22i2Bg4T4wAGaYiToYooVJT7Fl9aIPeeBVAYaioiI2m1NYWHjz5s1Tp075+fn19vY+27MluFZERISpqemmTZvWrl17+fLlgoKC3l7QMFxtv+kBpVIZFxfn5OR069Yte3t7JycnNzc3KGlwc3NzcHC4dOmSjY3Nzp0716xZs2rVKjMzMxMTkxUrVixdutTIyMgQu91YsmSJkZHR8uXLDx8+zOFwYPelJHbRx2auszf5frPV79utft9s85lt4TPL3MM1JFOBKpRKhVKdjvSbp0e9AOaBnu5uNptNo9HGlztTKBR/f38/P78RWPAPCAiAkEAkEgMCAkgjppL4E4lER0fHX375xcXFhUgkBgcH+/j4BAQEkH/TSCEkEsXTc/0t+wlOjsiNm38/c07np13/WbVq1pIl8zds2HrhwqXw8PBMNuiqzsvi8fi87Gx+Tg5fIBCKhBKxWCaWyCS5edJcgUwWIZPdkMl2FhR+weFM/uTjCZevICWjgGEYFTBgKCqeVCRfLJffkhcLSuS3S+Tl8qKSkpJymSwrIsLW2/tTP79/xMWB9nBZWUg2H8nhIwIBwueBt2wOkp4ORNgU6j/sb+ke2Pe+qcm8hQsXLzZcYmQE/p2XgsJmS5cYLdm4cePly5evXLliY2NjZWV19erVlJQUGFgIDg7+/PPPdXR05s6de/z48aioKDhdIpGoqieJRKKsrCwYbRhTYhViAwQGoVDIZDJhmtm9ey+07qT6H+uv7IG++oa6OKbo6FHmnG9IOrqqrnChWtqJkyeL9HTLpkypf3zYoclAv9VAv3OqAajC9PlnPbt+kdLt/1vzoapfmy5/igZJa1KYNqmW/FL8bHf5xQNDf/9ATHePWVu7dk+vZnuHZnOLZmnZJDZHI5bxQFQhOUUjR/DlvXt4hfLpK6p3NaOOS9BLn9yzNoLA0HXym0cAw/kZgCsufY4mOaD96gpsz3gNvhLAoAIDJWbwUEaPP9XBwRX7+/sbMKuvr+/s7Ozv71d9ylNt7S+1MHRRW1sbmUx2d3d3xczT09Pb29vX19fPz8/Hx8fT09PJycnOzs7W1nbXrl0WFhbr1q1TdWozMzOD0mdzc3MLC4udO3e6urqqUC1PXvvVBo9vtvrP2Rrw9Rb/T8w9F+zERyZJgZOVg+DE/aXcrT7Y3+EBuVwOBc2j4wljxsPCwoKDg+Gl64eZv78/gUCAwAAzjmDogIRJGoKDg6HCAUYVgjAjkUjr1q0zMTEJDg4mYxYSEvKoiklEMik0kHDd2fkNe3vEyQlxdUM8PBF3jwmOzv86fnz6po2fLl22YP26TecvXIykR2RxuTk5Odk5OfzsbIFACGsKicVi7O45TyotlMmK8vJyCwszOFzPGR//+8qVj0tuv19Y9M9CDBKKYWyhWFNevKSk2EkuF8hLyuXykmJ5sVxeJi8qEkt8WKwlMdF/T04BYCAWgTQnqKUWCZHsbNDzgcMF5ZXS0pCUVNBFLg50csDdste2sPjCaInhksVGRksAMywbsY0bN546derEiRNWVlY7duw4e/ZsUlJSbm5uRETEsWPHyGRyVlYWLHYE9QxjXoVCIcw1gsAgwWx0kAGOp6WlBQQEhIWFwZSk33GNqFdVe+CpPTDY0dEiFBZeu5ZuYkqdNp2CJSwFaWpRtLRitCdzdHWKpujVYQGHR4YdoNShb7JBwJ6puvew7s5V4FUrbrIGWesd5vv1isfod596T59uBTu74qnTUgymMh8c4v8IDYNS2dPfH9bds6KtXROiQlOzZsntSRmZw6gQy5gYy5jIjJ+YnDJRJP66ocEbRYF88Rkt9gJ6ecZdq2XDwHBqzqOBAVRSmgHyl4i/oC2jarM+46f+FVd7VYBhzL0ivOkfM/EJz88jSeORE59wg3+1xQoLC2FZVV/MAgMDg7H+uPBpK4FA8PHxcXNzu3bt2okTJ/bv379r164ff/xxx44d27dvhyM7d+60tLTcv3+/lZXVyZMnw8LC+rH8zjt1zd9t9Zq53uuztR7fb/U46RRXXgU6NCmUCoXyfr2mv5rD1cf7tB6ora1lMBgwtvC4CAOdTg8NDYXVveCV7Ovr6+/vD/ORYK4R5AQyZiQSKSgoCGqjodoBlmENDg62s7M7d+4cXJhIJOLxeBKJFBoaSqFQVFsgkSkUEtE/YK2XN87bC8HjEU9PMHh7I34+SEAA4oH/+4ULUywt3zMz/XbD+g0XLl6Mjo7m8/kgXwcz2LsgN1eSmyuRSnNlMhlohVxUlJMjnjnjs4vnT5WWJhYVXS4uWlVU/GFxsZ682FAud5HLRSXFFcXyUrm8BAwlZfKiAqHgWlKSQWoyhgoS0LFBJgPlWSEzAGDIAT0fOFzQMC6dBYAhEQOGmGgkLAzB4//5i+VHy1cYGpuYmK9ds3XrVktLywMHDhw+fPjChQsXL148dOjQf//738VLFl+/fj0lJQVyDownjIGE0W9hz/iQkBA+n//IIENubq5UKs3OziYQCKGhoZ2dnU97YaiXV3vguXlgYAAkLHn78Hbtivzv+yRNrRBNrSBMKh2upZ06qjwrKKD0YM5Sq47+VsI0nbsYMNyZblA0VSNU619hWlsSV6D1LydudulS0QsABqWys7+f2NVl2NGp2Y1FFRqbNOXySeksjZhYEFVQoUJK6kSJ5Jumpt+HCvBk18jQy5832CyFwFBvswS98PED+UgqMQMcsZuJeqxEq7Enlc/tcvlLbOiVAIa/hKdfk4Ps7+9PTU2lUCjw+SuJRKJSqfCeLCIiIjw8nEajhYSEBAcHe3t7Ozo62tnZnT59+uTJk8eOHbO2tj506NDhw4dtbW1Pnjx55syZ8+fP37hxw8nJKTs7G0XR5o4uy7M0ywvhPuE8ecUD35vq2MJrcoG8/N3s6+vjcDihoaFj4glj3sJqqj6jDAIDiUSCFZBUEQMyZrDTyGhgCMQsODiYSqWGhISQyWQqlero6PjBBx98//33p06dCgoKCg0NhbMw9cI1P9/p/v4IIQghkpBgIkIkIgQCEhiIBPgjgYG4YCISHIy4OGsePPDeKrNvNqzbdPHSlVhGrEAgkEgkYszgU3apVIoBQ15hYaFIJJg589Nzv/5aWlZTLC8tlueXyJNLbhNLSoQlJZUlt8tAFhKILRTJS8ry85O4WVtSUiezWIhIhOQXgCEvfxgYcnNBkEEkRHIEQCHNzQLAwGIhqWkgwsBkgo4N9HDQl9rD8x+X7TbY29v7+vgGBQXh8fh9mJFIJGdn56NHj+7cufPAgQNWVlY3b95MT0+HeoZH1k0azQwCgSAuLi4xMVFVXPXhCINUKk1OTubxeLCsqlIdd0TV9pI9MNDcfDcqSnzsOPPLryh6U6gYNgRjmoc47ck80E96Cgw7NBnot+jpF3+kPyt72pQaDBiqp+vypkwkaf6DruWye1rXBzM6du3sCQwcul2CDgy8sAO7+AcDg0LR2t8f2NU9v7NLs7sHJCA1NGkWFk1KTR+LCqlpGlLpNy0tfr8rqjDGcXGX+k4bFgPR88aSvevbT3yPXpgJWjQAPTTWpWEMM1yaidovQktAFRa1PbkH1MDw5L76SyxZVVUVExMThll4eHhERAQUIDKZzPgRYzKZMTExNBoNYoOrq6uTk5O9vf2NGzeuXr0KE51v3brl4ODg6urq4+NDIBCCg4Nra2uVSkV3bz+KDkJXqm8F/hKX1HM9SIVCkZubS6fTHxdYgNgQFhYWGhoaGBgIG4l4Y+bj4+Pv7w9jAiSs0wJ5lAUFBfn7A4WDylTAoFqKQqEEBgZaWVnNmjVLV1d35syZe/fuxePxGFHQvLy3uLv/LWAEGKgh4Gk9PQIMtDAklIqQSQAkKCEIiTTB3V3Dyurf5mvmbtxocdnuShyDKRAIxWLRSIRhGBgKCgqEQuHMmTN//fXX8rLyktvy27dLS0vLy8qqysoqy8rKSktv375deruktOR2RV4ek5XxXUYGTigAkACE0cWgV0Me1vdNKkVyc0HHaJEQ0zDArCQOkpEBZAwpqaDdW2zshAg6jkrFEQHbzCEEeZ6/cMnY2Pjtt9/W1dNdaWZGIpECAwPPnj1rixnUMzg7O6enp6vkCoJxLTs7m8lkjhNkyM3NFUskz1z04rlebuqNqT3wgAeGurpa+dlFN2+mGi0NnTadqqVFxsIOsDxrGlaetV57Cn2VgX719KlYMhLIR2Lo/JOkaRCuI/jKoF1PH6vNatDw7jutxsZd164NcDnKPz6YduHiHxVhUCga+/vxnV3fdHUPo0J9g2Z+waTUtAdQIT5hYmraxLy8b9ra/FG04wG3/v43ve295NNVBzdWHdhSeWBLxQGLlsub0dgLKPUQemnWSK+GB8nh0kz0xly0IPn3f/hfZwtqYPjrnOvfPtKenp6srCwGgxETE8NgMOLi4uLj45OSklJTU7GKkBlsrMILh8PJyMhISUmJi4uLiIgIDQ0lEolBQUEEAgFmgHh5efn6+gYGBsLMDRiXSExMGAQPVJSgMZtySK1Y+O3zoV7iIQ/cu3fvNysjweuNTCb7+Ph4jTJvb29VQVWIDeQRg6VXVToHf8xGF2CFC8LoRGhoKIlEOn/+/NKlS/X09N566y1z8/XXb5xzdJrh7ITzDwARBjIZoYUi4eFIZCQSEwN6qMXFIbGxSEQEoIjQECSMhlBpCN5Tw/rIv1eafbV9+47YGGZuLsjrxxJzADDIZLLCwkKhUPjJJ5/8+uuvlZWAECoqKipHrKKioqK8vLyssrS0Mj+Pnp75DSsTJ5MBTlDVUwLAkA+YAQIDKNgqAs2k+Q8BQ2Ii2MmoaFxIKHLhAmK2cuJ7772tqak9c+bM7du3Ozo6wn7YJBLp6tWr1tbWNjY2tra2KmZgsViqKqvjIINQKExPT4fq54fDC3CKRCIpKSnp6+tD1ab2wCvpAeVAf6dcXubtzd2yhf6f/1BAwpJ2MFZkif4P7e12+nr1o/KRqFp/D9Va5q53902DRn39BgMwYM2kATw0vTG9Ze63nTY2fVGRirt3/6DnaL+ef/7AoFTe7e1z7Oya3d2j2dUNogr1DZp5+ZOSUzWiYybGxIIEpJjYifEJE9PTJxYWftfRQXj+qKC6PAb7u/jJzWEBjVT/Tk7SUMdIqaWKbNRrHQg1jAkyQBn0jW/RohTVNtQj43tADQzj++cvMVcl8CgtLU1ISEhKSkrGLC0tjcVisdnsrKys7OxsoVAIK5nIZDKxWCwQCLhcbnp6elJSEpPJZDAYUVFRkZGRdDodVrqMiIhgMBjx8fGJiYkJCQl0On1Us211CtJf4tJ6vgfZ3d2dlpY2fh1VqISmUqkEAsHLywuPGaQGHx8fmIYE0+3Ioyw4ONgPq6QEX1XAAJcfteD90dDQUDKZ7OTktGnTpvfe+4+enu63c//v2nWQgBQU/AhgYDKRxEQkOQUMzHgkMgoJD8eFhyM0GoLH/+PkqQ9DqB4SsUwiAf9lUswgMIhEotmzZ9vZ2d25c6eysrKmpubu3bv19fX37t2rra2trqmrrJBLJNcyMz/hcHCyPKS4GPRkUA3DwIBpGEBKEgQGTMaQxUO4mIwhMxNkJSUmgXJJ537FffopoquHe+89xMT0v5evXAwODg4PByGdUMyoVKqbmxtMQbTBDMKDn58f/wmaQAsEguzs7LCwsNTU1McpGaAH6uuBwEltag+84h7oa6ivoYWJDh+O+fQzirYOSVf7y4Sp+nUj+Uj8KROJmn+P0D573KBbe5gWIDPAV6iTHm4mPeOTju3bu73wQ0WFyuda1/HU6YLnqGFQKKr7+q53dM7s6QWo0Nauea9eUyqblJxyv/wRRAVWhkZR8XcdnQQUfXmSpI56lLL/scxw8zu0lPuKX2OvyO6pgeEVOREvczfgI422tjYOh5OampqRMRxJ4PF4fD5fIBDA7q2FhYVyubwEM7lcXlRUlJeXJxaLYX/WzMxMFouVlpaWkpICeQM2q4JBCTabnZCQkJGRoVAoXuahqj/7NfQAvD4VT9CmTUULsJqqh4eHp6cnZAY8Hg/zkUgjRh5lgYGBPj4+Km00DDWoCrCOWnDsKJUaQg0NxXs6bd/20cJ5OHsHHCEICQ7GqSIMERFYhCEWKAQSkwAtpKaBFKC0dCQpCRcTA2IOUZEInY7EJ/wkEgnF4lyxBNRKkmIyhnzMIuj0LC636s6du3fvtre39/b2Dg4ODvT3d3V2N9bXSKWn01laQiFSVIAUFyPFo5q7FRQiBQWg+xvUPefmIrkSRCQGEQaoe+ZykXQWLjZ2QlIyLjl5QlLShOPHcD/8gDt9GnFzQ3z93goJuU6j0UNDaaGhVAgMNBrN39//zJkzkBNUzHDixAkikZgNij79hgmFwpSUlMjISKFQCJkBPokYHXCQSCQFBQUdHc87dQFVm9oD43mgV6FI7+gceCblzFB3d1NySrSX7dsl06dWY8BQNV2bqTORqKkZp+NvPKVWG9RmfVgkrSKHZoPhhKXGf7/V9sMPXZcuDbDZyraRh+Xj7fhvzDt6LO+5AMPQUHlv34WOjo96ejU7uwAq3L2nmZurkZj0QFQhIXFiRqaG/Pb8rq7gl4kKKq/0d6GRpx4taYB6hrpC1bLqkcd5QA0Mj/PMn3H68GN9JWiR9qA1NjVmZWXBYAKfz8/JyVHFE/LygPKypKSkrKyscpSVl5eXlpYWFRXl5+dLpVKYjZCdnc0dMRiXgPcOOTk5mZmZiYmJ6juABx2vfvcbHoC0gKLo3bt3o6Ojx5cuwGQkKNnH4/Hu7u6QGTw9Pb28vAIDA8mPMiKRCEsGjwGGxy3/4DaIFArNP+C0g72Ouyfi6zeBEIgjEnGbt+CsjuDo4YAHYmKROCYSHw+e36uAgZWBZLKRjEwkJQWTGkcgCQkLBAKWSCIZAwyFhYUVFRVVVVV1dXVdXV0PILeit7T014yMiWIRUizHaAFr6DY6vFCA0cLolCRJLiKTIAIRKJTEz0ZIpL99/ukE34AJaWlIQjwSFYNFP+gIMRjxxCMB/ltpofQRWADIQKPRyGSynZ3daGCAiUlHjx4lEom/gQsjsxMTE9PT02Uy2cO0oEpMqqioGBp6CV2ufuOiVM/+83pA0tOzobbOv6n5mQ+RMEgebvB8Z7qBfOqkMO1/UrQ+iNDz/c/kkIlaDO3JfF2d21Om3H08OYxOWGqcatD8zZzOw4f6QkMVNTXP3NvUykr2O4FhaKi4p/dke8d/VahQd1dTJNZISByOKsTEggSkxKSJbI5GWdmCnh4SinY/sxuf/4qD/Wj8NaBneFgGbTcTxZujnS+n4u3zP9I/bItqYPjDXPvKbHhIMahUDqKgu+SDT/ehmgBV1jc2ZGVl8Xg8IWawSKJMJisARR2L5HI5zJyurq6uqampw6waM5hUXVJSAqMNQK0oFkNygJsSiURYsXXwwufzU1JSqqvV9Y9fmSvj9dmRzs7OjIyMJ0xGgsDg6ekJgcEdMy8vr+Dg4Ef1TyDD5CVvb2+fEYPkQCAQyE9gFBLFxWWl/S0c3gvx88ERAnHBQciOHTgbawAMkZFIZDRghqRkUIkIAEMqiDBksAAwcLhIFhekKtEBV7zB59PEYqlYLIF3zDKZLD8/v6ioqLy8vLq6uqWlRakc/S/c39DgnJ2tK5Pibt9G5CWg67MKFeBIQeFweAFGGPLykPx8HIeD8/JFQmkAGLhZAGOuXMFFx0xITsXFA90zQo/EhVJBYpWXN+Lj9U1ISGAobTgfCQYZYLUoKGCAEQYbGxtra+v9+/efOXMGNnQb4YLH/uVyuQkJCQKBAB7sw68SiUQmk7W2PofHq6/Pla7e05fsAe+mpg13762/Ux3V9iydAZRK5baGbdPvYQ2eq6frCfUnkrT+SdcypX4cN3s2eRJoJk3U1CJpgmbSbB1d+ZQpdfpTQLu3BwuzqnKWVAlLLQb6TR9+2L7ZosfNbTAvT9nzdN3HDhyUPjMwDA3l9fRYtXf8u7dvOKpQW6cpFGnEJzyACknJE7lcjcoqw74+Koo+3e69oLOuVKDJjlhn6AcF0OdnoJdnouFHUYX68cR4p0INDON557Wep3o0i6Job/9AbUObrKSOn1vOFZaJCqvu3G1q6+xUooNtLc1ioUAgyIHP+aRSaR4o/w5QoaSkBBRkKS+HTzfvjbK7d+/W1dXV1NRUVlaqQg0FBQV5eXkymQxmU8gwyxsxsVickZFRWlr6WntVvfMvxQMSiSQ0NJROp4/frI1Go0Ek8Pf3d3d3dxsxd3d3PB7/SEEChUIJCAjA4/EP11MiEomwrRt5HKPGA5DNAAAgAElEQVSEBhFcbt5638kZAIOvL1Y+NRihUIGsOSwMRA+cnJGlS3DHj+Mi6KCGaVoGkpaOY7FwmZmg0XIWD2FlIFFRSETkPzPZ10WiPAlG2bm5uRAYCgsLYX+0ngdvERSKpJramaWl/6qo+H+lpROK5fe1zkVYVhKgBaysKuj1VgzUC+FhuEMHcV/OnqCrg9jY4kBlVS7C5uC4XByWIgUiDHFQmU0DBZ18fRF3d90g4kVaKKhtoDIajebn53fy5EnrEYPAsG/fvl9++cXPzw8+NXgsK2AzhEJhXFwcg8EYp8SqWCKpKC8fGhyuq/ZSrj31h/51PNCjUByqrll/p3pdZZXFneqszq6nPfbawdovamZNgwVV70yfnKQ7kailGTvZrdkd7e6uT0iQnj7D/ObbkCn6IRg2DJdXmqyTp6dXPS45wLBD+1SD9qkGjW+90bZ8Wdf58wNpaYrGJ3ou/oul+BmAYXBI3N29r73jzZ4+zY5OzdY2zeoazRzBJGb8A6iQkjqRx59UXbOifyACRXuf1mkvdnklYIZLj9JA232KSmNe7M68Zp+mBobX7IQ94e6qaCGv9O4tAmvLcfL8H32/2oT/eiP+yw34Lzfh52/3Wrk/aO8F+kWX8PDYNLFEUlJcKC8sKiwskhfLS0rkpaWlsB5LdXV1XV3d3bt3Gxoa6uvrVa+QGaqrqysrKysqKm7fvg2FDYWFhQUFBYWjLB8zqVSalZVVXl7+hIegXkztAegBhUIhEAhgA8GIiIjIyMjwRxnsEAKBwcvLy8XFxc3NzdXV1c3Nzd3d3dfXl/woI5FIfn5+UOcAtdGwBquqAOujVho9jeruvvva9b+7uSF4L8QHAAN4Nk8igzqq4WGABLy9cEuNkKnTcO+/j/tlDxIUBLCBzUbYmQiHA57xZ2RiWUmRSFraIaFILBaJYVxOKpUWYGZmZubs7Dz44H2zEo0eGFzQ0qpXdedfpWX/Jy+5Dwz5xUh+Ia6gACcvwRUUIrEM3IVfcfPn4954A5k5A7djBxLgh0tPxyIMWLNn2IohORnkTTFGSjkRSYifL+Lm9jdfv520Yf3CMDKEhYURCITTp0+rspKsra0PHjy4Z8+e3bt329raxsTEwM4Mj2MGmPTI5XIpFAqbzR5H/SyTyRoaGlC1qT3wx3ugsLd3Y9Ud84pK8/KKdXeqd92pvv2UpbpYvazptVh4oWr61LJpmvTJE0laOgl6wn7R/d0fHGzh84udnVOWLaO9+RYVIweiplaolnbS5Mm5erpV+lMaRxKWVKGG0SONBvptBvrtUw2aDPSbv5zdvmdPb0jIUGXlOAlLu34SPRUwDA7yu3t2t7VP7R1BhTvVk7JzNOKY91EhljExLV1DINS+e89scDAORV9cW4n7znyGMeUQGnvpERroSzNRV1O058HIkiQSJDKNDu0qFWj0OTSb8gyf/LqvogaG1/0Mjt1/BYoqFKB6aWVd02mH+O82e89c5z5rk/fXW/znbPWbs9X/620BX2/1/3qz/1ebfD9f7zXL3GvOJrz5kQCngHi+MK+6orSyorQcy5mGCUj19fVNTU11dXVNTU0NDQ01NTVNTU2NjY319fV3796trq6GxVtgnKFkxORyeTFmEBwKCgry8/MFAoE6JWnsCVO/fwIPDAwMiMXi69evX7x40cfHJywsLCIiAgYcIDuEhYVRqVQKZiQSydPT08nJycXFxRUzd3f3gIAA8kMGuzvj8XiojVaVYIUFWCF7kEikh9ZTTQghBfs7On1tb49zcwMZ/z6+iL8/QAISCaGGAGCg05GYWFxsLOLhjmzdjHv3XeTNt5DVqxAPD1xq6nBSUGYmSAQKD0eSksyFwhyhELRwgxE/GO6Lj48XiUTd3WMSgmtRdF9H55vVNRPLyv9PXoIbrqZajOQXIQUFE2R5iIcnztRkwn/+g3vnHWT9epy7K5KeBuquSiS4bAEIbnAfDwwkMug35+aGeOENQ6mU0ND7WUlQxnDt2jVVPtKRI0f27NljaWn5888/79ixw8HBASLB44BBIBDArMWYmBg6nf5IGQOkCLFYXFRUpC6xiqrtj/dARGvbhrq75uUVcFhfU3uktq7hQVAffy8c2hxU+Uj6UgMNipZGiNYX3C9b0Udn1nUWFZUTgtjrN0R++FGIJujqQNTUCtHSjteeLNDVLdcfTyQNEaIF00m3Gug3ffB+67p13S7Og1LpwwlLm7fkPBkwKAcGM7u6t7S1T+npBVGFllbNyqpJPP4DqMCIm5jJ1pDKpjQ2bR4aYo3Ndh7fR6/C3IFelLT3Ecxg9ynKJz2wg9W56JHJKCfg/sRsMmo1Ga35KzaKVgPD/cvgzzGGCROG2JJyE0vfmeauX271/XZbwDfbAC3M2uD7ubn77A3uX232/GYzfs4mj683eXyxweMzc4/P1+A/XeO+6CfPy17x0ryShnt1dXW19+4BVGhpaWlvb2/FrK2trR6zlpaWxsZGWNixpqZGxQwQG0qxblKQHYqLi4swKy4ulkgk6ozkP8dl9lKOQi6Xnzhxwtzc3Nra2tvbOzw8HJbxDQ8PDw0NhbQQEhISGBjo6urq7OzsMmJ4PB7mFz2cYgTbGHs8WE/Jx8dn/HwkEjAymUwN8L9685a+oyPi6org8YiPzzAwEElISAhoxUCnI5GRABjiE5DkZIRGxVlbI7O+wE3Rw333He7KVVw6CwQZEhMBMDAYs3n8RKFQLBKJIDNAKVFpaVlVZWVjY6NCoQAVC7CaBUq0WYlad3f9u6ZGo6z873I5rqgIKSxG8vJwBQVIYcEEsWTCxvWI4WLk6jUkLQWXDwQMiFSCCIUILwcHBQyckcZtqWlg92CEITICFHsFwBCAuLshrq4fkMm+WKGk4QgDlD47OzvDTCRra+sDBw5YWlr+gtnOnTttbGwSEhLGDzJAlsjKyoqOjuZyuRAPRpMDHIevdXV1qqjpS7n21B/6p/eAEkUv3L27vqZWBQzm5RXra2rPFOS39z9pS5BtDdtVwKDD1tMgaU0K1/6xYMfwP+3jndhXV1cbE5uzd2/MrNlkjBxIWCfpWO3JWTq6vymSHp2w1PDmG63LlnaePNmfkqK4dw9+5pq1/KnTkg2mMh8c4nk8lbx7aGAwtat7Q1u7bk8vaKrQ0qpZUTmJm6XBiBuOKkTHTIxjTsziaRQVTW9t26tUCh5/QK/8nNYa1NFobG7SpZmox2q0/0EBBssTPToNba4Ch9RRj556F01xGj68/i400xsN+gkl/ozKGGDiPTmaeBMd7AfjXU0o9SBaOeKlVBdURB9e8fX8owaG1/O8jbPXSkVPb9+uM5SPVrvN3RI4dwvh6y3+n673mLsF/9M5qndYVrqgjC8pZiRx4lN5EfFcP2raeRfG1mPBhj/6fLra/f2Vzkt/8gqkcxoaGjvam9va2jo7O7o6urq7uru6ujo6Otrb21taWpqbmxsbGxsaGqCSAcYZqqqqKrH0pHLMIDaoQg3FxcVSqbT3udaWHscN6ll/Sg/U1dV5enqamJisXLnSxsbGw8MjLCyMTqePBgZ/f3+nBw2Pxz8uUODn5+cxyjwx8/X1fdzy5BGjUEhkcqgnfvf1G//PAQMGT89hYCAQsI7OFHDbjQHDcO82JhOXlARygRhM3M2buEWGiJERLiEJPOlPSp4QGYXExBiwOX4iYa4IM3ivnJeXh6UIVt2pKunqpirQLKWyB1V2oGjkkHJFa9sbd2on3S77f8UluJISxN1two/bcDlC0OBZlofweDiJGJRblcpwsAODSIjkCBB+DsLLBqCiAoaUVFD1lckEKUmREUCAQaaA9CoPd8TeUY8QaEcb6cMAoQHqnmGE4ciRI3v37oW08PPPP//00087d+50d3eHYQQIBuO8ZmRkJCcnPyx6Vk2RSCSFhYWqAIuaHFC1/QEeaBoa+qmyal3VndHAYF5RuV5efEPIH3gCOWyLovW72u+m1Q4XVNVi6EBgcKpzfvL9HezoaMjIkJ49x5w3n6KrRxmROkRqa2fq6BRP0avTBwrpx4mkGzDxdNtUUJ61xUC/ZfasLsuf7nhRZ89OM5ia+CAtMA2mxvN47Sg62D/A6Oxa1dqmDVGhuUWzrHwSh6sRywCooOq/xs/RuF36Xlf3caUy/8mP6NVdUkwHAugxDd3sPkdLMh7Y56EB1NUY9VoPMr5CDqC3Fg7zgFKBem9AL36GZhHQhOuotR4qiURba9Fj09AqMdhCbjS6dwIaZgvG+7rQXz9ChbQHtvy6vVEDw+t2xn5rfxVKpWKop+pu82bboE/X47/c4v3ZBvfT9gnS4pohLMWwp6srP09WWJBfUlJceruksry0uqq8vKI8RyonRHB/Phfy1QaXD0xu/XSWWnS7drB/oKenr7u3u7unu6enR8UMra2tMDEJBhlqa2trampGY0MlZmVlZaWlpSWY5eXlVVRUoFir5986CPV8tQce4QF4p9jX1xcREWFiYvL999+bmZlZWVk5ODgGBARQMaNQKL6+vo6YOWHm7Oz8OACAyUsuLi6wnhIEB09PzycpqEohk8nEIGfnuVeu4ewdEFcXxMMD8fJC/PwQAgEJJiKUERlDRASQKMTGIHEM8Ag/KQm0Yshk41JZIBOJzUaysnCJybijxxA3j7+zWKdV6TqqIAM9IiI1NUmWd/Tevc9Q5ToUvYqiF1DUbKD/vaoqLVbG/8vN/VthMa74No5Mxp2/gBMIcHn5iEyGyPIRaR4C+rWJwSAWIQAYMFrI4oMaTWw2kpEBqjYBYEgEwBAbMwwMJDIABk9P5Kb9//n47KeFPaB7Dg0NdXV1PXbsmJWV1f79+1X5SLt37/7pp5927Nhx9OjR5OTkJwky8Pn8uLg4FosFgwwqTlCNQHCqrq5WlVhVM8Mj/j3Uk36fB0TdPeurax6gBZibVFG1jsf2leRggb3xPiO/P//t6nem3Zk+DRZUpWlrkLU0IyezOh68AR1vG/fnKQcHm3Nyip2dk1f8EPrGmypyoGtpp+vo5E/5bZF0g4F+i4F+91S9PIMP3zegTp6arD81/kFmSOZyRf0DK9vaNVWoUFo2KZN9HxViGROZ8RM5XI2Kis96em8oUfAj/iexwV5wxz9GAG038/+z9xXwUZz5++/Su//dERIsAtSuvZ4UWqx2lBYo0JZiCU6CO0VKAsEhQlyIb5zoetzd3RViECUu69kkK/P/vfNuNksSAgUq9PbLfJbZ2ZnZmXc2yTz7fJ/nwaKMx59gdz127R0sSBe7+rYUDGAYJhJiuf5YzyPpyl57Md+DcP7uWizVGc7Q9TCfg5jDt1AC8bgSu/UBxnw8fs+v1XMFYHitLtdzHKxYLEKmIo97mHsu+n+l4xGTWoV7qsL4BRaPW1FVWVVVVVtbW1cHlc2NjY+amppaW1s7O9r6ezv6enoKyxuv2Ud/tN3xvzp2jIQSoXBEODQkGBQIBAI+n49IBnnAgPuswgcZbEBK6NzcXOQIifTQZWVlHA5HgtdznIdiFcUIjB8B2W2iSCSKjo7etWvXypUrV61a9f333+/fv9/IyMjb25tEIrm4uNja2t7Fy87OzsHB4WkAwN/f38XFBQmjnXFt9BR+SuQni0Km+fvZW1m9bWYObG2BowMEDB7uUCjs6wsBA5kMaDRolBQSAiLCxwBDXCJISCIkpxBSUwkZmSArA+TmEOLiCOu+IVy5BpJSDhQU5ZWUlFRUVJaVQ3/V6ur7n32+8sTxf5aWqba1qvA4bwmH/zE4+H51pYa9w/S169/4x/vTfAMItbWgunra/fsE2JWEo4WKClBRAZPaSktBaSkBoYWiQpi9ANULuVLAAI2bUqDfa3y8FDCEhECGAbUkuboCG2tAdN0ja0mi0aQKaD8/v9u3b589e1YeLRw7duzIkSOHDx8+ePCgh4fHc9olpaSkUCiU/Pz8STFDRUVFZSX8rdX7fIYw4z83iueKEXiOESAPMOUFDGPIoal5e139ttiwyPraqXcTyg+b3yk1VFUtUVMiqShRVBbEvt0ixLtZpt54ylc5D2oaSaTMPXvD/vMhZbRhifGkSBrRDvLaaDTfq67WqTbn3JxD/5lDmzc3VgYb1DTiZqtlZGTcEon/xuYo9/Ur1zfMSM9QioySsgpR0dPjE1CuwtIBphuGPZcd05Tn8ft7Mcd3vJIBdSWJJgi4s7ywswCLMX3iHLi9WJIt5nMA89bBbvwdPmIYxBve2phoBHPeBPuUHL7DuuohC2G39oltX8MnCsDwGl60KQ9ZIhHBe3J8ncfdzPIaPPcALpAM8rlV1dVVlRAtyCxTm5ubW1paWluhFVJXT89Afy+fxx0eElTVtZ6/E75wi62xexyPPygcHh4c5PP5PC6XiyQNqCsJSZ87R0vWodTY2BgbG1tYWNjY2Pjo0aOKiorm5ubR45ryBBQvKkZgyhFAnyKxWFxcXHzx0sVVo7Vu3bq9e/fevHnT1tbWxsbGdrTs7e2fBhg8PT3t7OxkgAFhBoQ6yM8sCsPT84qJ6XRzC2BtC71Tke7Zy1tqlBRIguaqsCsJj28LjwBR0SAmBt6XJ+AJbskpMO85PR1kZsKJwYCyh5jYtUWFWSQS5caNW7jlaGlNTe3Spf89fpTw8OGfGh7+rahYyfve33bv/H///Ncbf/872L1zmivxjfx8QtV9nFKohI9SqFAOrVThhNMLJcWgSBbwnAsBQ5a8gCEZFzDE4H5NIYBOh11VPj7AlQisLYGd/XoqlTKKGSBgQLpnAwMDhBZOnDhxDC+EFg4dOqSjo2NoaJibm/s8mCE/Pz8sLCwxMbGyslJGLCBj2aqqqvLy8uzs7ODg4NraWsXvkCl/OBQvvuAI/N8fSIMJAoYxzNDcsi0ve084vayrY4o3MBwwkgoYWufPSpkDAQNd5ZuctYOvLpRA0NHRHhdXeOZM9Gefk0aRA01FJXbmrOI5cxpVoUj6CeSgptY7S613tnrdsg9KN38av27DAXVz9TmxM9VTVGfHbnjPpLH6PSZHqfbhjNTM6RGjDUjRMRAnxMZNzy9Y097hJxJNrtieYihem5fa72PGi5+IcjNeNGLxpaSvefwp9DzELszEmgrGlrO74baeu7DKKKwuHXPVwjx2wlcbC7A7SyClYPEFNiKAcKKQgvkdxWLNx7Z9PecUgOH1vG7PcdSyv6xoRiAQoJyE2trahoYGlMXW0tIii2Pr6enp6+vDRQs8Ho83LOAPi4TxWTVrD7udMmL0MlnCkeFBLo/DhTIGJpOJdM9IydDd3d2Fl4xnaGlpKS8vv3//fmNjY01NTUVFxfAwLgN6jiNXrKIYgeccgfb2dhMTkzVrVn/11VerV8NHTU3NO3fuIMxgg5ezs7O/vz95QlEoFBlgQGZKCDkgQ1USaQpzJDKZTCOTaU5OWkbGBDMzYGUDkO6ZiMsYvL2BLy5jIFHgnXcwDhjCwqWAAUU+JybBLqCUVJCeBpuCMjOhKjoIBr39oyA/0s7OccGCBW+//e7BgweDgkI+//zrI8cI/gGEgwfe+OcHBHU1wrffvmFmPi09fVp93Ru1dTCOrWIUJ0BWoRxvQxqFCohbKMbRAqIXUDNSZiaeC5EKjyQxUap4Dg+H5k40KggIAPe8IW1iZQHs7NZSKaRxgIFEIl2/fv0kXidOnEDqBXnAcOrUqbCwMNSVVFhYWPT0KiwszM7OjouLKywsrMCrurq6qqqqqKgoOjrayspKX18/ODgY5cTLfrM954dEsZpiBJ45Av0i0fGW1h2tbWMgYdQrCS6BJEPd9oSok7HhTxNASzDJnu49CDBoNM1XDp8FBQwhM49WHXvmu7/ACkIOpy8/v8rYOGHdeoqqGhmXOlCUVaJmzsqbg4ukVdV6Zqr1LpjXt/utgfC32M3vcQff5A+oskM+JH+7fffbVy0Wra72Ua5vnZ6RppRIUUoOVEoKnxGXpBSZOD0x8+2K6kP9AykSyR/9r/YgC7P++gnAYLSw+epSVk3O+IvSXY/pzsYeZo8tr4zB9OZg/AHpEvv1mPsOOD/Ew+6uwajnMS9t+DTLG+ofbFZhDVlj276ecwrA8Hpet+c4aolEIhaL0R/X4eFhFMY82obU2Nzc3Nra2tbW1t7ejjIW+vuhxBk1HfF4XBaL1dPTh2EjXT0sXYvIQ9cCWzv6hgVDHA6HxWIhwNCHF8IMKJ+hq6urs7MT+SbV49XQ0FBWVtbd3f0ch6xYRTECzzsCsrvG/v5+Dw+P7du3f/XVV6tWr9LW0bawsLDGCwEGNzc3MpmMPFLJchUQEODi4mJvby/zU0KAAdERUwMGEpkaGOhrbfW5oRHB1AxYWY8BBg8PKGNAXUmBZEDFE9zGpM9RIDYG3pon4KnPySkgLVVKMiQlE4KCQHDIrLR0x7z8wqioqMuXLy9evGTBgreUpit/8V/Ch/8hrFxJuHF9WnQ0oaKcUFvzRnX1tMoKUI5PFThIQFChXEYslEh1C8VFULqQXwCNXHPzoNY5KwuilPQ0aT/SmIAhDLZRUakwTcLbC7gQgaUVsLL5jEz2GXVWlTIMJBLp6tWrJ06cQG6qR/E6fPjwoUOHDh48uH///r1799rb2yNJxjMBA0qCT0tLq6qqKi4uTkxM9Pb2vnDhwpYtW86fP5+ZmakQPT/vz4ZivZ8/AlUCwS6YwCA1VB0PG5qadrZ3Hn5QbZGf1f9kfqLsrfrF/V+0/xcqnlvnq93XUKKqKJGhRZJ5m4VsnV9kRixmVVU3EF1TtbSC3n6brKwSMF3F788qMeqzHh1WZ6bPY3LmMAXKLJ4yi63MZCuzBDN4XcptuX+pzv5Lie+Mql2zmz6Z27FQtetD1Y5P1NrWqD768R99DGvs54fW/SJn90vvdJiPEbeMkz7XXV7WW54y/p27G7DzM7C6tLHlzUXYT8pQxtBWDp2RjBdjJkuxnga4AuUcdhpgKY5wvrMGbmiyFBM8mfAwtqPXZk4BGF6bS/VzD1R2RzU8PFxfX19ZWSlrQ0KiBflEtoGBAQ6Hw+PxoFJBIBAMDg2PDPX193LYbJFoeEQ44kTOPHCZ2tLWJeAPMtn9rAFWP15I+izPMyCSoa2tramp6eHDh5WVlbW1tbgd5M89A8X6ihF46gjIPt4YhgmFwtSUlP37969cuXL//v0WFhZWeCHYgAADeUL5+vra2NggkQOCDQ4ODkQiMTAwcCK6eHJrEplK8/GxNTZ++7YhMDGFt9S2d4GDA3BxAe4eMCDZxwemMQSQAAU3Vw0OAqGhMMQtMgpER0OpQHw8/FIfkQxpqZBkSEyEd+rBQX9OSr6ZX1BUUlxkYmpiaGhoY2PzzrvvvjENzJ8HLl8B6RmE6iqYpVBZAWFDeTm0Pxqb8O4jpG9GxALsRCp8Ai3AZiQcLcjkzomJ8HhioqGbUyjuqUqmAD9/lPQMzK2ApdViMsmbwQim02mIZ2AwGCQS6cqVK8eOHZtILxw4cGDfvn179uy5du1aVlbWM7uSCgsLCwoKUlJSvPAyMDDYsWPH2rVrV69efenSpYcPH8pf7qd+JhQvKEbgRUcghsWeXMDQ2LS9uWV3axuVxeoWiabYfdXIqOK5bf6cPFUkYJgRNjOC+euFBw82tbTRQ7P3bck+O7c1b84AbyZzUHmABQ1SB5gwp5nFVu7uVq6qmZGRq1RzfHaXmmrfLLXeOWq9c9V65uIzs9X6Z6v1qWr0f/E5/9bNkcxMCYczxVm/9i9BwLBVHjCIDRY+0F/WV5E6/tRY7ZjD91iLXACfRIwl2GDWX2LEzVDl3FmDuWzB4vC+o/p0zH077EqCf58EWLA+jJd+/UsBGF7/azjlGQiFwocPH1ZVVTU0NOASZym3gELZurq6ent7+/v72Ww2n88XCAQjIyNCvEZGRoaHhwUCwdDQ0JBAIBYPU2JK9l8mNbR0D3G5fSzYktTf3y9PMqDGJJmMoampqa6urqysjMvlTnmMihcVI/CCIyASidhsdnNzc0NDQ2Rk5IkTx0+ePGlubm5hYWGJl5WVlaurK3my8vLyunv3rp2dnb29PXq0t7d3dXUl4c1I6HGy7WCWA5US7Oamd/PmXw0MwB0TYGEJbGwhYJDmPeNpDH5+wD8QmpMy5EiGiAjoW4rskhISAGxMSgYpKfDL/owMQngYpCaiY/bm5mUVFBRevHjp+vXrlVWVX3+1bt0awqGD0xYsAFpahML8aeVlhLKyaZWV08rLpiGJguyxFGcVZGgBdSLljXILiF7A3w6aI8H4hWSoqUAJDOHhUKJNp0PFs58fdIl1dgbmlsDScskUgOH48eMyrbOMXkCA4eTJkyh1bmqGobi4OCMjw9TUVEdHe/369V9//fUXX3yxZcsWGo3W19f3gh8OxWaKEXjuEQhnsXe1d+xobYNTcwvsQUKPjxq3V1Xuykit7O6cemeeHC+pgKFl/sxYaKiqRFaZFT2neujXcyCVYJ0jmAtvZAVnaBaLPx4qdHUpV1TOSEyaHhYzPcdvBvvdOaw5s3vVVSfqpHvU1ZjqalwN9R51tf7/fsG7fHkoNlbc3j71CLyWrwo4mP238oBhxGBRqe5S9gO51iN0YhIJ1DHLRz6j5SODmPB5YzpeyyGSO2gFYJAbjD/crEgkamxsRGgBZSPIRAudnZ0ILchzC8PDwyMjIyK8hELhCF5DOOPA5wvEwiEXcvrO896Nrd08LndgYAChBQQbent70T47OjqQv+rDhw/Lysra8d8yii8I/3Afrt/shCQSydDQ0MDAwIMHD1JTUxkMhre3t4eHR0BAgJ+fn52dnfloWVhY2NraPi3j2d3d3c7ODmEGO7wcHBzu3bs3af8S+cmikBn29nuvXpl26xYwNgbm5rAryX40jUHWleTnD6XDNEQyBMN78XDcLik6apRkSAJp6SA1FYSGAzMLwjdrwdIlgExenZWTmJObn5OTk5eXV1JaumLFmkMH/19xyZ9Cwwl0Ote3+C8AACAASURBVCgpJJQUE3LzQEjItMwsaH8km0qKCSXFMJqtuAhKnAsLQUGhtA1pzBkJghPYB5Uyql6A/kgxuKFqmFTAEBgIfHyhS6yTIzA1B+YWS8ikewxGkIxhCApiBAYGXr58+ejRo7LsBYQWDhw4sH//fh0dnd27d+vo6AQEBDyTYSgpKUlJSTl69OiqVatWr169cuVKTU3NmJgYmY/qb/ZpU7zx/8YIDIrF9wWCPD4/h8e7/vjxjgcPthXmaaUlacWEaYZQtkcwfCvkvlqeMCbtoo71nd/Ob18ADVVrNGbQoaGqEkXlg/R/dUiegTQm7OxFFojFtfzBGyz2PwcFylyelFKQsQodnTPKypUSEqX5a7HJf0m78dewGf8yUj+eob6cqT7naakOvThyQJEO/Ys/5hw+JCCTRA8fvsgh/j63YXdj5p/Laxi4tz7K0fuc34yTA7/PY/7tjkoBGH67sf9l3lmCF4ZhYrG4ubkZT32qf/ToUTPuhoSIhc7Ozu7u7t7e3gE8xRl1IiG0IBQKxWIxQguQW4DsgmBwcJDL4/J4XMHg4DX7aJ2LAV1dvTwY4Mbq7+/t6+vr7+/v7u6urKy8f/8+AgxNTU3V1dU1NTWKP/m/zHX+39orApwikaizs7O8vDwmJoZEInl6errgRSQSXfHywMve3t7a2trS0tLMzMzGxsbf31/WYkTC2QMymezv7+/s7DxqpCT1YLW3t/fx8ZGtTH5qUUgksoXlt/r6hJs3gaERMDWVkgwyryQU+ezrC0kGEhlihqBRuyScZCAkxIHkBNihZHsX7No17e9/B6pzwRdfgNM/Agr1w4yMkOyc/Nzc3Ly8vKKiotVrvt21e3p+wZ+KSyEYKCgAJWUgLAx8uhze05fh9kelJTCjrbCYgHBCfgHIzwd5eVCxMAYVUCdSOuQWoJVqCqQ44uJwN9VoEB4BIQ0jCJAosJ/K6x4gugI7e2ByB1hYriCT/caJngMDA/X19Y8ePSpPL8jQgvbevbt37965Y4e9vf0zAUNxcXFeXt6FCxe++uqrNWvW6Ovr5+TkCIVCTFGKEfjVRyCyq3N7fKQmPUCTEagZRNIMJm2PDLLOz0a/heQPR4yJ/Xl+3/duWNqxbF4bntfWNl+1VA2iBdxTdXHl0j6JLEpZftNXNi8SFQ0O/sjmvC0YUuZwn4AKTLZye8eMklKl+IRRqBD3/1JS/xIaskR7+cV35tKUNVLMZn+ZOWtG5dy5rWqqvU/Pg+vFIx3YGupsDfX+f/2TqaU16OIiLC2RvO5eJs3FmNFHY4DBaGHr1SW5tzYIuaNS5ld2of4IO1IAhj/CVZQ/BwQYJBJJW1sb0i0gQyQkcYbeqXghtIBUzkNDQ7JOJEQviESikZGRoaGhQeSlyuNxuVxc5ACJhR0XAk7eonb39rOZrL4+GPnc19fX3d3d1tbW0NDQ2tra0tJSV1dXXl6uaEaSvzSK+ZcZAbFYXFZW5uTkZG9v7+zs7IqX24Ryd3d3c3MjEonOzs6IPRjnqUqhUKhUakBAgKOjo7W19UQD1mcCBgqFGhDgedvgw0uXCDduAANDYIJ3JVnbAHt74OSEBzLg0meoZAgYTX2mSzMZwsMAPRjctQcHD4L//JugoU5YvhScPEnw9ZsWHEKgw/4ltbRU35ycnOzs7JycnKKiolWr1m/b/te8vD/l5UO6IC8f8gaZWQQyhZCSBgqK3igsIoSGAxIdLi8uwYOccXGzDC3k4BltKKYtHQcMyXj2Akp3RuqFsDDoAEujwQP284MCBiIR3L0LjI2AldUqCjVgHGDw9/fX09M7cuSIDDAgrfO+ffu0tbX37Nmza9eubdu2GRsb5+fnPxMzlJaWWltbf/XVV2ZmZl1dXS/zUVFsqxiBlxmB+zzezpgwzWCybNoWRvsxPlIgHO/N3zXc9UHpv9QbNOY1Q60zxAw4bFCrVp+dOXdW2twFLW9eHLjYKmx9meOZdFuJRDgiTBoU7Gdz1AVDymzOk1CBpfz4sXJRsVJsvBQqxMX/NSX1b1Tq0jNnrv/7P+GzVJNVNRLUNOKMZ/6Xofw3krIKXWVm/KxZJXPmNKnCJqUnvFnV1eTblvrV1dga6lwN9b533h5Ys5pnaDicnCxhvp7Wq7l+8sFtIoOF5ReXlVrh7kaTjvv/9kIFYPijXX8EGNrb2xG30NjYiCTO7e3tHR0d3d3dPT0947gFhBZkUAGhBSRg4PP5vFG0gEe2cQYHufnljxZr2t5xi+NzeH29fb1w6u3u7kbcQktLy8OHD0tLSzs6oGX1xG9l/mgjrjifX2wERCIRl8ttb2+vrq7Ozc1lMBiOjo4mJia3b9+2tLQkEokIHkxADXCBu7u7q6srkUj09vYODAwkj1ZAQICrq6upqamxsTFSRSPYYGNj4+LiIltTxkWMbjf2P4VC8/SyvHxZXU8PXL0Gbt0e7UqyAnZ3gaMjlD67uQFPD3DvHpQOB+AkA4OBNM0gMpJgYAjenAf+8QHQ3kuwt4N9SilpMJMhNp5AoQEy5W/x8WZZmTnZWdnZ2dl5eXnXrt2+Y/qfvJxpOblSxgDChoJphUWQRsgvgN1HF34iLJhP2LQJOLuCjEy4JC8fhi3AgLZsOCHdAkILKSm4lWoSrnWOAdGRMFoO5rXh/kgBgVC07eEJT8TGFhjcBpZWa6lUshxgoDEYDF9f3/Pnzx85cuTo0aPIHAnRC08ABi2tc2fPpqSkPDPyuaysjE6nu7i49Pb0YHKl+AUiNxiK2V9jBPgSyanMVK1QqgwwaIVQdoRSGwbGcwUsIfuT2M+UGNANaWb87NnZqqrFamrVGhoP581rnT+/DaKI+d0LPmv/vEX0stltsjOXSAZHRkJ4/E1szqxBAdQxI03zABP3QWIpt7TOKChQio2VQoWExOkpqdOzstceOGj83nuRM2elqKrFS/Oe58WTLnoW6/4Uv249RV0DBTtQVWZGz5yZN3tOverczlHk0PskZkD4oU9djakBpQ696mr9y5dxz50dCgkRtb56dCQ791c8IxZh/kfHAIPRwq5rH+de+KTO78YrfqM/yu4UgOEPciXl/6x2dXVVV1fX19c3NkKJc0tLy+PHj5F3KgpbQCpn1ImE0IIYLxEMiYbSheHhYdSMhNACohc4eLHYTIFg0Ckw48MtlqEJJRwOCxqq9vR0d3c9fvy4ubm5sbGxoqKioaFB/pD+IKOsOI1fYAQmfk6EQuHAwEBzc3NJSUlcXFxISAiVSoXZB3jGMCIHDA0Nb9++bWVl5eLigmCD62ghtCCPIjw9Pb28vDw9Pf39/R0dHa9du3bhwoVbt24hMyUrKytra2u0K9JozxJ5skKvUigMouv1Cxdm6F4Al/XB9WvA0ADv27Eg2NrAHh4nJ5h35uEOcwx8fAj4/TdBV5fg5CwlGby9ppmZEShUEBdHgILjeKlpUlwC9DMNDARRkSfTMzKzMrOysrIgzZBbkJ2rnZ0NAYP8lJsL0JSXD+ITgPEd8OWXQFUVfPopweA2iIsnFBYR8vJAdhaEEAgqpOK6heRkaMqUgEc7R0eDiEjY4BQcDOg0qdzZ2xu4u0EBg5UVuH0L2Nntoo4GPNPwCgoKcnd3P3369BG8ULQzMkfS0dHZi/qRdu7U1NQ8fvx4fHx8aWlp0ZRVWFhYWlrKZr/2zoOYol7/EbB6ULUtgiEDDJrB5G0RjLD6molndrr4R6VgXLFAUpkeiHci0VRmMGaqRM5SLVeHgKFjwdLHy9qEeILqxO1/zhKJpHdo2IPLW8nmKPMHoeuRPFQYYCo3NSvn5SvF4FAhKnp6YtL0lFSV+oYdIyPxmVnds2anj0EFjVgcM8TllfBgA7NgiPvoUYO7e/bBQ8Hvvx+grELBp7CZMzNnz66eO7cdb1jqV1ebFDmghiWOhjpTQ63vP/9m79836O0trKrCfuddhfcTsTtjqW0ig4UlukvzdD/pyg39OZflf2hdBWD4o13s3t7e+/fvNzQ0ILQgn7SAeoeYTCabzebxeKgTSSQSIbQgFotlgAGhBUQvcDgcBBjYbA6bzWKzWXwOr6u3e+tZ39VH7lXUNfUPQMjQ2dnd3t7e0tKC4uFGRsazt3+0gVacz8uNwDicIJFIBgcHu7u76+vrc3NzIyMjGQwGlUql4EUdLRoNfrdNp9MDAgKcnJyMjIwMDAxksGEKwsHV1dXFxYVIJLq4uLi5uTk4OFjgZYkXQg5EInFqwECGRaFSGLY2R3888xc93dkGBousrb6xsVxocucvZqYES2tgawfvs91cgbsncHN74949gp//Gx4ehBUrCJf1oaQ4KAiEhhHCI6eFR0C2AZkmxcZKb98pFNjFFBa2LT09NT09KyMjIxPChrzMrD2ZmdOysqAjahbOGEDeIAdOCELk5UNWISMNuLlP26pJePsdwr/+DU6eJgSS4MKMdMhgIJVzEo4WoNA5FkTHSNECVC8wAJUC+RAfH+iPRCTCDiszM3Dr5hsuLqeCgkLpeCHYhtieY8eOHcbr0KFDCC1oa2vv3bNnz+7dO/HaunXr0aNHY2NjnwcwlJeXo2g2TFGKEfhNRyC+p3t7XDguYJA2Jm2PYBhkpoz7rYVhGO0xXYkB9c1jExn6IykFqMwtUpv3eP677X9PFiS/5NmIxU0CgSWb8zF/UJnHnwQqPGqckZOrFB0DWYWo6OlJydMzs9QaG48IhrIxTIJhWERkl8a8JCmxIEULseoacdk542mToe7uttDQwp8uRCxeEjhzFgXPkw5SmZk8a3b53LktarBhaQrkgBqWOBrqfe/9nfndtzxz85GcbIzPf8kRePWb97dijt/L0wutVz7OPL8898pqQfcro4Ne/WH/pntUAIbfdPhf9ZsPDAwgtNDU1IQMkWT6ZuRohDyRuFyuTLcgQwsyrTNqRhocHJRvRsL7kSBeYLFZTPbAII9Djyr493aP8yaM7p5+SC+0dzQ3NyPpgiKQ9VVf2D/s/oRCIYfDaW1tLS8vT01NDQ2Fd6XIpwh9k42QApqn0WhUKhXNo5vXwMBABBtQk5Kzs/MUmMHNzc0VLyKRaG9vb2lpaSFXNjY23t7ezwEYINXh6HjF1HSXm6sRjRYQExMTH0/389tvZv5Xa2vIMFhbE366AFavAdu0pnl4AD8fgq8fcHMn+PkTqKOxDCEo/jkMRERAe6LoaBAXC2JiIMPg5w+CglampcakpWWkp6elp2ekZ2Slp+9OT5+WkSGNhc7MhC1GmZkQP6AUNhxLELJzCPn5hNwcQCGDEyfBB/+YpqFB0LtISEuTZj4kp0BkAlMXYmDydGQUbIgKDYVIhk6DW/n7w+A5dw9oqGpjC+7cATdvKnu432AwQtCYo0cGg3H37t2jR48ewgupF3R0dBBgQHLnbdu3b926RVt7b2ho6NSAoXC0FAmPmKJ+ByPQMjysnRSrFTImY9AKpWqHM7p44y3CWwQtGmELxtACQg54ZJtGw7z5XQtsmLYvc0IicSWff5nFfn9wSJkjZ3+EGpD6B5QbHs7IylaKioZQITpmenLK9Ny8+a2tPwqFZfLvy2C0PydgkG0lZLG6MzLKbt2O//prMh4mTcalDnGzZhXOmfNIVbXrWQ1LLCR1eHPBwIr/cvT1hyIjxb+TCNdBNuZzEDNZhBkuhJPRQt7NRTnnIb3QQMeDFGSjoJiRGwEFYJAbjNd8lsfjoU4kJFqQtSGhpIX+/n6ZJ9Lg4KDMPlXWjIQ6kSaVLkjRApvNQhnPzAEmc6Cru3e3PmXJHq/whKKB/p7Wltb6+vqysjKFYPE1/xz94ocvkUgEAkFvb29TU1Nubm5MTExISAhiEigUijwqkOGEp83Q6XQGg4HYhnFNSlMgB6RtcHR0tLW1tbCwMMfL1tZ2nDya/JSiUChhoWHxcfFp6Rk5uXlFRcWVlQ9KitOIbl+ePUvYsBH8/T0wdw744H3C/gMEVzeCjze45wMCRsUMFCqgM+ANejCOGWCaWyS8d4+Jgd6mIcHA1w+QKf9KSqKnpqanpaVlZmXv33/kzOkFqWnT0tNhZxGCDfKPCDxAs9QMkJkO+YTsLJCdCZUJV68SrG0IsBMpGeKEiAjYARUTC2KiIFCJiCCEhBKCgwGDDigUQAqQ0guurjBWwtIKGBqCmzfn+/jYMIKC5AEDnU43NDSU5TrLmyMhufOOHTu2bdu2ZcvmXbt2hoSElJWVTdmRBF8sLCx8/PjxL/75U7yBYgSeNQJiDLtUWrQtjPZEV1I4PbFxvKPoiGRkU8YWpaCZT2AGksqs1DnzuxdodW/jS17sy3WxSJTL4x1jsuaNsz8awPPX+vqV6+tnpGcoRY5ChZTU6QWFb3d2XhKJHkw8v0BS688FDLKdiAYHWVXVNXb2aZqa1PkLkNSBoqISMXNW9uzZNapzO/BWpad5s/bhDktQJK2u1vfRIu7xYwJ/P+jN+ls1LA3zMLruGFowXAi1znpLMs8vz7+xTtCr+BUku/LjZxSAYfyIvL7P6+vr79+/39zcjCKcu7u7keHpAF7ynUjDw8PIPhUJnRG3gHQL45yR2HghwMDCi8lkDgz0d/X0dDxu8gvK/kiHulOP9OhhS1Pjo4qKiubmZqS6fn2HUXHkr3AE5Bl8oVDI5XI7OztLS0tTU1PDw8PpdDrCCTLe4GnA4JnL6XQ6Yhvu3LmD1Myo9QjBhknBA1JFOzo62tjYmJub29jYPA9gIJFINBotPT29uLi4FK+ioqKYmBhDA4PPPvnHnNmEefPBmm8I585Ns7EBru7AzR2KGVD2MxJAB+Lxz/RR06TQUKgfiIgAEXigW3AwzG4jBb6VkOCTkpKWkpKSkZ512+COhcXSlNRpqakwtCE1FfqiTpyQSgFapqIJXy0jE6SmEZKSCKmpBFe3aWu/IXh7QzYjMhJO4WGE4GACbEaiwmYkfz+cXnCD9MJdW9iPdPMGMDT8mEzypsOYZ2mhmOeLFy8ePHjw0KFDMnMkHR0d2I60a9fOnTu3b9+upaW1adPG3bt3hYaGPidgaG5uFovFr/ATqNiVYgRebARIbW3bY5/sSopkWOZlTdybSa0ZlDHIupLwBAa1Bxr/6fywYqRi4vpTL5FIhkeEMfzB7Sz23HGaZhTV3NunXFs3Iy1dKTIKsgoxsdNT06YXFb/f03NLJGp42s7v+bS8MGAY26dIxG9peeTtnXPoUPD7/0BSB7KySrDKzNTZsyvnzml7joYljoY6S0Ot9x/vs7S0eJaWI+np4oFf0cNUwMEYupjJR1JuwXChxHBhg/7i9HPLci8s78igj52sYm7CCCgAw4QheT0XCASCwsLCpqamtrY2WRtSf38/Ey8Wi8VmszkcDp/Pl6WzybgFJF2QRS7w+XwuXjJigY1zC1J6AYcfrW2tOblZWfnFG05Rlh6kuwemNNQ8qK9rwFMXYMek/J3i6zmiiqN+NSMwPDzMYrGam5vz8/NjY2NDQ0PJeMl6jdBN6DMhwdNWGLcfEonk5eVlbW1tYmJibW3t7Ow8UQONepNkjy4uLpaWllZWVpNGvKGjlT0GBgYGBQUVFRVVVFSUl5dXVVXp6+vPnz//nbff3bxpw4nj7968RbhrC+wcAMxkcCQQiTAqwcsT3oj7+MB2I2S0SsUToBHPEBICMUNYBIiKginL3j7A318tJtopKTk1GVZaekZmWurRpORpSXh4QjLucZSSCjUJ8lPq6NPkVJCcCmMWkpJBUiJISCDEJ0B1dUAAQVdvGpU6LTIahIWDQDKBRieEhk5j4NHOvv6QCfH0BC5EePBW1rAf6coVYGGxkUaj0OljDENQUJCPj8+ZM2cQYJBXL+zes2fnzp07duzQ0tLaunXLDz9sOHBgf2xs7DMBA2pKevjwoSJ+AVPU72AEKvj8HQlRmsEkGcmgFUo9EBnMFAjGHV0Jq1QlaNYYYCCpqMTMmt+5wJXtNm7NqZ+Kxazh4UAe/zsWW4U/+IT9EYIKPb3K9x/MSEmVhwpKZWUf9vWZSyTPaLv39GqeHDBkv2CS+nBfX1toaPGlS5FLlpJmz6HiUgeGikrCrFnFc+Y0qqp241KHKWgH1LDUO39e35LFnCOHB91chaWlkl9U7cDtwyhnMNPRTiS8Gan1ysdpZ5fl6X5S7aknnuCcO/Ul+197VQEY/iBXvL+/v7KyshMv5Jo6MDDAZDJZeCHhMp/Pf5rQWaZbQELncYAB7QTnFgb68erofNxQV//oUf0V68hPj0btuhDwoKZWODKMy6v+IEOqOI0XHgG86Wiwq6uzrq4uIyMjOjo6KCjoVZEJT0MOaDmCHxQKxdvb28bGxsTExMrKysnJydXV1U2uUFcSenR2drawsLh7966/vz/5WUUikfz8/AICAmAGM14ODg63bt2i0xlxcRGOzp8b3yFYmANrK5hgYO8AnJyhY5K7O7wRv3cPsgcyo1UKBdDoUGocjIdAh4ZCnoFGh9DC12dGRJhpYmJqYmJiUlJyakp6ctKxxIQ/JSaBpCQcBuCPCDkk4fhBNg81zUlwSoBQAbonxcVDSiEWtjwR4uMIUVEwo40RBLZpEpYvB7oXIQdCphAgveBFILoDRyd48GbmwMAAXL70Z0fHs6OGqnB0ke4cCRgO4oUAA9Q6Q7XzbtSMpKm5dcuWzd99993Bgwfi4uKeaauKWpLq6uoUfgkv/HOn2PAVjsCgBPuxIGebnLkq9EoKp6e3NI17F66Y+0XiCiX6qO6ZpKJapb6nd69AMh5ajNtQ9lQs7hwaduLyPuPyxtsfIajQ1a1cVT0jOUUpIlLKKqSlK1VWfcSEAonnSpL28Gx6tYBBdvBCPr87La3SzCxhzTfUefORSJqqrBI1c1bu7Nm1qnM71KSpDk9zWOpXV+PgkXA9by4YWPU1VDswGKLmZsnQsOxdXsFMfwt2T0e+EwkzXNh+5eOMs5BbKDLeOsRUxL88Y5gVgOEZA/T7fxl9l9/b29va2opcU5FWAbEKiFjg8Xh8Pl8gEKBmJJkzEuIWJkULo85IULfAYrEGBiBUQMrp3t7ezq7Oh48edT1u8w9O+/xI0OeHaJlF9dCdTSSSQIJBUf/TI1BfXx8REREcDJtYyHi9fNPR1Dhh4qvIZAnBBgMDAxMTExcXF4QQiE8WAgxEIpFCoaCjlX+kUCjoLhm9SqVSTUxMFi9eTKVSc3NzU1JSkpOT4+Pjw8LDAwN9LCw+NjAioOBna2uY4uzogEe5uUIZsafXE5gB9SbRaFLMEBwMeQYKFUILb+8/hQTrxSckJSTEJyQkRUbGhoXrxMa9ER8PMQCaEhOhNSp0Rx2dgfOjr8bjymaU4hwTA2KiIX0RCUUL8F1CQ2HytJEh+PIrwuxZhPc/AHv3EqytgYcXwc0VmiNBesEEXLsGrl19GwoY5PqRECQzNjZGaGH//v379u3T0dHZi6OFnTt2oGakrVu3bNq0cd36dcePH3+eHAYFYMAU9TsbAc+mxu0xoU94JcHI50m6ki5WXpLKGEgqylGzFrYuejAyiQfrxPMTiR8Khkw43IX8QWUu7wn7IwQVOruUKypnJCZBnBAROT02bnp6xvTq+4tZbEcM6564w6ctcfd4GmAY75L0tD08c7lkZITzoKbewyNzz96g9/9Bwo1ZycoqoSoz02DD0typG5ZQWhxTHQY7QBem999jbdnMvXNnOCVF3P/SB9lSgjn9MA4ttFz5OOPcsuyfludf+ZrVUPzME1SsoAAMr/1nQCKRiEQiFMrW19eHWAXURMRms7lcrgwtCAQCmTkSUi+M4DVOt4CgwgTdwoAMLfT09HR1dre2tbS1tSZnFq084vffE/EGTgl4I5JQIpGIxWKJXI0bYlm3kmxNtAJajrYbt4ni6es1Ajk5OTdv3rS2tr537x6VSkU33BPv6X/pJejWlkqlenp62tjYODk5PYkUpM8mBQxIfo0U1ZaWljo6OoaGhggz+Pv7e3p6RkdHR0VFBQcHk0gkb28vRweimektg1vv3b5NMDIGEDNYAGsbYGcHnBygJMDVdYxngL1JfiAAD4EmU6B+AMEGGLSMMwzuHoBC3R0bGxUXF5+UlHz06Oldu+dFRrwBiQJ8iouDCua4ODjJZtBTBBJicZAQHS21QoJyhXApVAgOhmiBAckNAok8zdwCxr0tmA/UVMH6dQT9S9DlycISJlhfvAjuGH9DpZIYtCA6nYbGEz0aGRkdPHjwCa3z7t278Gakbdu2aWpu3bx508aNP6xdt/bKlSt5eXnPTHpWAIbX62f8f+FoS3m8HfGRWkFPdCXtjwzuHRyvY07vS5+BdM8klblFqr5832eOj0hUyh88z2a/MyhQ5nDHEhUGmBA2sNjKHZ3KZeVK8QlSqBAXPz0zS6mmdhmH64xhvc/c/7gV3N0bn8IwvPS9+Lh3wp8OdXe3BgcX6+lFLFkaOGs2oh0YKjMTUJi02jMalnpx/yWWhjpHQ71XQx31LPGJxJGyMskLRLWUhmE2X445qBouFBosqr0EdQtZ55fl6H3RVRA92Ukolo0fAQVgGD8ir9dzdHvN4XC6urqQYkGeWECdSDwebxAvxDAgDQOCCsN4CQQCmYnqRLTAYrFQM1JfnzTRubOzE2W01dfXlZaWbvkpYOXZtI1nKUwOF8PEYogXxKgRGWU7yIYUUR/ygEEGEoRCIa5/gOvKVpBtqJj5NUdAght3I6JIgonFmBheE0wkwh8lmOSZV6iwsPDSpUsXL168devW3bt3fXx8xsGGXwhCTNwtFU9yoFKpgYGBbm5uKIphHGxwdnY2Nzd3cXEh44UONTAw0NHR8dSpU5988omamtqCBQsuXrxIpVJJ5EAqlRIUFBwYGOjp6eHg6GhqanblyrUfT/94+tTqK1f+eusWuG0IM9RMzYCFJXQmtbeDPIOzMwxn8PCABIIX0jP4SfUMJDIgUwANd1xlMKDUwc0NkEjbYqIjYmJik5OTN27c/uWXfw4LW4cggAAAIABJREFUeyMmBhqwwuiGGHyKhtQBIhCkM4hMwHECpBSiYMwCggohIbD3KSgIQgU6DUYukEggIJDgHwB9mWysgc5+wvvvEWbPAR9/DE6fArduAV29Pzk4nGYEBY+2JMEEDDqdHhUVRSaTT548uQ8vKb2wa9covaC5ZcumTZs2fv/992vXrjUzMyvG65kuSQUFBTU1NYqWpF/zh13xXlOMAF+Cnc6fpCspsenRuK2YYubi+KVKVBWVmNn72g+MYFPEEI2MCFN4/IMstvqgQJnNGQ8VmCzl9g7lktLxUKGu/jMu1w3DXlAf7O4+OcOQlfWCGoZxI/C0p6LBwZ60tCozs7ivvqbNm09VViEpq1BVVKJnzcqbM6dBVXXqhqUe3H8J9SyxNNR7FswfWPklR1d3KDgI9iw9M+5pRIAl2mDGH2HGY7oF3q1FZbpLUs9CtJB1fllbSuDTDl6xfNwIKADDuAF5/Z6OjIx0d3cjxQJqQEI4gYsXohcQYBgcHBRMqMHBQZluAWU5y7gFhBNkzUgwnq27u7OzEwW01dXVVVRUsJg9Z80ivj6T+vlhcmFlM4ZhIpFQLBa3traeOHHC2dlZ/s9/cXFxRcUTrhGI3ECwp6WlxcTExMbGZnj4lXYuvn6X9PdyxCJMKowXSkaEmEiMQegAJ4lIPKVapaCgQF9fXw8vGWzw9fWVNSbJZl4tyYAE0OhRvr+IQqFMDRhMTU2dnZ3Rhh4eHrq6uqtWrVJVU1VVVV29erWenp63tzeFQiGRSGQy2c/X38HR0cTU5OqVaz/+eOLA/q17dn+is2/Bj2f+clkfXLsOb7UNjaBi2MwcOpMingFhBiLem+ThCX2T7t2DPIN/ALQnCiQBMhliBhoVeHsBogsICNgZGRkRHR2TmJi4adP2lSv/Ehzyp0i8rQjasEZNNkVLvY8iEE7Au49QAxJEC0GQWKDTYdcTiQzfEUIFXyiZ8PSESIboCg/18BGwaBFBW3ua/mVw8eKbvn52ISGhdDwsD6EFOp0eGxublpZmamqqo6Ojra29dy/UL4w6I0nphR9++GHdunUbN24kkUjPQy8UFRUpAMPv5SdfcRyjI+DR0rQ9Nky+K2lbBMMoO33it1qXqi4rhc38qGjxQ9F469XRnQ2OjIRyuBsHmCrj7I9QqAKTpdz2eEZhkVJs3BirkJun9PDhMj7fE8NeKgTdzW1yhuGXBgyj545JRCJWZWW9h0faVs3g995HnANqWErHw6TbpgyT7lFXQ+BhABc8DKCepa1beMZGI8nJ4p4e2RuNzXQ/xAKOwTYkIzxsAY9c6L72Ud5PS9NwtJB9fllrvPfY+oq5Z42AAjA8a4Re9HV0Ezzx18qL7m/y7SQSCZIWsFgsWSSzDDDI0AKfz0fAgC9XPLwQrpA3RELtTEy8kMS5v7+/By/ELbS0tDQ0NJSXl6OUpZtOCV+dSvzsaKh3cAH+VTQ8dQzDjh8/np+fj2EYajaQSCQtLS19fX0tLS11dXWZmZmDg4NmZmZeXl6IW2hvb2exWGfOnGlvb5/8bBVLf4UREAmxB0VYVT5WVSDmMUWYWCSRCCHPIJI0Vour8kUVWaK+DsQ/PO1wiouLr1y5oqendxEvPT29S5cu3b59++7du76+vhN5gFcCG2Q4QR4qkEcrICBgUobBFU+ANjU1dXFx8fPz27lz51tvvaWurv7ZZ5+dPn0a0Q7I/hXtiUKhOjg4nDp1Slt7yzatz7S03tm5a/qB/YSTp8FP54GeLuHKFXD9Orh1GxjhmMHcHPIMUszgCHkGIhH3WkWSBrw3yd8fticFBELMQCIDTw/g7AT8/DQjIkIjI6Pi4+M2bty2cuVfGEFvhIVDuiA8HLdhjZjkEZIJeOtRWCgICZFOKF6aTodohEIFgWTgT4Laax8fCFo8PaAs29kZSrStbQnWFsDQgHD1Kjh/DhgZfWtsbKCjs9/f3x8FbNPp9JCQkJSUlJycHBqNdvTo0T179kD1wq5dO3D1gqamJlIvfP/996tXrz506FBqaurzKJ4VgOFpP02K5b/hCJTxeTsSop/sSqLsjQh6zOWMO6oiVvH7OR9EcqLGLYfSPnH/8LAnj7eazRmvaZZBhZbWGfkFSjGx08MjIFqIT5ieX6D0qHHpoODFWQX5I3F1/Y0Bg/zB8FtaWmi03MOHQz/8EKU6kJRVGCozE2fNKp07p3nUm/VpDksIPPSpq6GepT51tf6lS1iHDvGJRGF5uZjFxsQirCICs/tGXrQgNJDap6bhuoWsC5+2JfnJH5Vi/pkjoAAMzxyiF18BfT37C2EGtFs+n9/V1SVzTUV3/0i3gNCCHECAZqkykIDIBPmYBfl5GbeA2pB6e3u7u7u7urpQJ1JDQ0NFRUVHRwcaGjPv1BUn4748HX/bORH/Alo6YufOnbt//35wcHBkZKSVlVVwcLCTk1NcXJydnR2RSDQxMUlOTjY2NqbT6TLn9bCwsIsXL8qTEi8++ootX2wEhgTYue8lywnYMiC6ZyqCLWYiEYaJ6yuxze9JlgNs9SxxQcrUn+qmpiYjIyNdXV0EGORhg4mJibu7O5lMflWwQYYTyFOWjGGQ70pycXFxdHS0srKysLDw8PBAgEFHR8fGxoZEItHpdCqVKtsrzi6QqVSqkbHuuvVvfrN2+vcbwFZNsHs3OHQQHD8BfjwNLvwE+/6vXIaY4fZtnGcwwXkGGWbANdAuLsDVDZc0eIF7eHsSdE/CVQ3+AbAfydEReN9bHxpGDw+PjI2N2bhRa+WKv1Bpb8gwQEgIFC5PnGQrBAfjlAJiFWgQKpBxqBBAglnOvr4QLXh5AQ9PKK5wdgIO9sDGBpibE4zvgBs3wUVdcP7cLA93g9u3DbZu3eLn5ycDDPHx8dl4xcfH6+rq7t61aw+uXsC1zppbtmzevHnThg0b1q1bt3r1amNj46KioudkGAoLCx89eiRrTXyxz69iK8UIvMIRGMKwc4V528KoMnNVzWDy9qhgRt14TbMYE9eN1MGOTbkSix8PDdtyuEt5fGUe/wlNM4IKA0zlpuYZuXlSqBAeAaFCYZFSU/PiQYEzhr2yfqGnAYbsF7VVlTvLF58d7u9vj4kt1deP/vQz8uw5FLxhiaYyM2YmbFiqf5bDkox2GOtZmqfB2rhG7H4E9iDJ2pCMFrJuLirRXZp6blnGuWV5F5bn6n/VmRfx4sf9v7qlAjC8gisvgtZAknH24Z2dnb29vYhnmPQ9pr7lmnQT+YVI69zT04OEAQgq8Hg82Yw8VEBNR0+jERCZwJKrgQEoce7v7+/FS74TCaEFaSAr/rvxtnPClycTV55NPX0nTK77HTt79mxJScnZs2e7urpqamp0dXWpVGpKSkpgYGBaWhqSk7q5ueXl5cnOq7y8fP/+/VlZk9hQyNZRzPzSIyCpzpd8oyZZBMRr5ogrciUSiXh4SKK7VbSEIFoIxE5XRTBa64m/i+MOqb293dTUdBxgQLDh0qVL165dMzMzexnYIAMJk5IJ5MkKAQYikejm5ubh4UHEy93dXVtbe/ny5VZWVjJPVUR3PG3PVBrN5M75b9b+efU34NvvwObNYMdOsG8fOHwYnDgBzp4BFy5AzHDtKrhxA9w2AEbGeG+SGbAcxQwODsDJEbi4wBYgaJ2ESxq878E7eF9fOBGJwM4eeHp+GRJMDg0Nj4qK/OEHzRUr/kyhvoG4AgQGUIYDnMenoCA8PRp/ZOCaZjoNaqmpVJy4IEHuIiAAJxZ8IbHg5QndVIlEaOLkYA9sbaFK+44J5EauXAY/ngGXLy+hkH3p9CAaDcqdUVjbrl27zpw5ExISkpeXl5ube+fOnV27dqGktm3btyMr1Y0bN/7www9r167V1NSk0WjPSS+gHIbW1lbZ1wfjPlSKp4oR+E1GAE9we6IrSSuMdjYpbnjKhEGR6IFAcJPD/fdETbMMKjxqmpGdqxQdI2UVEhKnF5fMaGn9SCBwwLDJemxe4vyJT2EYsnNeGSZ5iaPDxENDfTk5D6ytE775hj5fKnUgjTosVeMOS714sMOk3qwIOcDHuapD372LmX0kbUMyWig0XNh8dXHmT8tSf1qedWF5nu7yMmttVn3Ryxzt/+y2CsDwspdeIpE0Nzdra2u7u7vL9iUWi4uKimpra2VLMAxDX/mjJUVFRT/++OOVK1e4XO4LIwc2m93X14d0C/LsAZfLHYcWuFyufKMRnr029tDf348kEEjfjOgF5InUjZesE+nhw4eVlZUtLS24EZIQPxfJj6YRK8+kfH0+U+cKY0Q4jGESgUCQmJh47ty5hoaGmzdvRkZGFhQU+Pj4+Pr6RkVFeXh4REdH+/n5eXl5OTk5RUZGjoyMSCSSzs5OgUBAoVCKihQ/zPIfnF97XoRhEl8L8bJpko+A6ORq8SBbQraXfPKGeCGQHPyvuL9bhElEUMvw1Oro6DAzM7tw4YI8wyCbR9qGq1evmpmZubm5/Sy2AYmYyT+z0N2/j48PorZOnTplZWWFkhkMDAz09PRsbW39/f1JeD0NKqD3pNDo3t7Evdr/+no1Yc1asOEHsG0b2LsXHDgAjhwBJ0+Bc+cgZrh0CVy9MooZkJ4BxwxWuNeqgz3EDMg6yc0dwgYPPNnt3j14K+/iBP1Y3dy+YDD8g4NDw8PDNm3a9sUXfw4IfINGg3plNEGbIzzGAcEDBk2qZoaCZpxPIFOkWgXIKgRCqOCLoIIXLloYRQt2OFqwtAQmJjB44do1yJOcPPlnW9vTdLpM7gwBA5lM3rdv31tvvfXuu+8eOHCARCI5OTlpa2ujZiQtLWkz0g8//LBnz54NG77X19fPz89/fnqhoKCgQ9GO+NSfKsULv80I1AuGdiZEaQWTx0iGEPL2yKDc7skDEITCPP7gCTbnzYmaZgQV+geUHz6akZmlFBUthQqJSUqlZTPa2pYMDTn8LLPU5x+Rp2kYcnJ+EZek5z+wiWuyKysbvLzStm4NfvfvSCQdqKxCV5kZ/yyHpV41td65aj1qqoIN74iNFjENPm6+/HHF+cU5J5fkHfm46ODHteYH+sqSRJho4psqljzPCCgAw/OM0lTroO/DTp8+XVVVJRQKU1JS0tPTRSLR/fv3e3t7a2pqysrKYmJiuFzutWvXfH19hXhVVlb+nzDgzJkziYmJU+396a8NDQ319PSgZiQZq4AYBlkzEkIRCC2w2WwZREDKBMQhyFQKCDOgdRC3AO1Tu7qQyrm5uRmhhcbGRnjKEglXMFj9sH1oaGi7Hmn1Txlf/5S59zJNMAyjaoaGhqysrMLCwsRicVdXl5OTE4PB4HA4Tk5Obm5ujo6Onp6eLi4ubm5u2dnZfn5+w8PDEokkPDycSCQWFxcrvmJ8+mX/NV6RYJiov0u08yPJYoJkyTSRm4Fk10LJIoL4E4IkLkAiEeO0+1QMQ39/v52d3dMAA0IOurq6enp6CDZ4eHjI4g7QF/yyR0QmvBhOIJNhBxHqffL29r5w4cKKFSvmzp07f/7869evubm5uri4uLu7E4lEOzs7Pz8/MplMQo1HcjPkcUWhhoaHGxsdX7P2z6vWgPXrwaZNYPsOsFcbYoajx8CpU+DsWYgZ9C/BmGTIM9wGhoaQZzA1A+YWUAZtg9utQqrBSapqcHWD3/d7esLJwQFY2QBX16/pNH8GPTgsLFRTa9ey5X/y8Z2GRA4kXCFNoQA0UUdnpK/iguZA6IAEZc3+/rDZSQYVPDyggsLVFVIcjo6QyrC2gSoLE1OIFq5fJ+hdBCdPgBvXV4SGUkJCwmVCZxqNFhISkpqaGhIScuHChYULF86dO/fTTz9ds2aNlpbmjh07NDW3btq06Ycffti2bdvhw4c3bdp079690tLSouerwsLC4uLi/pc3XP81fkQU7/E/NAIiDLtWVrItnC4FDEEkzSDStgiGQV7WuF+CQmEWn6/DYs+dqGkewJ1S+/qVGxpmZGTCqObwiOmRUdOTU6aXlSm1d3w+NOz2ChuQJl4eL+/Jk55zc393gEF28LyGhsaAgPTtO4L//h4N71byV1ahqMyMmjkzZ9bs2jlzH89R7Zmr2j0Hn9TU+v71LmvZB8zvljbvW1Z1dvHDr//etujN9n8v6PxgfvfbGr3z5g0s/4Sjs49vbjacnCRubYW2f1OWuLNzJD9vJC9PAttyFYUpAMPLfggkeJ0/f762tvbevXsZGRlWVlZBQUGWlpbp6emWlpYBAQGGhoZpaWl37txJSkoS44Xe1d3dvbGx8QWOQCKRoDhnWS7bOJCAGAYOh9PX1/d/SmJkcIQajRB1gMQJaL63t3cceECiBVknUmtr66NHj6qqqhoaGlCHsQQT8QX8UwYMq3upX5+grruYu+pc2r5rDLEE2snJ7vifkzyRHxMM6qaf8WP8AiOm2OT5RwAa42KYJD1C/JWyeBmQrPir+NM3xIuAxOAIJprCLnDsHZhMpqOj46QtSTKeAc0gYfTVq1fNzc09PT3RzT1CCwgkUPAi//xCCMTX1/fWrVubNm168803NTQ0lixZcuDAARMTE5TJ4IJHuTk6Otra2vr5+U3NLcgOISgohEYLOHb8i9VrCN98A777DmzdCnbuAtraUMxw7Bj0JD13DujqQsxwFe9NuoXrGaReqzhmsLYBd20hNnDEqQaoasA7lNzcga0dvIN3cf2OQg2k0aDIePOWHUsW/9nbexrCAAGBEAygKZAE/Y5kTwMCcJAQAPzw7iOkVYANSLgVkps7FEgQiVLRgp0dRAvmFpBbMDQEN66DS5cIZ34Ep07NDo8g3q9+kJefl52dnZOTk5qaGhERERsbm5GRkZOTk5eXFxcXZ2ZmtmrVKmVl5blz5y5duvT777/ftAk2Ix09evTQoUM3btzIzs4uLi4uLCx8HshQWFhYXl7O4YzXko59qhRzihH4jUYgurcH90oK1IwJ21ZWohUdqhlM2hEZVDmA7rYlQmEmD0KFOYOC8UIFFKrQ26dcVz8jLX0MKqSmKRWXzOju/m5ESMGwX/xj7+fXMmkOQ37+C/q0/pqXgvfwYbOff8bmreEf/JumvoA2f0H4P9+J/fDNvI80Wle9w9/78dCRRSMn/im58cmA8Yoy/eVpep9Uf/OPvllze+eq9qiq9qiq9ahBk6V+dTU2HinNVFfr+/A/zK1beEZGwwkJoubmceBBwuHwDG4zly0dgFupc0+flii+y8AUgOEVferPnz9fXV29f/9+LpdbWlp648YNLy+vnJwcHx+fkpKSgICAkJAQJycnZCqK7qcbGxsLCgrkb6+f51jQzTSTyUT0AofDQTrmSR+5XC6LxRoYGOjp6bl//35TUxPyO+ru7kauR7JHpFXo7e1FS5DEGXUitba2NjY2VldX19bWysuRJZKR8yZR/9Zy+urHyG/187/8MUHPMlpe9Pw8p6NY5/c2ApBekIgkmAjmAdrqipdOg5hhMRBv/7e4uWZqN1XZuXA4HBcXl+cBDPJsw61btxwcHPz9/V8YJJBHi0KhuLu779mz55///OfcuXMXLlyIcIKLi4uHh4ebmxtxtFxdXZ2cnBwdHQMCAka3nup/EolEoVITEpPv+dhu3qr+9Sqwdi3YgKufd+0C+3TAocM4ZjiNY4YL4NJFyDNA36RbMArN+A68OzfDrZNQe5K9nRQ2ODnBW3kiEd7Em5kBF5dNZBKJQqEwGEGHDp349ttZnp4EX5wr8PXD/Vj9YZeRbIIvoVdxQfM93AHJywvGS3t4QIG1mxtwwd2QHB1hNIStDYxzNsd1CwYGEC1c1gfnzoIjRwgODgcqK8tqa+tqah7U4nX//v3S0tKsrKwMvDIzM3NycgoKCtLS0o4ePfruu+8qKyuvWLFiw4YNu3fv/umnn/T09EJDQ59TvYDgBPJUVVgqY4r6/Y1An1h8OCMFdiVFhmyvb9iWm7WVEbg5LNS0qEAoyuTxdrPYs6eACjW1M1LTlCIip4dHTo+Knp6eoVRWPqe3b5tQGINNldjwKgeCRG6dFDAUFLwGgEE6EKx2QaQr21iTe2mZ2OhzzHQ5zGKD00LMZKHAaFH9lSVZPy1LO7cs/aflzR+9BdECbsY66eMYeNBQ7/vPv5mbN3FvXB+KjRU1N4uZTPaxY3wN9YFRL1euhjp7yxbR48ev8pK8hvtSMAwve9G4XC6NRtPT02tvb79+/XpsbGxSUlJISMjdu3djY2Pt7e2Tk5PR7cjdu3djYmKQQrqystLPz6+kpKS2tvbnfqHOZDI7OjqYTKYsyFkeLaD2JPSIdAsoSKEdL0Q1dOGFUEFXV1c3XuOgQkdHB8pbQGjhwYMHQ0NDcoMF7fiNnFO+OB617lL2Ov28z4+Gu9Ny5VZQzL6uIwChAvRRxSQPq8Wr54iXAtEiIPIwlhFH+IlJJBjqTRo9TTlaSCgU+vj4TN2SJE81oJQ3PT09fX19W1vbwMBA8tOLJNc1hOZlS8hkMtqWSqXa2dl9+umnW7ZsMTY29vX1ZTAY/v7+rq6uyCXJ1dV1FDIQEYqQ38nT3xy+QqPR4hLjE5MSr9/Y+83aP61eDdatg2IGLS3omLRvHzh0CGKGUzhmQBroK1egNuDmTdj2Y2QMjE2k7UlW1hAe2NiCu3bA3h6yDY6OUHxsYgIcnX4ICPANDITH5XPPj+i23suT4OUNYxO8vaHUwQcXPCDZg+zR+x7wvgftj1C0AsQJqAFJBhUcgM1dYGkNSQwzcwhgbt+GeEb/EvjpJ3DsKNDV/U96elR9fUONtB7U1NTU1tZWVFTk5ORk4pWVlZWZmZmdnV1XV/d/bYSbN29av349YhhOnz6tra195cqVgoKC5yEWZOsUFBQ8evToyc8Y9vsulHCIjnEiKwpFPvjPhGzm9302iqObcgTutbZsj8Glz5EhG8PCNUOo13IsMtrWM5kzBwcnZxV6epUf1MxITZ0eETU9ImJ6dMz0jEylyqp5/QPHxJIxq48p3/aVvUinP54UMBQW/r4BA7Mda8jCkh0w30OYzdcQHpjgIMF4IZQ149PQ7UXNVz7OxdMVMs4tyzi/LOvM0uZ/LuhXVZ3amFUGJPpxh1a2hjpLQ71v4Yf9K/7bNwFpcDTUWd99K3yhlpBXdhV/6x0pAMPLXoHh4WEnJ6e0tDQMw3p6ery8vKKjowcHB5Ga09XVlUKhuLq6ovSi0NBQkUgkFov9/Pxu3bplYGBQVVUlgX9rxv+9wf/SyN2CjR4mi8VqbGxEoc7jAAMCCbLwNQ6Hw2KxmEwmCmpAXUYIKnSOVkdHB5pFsAEpFpBo4fHjx4hbuI8Xn8+Hf//GDhMem4Nf+mfHo77Tz1l/Kfe/hyiltTC4bZKDHj14xf+//xGQQJJIJMK7ksT3zCSf/km8DIgXA9GRFZJeqZEufpUlYvhPLJLAnD58/bErPzIy8rMAgww86OnpmZmZTQ0YyKMFv+ynUGD6MgnmqVEolM1bthw/fhx1FgUGBvr4+NDwkhc9y3CCbMbFxcXT0xPtZHTfU/1Po9HjE+IzMzOjY0KOn/h81WrCmm+gmOGHjVAAvXuP1DQJYoZRDfTFi+DyZYgZkHWSoREwNoaoAKU0WFrBL/ttbSFmsLeH9ILxnWnOztp+fv7+/v4BAYGkQIqPzwZ3D4KHB6QLPDwhb+DlOX6CIAGPYHP3gHyCqyvkK1xcoFLC0RHyGHfvQnBiaQUxiakphC63b8OkOf3LOFo4Bo4cmxEUZFtXK0MLNQ8eQMDw4MGDwsJCGVrIwgFDcXExj8ej0WibN2/etGnjhg0bdHS0L1++vGXLlt27d6NOpOLi4vLy8rKyMgQMpm5PknqvYa9NQXsA+PMCPSCEeAI6OnRoLIZhQnyCsxK0YOwH5LU5Q8WBjo7Ao+HhXcnxmxh0zRDKjaw76S1r+lkzuey/Mlnjo5pZbOWeXuX792ckp0jz16JjpmdlKVXff4fJ0pNIqkZ3Ofq/WDTh7//oS6/u/5DQ9skAQ2xpGevVvcmr2NPwINZVh1XHYYm2ECTYrobuqAgkGI2lNWNGCyWGC9k3PmrQX4ygQjpumZp1fnme3iep2gvDVGcnzoSpDo2qqt3qan349AyHJRwk9KurMSegBQQtIGZYs1pY94SZzas459dmHwrA8Du4VBL490YM/6bInGckMExXIhy7P8cPs6+vr6ampru7G/kaobA2WVeSLFqBhReCCki+jFMIMEihAy/ENqBHtAQhh/b29sePH7e1tbW2tra0tDQ2Nt6/f7+qqorH440bJvwPopgSWfjZ0fD1+vnrLuV+fpBMiiqCiljFn8Vxg/VaPcVbkuDtjrgyG5qrLgXiJQBihg+BxPGG7AtgSC5IJCIM+k3AeyZ44WWfXmxkZMTPz+/5GYafCxgoFAry+vTw8LC2tkbdRFQq9ezZs9evX0fgASmeyXKFXJJkOEE287MAA4lEotFoyUnJubnZxSVlZIqLptb8VasBxAzfgo2bcMyA8wwHD45poM//hEsa9KXtSTdvQarBEO9QMjWF3/TLxNBWVhBLGBn92cX5jI9PgI/PPV9f34AAso/PRiKR4EqEMMDVFeKBcRNaThwFCc7OUKiAcIK9HYQK1jYQKphbQHLD2AgewM2bEMNcugQj506cBNo6wMJid0V5WV1dnZRdwP9D9EJ2dnZGRoYMM2RnZ6MklqCgoI0bIVrQ0tI8d+4ckUhMTExMTU1F8QsZGRl37tyh0+klJSXl5eVPkzQUFhaWlpa+ZgKGhkpReZaoMkvc1ynGMPg90Oi3JRKxUFRbIq7IFpVkSthM+LMy+tJr9ctAcbBPjMCd6uobRXfTW1YPIKjAnDHAHEMLSKvQ1a1cVT0jKVkKFWJip2fnKD2oeZ/DuS7BGp7YHYZhI4NYaQjmvQeLuD3+pVf9PCq6awJgiFPXiK2seqkA6VdwmCIh1t+CPcrD8vwxui5G3IKZfQoRgpRJkAMJhjilYLhw6Paijqsfl+suyTi3LO1YwiTJAAAgAElEQVTssvRzyzLPw3SFvAufFN1a3xJh+/CeG01NnaysQlZWoarMjJg5K2f27DrVuR3qkHN4TtpBxj/Iz7A11JlfrhBWVPx/9r4DrIlsfT+693fv/7qrW9zmlrt7t+ju2nV3XXet2LAD9i5YsFCCgPQiRVFEkSpdlN57b6EEAknoJaGF3ktoIWVm/vfMgTE0Bbv38j3zJCeTmXPOnJlMzjvv937fCzjwt7CKacDwZpw0MPUCE20EEXT39lU3tvMG+eI9EwqFtbW1eXl59fX1bW1tMLQRDJFE5HWGaRYInCBOLEAaQRwP1OJWV1dXL2YEVOBwOFVVVSUlJbm5ud3d4z6BAKxITAbr5/0Ov8n6/ynrs/rkw41y9pm5leLdni6/jSMAAAC3A7m4FVsyA10xE7uphEr9KFpIQta8j2XEwIdhKIYJhQJRViLqb4swKaiAP2pKFBYWBgXNBBiYTOHJDAPECX5+fq6urvr6+rt37/7qq6+WLl0KgyxBhODj4+M5gT0/YPDAzcfHJyWFwmTmZufkhISEqpD3bdg4c/160gaJIcywV5p04ACYf588RZKVA3PxCxeHZNCAarhK0tAEmgGoaoCJGoxxJyXojKSjTdI3eNfqnoqzk6uzs5Orq5uBgYnCxcV37s68ZwXoAhBbyYZkY/14sbbGV1qBaK1W90iW94bIits4pXDzFg4VroNQSNeMSfqGIDWbtvbfNDT+RlYBGZ3lz5GOHScZGmzLplHKAFooKS0tIzDDKHohNTWVQqEUFBRARZOvr6+kpOSWLVuUlZUfPnyYlpbGZDKheoHBYMTExPzxxx9ffPGFlJTU/fv3qVRqXl7eWG0DTNkmFArFOMw3/aeD2umhq99DfnsHVduPDA4AwIw/KwEv8f6i9R8iv/8dPbgIra8EN3cxOP2mH9h0/8YbAZEos7HvaGf/p73d/+iaACrkF7wXFw+gQkjorMjoWRnUd1ms+X1911AUcO8jrIODpbtg9tKD+ou69ZYghguxIE2s7yUGLIqJaRkXMJSW9o7o2Cv4IBzE2ioxNgVLd8b8rmB20titNcC5yGQYJBj+ghmMXPA1Av2F7VqLylSXZuLeR0CroLCcCnDCiqyra4vsLrVkBPA7h2hwbmlZhYMD5eChwB9+hMmkvWbPCZzzftKHHxbM/ajuk4/bnhU5dH/2adfSpXwK5RUM1ZvWxDRgeCPOiEAkZJTU3XtEOaPvt/W8y5/H7sekMVuAiKABagng3L2mpgbqlWFQo66uLgIzQHqhuxtInKF2GbIKBFRoaGiAIIGDW3V1NYfDqampIZBDXV1dTU1NdXV1VVVVRUVFaWkpk8lsb39CVheE09Bu750eEl+Qyayq4LS2dnL7Bvrfor/8N+Lcv3mdAPTCQzN06UxsIQk5vgrh89AHN9El76ALSeipP5GeLsBCYJjI1QT585/IMhL212zkoTlwyhCzpKQkVVVVAjPAgoqKijJuE4GHcQGDh4cHjI7q7u5uZmZ26NChBQsWzJ0797vvvzt48KCpqekk9coTAQZbW1srKytHR8enRkmCgOE/U+SsrCwGnRESEuzs5GJtbSEn9+fGje+s30gCQZM2k7bvJO2VwjHDEdJJPD/D2bN46KTLwPNH5Qp4qH8V91DSwYOu6huAnNCG18CE3tAAJH3TM/jM2trQ4b6Tg4ODq6vbqVNn//prjpkZDgNwtYOlJci2Nmq5e4d0xwIst28DPgFSCjfMgOOTiQmI62poCFCKnt4/rG3WeXtfdXW5clVzwdmzMw4fJsnJfR0X58NmV+L+R8ANibC8vDxxegEGSmpra4Nn29fXd/PmzVu2brG1tYWhUQlZQg6e4zk+Pt7ExGT16tWfffbZ6tWrTUxM4uPj8/LyRkVcfeKtRuzCemOKaHsDevw35JcZyJ/vopmxgCLGYQHKG0AvbUF/mYGunImGuQ1BhWne9Y05cVPtiFBEH+DJcXs+6e+b1dX17lhWobFpdm7eu7FxQ6xCdMwsaua75RUL+/tvYthImWxPK1YQgfkqY7f+wkwW9uktzlBcnnh5ebPmYuzGIizZZqp9m/z2CYltYwHDZ59HsctHuw9Mvs5JbSkSYJ11WHU24FJib2GPzmHWuzCzP4Cjkcmi8WkEAi3gOEGkv7BTaxFbdQlNaVkKTikkKyxPV1pBI6+gKS8vu3e6Lc1nsK12os4M1Nc3xcZmKypG/rH6Ic45eMye44dndWB+9BHnEyCMnirn0PXZp53zfxwMC5uo0f/W9dOA4TWcWfyfZWheLUKFCRllp7S9Vx62WbzPZukhh5VHnRfLWLv6J7KKi3JyGNAJOD8/v6qqihArw0ConZ2dBGbgcrlduLW1tY1yPSK8jDgcDsQDlZWVFRUVVbhxxKy6urqioqK8HHgwMxiMpqbxE9O8hiGbbvIVjoCoho3s+RFZREJWv4ckBQN3o45mRHadaMkMdBlJ5G2NoKiovQnZ+S0qtQClhGK7f0B2/BvpaBHvY0JCgjhguHLliqqqqqKi4rFjx06dOjWRt9JYwACFCjY2NmfOnFm2bNkHH3zw5Zdfbtu2TVtb283NzRc3z8nZEwDDrVu3rK2tnwoYPD09PTw8fH19ExMTg4ODHYA5Oju72NrcPX36t3UbZmzYgPsm4XqGvXtJ+/eDWKsnToA80FDScOkySVFpKBu0mhrpqgYQHGvrgPzKMC20ri5QO+jp/mBtZWZnZ29nZ2dv7+hw3+mu5ZZbN2fcugVyOIDl9nBB7OOtW6SbN8ECQMKNIe8j42GooK9H0tEmGV371NPjQi4zs6CwmJqZ4+Zmdu7cF0eOzHnoblJSWlZaNkQsQOlCaWlpUVERlUqFzkjE638SMhKJ7T09PSUkJNTV1ZOTk8dSBzk5OUwmMy8vj0aj3b9/f8+ePZ9++un8+fMvX74cFBSUlZWVk5NDo9GKiop4PJDC5e163IBkxqGr/oH8QkIUJbGeLuCYhGGisAfIb39HfyYheifQkSha/AcyXX7zR0Akyh/gXeT2fDo2rwJ0QGpsms3MfTcmdgRUqKpewePdG5F/rasBK47FQnQwi41Dst1rC4X6v+QoL0u4vDzx0vIq9cXg+Xr8nZc3Junp7WMBw9f/iqmqAurEF2YD3VgzG8iU6X5YrDnmeRGz2YXdXIMZDMODITXCSAKBQAiwgOOEQb2FbVqLWGpLspWXURSWJ10G3kdpOE6gk5dV3j7QFXVXUFuATfonJuRyO+j0gmvX4rds9fhoLqQdfOa8H/HBBzkffVSFcw6TETlADyUQbvVfX/MeuL2woXsbKpoGDK/hLBF/ipz61iu3wpfst15yyOG3466rjrv8ji/L9ts5esdXsMuKh5/0lZeXQ7ahqampBQ+KCh2TOnHr6uoiciw0NzdDMqFu2KAggcPhQJxQXl7Owq0CNwgb4GtlZSV0X2YwGNXV1UQ/X8MYTTf5mkYA5Q+i2sfQX2YiP5OQW4pAzwyiIWEILUG04QPkF5Jo65ciVi4S7wNSuVlpgWBKN5WQJSQ0IUBcupeZmamhoQGJhStXrsCCioqKoqIimUzW0tLS0NAYyzMQgAGqmeEE3c/PT0lJ6fPPP1+7du2VK1fs7e0JKbOHB5A7T9ImAgx2dnbm5ub29vaTAQywLQ8PD2dnZwAXHJ0cHJ1cXFzv3jE5dnzJ+g0z1kPMsAXETdq9h7RvH8gDfew4CJ0kJ0c6dw64J13GqQYymXRFFQQz1cBjKGlrk3R0Qd6GKyokfb0VVlZ3rW2sbW1tbGzt7Gxtbt9ec910BhBJ3wB4YMRyA195AzAJpqZgAXyC8RCloG8AlM06uiQtDZKR0Q+BQRbZ2dmFhSWFRUW5eQUplBRbG333hzdKSotYII7qCGOxWMXFxURwpFTc6HR6b+9jN4b4+HgLC4ukpKRRjIE4zwDL0BkpICDg/Pnz33///aJFiwIDAyHGaGkZgTaxt8EAEYehmJU6unQmspiEeN9FUUTUWo/snQ+4OKkFSPVkwxC/DYf7X9hHkahgkG/A41kIRfkoCvAqYSJRYf/AZW7PPN7g7G7uY6FCZxcIiNTNnT0KKkRF/oOa8c+q6l95vPsY1jVUj0iAFcdhPgqY+ToCJww52xj+0qq1iKq0jK22pObqkn7dhZjpIizHl+jACy8wGF2ffhbz6WdRYkvMj/Pj6uqeCTCIhFhPC9ZYDDyL6AFY4j3MXw1zPIjd3Yxd/x33L8LZA0KHMNbLaDyQgBr80quzsEFjUeGVpVScTwA4QWFFhtKKHJUVeSrLqm/s7A68JizPwAYf34KmOlYon99dWlJmeS9JWtpn3hcQOXjPmRP3wYeVeGAlQufwZPzQ8eknnfM+779jgccUnGov3srtpwHDqzttw/NvGIwPic8o3Sh7f5GM7e9HAVT47bjTb8dclh9xXHbQYcFu6zuu0RxOJQt/2F9eXl5XV9eIGwEYWltbIWYQd0+CkZFgsgUIG8RZBQgVSktLi4uLS0pAhHU2mw1hQzluLBarpKSEyWSWlJQQTxBf3QBNt/TiRgBcbGB5LER+et0oBuZA1BhEeoFoxzeo3FqstQHuhcIoq/f1sO3folvnIdYaqPtN7McZqJMxuJrvqWM/kTCPEffN3NxcbW1tIhUDBAyGhoZWVlZ3cbtx44aamhrEDASuuHLliqmpKcyiICMjffPmTTiJd3Jysra2ho5Jo7I0TB4zPAEw3Lp1y8rK6qmAgWgLKiWcnV0cHR0dHB0cHR2dnF0sbhsdPbZ4/XocM0iQNm4ibZMk7dpN2icDwq0eOUw6fnwoS8O58yDo6uVLQG1MJpNUVEiquB766lXwUUmJpK+/2fKu5d279ywtLe/ds75nZXb9xvxrRjOu4TDA2BhEWBq1GBlBtTTwOzLANc0AJwCtAvBxAmyG+iy7++SY6Pj09AwGg55fUFBYWMhgMFJSKJlUWgnOL0C4AOmFsrIyeMfIyMig4AZDqRLEI4IgXV1dRUVFdNyeHAGJCJqUm5ubl5cXExNjZWVFoVBycnJKS0urq6tbW1uHLra3JGwCHh8JQ1rrkNOrsQUkVOontJYtstZEls7EVs5Eoh7hESCmXZGefuN5LVsIBFHcnm/4gtm8wTld3Z93c5f29p0XCtOFwrj+Aflu7heTgwr/jIr8e3raPyhFP/tX6AkxMb0fjwuer8MwoNdGKndx8W7N1cWd2ouGYoNCOe/9fViQFhZ/F8vywAqjgA64mYVxmzHhIJQ4Ps9AFRUPfDIv7oOP49//OO79j+PmfBw/e27S/EXpXcPoZnTliAjj94PWm1lYVRZWFIPRPEF402BtzF0Os94J4heZ/jqEDcSdi2C001F4YNyPOIpADX7p113YormIrbaETl6WrgiYhCQgZV6RSV7BUFlRqLqi8e6BvnAzpDwD63/BQWD7OJxKV9fUY8f8vv232+w5Ye9/UP3Jx5kffVQwd27Vxx834uneoNqhA/dcGuW81I6HVOrT00VHik4xDOMGBLTr6vbGxCDc1y0rH31qn/3zNGB49rGb6p4obhiGDfL7HoVm/HHQatnh+6tOuP5+zGXFEZfFB+7/dthm10U3eUNvzbtBkfHZlRXlZSwwp6+pqamvrycAQ/Nw5gQIGLq7uwcGBgZx4/F4AwMDvb29XC63vb29ubm5vr6+urqajVsZ7m9QWFiYn59fWFhYXFxcWlrKYrHYbDaLxSorK4NKCSaTOTAw8Na5B0z1dPyPbD8oFLZ391RwajNo2ZHRURFhIYH+vg+cnR46Owb4eMfExlOo2YUlrKb2dp4AFyEIBzFeD9rPRfmPH+HgYbwQkYiH9najvd0gfpe7GfYDCXU0AteJpSr2MwnzvCv+oCUvL08cMEBgYGVlFRAQ4Ovr++DBAxcXFyMjIxUVFVVVVTU1NXV1dUVFxUuXLhGAQVJS0sjIyNvb2xMPmQoLns9hTwAMN2/evHPnzuTr9vX1DQ4O9vf3f/TokYuLC3BNcnJwcna5fdsY5xneATzDBlzSsJW0cxeQNOzbRzp0CGR2O3lyyEPp/DnSRRw2KCoAJyUVFZIKGaROU1D4h9G1U7dvW1pYWNy5c8fS0lKZrCAt/aGWFkj9pq+PLwYgyBJc4Bo9fcAk6OoOgQQtLUBWXFUHDMaVKyQlZZKq6sq7lrd9fHxiYmKoVCqTySzAMQOdTk9OTk5PTy8qKhIHDCwWq6qqisViQZckiBlKSkqg1nlwcLCurm5cH6ScSRiTyYSxVul0enNzs6Kiop6eHhGD6634bQJXPVy4g6SEoOs+QJaSUHkJZO0c4MundxwVADWX6C0BP2/FgL/ATiJISU/vD339Q9RBN3d2b9/s/oHZnV1zO7tmD/Bmc3vGsArdsxsaZzNy340eckD6Z3Tk/6Wnz0rI/cUi89LRMJe9MfHpPeB/c8i4TdgdCeBoNN5cGdUHs+TRX8H4oSaLgHO/0ULMcBF2/Tfs1lrMcivmcBDAjxAdLM4CS3PCsr2xvBCsNB5jp4LZfA0Dqy/AmkqxFvbQ0l6NdXCw1gow3QcrWfXZ9P2rnXYvs5NebnXo17snV5mf/8vMSObGQIonRnXFkm1BGNMII4AHfBQx1xOYzW7szibQ+vXfsWuLQX9M8Y5B0sBoIRAhTB4bwEEY3l6k/0svDhIq1JYwlIdAQjJOJqTjZAJTZRlLd22b07nBRDusLh/jT0Fo8Ww/uv76hoaExK7ExLSlSx/MnuM+e44XLnUIe//9hA8/oH74Ye7cuSUfz634+OO6Tz5u/vSTluGcbh2fftL32af9igqoeJZ6BOFISlZ9803lr7+yFyyoP3q0y82NX1kpTsIPXyhv0/s0YHjVZ6u7s93dP3b5gXsrD7utOuH261HnxfttNp91MLYKj0rMKigqqaqoqK5iV1WUs8vYLFZZdXU19C0CCmjcGhsboZihs7NzYGBAIBCIRCIhbnw+n8fj9fX19fb2dnV1QT0DxAwsFquoqKigoICBW15eXkFBQVFREXyaWIxbfn4+jUaD6sNhPuRVj890e88/AkIR0tDaTsvJcb5vrX/plNreP69tn2+9Za6bxDs+EiRvCVLQJlKMJMl3I8l10z/vb/vIfPvXOlK/6Zw/Yn3rWkxsTHVDM08kEu+GCEMREUi2IALuSQhw1HYzxb4noc6mADDcU8d+IWFeVuKAobKyUl9ff5RWwcHBIT4+PjExMSwsLDAw0MHBQUNDQ1lZWU5OTkJC4suvvvr999+JPAzE43zPF2RPAAw3bty4desW1EtMpjUvL6+goKBQ3Hx9fd3cXJ0cnR0cHJ2cnO/duyV75i+JTf8HY61uxFM0SEqSdu0iyUiTDuzHqYZjADbI4sIGyDZcvExSVAScw7lzpMsKn5gYq940N79565a5ufndu3f27JGe9+WMKyp4dCMdAAl0tPEFL2hrgZhLWlogRqqGBnBwUlcHuuorVwACUVbG4yCdf+e07FYtTV07Oxt/f//4+PjMzMzc3NwC3Gg0Wnx8fEpKSmFhIcQMpaWlFRUVHA6HAAwpKSlZWVnd3d2Dg4NNTU1FRUU5OTlPZhWeChxoNFpubm5/fz+8/8B7Tn9/P5vNhs8ssDfYUDz7CB7eDkVMLiBLZqJL8ZzoW+chVUUgOQOIiz0Viu8NPtj/pq6hKL+3bz9vcAQkgFLmbu44Kdi6IFRgvBsdA7QKYeGzYmP/X0rqe2G5q4zS1A/F+e2k5+8tKJROjiOXFA+IU0pNZYjjIcxoEQAGT/XJGYsr4PT6Gj41h0yFMa4EMFkE5u7GcL6+EMzmjZeBtMemv4LJPVxurALC4hurhj6a/oaarBzUXcrXXSLUW4ToL8QMf8auLcCM52Omw3VCoELEM4WQ4BlQgfiBDCME1OAXnh6IcVR7dXHJlaU05WVpOJOARzpaka60IltlJU15eYH67zV3DnJDTEQFURi3cfL6BHh9ckUim44OzVx6c2/PM1+x6WfO+sx5/yGOGdxnz3kwe47b8AI10z5z3g+Y837Y++/HvP9B0gcf0j76qGjuR9WffNyhqYEN/28KGhtL587t9vYWcbl9GRmtBga1W7ZwJCXfdrZhGjA883U1hR3hHyGKYU3NjRHRyVvP2P56yOWP427LDzn8dthW83ZAZnZ+VSXQFpSWFBeXFBaD15KS4pJydnk1bhwOZ1iSUAfFDO3t7TweTygUwtTRIEAHgohEIj6fPzAw0NfXR0RMampqqqurq6ioKCwspNPpNBotKysLBj7Py8srxK2goCA/Pz8rK6uiomIaKkzh1L7WTYHLEfB7AFpR2BFu70B8Yoqptore3hVmf/0zZAup4gCp/yRJKEvCzpIweRJ2YeQiT8LOkRA5Eu8UqeEwibKDZL32Hf1dP18jnw0KCmpu7xj++0MQVCRCBaA52FKMF7pkJmIgBz5qHcF+nYEVUIluYBhWU1NjYGBAJpMJdyMymWxnZ5eZmclgMLKysmJiYmxtbXfv3v3jjz/OmjXrww8/XLZs2aFDh/T19V1dXUf5HXm+CHsCYLh+/bqJicmjJ2aYHtWFwMBACBhCQkKCgoK8vbweuj9ycXF2dnaxsblLJkvt2vPxuvWkIapBAkRP2rmdtGcvSQZ6KB0B+d0AbJAF6RrOniOdv0A6ewZ8JJN/MzUxug7M9ObNmw8fuh8+cnLevJmXLwMYoKYG8ICGOg4McGxwFX9VUwXZmq9cARII6Nd0WQGIJS5eAHFdT58i7d274NixEwb6hg6ODoGBgYmJiVlZWfAOkJ+fn5mZGRUVlZycDJ0VIb0gDhjS0tLYbHZDQwMBFSBayMbtqdhAfAMCZmRnZ5eXl4uG/2XhlZWVlfXjjz9KSUkFBQXBfJFw/Zt3X0IRPNk5cOQrL0D2/SRcPgNZMgN1NQPIesiJBNAM+GYIionwH+s0hIDn87W98vlB3J7RwEA89hEsD2kVGmfTGe9GRwOoEB4xKy5hVirlfVqFlF6V274o/x0BAVJFJTKcWhlOjUxZ6b6crICRLj5IT1uF3fkqjeVtWouE+s8EG8Tn36PKcDo+/utCzHCYAXi8wc8AJ8DF4OfR/Maoyqf6kWjFAKRUE+ov7NUBjkacq0tKVJfQlZdlKC6H2mU8bcKKDGUAEtIVlzOu/lVuebw1+MYgIxRtq8IEz6SpwLAWoVC7selAQ+P+lDjv4mfPk1Bget179pyH4y3uwytBeofZc7w/+DD6u++ydu8p1tRs8fUV1HCIf0BuQEDp3LmCurrHVzmCCIe9Lh+vfNtK04DhVZwx+FfX0tacRkk5r/NwyT773487LT1wf+v5+/4RaWUlRSUlRdBTqGDY8vPzi4uLy8vLKyoqKisrq6qqCORQU1PT2tpKcAsInvMNz+GAAIpcKBwcHBwYGCD0DC0tLfX19VVVVWVlZQwGIy0tDfoiE5ghDzf4kc8fkf/hVYzOdBvPPgIAJ8I5fVVNjYOtzdUDG+5ufI8hReo5SULPkzD5GWA5T5rcMgO7QELlSQOnSOx9JM+t/9DavfTuDYP8oiKhSISjEqBwhp1F+rjIiT+QP99DTc4jq94Tya1F+0Y81OFwOAYGBgTDoKKiQiaTr1+/Di8/KyurHTt2zJs376OPPvrpp59279594cIFVdzU1NRsbW2fKifwnLo9ATCYmJjo6uq6u7tPvt3g4OCwsDCIGUJDQ0NwA05KHo8euLs7OztfN7169OiyTZv+3/p1JJClYSNJYhNp2zaQ3G3vXpLMPtLBAyCG0rFjIIzSqdOkU7IgBuvJkyQVlS1GRsbGJiZGRkY3btxIS0tTVVX/4fvPbt7aZmj0o5rabDL5b0pKwMtISRlgAzJOIygrA/GDogJwarp0kXThAiArIAI5cYJ06DBp5+7/k9y2RFb2tLGxsbOzc0hISHJycnZ2dl5eXlFRUX5+fnp6ekxMDIPBKC8vJ244LBYrMzMzPT0dogv4xEF89v/MZSqVWlhYOPaGMzAwEB0dfejQoW+++WbVqlU2NjY1NTXiaEG8/Ow/nRe3J4KBNDqI4Ulk0Qzkj1kiDks8JwmCCkFGdAwV4jGUECAUmrbXOAID3J4NhDPSWJxAUA1NTe8xc4dYhYjIWfEJsyiUT6qrZQX8dAGGuHT0S6em7A3wlC4skKnmyFRVg9eiwlMsVr1wRIBpfm9X4X1yFnlljvKymquLBXrDnkiGv4j0f+HpgnBJiP5w1CCxafcLntBPFQA8YXuxTqIGoPODegu7tRc14/CApbokjwxSJaQpgjwJuBphOUUR0AhZ5JVZ5JVpyr/SNDeU2sg3hN/ry48TdTdjosHnvCA4fL5KQ+OBpmaZ0lLp8ADFuHDeyLMw+fpbMjIe4QzDKMzggYMEz9lzfL/4MkVGhnXzZk96GtreTrAK4k3Unz1btWoVNkEfkP5+dORTEvF93+TyNGB4FWcHRdG+vj4qNdX5UcSKg7a/HndZuv++zCWXuOSM0sLc3LyCvLy83Nxc6NebixtUHuPRjFjl5eVQnVyJW0NDQ39/P5/PH/VkDmokUBQVCATQMYnL5cK0DE1NTTU1NRUVFQUFBampqQkJCSkpKVQqNTs7m8FgMJlMOp2ekZHR0fESE8e8ioH+n2yjsaXV1tpafcfSAIkZLUcAYwDn/dj5Gdg5gBYAcpjEgoLtSWAvSEScJ/WeJFG2kww2fWWsq1FUVi6exBsoPnPi0VN/IKv+jsitRXKSxb/FMKytre3WrVsQMECVApQr+Pr6njlzZu7cuWvWrLl48aKioqKKioq6urqqqioMpqSqqjrJCKeeU7QnAIbr16+rq6u7uLhMHjAEBgZGRkaGh4cTmCE0NDQsLAwSDr6+vl6e3g73rQz0z587t3b3ns/Xrp+xdh0gHDZtIm3FYcOePaR9uJPSoUMAORw5ioscjr+rpnZc38DQ0NBAS1PTwMCATqfr6en/9NPPsbERFEpISPAdN7fLt8wlNDW/IZPfu3iBJC9POgfRnsMAACAASURBVH8eLOfOkc6dBSBBThb4O506CaIzHT4Mkk8DlCI1R1l525kzp86flzczM3vw4EFoaGhKSkpOTg4UMxD5FgoLC2FENTabXVBQIM4hiJefGSrk4PkZsrOzGxqGVPXED5cAAwiCMJlMDQ2NH374YcGCBVeuXMnNzR11xyP2er0FPC0JiuifRn+Zif7xT0QcMABaAUERPsJMQ7wsUWYqIhK83t7+j7cuEAT29I7jjCSOHDo6QbZmGCw1InJWXPwsSurXHM4VgaCQGL1BDNNhs6RD/aVSEmSqawBgqKqWqaw6UFdv0dI66mYoGuxnexpmKK1IUVhBV142oLdQqL+wUXNxrsrSdMXlVKVlDPKyoitL2WpLOOqLmzQXd2gt4uos7NddKNADoALRB8/sH+MHYr4+tiC+2VTLY2sbdqOCqECEAwOuzqI2rUUNGour1RezVJcUqCzNxtmDVDyNGh7UaDlFYUWqIghtlEVeSSOvpCgsT1f9k24izXqo2xDv2lPBEPZ14Z6txHA+V6GSz79Y33CgoVGmukY6l7HX30M62Ce/pfnZKhX29UWsWesze84jfPGcPcdnzvses+f4/OublCNHKh4+6q+re/J0H+nrq1iyhP3++9Xr17doafUlJ4uG5Q38ioq6Q4eqVq2ql5bucnYeF2w8W7dfzV7TgOFVjDOKogUFBUmJCSc1Hyw/ZL/0kL30JefElAxmLp1OZzAZdPjXS6cPFWDw8hLcSktLy8rKCORQWVkJ3YgFAgHx5yp+DNA3CWKG3t7e7u7ujo4OqH6uqqoqLS2l0+mxsbFxcXGpqalUKpVGo+Xk5KSlpZWVlY1boXjl0+XXOAJiZwfEP8IwrH+Q7+npobLrV99NMzuPk7CLOKVwHscMYOo/lWXU9vAj7r8kOE1K2UG6uuFry9tmLZ0g4ANMaQs4h74erK0BHezDeYch8gEO0cDAgJ2dHZlMVlVVPXv27M6dOxUUFMhkspmZGQxgmpyc7Ovrq6urC/XQ8BWCChsbm8lP3D0nbU8ADObm5ioqKvb29pOUVsNsDJGRkVFRUZBegLAhJCQkLDQU5x0gcggOCg728va2sjLV1DwiK7tCWvqrTZv/ISExY6MESUICEA47d5D27CZJSZGkpMFy/PhXmhpK2jq6ujo68vLyhoaGeXl5enp6C35akJCQlJubz2DmMeiMrMy0+LiAoKA7Do7yN8wkdXSWKSp+ce7cu6dO/e34ccAn7N9PkpYBHlC7dpNk9n2kTN5kY2sQER7s5OR4+vTpCxcu3Lx588GDB2FhYRQKhU6nw0AITCaTQqFQqdT8/HwiYfPzAINx96XT6XFxcUwmEzx5x03s2oYrHr82NTXZ2NisWbPm888/P3PmzASJ5x9v/+pL4IeAYdi10+jimejq/4fVlIkxDIB8QN1uIqv+iS0job/+HXG9geHibrFtXn2X/0dbRDF+T+8OXNw8IWbo5s5uan4vMgpoFeLiZ6VQvq6u1hQIK8cOGXuQfzg5XirEV7q0dIhkwHmG/TW1mX2jRbooitbFu2Vc+SNVaUWp6pIq9cUwsUAKnoMsRWFFisLyDOWVmeSVVOWVVOUVYFFanqm0nKa8jEleVqCytPjK0jLVJeVqSyrVl9SoL667urhRY3GL5uJWrUXtWos6tBZ1aS/i6izq1VnYp7twYHjh6S6EC1/vF74eoALgR7hBv+7CXt2FPToLu3UWdmqDelq1FjVrLm7QWFx7dXGV+mK26pIS1aUFKkuZ5GVZSkCdnKYAPIuAQBkPZASxQboS6DCEB2mKK9KUf8+4up5hdqTMXacmyqE9L5HXXi8afEZfo7EjL76GPQjQwv76BgDYODVS6ZS9fo+kw/xtGTTxzaZUro+LC/xx/sPZc7w++SRk+YrMywqV3t799SNz8E1c40BOTsmcOT1BQd0PH9afPFm5bFn1woXdnp4oj1f1xx+crVsHCwv7EhIqf/utRVsbVIMgvKIiwp1p4opf/zfTgOFVnIP29vakxET/4Og1x21WHnZdfcwuIDwpl07Pysqi4ZY1bMRH6CcA459CGSKEDQS9MFHYU8gziESiwcHBvr6+np4eqH6GJEN5eXlBQUFycnJERERiYiKcHGRkZFCp1DdfZfgqTtXb0QaYZhWWsDQunnaQeLf5GC5OkJ8KPJgSlgCuTaSB06SAzTPVD21KTE4REfO8iYerp6fH2tpaWVlZVVX14KGD8+fPP3v2LKQR7ty5ExAQ4O3tbW5uLo4WCIbhJQEGNzc3Ozs72zEG8zBcunTp7t27kwQMnp6evr6+ERER0dHRkZGR4iTDmDKAD2Fh4cFBIb7ej1ydbczMrhjoy6qqysjLrz1x8pcjR/51/Pg3x098feToF8eOfSUvL6Gurqqpqammprb/wH5LS8vCwkIAGBYsiI+PH45KRAdPGuhMJiM/JyebmpmemhoTHe3l53vHxUXLzl7BxHS/2tU1lxRWaGhK3ra49PDRvajo0Pj4xPj4xLj4WEtLy0uXLhkbG7u5uYWEhCQlJWVlZdHpdAIhEOKEcaf7z7wSVstkMuPj4729vYnwrBNfRI+/6e7uDgkJuXnzJnc4RiGPxyO8MR9v9xpL105ji2Zif/wDqykjPPcA+dBSK9zzHSLzM5rgCxId7vxW1FQJUrwBkDTtnvRKT5hQmN7V/UFX9zhoASoWOjpnNzXPrm+YnZn1bkLSJ5VVanx+xRO66FZXJxMZNIJkqKreX1evUFfPHc/npL0gJUd/B428MkMJuOtQFJZTlVcU3TvTUZjSlpdQl+xZFWJZ6q6TZ3mWZrg74+qGtCt/UpR+pSiuoCoDZx4qeWWG8sp0pZWpSuAR/vCyPFURLGliS7ri8sksxC6pCstT8f5QcKIgBbwCPiRFYQUFbyhNCTAGVBzS0HDeIF1pBUVxZZrKH1SNDdlGUvlW51meBjXRjs20CG5VHr+7DRG8dPfmisHBC3X1+xsahxieao5UXORev0dSwT7nokK4/Gf3dOqtrauNiGjPzRWMwX5PuB7gV60mJuwffkAFQ1wi0tPTFxfHy8/vCQkpfvddfsXQFcUNCWF9842IyxVUV7Pnz2+9caPh1CluSMibLHWYBgxPPfsvYIPCwsLEhDgzG79l++2X7be+aRfEZGRnZmVlZcKXzLEGAUMRbgRsKC8vh5GR+Hz+k0MQIggiEAig+rmnp6ezs7OlpaWhoaG6urqkpIRGo4WFhUVHRyclJaWlpSUnJ//H4/wFHOd0Fa9kBEQI6unlo7plQcaOGSiUMgPXo8lrFaYMLVD5IUxStZ9ksO5jO2urXt6E9+KW5mYPD48DBw5AhADVCwoKClD9rIKblpaWjo4OkdCNgA0vlWHw8PBwcHAYgxds7ezsbt++febMGRMTE8+pWFBQUDRuUVFRQUFBgYGBozyUxMBDWHhYWFhYeHh4VGRkVGREZGhoSGCgn7eX+6OH9x89cnB2unfP8vrt20ZXNVTJZBBq9uTJE1JSez08PIqKinR1dX/66ScxwJCTnQ0W4NuTg4MHRi6Dkctk5jNz83Jz87JptIz05OTkmMzMVCYzl8nMo+XQs7JgtAMalUqNi4uDv/1M/BYkrkIW5zmJ9c8MEsR3hCqphIQEU1PT7OzsJ7AKT/0ZCQSCS5cvBwYGPk8lT21lahvon8QWzMRW/g2rKRUHDEicD7ZkBnJHBWij76oiS0nCWB8QPkmETJMMUxvh5966r19xgDcaLcDISM0ts0vL3ktLfzcqelZi0qyGxsUDvIynNtiJIOepqdKhftLFRY8dk6qqDzQ2ubSP79870MIpcVFPJ/+eoQTyDNB0tw621Y5qCFwcfJ6gt6OvsbyLld3KiK1P9uJE2pf73ih20ci3lqffOJSlJ0nV3JiutjbtyupU5d+gSCBTeSVNBXgBZZEBWZEJyArAV2Qor8hQWp6utCRNcWma4tL0odcl6UpLM5TAt1TlFZk4JqGpgB0BAlFamUpela76V8bV9VStTdnX9jJvnyy0Vyp1160Kvlub8KA5K6yjOL23tpTPbRPxeQRbOOpYXt5HDp8P0ALkFgC3Uy3DLt8bHrjX32NvgKdUiA+tQUxz/PL6MbJmVCis/OOPelnZkavBp/rTp6slJIj13MBA1hdfID093IAA9j//2aik1G5u3nDxYs2ePdDlCRUIkKnDFaL+l1GYBgwvY1RH1Dk4OEhJoSQlxl/Qc/9J2uaAsiuVSs3MzKRmZg4VqFT4jJ+KW0ZGBo1Ggx4CMFtC8bBxOBwul9vX1zc4OAhDqcKY6OLtwX9QEJtjJMkA0zLU1tay2ey8vLzo6OiQkJC4uLjExEQajTaRg5N4zdPl1z0CwPGhb2Dglqnx7fVzWo+TsAtQdTABABjLOcA1xHoIA3C2AUCCJ9IOhBBi4DTJTeLvBmT52vpG8QHp6GiPjIy8eOHCAtwOHz6sra0NQQLEAzhSAC8EThD/lthGVVX1JTEMHh4ejo6OEwEGOTk5XV1dj8mljoabBQQEREZGRkdHx8TEBAQEuLi4eHp6Aq8kMTH0MGYIBrro0JBQsOA2hB/Cw8MjwiMiff38nV1c7ty1VLmipqioIC8vv2vXzn37ZEJCQkpKSnR1dUcyDMQ8HAcNOTnZOTnZ9Gx6Dh0gCKAQyGEwmAxGbg6dAb+EK+FuBJkwFhu8WIRA9BIWmExmQkKCoaGhv78/1DrDm5X4VTRuGcVN/CuRSOTk5JSRkQFrgB6Y4hu8yjKeHxFBbHRQqR+xo8vRxhHuKyKve+gPM1AbLfDrtdEEwYg97ooQYCN8+F5lj/8n20KQtm7uT0SCBUgpdHbN5tS8l53zbkws8EGCUVNjYmd1d3tPcpASOzulI4OlEqKHHnJDMUM153BtXe7AiOzR4hV2FqfnXNudo7+9ozBVfP2kyiiCigQ4ougaaKvrrSvtLqd3FFJacqIa0wPqEh9WR9pXBJizPAxL3DSLHFUKbS/l3ZVLNzjkeXKbn+wmuPjLSvie3uwjuy3L6EDeHdkCm0vFLlfZPqacKIeGVL9WZlxnSQa3Kr+vsXyws0k40IMI+YhI+Oa4zbQJReSGRqBbgAOOO4NJFxXuDfDcGwAAg0yY3/N4JU3qRIy3ESoQdN6/30+ljv2yasOGJhUVYn0TmVz5++8YhtWdPl3911+Qkeiws+Ns2QLHGWywevUbFYl1GjAQp+9lFTra2mOioxMSEqQUnRZJ2dq6h+fkZEFsQLxCwJCBW3p6ek5OTj5uMFUC5BmKiooaGxt7enogYIDhU9vb27tGxnGD/6AQMMBtYB43qH5ubGysrq4uLi6mUCj+/v7R0dHx8fFtbW0gCOBwaM6XNRDT9T73CLR2dhqqXXLd8PdBOfyR/7mZTyQWZuCBkvA4qhdxJfQ50uBpUt8pEGh14CSJL4urHQBsgMGUZqDnSED6PDFyGKrwPClmG0nz2Pay8goerz8jI0NDQ+PXX3/99ttvJSUlHRwcysvLBQKBr6+vkpISRAKTfH3ZDIOjo6ONjY3tSLOzs7OwsJCVlVVQUHjw4IGnp6fHJGCDh4eHt7d3eHh49LCFhIQ4ODjAjNTh4TijAFDB40hKw+BhnHcfHx8HBwdtbW15efkLFy4cPnx4y+Ytx48fi4qKKi4u1tPT++WXXxISEoZdkkZNxd/0jxCiJCQkXL16VU9Pr6Wl5bl/CiMqiIuN3b59u729fXPzMyodR1Q3xQ94qDJE1NOJtjeiHc2oiC+OBFAfK+zHGZiNFqjVRgP7gYR4WQoBXBiljJ1iq9ObT3EEBIIQGE0VQoXW9tksNqAUIiKHcEJ4BAicGh4xKzZuFo9XNsnq/+N3YlCYLx3mL82ki5MM++sbyPUNPcPem2Nr43PbB7te8A9hbCv4GhBzPSW5+csvYr78IuzxMi98/o9RNdU9eBAv8Wt2gmremNUCFL3e3AJiIhFoARcwSNOy9vo/wjEDYBgux0UMCN+gGAMNZ8/W7t8PRxHp6ir74os2MzOUz2d/912nszNc337vXs2ePUCgSKWy589n//STqL0dw7ABKrVeVhbpBdlUCWenV39CpgHDSx/zWk5ddHREVEzCFjl7CVm7FEpaFs4tQLRAQAUqzjNkZGRAwAATq0HAAFMllJSUtLa2crnc3t5emNoZpnNmsViEXy8x74eAQSAQwBCrhPq5paUFkgw0Gs3Pzy80NLSoqIjAGC99LKYbeI4R4Av4KhfOeq+fieCKZDB3H6VUFp/ow0hH8gAhlMiQgjaQbNf9w1xynuneRdf3LTfdt9xUZrnprp9ubfvy3sZ/+m4g0XeDSKwAKlzAoYh4VSPLoF18s5w9pEs7V63fvO3rf329Zs0aExPT/Px8IkomiqIhISGKioqThAqvjGEYBRjs7Ozs7e1v3Lhx8uRJWVlZOzs7z8kZzPIWHBw8jBeiY2NjIyIirK2tDQwMrl275ujoGBoaCiMpTeyqNAQefHx8oLRATk7u5MmTu3ftltgkoaCokJqaymQyQ0NDbW1tMzIyxDmBNx0lDPePjltCQoKWlpa6ujqLxXrhNxw2m62goPDvf/97wYIFOjo6BQUFT/bYfI5f4Ti7IhgIboygKMhviGEIyO/8ePqFet/DfpiB2emCPe20AGDwuCNCEBEmnHZJGmc0X9YqtL//PF8A0i80NM7OL3gvIXGIUiBwAixERM6iZn6NoqPjdz2hX+WDvCMJ0VKRITLlFY/Vz7hjkjM+23vCvq/sq4iolk8/S/jk02ixJeaLr2KqqsSyU7+y3jxfQ7kDA2CcYTRbAjNwaqSS4/b6DQEGQDIE+xS3vRpINqnj4ZeWVixb1nj5cre7O2fz5srly5Hu7n4qtfTjj0ESaNza796tO3wYFQo527e3mZtXrVkj6u1F+fyqNWsalZREnZ1Nioq127bVHTnS/ejRqw+yNA0YJnWmn2ejcjYrCvgtJ6w9YXXZ8BGDnkOlDjkjEZ5IkFuAaCE9PZ3BYOTl5eXn5xfgBgEDi8Vqb2/v7u7u7e3t7++HSKCzs5PNZpeXl4v/RUHYIBKJBAIBJBn6+/uh+rm1tbW+vr6iooLJZAYGBsbExIinRnqew5ze92WPAIJhD9xcrTbN7ZMFzkhDE/eRs3kwlccdjfiypNw9pPub3tXcucRQ4aSHq3NcYgotN7+6pr6+uQ0u7CoOLbcoKY3q6/ngpraK7oG/bLZ8lLqTxD35NNhwDm/lAilXirTxm9mOTi59/eNEwEhNTVVVVSWTyRAMTOb11TMMt2/fvnr1qpyc3MGDB48cOWJmZjalAE1+fn5RUVEQM8TExMTi5unpqaKicuL4cRUVldu3b7u6uvr7+8OgqxEREeHDhtMP4CUiIsLHx0dTU/M4bjLS0tu3b9+8ebOGhgaVSgX6ZvyGMDwDf5veGQwGjUZzd3dXUlKSk5NLS0sj0MKoW9Yz/HzEa0BRtKKi4vbt27///vu333574sSJ2NjYvlfiAQwYBjwpG0AOADg8RgsgAkqsN7p4JmYoC0hcw5PYqn+gOYkosMdZTZ7h2Kd3mdIIoGhPb9+Gpub38vLfi0+YFREJ0MIoqAA/JiTNqqklEyqUSbbiVlsD1M9pKSNmsdWcQ7V1YyMmTbLOF7tZYFDjZ5/Hf/pZlNgS/fm8qPKK0QGdXmy7L6M2Rn8/yJTHGY5mCwUM5RVSEUEEwwBSZIT6eRQ9ewa3l9FzQV1d261bDWfOtOjowLRuLdralatWocMZG9otLBrOn+9yd+fs3i2or6/89VcMRTudnct//lnU0zOQnc3Ztq03IqLb27ti0aJmHR3YSWFz80S0A9LbK5x0fKenHvI0YHjqED3vBmWlpZFREZHRsetP3rNwimDSc4CCATcY2BRSDQThAAFDvhhaKCoCad3YbHZHR0dXVxf0SoKJ2/h8fkdHR0tLi/i/FPgvwjO48fl8iCsgYCC8kiorKwsLC4ODgytxXCu+7/Me7fT+L3EEwETExdHx7ob3B2ShS5KY8AB4E4GJ/qAsKW4byWDLN9fI8kFhUZzGZpB1dhLW1s1NTqOaG+lr7Vzovfmd9qNDrkrjI5NzOL9xkZQjRdI4ubOucRxXkJ6enuTkZFNTUyUlJbJYyueJkIMKboSGwcPDw/O5DVbi5eXl7e3t5eXl7Oxsg5s1bjY2NteuXTt06JC0tPSB/fv37dunpaU1pXYfPXrk7e0NMUMMbjBmMaQa5OXld+MmKyurpqZmaGhoZWXl7u4OD8t32AICAuzt7U+cOHHw4MH9+/fv3LlDUlJy69atZmZmkFKg0+m5ublvE1DIyYEJXiIjI83MzGRlZY8ePWphYZGTk8NisQYHgWL+Jd12Ojs6/Hx9d+7cOW/evA0bNnh6eMDsDURzRGESv4nn3QREServQWX/Qv+Yg2odRla/i5z+E+vtet56p/cfHoHedrfBPqBOJnLQD38z6h1F0drOrigW2yyDKhEX/01UNIiaGhM7KzpmVmTU0JKQOCs7Z60IAU4gU7IuBLmYmQ7UzyCP2+OJ7P66+gt19W3CSd6Dp9Tm1DYOCGgYCxg++zyKXf72AQYhitq2th1obHoMz6o5QMAQ6An9keCrdKifXmriG+7513j5cruZGXEuO+7c4ezeXbV6NS8nR9TUVLV6tYDDYS9YwA0KAhc5ny9saEDx+2eniwv7hx8woRDl8SpXr263smq5erUvKYmoChY67Owqf/tt1Mpn/jgNGJ556Ca7I5vFiogIi4qJ2XHO/mFAHD07OzNrKDgShULJHGlUKhUChgLcCgsLoYChsLCwvLy8o6Ojs7OTAAw8Hk8gEIhEIxlwHC1ASTSfz+fxeAMDAwRgaG9vb2pqqq6uLiwsZDAYYzXTkz2q6e1e+QjAf0QRijnet7OUeH/g9HDWBUgyyAMBdMYuksGmb28b6RWUlAlERNDGEY88J+442AxFser6BhdHB4P9f8bu/b9BSCaM5TFwwAAEDxdIsdtIV+WPd/U+/uMh5mQoipaXl3t5eenq6iorK49VOY8FDzBxm+eLMA8Pj0ePHnl4eLi6ulpYWBgaGJibm98baRYWFnJn5KSkpPbJALt48eKU0rc9evTI1tbWz88vNjY2RsxiY2Pj4+PDw8PNzc0PHDiwbt26v/76a926ddu3bz9w4MDhw4ePHDly8uTJU6dOycnJnT179sSJE3v27JGSktqzZ/f27du3bdu2a9cuBwcHBoNBp9OTEhMD/P2pVOpbgRkgJZKWlvbw4UNVVdVz585pa2s7OzvDyGxRUVGFhYWClxNykbjw+Hx+RkaGnKwsmUwm3OSIK5/YjFjzkgrANRTDRDnxyJm16MaPULk/RVkJb45y9CUd9ausltti3sL+taPmrJBXMvl2BwZqmprDWGyjLNrupORFsXFfRcd8nJj0r8Ki43z+FJyRxFuk9fTIRIdKxYTLVFaJ+9YfaGg0a24Wim/6Osp+fvVjAcOnn0WxWMAt/q0zPoratbXvr60DQ11dLVNTK5WZLk4vgEBJwT6ykcGdvAml52/EUSMIQS9gGNZuaVlNIjWrqWEYNlhaWr1xY6O8fO2RIxiG8fLzq9asqfrjj9rduxsvXarbtat6/XqwPje3dNasGmnpVmXlntBQpL+/w96+1dBwgAYyUdTs3j1uyKZnO/ZpwPBs4zaFvTgcTkRERFxc3N7Ljl5BsYzs7MxMKszAkJ6eDsMaDqdhyMrMzMzIyGAymQW4QWck+ApjqoozDIODgwKBQCjEFXQ4yQ25BSKAEgEYYEIGIuszBAxQ6zyFI5ne9LWOAIIheOB24PFgb3PPdtNsnhzOMOByhdbjJJuN7xmR5Rn5hcPzIZDfDV+e3m+QuRn4SAwZjy/wDw5bPP/7c9/NqDxIQi+IURk4eEDPk1BCQSFPcls3w9ryNszPANHJcB9AhXw+v7i42NHR8cqVKzD381icIL7GxMSEeAzvOXXzwO3Ro0d+fn4REREMBsPe3l5SUnLLli1SUlJaWlrW1tZWVlb37t2ztLS0srIik8l79+6VkpKSlpY+ePDg7du3J5+NwcvLywm3yMhIMbwwVIyLi4uNjfXy8jI0NJSTk9u+ffuaNWv++uuvNWvWrBOzDRs2SGyS2LZt644d26Ft27Zt7969vr6+8FZw586dNWvWxMXFveGiZwIqeHl5GRsba2pqWlhYeHp6hoeHx8XFxcTEREdHR0VFxcTEsNks4XCc8qFr7iW8iUQiIr1MX1+fhYVFSUmJ+JX5EtocUSWe0w1gBqSnG60uRXq6Hv/GRmw4/eEpI4CIOgd7KaOxFspr5xzpKCN1N11DkHG8Ip9SKf61UNTX18fu5jIHeVWT2X6ibcBtuapSJiJQipomTjLIVFXvr28IHBmbZKJKXt56H9/xAUNJSc/La/Sl1ixE0Xtt7SCsakUFoBeAPxIeUDXQSyrYB7A9Yf4W2Rn88RJivNSOPU/l/MrKjps3hY0gAiFAAh9+yP7xR0F1NYaiNbt2cbZsEdTU8MvLub6+7LlzW42NAca4e5c9b56wFgTnRRGk9sCB8u+/b1FTq9mxo8PWlvXvf0N24nl6Rew7DRiIoXhZhba2tsjIyPj4uJ3nbS1cIpg5OVk4w0Cj0aCGISsrC1INNBotE7fc3NwC3AiGobi4mHBJgrpnKGPg8/mQZBANm1AoFOBGoIX+/n4YKKmrq4tgGIqKit7AnKkv6xz819UrRDHbe3esN7zHlwMJnpl7SBrbF3r7+vEGnysohEgkKi4uuXv37qZNm7766qv16zeqa+uSpf+K3zEDBakeRsMGsAanIARyJKN1H8XGxwPcOkE6qs7Ozvj4eDMzMzJuE7ENKioqenp6bm5uk9QSeAx7LkE+AfoIRUZGZmZmVlVV9fSAv0MOh3PhwgU4TT9y5Ii5uTkEDPfu3bOysjI2Nj5w4MDu3bv27t27Z88ebW3tSTbtidujR4+sra0hyQBDrI5CDvHx8XFxseD++wAAIABJREFUcWFhYXZ2dpcuXdy1a9fGjRvWrl27bt269evXb8Rtw4YNGzdu3LJlCwEY9u3bd/v2bX9//8TExPCICHd398zMTEg4EDxDNm7Ex9dSoOPGwC0tLc3Hx8fCwsLU1NTS0jIgIABCJggVoNIDYobY2Ni6urpXOXdva2s7duxYenr6uI2Ou/K/7rbxFh8Qvy+zLv8D/kAehmEiYUt/p0tfxyMM5fEHmC3l29o5x9+EY+tGkEtZ6SBiUmHhY2+Zqup9nJqjNbX5A69TXuztMz5gKCp+WwEDhmE3a2tl0pKkIoL3BnoPowXvQ2EB52MjDKipqfW1womjVL0JF8yT+yBsaqo/cqTT1RVIoQYGWD/80H7vHtyln0IpmTMHBm+t3rSp4fJluL43LKxk1ixebi7Ypaur6q+/yj7/XNTa+uSGJv/tNGCY/Fg945Y8Hi8pKTEuNma/kqO8vjuTTs+igQTP2dnZWVkgviqNRktMTExPT4drEhMTGQxGAW4QMBTjVlZW1tra2tXVBQFDX18fj8fj4wZ5BiFuUOgsrl7o7e3t6enp7u7u6OhobW2F6dtKS0uJZ2/PeGDTu72uEQAcAyZAEJu7t102vxu3haR6cDOzEES7QsBkfZIOSKN7z2KxDhw48P333y9ZskRFRSUlOZnL7cYwrLquTlvxjOfGvwnPjgcYIIq4MKP2IElj/7oGEDFznA7ACRmCIA0NDYGBgVpaWsrKyuPqoZ8BMDzCzcvLKygoKCMjg8PhdHV1jYqTExYWtnnz5rVr127atEldXd3KysoSN+igdOHChZ07d+zevWvXrp1nzpxxdXWdPGbw8vJ68OCBr6/vKJww6iN0UgoLC3NzczM1NT179qyUlJSEhMR63ADJICEhKSkJAcOOHTvOnz+vr69vYGBgZmZmZ2cXGBgYFRWVkpJCo9HgHP1NiJgEe5KdnZ2cnAwDPVlYWDx8+FCcbyFwgnghKiqKQqG0DsdXfTXzdZi+BrYVHR1948YNFosFFQ6jfwzTn9+wEUBRQTN7Q0fNOcEgq5m9rrViY13+nK76K+BBbG9KXf4ciCVee69zenv3xYRJRYfJVFSKOybtr6u/XF/fMqxtffX9nAgwFBZxX31nXkiLHKHwUFoKCIuEEwvQB+lSXERlb0/P6xvnF3JoYytBUbTVyIj9ww9tBgat165V/vYbe/58IGlobCz77LPe2Fi4S92hQxxJSWJ3zvbtnO3bR/NyxNdTL0wDhqmP2dT3YDAY0VHhioYPVx+9F5eYlpNDo9Fo8HEgDJVIoVBSU1Ph88KoqCgajQahAgEYSnFrbm6GLkm9vb19fX0wuOrg4CCEDcTrIG48Hq+/vx86I3G5XEgvQMBQWVlZXl4+/U859TP5ZuyBAuckDMOECGp501jp3PHKmlrcpwhk9Jl8F1EUbWtrg8/gMQxjs9lqamr+fn7ivmpw7t/d02egqeay8e8imFh6FNUAyQd5UthmkuXN69Ax6QndGBwcZDKZ9vb2V69eHStsmCRgIPgEDw+P4OBgCoVSUlLS0dEhnOCvoq2tTU1Nbc2aNWvXrj18+LA4yWBjY6Orq7tnz+4dO3bs3LkdPtqfPGDwwCkOPz8/8VnyKLQAmQeoh05MTITyBnd3dzMzM0VFhYMHD27HVc4QLUhKSh48eFBbW9vAwEBfX19PT09fX98IN3Nzc2dn58DAwLi4uLS0tBxcW/yK8QP0O2IwGNnZ2RQKJSwszMnJycbGxsnJydvbOyIiAso5xOHBuOXIyMjU1NTOzs4nXCov76vAwMCFCxd+991358+fT09Phzrsl9fcdM3PPwI8bmR94bx2ztEBbhSITN8dWpv3rmCgAEORlvKN7ZzTz9/EC6nBhVMlHR6AR0x6rH6Wqao+0NBo1NQ8+JqeeU/kklT01gIGj9ZWmbgIAi2AsEhh/vdzc17ISXwzK+lLSem0tOyJjOx68KD0k094DEZPUBDriy+EOIeA8PkAUdy6BTuPDg6WL1jQZmHxAo9lGjC8wMGcsKrm5uaoyHDDO54LZBxNrEJyGTnZ2fAxIQOGTaTRaBQKBfobxMbGUiiU4mErKSmBaKGsrKyurg4yDOK6Zx6PBxEC8crDbWBgoK+vDzojdXd3d3Z2tra2trS01NfXs9nsRtxJbsIeT3/xloyAQCjiC54EEqhUqqen51jdJ9BUDQ6eOHHCxsZm1MP4cQ99YJBvpK3pIfEOzAIxjnuSPMgEp73+S0ZewbCgZkKqAcOw3t5eCoVibGysjBvhoQQBw4MHDyaaskM+wdPTMyAgANJxra2tk0G/+fn5x48fX7t27caNGy9evHj37l1IL1hbW5ubmx85cmTr1q07gIxA8qq6uuewQTww/GnCdy8vr5CQkFE4YaKPkG2AyCEoKMjBwUFTQwMKGCQlJXfu3KmgoHDt2jUD3AwNDdXV1aF0WE9PT1dX18DAAPr8uLq6+vn5hYeHJycnwwzxcDbPZDKhjxDEEuKuSk/O5Sz+LcFgEAiByWTS6XQqlZqUlBQQEODs7HwLN0dHx+DgYBhVdlxsMO7KqKioyKiotLS0zo4O7HVYR0fHgwcPNm/ePG/evB07dvj4+LS/MVHzX8d4vLlt9rY58fvpGCZsZv3ZWLpwuKNIM3tDO+cUDh4i6vLf5/NyRYIaDOUPb/B63nkYpkGnyYT5S+cyR4kZDjQ0urZ3jHNbfPk99Z1Aw/CWAoY+DLtYmC8V4jMiLFKIb07TM2rWX/4ZeJEtoAJBT3Bwf0ZG6/Xr9ceOwapFXG7Zl19y/fzgxz4KpWT2bF5BwQtseBowvMDBnLAqkUiUlZVh4+yz6oT7pjNukXHJdDoNj64OntJlZ2fDf2ImbmlpabGxscXFxQROKCsrY+FWVVXV3g5SMfTgBkmGgYEBnpjBmEiQWyDQAkEvNDU11dTU/OdZ8quJUD7hiEx/8UpGoKysbPHixcrKyvC5O4/Hy87ODg8PJ56nUigUNpsNnTSe0CM81gvW3duvo3w+bsfM8QOtghQQM9K2k25oqkxm+o5HZEKrq6v9/PwMDQ3JuF25cmUiwAApBQ8Pj6CgoNjY2Pz8/Pr6+sHBwad2njguBEGioqJ27NixZs2anTt3mpiY2NjYQMxw7949FRUVye2SW7du27Zt26lTp5ydnSdCLJ7jmYeHh4+PT3h4+EQggVgPH8BHREQEBgZ6eHg4OTnZ29uTycoEvXD48GEdHR1DQ0PokmRkZHTkyJEvvvhCWVkZrtTDTRc3fX19Q0PDmzdvWllZOTk5eXl5hYaGQkCSnp5Oo9EIDEBgCQIAQFABn1MQZQg2CF8jGo1GpVIhJeLt7e3k5HTr1i3YDSsrKy8vr7CwMHho46KCp66MiIig0Wi81xfJhMcbSEpKOn369L/+9a9ff/31Pyn8qqqqJn9REVfXdOF5RwAVdNYp83riiXqEvNKuBvWuhqsNRd+2VYNZUX+Xf13+B8LBCrjNQHdEfcEHwkE2igpbK3a2lH3SUr4ZEb0wj22iJ1MtVPN4J5NipSKCZMrKxMUMMtWcfTW10dzX4AXk7z9OWNVPP4t6SwFDen+/NCVRPCySdIjPhZiwvpcTfm2qF8Ar2x7p6xMRJC2KcrZvr925U1BVxS8v52zYULF0KYzB+qL6Mw0YXtRITlgP/O9pbWkJDgndfcl15emQs7ru2VlZ9Jzs7Gwag8HIyMhgMBi5ublMJhO+pqen5+fns8SMzWazWKzy8vKGhoaurq7u7m4ul9vT0wN9k/r7+wmcACXOvbhxuVzILXR0dLS1tTU3N9fX11dVVdXV1U3mofKEhzT9xdswAh0dHVBE29bWVlFRYWlpKSkp+fHHHx87dmyU3v2p0yME5KMCPEZNU6vK/k2l+2eMjZuEcw4z0DMkg/XzGLn5wyTD00dKKBSWlZXZ2NioqqoqKyuTyWQ9PT2CYSBcj3x8fCIjI2k0WmNjI4/He2qfx224v7/f3Nx8/fr1a9askZeXJwCDtbX1zZs3Dxw4AJXHkpKS+vr6UwIMnrj5+vpCn5zYCSwqKiooKOjhw4fOzs62trbW1tY2NjbXr1/fv3//tm3bJCUld+zYoaioaIibAW7GxsaHDh2aO3eukpISnKnrDxskHHRw09bWhgUDPM+0sbGxhYXF/fv3XVxc3N3diZhRUVFRCQkJycnJKWMsOTk5ISEhMjIyLCzMy8vLzc3NxcXF3t7+1q1bOjo6V69ehZoTY2Pj+/fv+/v7R0dHQ/ADdcxPxQZjN4iPj6dQKFlZWd3drzo1wdjrp6yszMDA4McffySTyZNEvONeY9Mrn20EUFTY02Ih4BXC3QUDuQ3FP/S0XOf1xLdVStflzxIMlqEov7F0SVej1nATwuay39trzgGJp7CVxw0R8kGsmDfBEtrbZCKDpRJixJUMMlXV+2pqj9bWMl+5ADowcHzAUPwWip4RDDOuqZGJChH3R5IJD/AoApr4/2UTcDh1R47Ubd1ad+IE++uvGxUUXuxoTAOGFzue49QG/5lQFM3LpauaPFx9NuL30/56FgF0Rk4mlZoNoqxmZmVlwgd7EDPk5eXl5uaWlpayxay8vLyiogKSDF24QcwAYQMOEMAL/AhVzl1dXR0dHe3t7RAtNDY21tbWVlRUQLnz2L/McXo/vertHAEejycnJ/f5558bGBgcO3bsu+++W7x4sZKSYlJSUndX17OeesClUzIyDbf8axBEdB0dNwmVB2kZ0neSzHTUBYhoSsx7R0dHcnKyhYXFf/gQHR0dV1dXCBW8vb3Dw8Ozs7M5HE5PT8/zA92mpiZNTc01a9Zs3bpVXV2dYBisrKyUlJQ2bdokISEhsUni+PFj8IG951TMy8srODg4IuJxOufw8PCIiIjIyMjQ0FBfX193d3c7OztrMbOxsVFSUhpLLxgYGBgaGhoYGBgZGR09enTu3LkKCgpjAQNkG+Ar5BwI/KAlZpqamjo6OpCOMDExMTU1vXHjxnXcTIfNxMTE0NBQW1tbS0tLTU1NBTcymaykpASBnLGx8aNHjwipxlgA8IQ14oGSYmNjk5KSUlNTMzMzc3Bjs9mvkWQgfuIoijY2NhLhm7hcbkZGBiHyITabLrzAERAJ6gQDhSgCcvmBWA68IuEgCG/aVa/WXL4JrkSRvhbWyo7aCxiG9XW41Rd+LhI2wa8GugLaqmTQZ42pCit5Ga8ohtlWsEGU1YzUUY5J++vqz9XVV+Hpt15G0+PWGRj03wMYKgTCA1kZ4s5IUkFeh8P8a/AoHeMe/v/USmF7O8rjdT182J+W9mIPfBowvNjxfFJtra0t/sHRG+UebCBnrDrldsM2MJNKTUvPyKLRkpMp2dk5jBx6NoNOZzAZuAdyQUEBW8wgYCgvL6+rq4MZ3CDPQLAN0E+JO2xdXV2dnZ0QLbS0tDQ1NUF6YdpP90kn6b/iO5FIZGhoOHPmzHfeeeebb745ffp0WFjYC1KXAhRgbX7Td9NMBKSKm/k4G8N5EgAM50n9p0kam78rr+ZMEjAQ6AVF0aamJi8vL1NT0wcPHoSGhqam/n/23gOurfve+yd97n3+dzxNb9okTpr2dt3btL1dcZukbWI7cRzHbuwaAbaTeMQxS9jsaWw2BswwYPbeAgQSQ+y9p9ggiam9EZIQGyHpf3V+5kQGjJkGzPm+sHx09vn8juC89V31Y2Njk5OT2+cEzYH9X/fdl19++Ze//EVbW/v+/fuhoaHBwcGhoaF+fv5Xr14BpU5PfXrKyclpC06GjIyMpKSkkJCQIMhA5dbQ0NDw8PCQkBDQAgLmhfDw8AcPHly8ePEMZJ9//jlwLwBacHFxuXfvnqur62pggPOhVwMDwAYnJydADvfu3QMM4OjoeAcyBwcHe8jsILNdNhsbG2sNA2+tILO0tLx9+7anpyfoqLAOGKxeBNpgg/nl5eUg3UIzUIpIVDtbh4eH4Ug5zcHak2lwT/b09Bw/fhy0boDvUvh8eDwekgkGq7GFifnpejHzC/7g2zzyj4QjJ5YWGCqVapx6UUS/rFKppDxX3uC7quUazTMSLKvvP+QLNKVimkP+bynXBRxRHS2p3FYt6S2c+QY3mVGq7nZ36BTgUF0dK5mBzbHgcIRPqdCwwf1varXcPO6ajdtIpD2Ij9rUma9eOUYg1Kks1oxH0iFk32+qXb0mMmdnFUCAYWf1XG9v8/Pz7a3Ntt4Z7+njPrFu/eu1dBuftPLy0np1eEBNZVVlc3NLU0tzZ3trG1EdedzR0UEikYaGhoaHh0c0bHR0lMvlAmYArgbpKgO0ACKRBAIBCEai0WgCgWD1X771ThpZdgAV6CASjx49+k//9E9aWlp//OMfbW1tk5KSNpKr8MxrVUIlU9l8oQ3qQ84XajxQN3uGiyaBVm4mWpkfaWESYjb1lA/flnNzc4ODgxQKRSKR7HgncnAUhUJRXl6ura39wQcfXPnqqwe+vqGhoY8ePQoNDXVxcTl79uwJtR2/cuWrjbeDwGhYSkoKCDTy0LD79+/7+voGBgaCAwFmCA8Pd3R0PHfuHAAGOHvBDTIXFxcbyHR1db///e+j0WhnZ+d79+6BukmaqKDpYVgTGGBmAF4HBweHO3fuOEAG4MHe3t7Ozg6wAzgoYAcrKysLCwszMzNjY2MjIyMfH5/8/HxNd8FqQlg9p7S0tKqqqr6+vqmpCUYFooa1Q0aj0dbMzn/mbblLKywsLDCZTPiU2tvbm5ub4Wil4ODg48ePs9nsXTr6i7pbpXJhXlYlpl8YH/3dJM95cW5gcbabR/kVSF+en6pk9nxvcY6yMENkdv/7/EwL0GFpgc7s1pJwHNT1EkQxEwwjdRHpfW/shQWj+mpUIV6HRNKhryya5MTlTT6vtmJ5+S8IMAgUiq+7OrRz0r/1MODTdfOz2rjIJ3HXPw8IMOy6xPABlErlQH9PUXHFP4zjj9+q+cS26d0beN3b8SHxGcVlpRUVJeWlZd6hWFxhVXNTQ2NjIyh7QiaThyEDyADzA5vNFovFMDaInzQ4EkkgEHC5XA6Hw2AwBALBpp7h4DNHJg6WArOzszQarbW1NTEx0dLS8rPPTv/yl7+8c+cOeNaBH823c1FpqWnJp/5FARq3wcAAJoy1eF9oOenrymbmtnOI3d62tLQUMAMajQ4ODn60bPr6+hAwnPj000//N6gds2xpUPnU5Xfr/Z+enp6amhoZGekL2YMHD3x8fEAgkLe3N8AG4G0ICwszNTU9e/YsKI5kZmYGiiO5uLi4u7uHh4eHhYUFBARcv379lVde0dfXd3BwsLa2trOzc3BwcHR0hNkA+BPgt7BvAbgX7i4boIU7ywa7GgAtrAYGS0tLCwsLU1NTNBptBJmxsXFgYCDI7V4NBppzgGOhrKyssrKyoaGhtbVVAxDWnuzo6IDTq3bkLlXtqDk6Or711lsoFCo/P392dpbJZB49etTAwAAQxT484R29+h3b2YwYx+3/PzL+vSW5AN6pTBjCJf1SqZhTKhX84RMihoFKpRKMnBKMnAZOhtnJEv7QBxzST5cWWCo1KhwAWgBX1yWTXS4rVHdmGB5+IgGaSrvI5XkLBDPPpdAqoYC3pofhwHV6zpiY0Kkp08xeQOVhratKFp6LjPAdezgnEGB4ruM+OjpaV1MZn5b34fWE4+b1p2xaPrxd+uH1xK+soh5GYvDZWd/YRX5uFJaNL6isLK+sLK+GGroNDAyA71wHIaNANjQ0RKfTRSLRxLKBafgVFFHl8XgsyCRbj1x/rhIhB9txBRYWFkDLvx3cs0Asu6t7THRNw70AY4OxluIbrfuf/XRgcGh/PkWBs1IqlWVlZefPnz916tSdO3dgJ8P9+/d1dHROnDhx7NixS5cugUyGNMgwGzCwJmAGEHHk6+sbEBDg7+8PXn0g8/f3DwoKCg4ONjAwAOnOwL0AEp1dXFx8fX2zsrLy8/MLCwudnZ2PHDni4eERExMTHBx8//59F6g/g5OT0507d2xtbQE/3Fk2GBhgxwJYAnkU1C/Aq2BnZ2dvbw/ewrRga2trY2NjaWlpZ2fn5ubm5+fn4eEBgMHY2NjQ0NDU1DQ+Pr68vFwz1mg1KpSXl9fW1jY3N28EFYhEYkdHR3t7e2dn57hQuD9vm8nJycLCwsuXL7/x5hvvv/9+dHR0UlLSG2++ERsbq1Ao9uc5q/afLcm57P43ZYIgjVNTiujXxqm6j9lASmD1viJfoC3O9nH6X51gXJLx3QSjny3MdHDIv5oaj9XY8GBMFvB5OsW52pUlOmPU1cwQJBQuKDcYv7n16y0ofBGAQaJUGvT1audiv3Uv4DCo/Kxy2uPCWVsXCNlyAwogwLABkXZuFR6PV1FRUV1d9TAy66/XEk+Y13xqS/zEquVvJuV/+TrttGH0xzdC37uC+Qc6Ji4xnZCTlZuTn5eXV1ZW1tHRMTAw0NfX19vbq/lKoVAYDAaMDcDnMD4+DrdcYLFYAoFgPyQU7pyKyJ42qsCuPsTEhgUXnvk/SuO12j+baCWd0MrNwSl2/w/hRrXQWA+WRalUlpaW/uMf/zh9+vRdR8cQyEJDQ+3s7E6dOnX8+ImTJz92cnLKyMjAbNjSNBwRqampUVFRAQEBPj4+fn5+gYGBQUFBAQEBXl5e3t7ewPNw9eqV06dPf/7530FOMwAGNze3mJiYPMiKi4vv37//xptvhoaGFhYW5uXl4XA4LBabmZkJmkxHRUXFxMRER0eD9tVBQUHh4eERyxYZGRkcHOwGGdg5zCQuLi6Ojo42NjZWVlZmZma3NczExMTOzi4pKSkrK8vd3R12LxgZGRkaGtpBfSrKyso0OQGehlFhzeijtZ0L0Fx12bi2tp6enh3Kt1Hthi3J5T09PfYO9u+888ff//53r7766ltvvdXe3q6EbDeO+OLtU8p15fT/SLGkjp6fn26coJ1n9ryyOEcGV6pULvCH3hMzzaGlLROMq1K25eLckEql4g8fk/K9DpwgSpUqnkbVKcSj6qp0aDT1D3X5h0a/yOWFjY8v7jIyFBXxXwAPQ5ZEqlNXqZm9oH0oq6nu1UcAAYbnqvzk5GR1dXVlRUV1VWlwdNZHX8d8iC47ZdvyqU3LKZuWkxbNx2+Vn7So/Su66PTNcG+/kIjw0Kio6Li4uNTU1IqKCpAd2Nra2tzc3ARZfX19XV1dc3NzT0/P8PAwjUZjMBg0Gm1kZAS4IGQyGfx49FwvFTnYi65AVz/F78yPl/TXAga0Vts5rSAXW4VCXYx1P9+BSqWyvLz88uXLZ86cuXv3bmhoKMhXNjAwOHny5PHjx1EoVHBw8KaYAaNh6VB9Un9/f09PT29vbz/I3CG7f/++s7Oznp7uZ5+d/uKLy6D3AqiD5Ovri8Vi8/PV3xcUFxd7eXn94NVXgx89IhAIeXl5+fn5BAIBvIIJeA7YBJAGeM3Pz8fj8RpnpJ5Mgyw9PT0qKsrc3NzAwEBfX/8mZPr6+sbGxiYmJiA+6uHDh7dv3zYyMjLWMENDQ3t7+4yMDM1khtLS0srKSuBV2CwqwBTRDtk+Z4aZmZmhoaGkpKSTJ0++9NJLWlpaH374IYdzKDpGqXbClhbZrL7XxSxTEe0Sb/B3YpYBj/LrCdrfoe5s6gPMiDNYfT+Qz9PV9VKXZAsz7YqlCZkwgkP+5eLcwE6cwvPex6JK5UseUDNDc8MKJ4MOjX6Rw4sYFy3u5tcrxcVrAwOFMvW8tdjq8SYUCv2BflQe9gIuDfYw6BRk44ceo+ZWd4xst1EFEGDYqFI7st7CwkJDQ0NFRUV5eVlVZVlSeq6eafyfb2BPWNR+atN6yqb5lHXzp9Ytp2xbPjQp/ehL/68NjE3QaHNzc2tr63v37oWEhODx+MrKyoqKipKSkoKCgtzcXCwWm5aWlpCQEB0dHRUVhcFgSktLBwYGkBikHRkyZCdPU2BqdsHlxj94X72kWu1kMNbif6HldO3vk1Aaw34GBsAzvb29dnZ2f//73+/cuRMC1TLy8/O7cuXK8ePHjx07ZmxsnJKSkp6ejtmkpUHeBsAMoaGhIJPB09MTZC27u7tbW1ufP38e9F6AsxdcXV0TExOLiorAEz+BQIiIiLh8+XJ8fDwABk0e2OB0/iojQJaXl3f//n0DA4ObN2/qQ2ZgYIBGo83MzMzNzU1MTNBotAYpPJ4EfgYnJycsFlsOWXV1NegTBz/6b3kCMMPAwMDkXvS3etrdrjnf3Nz8rbfeevvtt995551jx47p6emZmpp2d3fv8/tc8xL2fFrMdhANakm5zkuL6gKp8gXaBOMKs+e7Eo6VfIGlVExPi2IVchHUxbmIR/kZj/Lr8THUwkzbnp/5lk9gWql07unUKcSh2ltXFE3SodH1ONyI8fHdi00qLRWs6WE4QMCQPDGhW1d1ISsVpgXt3MybxXni+X2dLLflG2YfbogAw3MdFIVC0dHRUV5eDjFDRWVlaWFJkYt/6lnj6L9cz/nApPykZf0p6+bTNk3HzGrev+h39pzexx+d+CtkJ0+ePHv2rLa2tpGRkYuLC4iB9vf3f/DggYeHh7Ozs5ubW2lpKZ1O3z/VCZ+ruMjBnrsCwX4+VZ9/Z+2oJH0tl09/PkZj7bKnfceumc/j+fj4XLp06e7du8HBwZGRkR4eHp9//vmxY8fOnDnj6+u7BWDALBvYNiEhISwszMvLCyQfOzs7m5iYnDnzGVwcydXV1dnZOSQkpK6urqmpqbq6uqKioqioqKBA3cwhLy8vF7INQsJGVsvPz09PT7e1tf0GMgMDAyMjIzQaDYojwakLa2KDoaGhm5tbenp6TU3Nll0KT+OKtrY2CoUCOsbs2Bjv0I66uroqKyt7enqYTKZUKoXLKO3Q7g/FbuTzo6ze/5gRZ2he7YwEN0H/dEZC0JypUqmW5IKlRZYKKtG2YtHBejsul1u2t+gU4FGdxBVFk9R+Bi4vRDg+vzt+hrLypwGD7EBoyJCAFFTNAAAgAElEQVTLr6izFzI10511CrIzyY87/R2IqzjoJ4kAw/MeQQqFUlFRUVVVVVlVWVlVWV1RVVNTmV9Q7BmcesUy6uMbMceMiz40rTz2VdA3+ugrl788DwVYgxTMDz744G+Qffjhh8ePH//ss8/OnTuno6Nz6dKlK1eumJqa0mi0fR4B8rzlRo63OwqAP2qlZeXRH/2zwlhLZQj9wHnP0ETgR99taVXHdh+Ue1Imk+Xm5qLRaDs7u7CwsLi4OFdXV8AMX3zxRXh4+JYDkzCQAWyIi4vz8vJycXFxcnK6evXq559/bmpqCrsX3NzccnJyuiDr7OwkEonNzc2gZlplZWUpZAUFBSAwCUYIABLw6zqoAK8DJvLy8ggEQkhICIAENze3gICAmJiYxMRET0/PFZFIwL8AzwSJDTY2NmlpaSBr+WlP/1uYDwhkbGwMeRzfnU/w3u91gmnMo/xOqZh98lQOyjcMT571ht8xFhaMG2pQBThUd9eazPBQOD61CwV/yisOMDAoVSp/vkCdvaDhXkDlZhqU5IvnVtw/Gx4JZMXNK4AAw+Y12+oW4MmJxWJBec/VNZDV1tbU1tbWVFeXlRbn5mS7+0f/6VL8ia9DzO2dbawsjIyMrl+/fvnyZW1t7c8///z06dMgrvrYsWMnTpw4efLk6dOnz507p6ure+XKFWNj48bGxq2eHbIdosCmFFD/Xe8fHHX76HUFaL8AXmFmMNZKOfXPefn5BytOQ6FQDAwMBAYGWlpaenh4REVFOTk5nT175sMPP7xx48bW2jJgnrTExESQJuHt7f3F5ctXrlxxdnZ2hQxUU42Pj8/PzwfJAB0dHZ2QgfwlkBZcX19fo27eUltdXV1cXFz4pBEIhBVUoPmWQCBorl5SUlJTU1NaWorBYIqLixsaGlpaWmpra3Nzc52cnAwNDVfHI8FzwFJzc3MXF5eioiKAN1tgg6dtApiBwWAsPcf+VirEnpcCi3MkRvf/nZ5Ie14H3C/HGZ2bN6ivUjNDz2pmUNda9eDzRTt9z1dVj68ZkjQ4eAByGBpnZvX6+7Tx6ZruBRQhizAyuF8G9XCcBwIMz3ucJyYmqqqqapetsaFR/Q1iQ2NFRQUhL/dhZOKZb0JdvB4+8Lp/756zhZW5kZHRtWvXLl68qK2tff78+TNnzny2bGfPnj1//ryuru5XX3118+ZNExOT7Ozs5309yPEOqQJqYGDwhC6od2ZvQGkMGsCghLCBcForJV5dcfLAKSSVSvPz8x0dHS0tLb29ve3t7c+cOfPJJ5/cu3cPg8FsJzYpLS0tKioK9G578ODB1atXLSws3NzcADCAV2dnZ3d3dz8/v5CQEAwGU1hYmJycrKenl5uTCzkeujo7OzsgIxKJLausvr4epDlVPGlgZn19/YotgHMA7Lm7u5tIJJaXl4eGht66dQt2JsCQAE+AyCVzc3MQvOTh4VFSUtLR0fG0p/+tzQf9KxkMhnynn58O3D35Qp6wTBg4J6t8IS9t/YsanJ29WVcJMUP3Sj8DlabH5TlweZzFnexgXVe3BjAceaN4aHi/A4NUobjNZKGqSp9wL+RjLatKZ5FfC+vfZzu9FAGGnVb0WfubnZ1taGioqampr68H/Yza29tbW1tramoIhXnJKdkhkTFR4RG+/j6eHp5379yxsLQwNja+cePG1atXL1++rKuri4JMW1tbV1f30qVLX3755Y0bNwwNDc3MzMLDw5G/rM8aAWT5FhRQQgHEIBBJqVIpoJbPSrFs2u36p4Ira+c9V5/Rig72O4jAAATicrlYLNbCwsLe3t7MzOws1F7Nw8NjO8CQkpISHh4eHBwcGhrq5uZmZGQEuxcALQB4cHZ2Br3YADyAMq/e3t7p6ekFBQWVlZVwhjFwQXR1dXV3dwNfBMAJMK35BA+m4XXARFeXGj/AJiD8qby8PCkpyd7efn33gpGRkampKSjJCpjB29u7srJS84hbgwTNrdqXjcPhHNwbaQufN2STF16B/pnpr2sr1vYzUGl6HO5tNocyt2PpvE1NE6s9DEfeKB4e2e/AECaa0OvrgXwLj4sjaeMxuvlZrUhr5+f+IUGA4flJDmIzlpaWiERiTU1NQ0NDU1MTkUgE3+rV19cXFRVBBRDTYmJigoKDHjx44Obm5ujoaGVlZWpqamxsrK+vf/369atXr16B7OrVq9evX79586axsbGZmZmtra2Pjw8oYX6w4kCe3xggR9qiAmsHFs8tKl0NdGmXX1Kinyiuqk6DRms1nNWKCvR+Wk+rA3GLKhWKnp6ewMBAMzOzr7766uTHH589e9bd3X3Lfob4+HjQUTokJMTV1dXW1hZwwopXFw0DJZVcXV3v3bvn5OTk7u7u5eX18OHDiIiIxMTEtLS0zMxMAoEAuh/UQ9bU1AQqL4NWaMAdATqjrajIXFVVVVRUlJmZmZSUFBcX5+fn5+DgYG5uDrKc18x1BjNv3bplaWlpvWxWVlaWlpYPHz6srq7ejdikzs5OPp8P7pkDceds8XOGbHaYFOiZnv6mFvIzdHWu4Wdgc75hcxqnpndEktZW8ZrAMDK6M/vfkZNcvZO6mZmLVKp2AV6z94IOIduntWHtv0mrd4HM2TkFEGDYOS03vCcymVxTU9MMWXt7e29vb1dXV1NTU1lZWW5ubkZGRmJiYiTUbsnHxwcwg62trYWFxe3bt0GhQ1DWEAQM3L5929zc3NbW9t69e+7u7iMjIwclx3TDgiEr7rECExOittaW5ta2PjJlhEofHqOO0ll0Lp9CY1t8+bkaGJ6srAqAof6sVqC32wvwxfD8/HxbW9uDBw90dXX/+te/njt3ztfXF7N5S0tLi46ODl42Ly8vN8hcXV1dXFwAM2iQwreTgBmA2wF4Hu5Cdu/ePdDL2dnZ2cPDA7SUDggICAoKCoSaxIFuxImQJSUlxcbGBgYG+kPm4+Pj7e0Nfr1YW1tbQHb79m1DyMDvljVDkkCus7m5uY2NjbWGAWaIjo5ua2vbWT8DkUhsa2vr7e2VSqV7/ElADo8osKMKUGZmDeuqUAXZqI62lbVWqTRdJusyk4WVSLYf1kkkil8/Uvb6kWKNn5I3f1gyuo+Bgb6wcJPF1mmsvZCV8m0p1ZyMrwpw9EnkV8GO3ogb2xkCDBvTaUfXYjAYNTU1jY2Nra2twMMAgKGioqKgoCArKys1NTU2NjYsLCwgIAD8Ub937569vT34u25mZmZqago4wRIyW1vbu3fvuri4uLm51dfXI8Cwo8OF7EzV19d//OjvbI7+s/ex7/oce9npb99zPPHm/bM/tz/1s7t//a7kay11K4YVOQxorV6UlvHf33/gZOvnfi/i4YOwgAeRjwIxSQnpibFpqWn7uZvvmkM+PT3d3Nzs4OBw/Pjx8+fPP3jwIB0yDNQHDbMBg+ORgJMhODg4MDAQ9IH28PBwc3MDiOAM2be44OLi7Ozs6Ojo5OQEFjlp2D3IAD/cWTYHyOzt7eEJe8hsbW01n/KtnjRLS8vbt2/DWQpPmzAyMjIxMbGystKAhceTVlZW9vb2KSkpu8EM7e3tAwMDU1P7PYJChRiiwGYUGJmdM2msRRGyUa1NK3u6UWm6dIYemxMkHJcsqZtgbtm6uqSvHynRoIXi14+U/PCt/QsMUwqFI1+gO9APocK3ndpQ+Vm5w5Qt64BsuB0FEGDYjnpb3HZ8fByEJAFgAAHEzc3NIDwAj8eDTk/AyeDv7w+YwcnJycHBwdbWFvyVB/EAdnZ2Dg4OTk5Orq6uHh4enp6eWVlZiMt+iwODbPZ0BQoJBS6f/FD0lTr0SGmotaSvpdTXUuirp1VwZaQnJxYNtcYuafVoaxH/odVxQavlnFb5Ga2qM1rWv9L68pKeaGL86Ufbv0v4fH5ycvLXX1/X1dXx8PBIS0vbSEpDWloaBoNJTk4OCwsLDg4GwAC/AnIAPVW8vLzc3d0BPAA8cHV1NTMz+81vfoNGo11dXTVgwQnQAvwKsMFx2e4smyY22EEGyMEGMmtra+AcMDMzWzMGSZMcAC1oBiOtwAZLS0s7O7u0tDSQr0zcUWtvbyeTSdPT+zqIYv/eu8iZ7VcF6PMLFm1NOgXZ2o11OlTaSmyA2rpZszmD8/NbvoKeHumTtKAGhrd+VDI2th8/TXKlMnB8/CKNdoGAWxGM5NlUu3gAC2lseeD21YYIMOzBcMzMzDQ2NtbX17e2tsINj1paWmpqakpKSvLy8rBYbEpKSlxcHMiP9PPzA48R4ItGOzs7W8gcHBzu3r0L0iK9vLwePHjg6+sbEREhkx2MVix7ID1yyC0pIFeqXeKZOJzrsVcnr2up0C+pDF96zAlPBiN9OxP4HKBkBhUa2sTkJRVaK+8TrbvGV3l8wZZOZF9spFQqWSxWbGysgYGBq6trSkoK5lkGgCE2NjYYMhgVVk9ouh3u37/v6enp4eFhbGz8r//6rzdu3FgNDHfv3gWBSStowdHR8c6yOTg4QA4Ge5gWYFeDpaWlGWQg4lGTDdacNjIyMjMzWwEJK95aWFg4OzsTCIQdD0wiEont7e2Dg4NzO5cMqkIMUWAfKCCUL93rIuoU4LSry3RGRlcyA5Wmx+Z8zWKXyGQKUHtik+fc2zv5xptqSNDAhpIf/biUSp3Z5J6ex+pJE2I9Flu7vEizMpJ2HtaghMCb2Y+E8zxE2QfHQIBhDwZBLpcTiURQ3xAwQ3t7OwCGsrKygoICHA6HwWCSkpJiYmLAV5IBAQG+vr4+Pj6enp5ubm7Ozs737t0DqODp6enj4wPCl4OCgh4+fDg6OroHV4Uc8sVVQKFORFAzQ1Jy4v0Tr0zfgGKQjLS+xYYnfQvq8CQAEoYvqQwhL4ThS0uGWqkfv+RsYTw+PqGOmjvgfVsVCgWZTH7w4IGXl1diYiJmXUtLS0tNTY2KiloNDKvnAIoA80E2grW19csvv2xoaOjp6ekOGej7pulbAH6FOxqm6Viwt39MC+DrBsi1YGNnZ+fi4uLn55eamlpWVubv778mJGjOvH37NvBwWq9rVlZWPj4+FRUVO54ADTwWo6OjoKEb4k1VIfaiKDClUPiRB1AFOO3SAhSFsjoNWpfB1GWyAoXj45svJ9rfP/nWWyVrAcO+e/7OFkv0WGyUOnUh9dvUBXz6xfysDgHvRRntA3kdCDDszbD19/eDxkxtbW2tkLW0tNTV1VVVVZWUlIBOrnl5eWlpaaGhodHR0eHh4aGQPXr0CFCB37IFBAQEBgYGBweHhIQAj0RVVdXeXBVy1EOgQFRkxMPj/75wU0uJfklhtDLdeVV4EuSIMNZSGGrFnviO2x1bsfSFcn+JxeLm5ub09PT4+HgMZOnp6cCfAN7Cr0lJSSEhIavjkVY7GVbMCQsLc3R0/N73vmdhYQE+/kFBQX5+ft7e3l4a5u7uDqKVNCkCpEQvxyg5urm5eULmAZmXl1dpaalQKFQoFMPDw46OjuuXUjU2NrawsFiXFL5daGlpGRAQUFdXtxvM0NHRwWKxlpajuhFsUCH2QiiwqFIl0amoArx2AR7V2706DVqHRr/I5d1ic5o3GZhHIsl+9OPSFcDw4/8spdH2l4cBL5VeZHNQHW3qOqrZGqkLhOzMQdILMcgH+CIQYNibwRsbG4Pzntva2lpaWlpbWxsbG+vq6kAD1/LycgKBgMVi09LSUlJSkpOTEyCLj4+Pi4uLiYmJgiw6OjomJiZ62WJiYsLDw1NTUxGX/d6M6yE4qkKlCgkKfPTBvyzqQ/kMK3wLT75VGn1HZfydRQOt0A//ydv97tTMjpUV31dKi8ViIpGYmZmZmpq6Ogc6DbK4uLiQkJAVMLCRtzAwWFpahoaGPm0T8D3CQw0LWGWPHj0KCQkBXz2EQNba2qpUKmdnZ+Pi4vT19TWdCSumjYyMbt269S0QPGsK5EUEBgbW1dXteGwSSJDg8b79uhFhhn31iUBOZjsKFPN5X5YVqtOgW5rWSGmg0vRY7IsMZoRINLFhVwOFIvvJT8tWAMNPflq2f4BBoVRmiCUX2Vyd3u4LTzZ11inIfkhskSOpC9u5q3ZiWwQYdkLFze+Dw+HU1tY2NqrbPLe2tgJgaGpqqq2traqqKi8vh/0MOTk5OBwuKysrMzMzHTLwRJKamgoeRDAYDCCK5OTkxMTEuLi4yMjI4eHhzZ8UsgWiwLMUgMJn5xeX/Lw8Yj76/5aenvEMJzPM3dQKOPavQf4PZucX1JFIL2j1bIlEAoqbAWbAPGmpqamghsHTHvfXmR8WFubs7Pzyyy+bmZmFhoYCH4WmpwJmAEAC8GtYWFhoaGiYhmmuCeilpaVFpVL19fVZWFgYGBisgATNtysaLzyLF9SJ1IAZIiMjm5ubd4MZOjs7BYLHzRmedeMiyxEFDpICpOlp85YGdRp0VZnO0NDTXA0mLHa1TLaR8kmDQ1M/+/lKYPjpz8ro9H3hYVhQKmPGRbpsLmqg/0IuVjPRGZWfdaeucnpH+14fpFthP50rAgx7Mxoikai2trahoQEwAwCG1tZW0FYJOBkqICsvLy8tLS0sLCwoKMjPz8/Ly8vNzcXj8TgcDrzicDgsFpuRkQGqtaSkpMTGxhIIBLlcroRsb64QOeoLrcDM/KLHvTuJJ76jULsU1IkKmsFISuPvqGeitea+0fL627+Hh4YsytUpEC+kge+26XR6TExMSEhIVFRUQkLCikzopKSksLCwdahgnUVhYWEuLi4vv/yyqanpCh8F8BLAhLCpCRgYlErlhGgiNDQU9jCsaL9gBNnqxgvPZAZra2tQNCkzM3M3ApPa2to6OzuFQuELeV8hF3XIFRAvLfmR+lCEbO2iXFQPFJ5Eo6sdDho/ekyWLoPpIxSOQl/HrKPYyOj0L/6r/PUjpZpJzz/7eTmDMbvOVs9nkUgu9xYKL/L4alrIz3qCFvKwRuWFXCTR+fmMxLOOggDDsxTaneVTU1N1dXX1kDU1NWmWS2pvbwc+h5aWFsAPDQ0NwPNQWVlZUVFRVlZWCllJSUlRURFIeMBDlp2djcViU1NT4+LiyGQy9IXuC/qN7u6MC7LXjSswOT3jYn0b+wlgBi2lRjDSkpoWXpJc03L78D8S42IW5Rv5CmzjR95fawJgIJFIYWFh8BN8ZGRkfHw8jA1bjkd69OgRDAzAwwDCiuADbQoSNFfWBAaVSsVkMn18fEAOw2pgALnOmj0cNkILYB1LS0snJ6fc3NwddzIQIevr60Mauu2vjwRyNjukgEKlKuHzrpYXqbGhsVZndGwtVwPtIod7jclKFktET49QGhub/uXbFSuA4ee/KGcw9tjD0D83Z8Hhqmmhr1dNCxqJzqg87PWi3CGxukgGYvtBAQQY9mYU5ufnGxoa6uvr6+rqmpubiURiJ2RdqwzMBxTR3Nzc2NgINqyFrKamprKyErggCJDl5+fjcLiUlBQ8Hg8KiezNFSJHfcEVUIOoUCK9c+sbwmffUdOCZn1VtJbgitbd42+kp2fI1ZmpyoNeE2mdwQTAMDAwEBoaCp7j4af5iIiI2NhYUO5sHR/C+osAMHz3u981NzfXZBLNp/8tTK8ABpVKRaVS3d3dV+Q9GxkZodHodRovrEMOMGBYWFg4OTkVFBTsIDN0QFZZWZmTk8PlctcZIGQRosCBVoA6O+fU0aZmhtICVF+vmhlWuRp06YyLXJ4Bk5UnlU6tFetPo828/auVwPCL/ypnMvfMwzCvVOZIpFeYbD0OF9XddSEPq5nlrJ2b+VUBrpv/bZ7SgR7EF+PkEWDYm3GUy+Wtra319fVNTU1tbW1dXV29vb19fX39/f0DAwPglUKhDA4OUiAbGBjo6+vr7e3t6uoCzv02yJqbm+vr6wE2lJWVlUBWWFiIx+OTk5OZTObeXB5y1BdeATUBqJmBPy62u3mx4vRLamYwhH6MtbhfatkdezMnL39rJcMPonhDQ0MRERGaX//D8BAGGXhAX58N1lwaGhrq7u7+k5/8xMbGBgYGeOeaE5vCBlCyqaW5GQAP0JxIJFpbW4NkBhCJZGRkZGpqug4VPHMR6AltaWl5//79kpKSrcUmwf1qwHcrXV1ddXV1SUlJLi4uOBwOqfFwED8yyDlvXIE5pTKHxbhSVqBTgEM11OgMj6wuuqpDpekymHocjhmbUzwpW4ENTObs//y26kkPQ+l//Xc5k7k3Hobh+QVXvkCPw9VlsFDE1gs5GZq0gMrDXinEd/A4G5cIWfM5KIAAw3MQeY1DKJXKjo6OhoYGIpHY0dHR3d3d399PhmxwcHBoaGhkZIROpzOZTDqdzmAwaDTa2NjY6Ojo8PDw4OAgiUTq7+/v7e3t7OxsbW1tamqCsaGioqK8vLyoqCgjI4NIJK5xbGQWosCOKsDgsK2u/L3hPMQMaHV3Z+uPf1pQVqL5JLqjB9yPO2MwGNHR0aC6MXiIXw0PmnPWZIPVM8GugoKC3NzcAgMDNfFgm9OPHj0KDg6GgQEM1uLiYm5uLhqNhmOTbt26ZWVlBfsKnokH66wACq02NDRsgRkAMHR1dXV0dNTW1qakpLi4uBgaGkZFRSHxSPvx84Cc0y4oMDY749LRisrPRhXnoTqJUAElhmZKA5hW11Bic8zY7MLJScly6WEOZ+4Pf6x+/UiZRg5D6X//soLFet4eBvHSUrJYfI3F1uPwdMao2g21F3BPVlDNzzIozR8QIblJu3APbW+XCDBsT79tbN3T01NfX9/V1dXT09Pb20sikYBLYWRkhEql0ul0NpvN4/H4y8blcjkcDpvNZjKZgB+Gh4cHBga6u7uJRGJLSwuIVoKrsubm5lZXVx+qh7ZtjAay6VYVgMoejTBYlrqfdF54aURHy/rUrytr66AbT3F4EmhWA8M2H+hXbA46sayYuZ23K4ABzneamZnJzMw0MTEBzGBubm5trS55tA4JPHMR2By8RkdHb7ZoUkdHR2dnZ0dHR1VVVXJysouLCxqNvnnzpr29PYVCAb/ikF90W/0AI9sdJAUWlMoiNtOgsliHkK1dUYLq71OHJ62KUNKBSq/qcbjGTFa6RMJaXBSNL/z5zzUrgOGXb1ew2c+v1PWMQlk0OWnCYl3k8nSZLJ1BinZFMZS0sNxvAY/RIWQ71FYwZNKDNCqH5lwRYNiDoQZ/2/r7+wEw9PX1DQwMkMlk4FigUqk0Go3JZHK5XIFAMD4+LhKJJiYmRCLROGR8Pp/L5bJYLBqNNjQ0RCKR+vr6urq6QJ5DU1NTQ0NDTU1NUVFRWVnZIlKMbA9G+HAdElDBwOCI6YUPb332TkuburQ/yFs4PMDA5/MTExM1PQzbeZpfvW1ERMSKHInV62xqDgCGpqam1Y/a09PTcXFxcDDSNmlBEyesIIuJiWlpaYEyltd4AXkOcAwScEdUVVXFxsbevXvXxMTEyMjIwMDAwsKisbFx9ckfrs8ecrWHUgHh4kLIQI9OHlbdq6GmEkUmPw0bdJnqp/PrbJb7IOf371W//lqJpofh7V9VsNnPw8MwuaQolcks2Rw9NkePxdGh0VGdxAuEbM0UZ+3cDO3czIhuIlJBdd/e1Agw7NnQkEikurq6rq6uvr4+Eok0ODg4PDw8OjpKo9EYDAaLxeJyuXw+f3x8fGJiQgyZRCKZmJgYHx8XCARcLpfBYFCp1OHhYQqF0t/f393d3dnZ2d7e3tzc3NDQUFpaWlVVhQDDng3w4TswZXiUPDh8OJ/hxGIx6MsevnkDDROeloEQFhb28OHDq1evurm5AT8DvOamCGH1yoGBgSUlJfK1KquwWazMzExXV1dLS8sdBAZQaNXe3h6DwYDma2sQAzQLeBWIRGJ5eXlMTMzdu3eNjY3hhGwDA4P4+PjpTTa7PXyfSOSKX2QFiKJxm4YqVB5WOz9bu75ah0JZMx9a7W1gMrQpzJ++X/XaD4qeAIa3Kzi77GHgyeU4ieQWi61GBTZHnXpBoWhXl1/AYTSTFnQI2TfLCNVM+os8YAf/2hBg2LMxBMDQ3d0NgAG4F8bGxkDqAgwMQqEQeBgkkE1ABoCBxWLR6XQqlToyMjI4OAgSo4GroaWlpaqqqqamBgGGPRvgQ3ZgtU9BfckvckGkdYYUAENYWFjEVu1poBEZGenn5/f2229bWVlFRkbCq0VERIB06s2+wp6KoKAgPB4/O7v2V4xyuby6utrOzs7CwmKnmMEKMgsLC2dn54KCgjWTGTo6Orq6utra2ioqKqKiou7cuWNsbAyKNRlDpq+v//DhQ6Qy0jp3I7LokCgwp1AUMmgGFYU6+RA21FWhSCTI2/BkbgOddoE09rMPngCG135Q8ps/VDH4a3/8tyngjELRNTsbOj7+jdrFwdVjsdWoMDqq3doM1U5NUQMD9KOdm6mdj31AbOZMT23zoMjmu60AAgy7rfBT90+hUOrq6gAwkMlk2L3AZDLZbDaXy+XxeAKBQCgUgkgkgAoiyIRCoUAg4PF4HA6HxWIBVwPABhKJ1NPT09bWVlNTU1tbiwDDUwcAWbDDCihVSuBdODyBSN8qCAMDeKCPjIyM2DkLDw/39/cPCQnZyC7Dw8PBavDE6q3AST569Ki0tHSdXxHz8/O5ubk2NjY7BQxweJKlpaWnp2dZWRlgBhCABLwKLS0tBAIhKCjI3t4eRgU0Gm1iYgKysc3NzTs7O8GtpkIMUeDQKyCan4sf6P2yEK/2NuRmaleVqRu9jVG/dTjQaKhR6i8+rnn1+489DK++UvTTv1SdxpNvsVjBQmGpTEaem5MuZ0hvTdE5hWJ0fr5KJgsXiW4xWTo0mjpXgcFSo8LYGKqz/UJRnjoGKftxxoI2Pl2nIBtdVVrNZrywfT23JuV+3QoBhj0bmcHBQQAMoD4SAAYqlcpkMlksFofD4fP5QshAGoN42UA+g1Ao5PP5gBlAMaWxsbGRkREKhdLb20skEmshW+dpYM+uHDkwosALpwAckhSxOxYVFbVBCIE5ITo6OqyqSBkAACAASURBVCoq6mmnEx4eHhYW1tLSsv6T98zMDBabuf28ZxgVwISVlZWFhYWXl1dFRQVIaAYF3wgEQmBgoKWlpYGBARyAhF42kFZRVFSE/Fp74T5AyAVtVwHqpDSwo1UvF6uTl6mNT9cuIaDaWx4XYKXRdZn0/z5b++orRa+/XvzqK4W/vtRwvmNEj8PQY7IucrgXubzLNLo+g3mHww0bH8dJpfXT072zs9SFBdHS0oxCsaRSKaDfFErodUmpXFAqRUtLwwsLjVPT6RKJJ4+PZjK/UreDUO9NjwlxAp2hMzKK6mi/UJyv5oTs1MdeBXy6DiH7y6LcBApJjKRZbnfkn9/2CDA8P63hI4G/0IODg/X19XBB1RUeBuBegDOeNdMYxGIxyGQAzMDlckHpJCqVOjo6CvIZOjs76+rqEA8DrDkygSiwqwqIxWIMBhMWFgaihiJ22mJiYtZ5+oePFhkZGRUVFR8fn5SUhMFg4uLi4EUrJgAwtLaCDHXVOiYSjYeHh1tYWKx46N/mW8AMAQEBtbW17e3tBALh4cOHlpaWhoaGoNX0MiZ8+/8333wTHh4+OTkJ13Ra57SRRYgCh1AB0sS4T2uDTk6GTh5WXa6UgNNurEUNDOix6b+52vTqfxS99oOi3xs3o4aouky6ZklWXTpDl8nSY3PU/MDj67G5elTaFTrjGwbTkMG8xWJbszk2bI4Dh2vH5pix2MZM1g064wsaXV3FVb0+R4/J0qVDTeXUrzQUmazdVH+hMGc1KugV4AJ6OsamZIdwgA70JSPAsAfDtxoYKBTKyMjI2NgYjUYD7gUejwdnLwBakEAmlUolEolYLAZFk0BgEpvNBvkMY2NjQ0NDAwMDXV1dDQ0NdXV1yFdxezDAyCEPnwJSiSQ3Nzc1NTUpKSniSYuE7Ml5m3gXGRkZHBx86dIlZ2fnqKio8OWIo4iICM3pqKgo0FUag8Fgsdjs7Oz09PTo6OinHWnjwKBSqSgUiqenJyixuk1O0NwcRDoFQWZlZQVQ4Vs+eHLKyMjI1dV1ZGQE/P5UIYYogCjwFAX6hQKflvqLuZk6+VnaeMyFnAxUdeEfblb84LuFv/umUYdG12Wt0S5akx/U0zS6miIYTDVIMFl6LLbmjy6TpV4ECEFd2pXxuJfc4CCqvVW7ouRCbuYTAUg56gAkvQKcf08HeRKpmvqUkdvfsxFg2IPxAX/whoaGYA/D4OAgAAY6nQ6nO8PuBRCLJF02QA4gpQGumMRms0F/t+HhYRKJ1N3d3djYWF9fjwDDHgwwcsjDp8CkVFpcXJyVlYXFYlNSUpKSkuLi4qKjo9eJI1pnUYSGRUVF+fn5vfbaazdv3gQAEA4ZcCbExMQkJiYmJyfDnJANWVZWVkJCgsZuVk5uEBjgp/Pe3l53d3dLS0vNJ/7tT1tZWd2+fdsQMmNjY5CoAAoigURnQA2GhoYmJib19fXgfOCzUiGGKIAo8BQFRiQT0T3Eb4pytXMy9Iozf4vK+f7Lxe/blugRq1GdRHV69OiYDpWqTpKmQ4/73z7901bCAxWaA3o+wHgA1h+j6gwPo3p7tFsatcsKL+Ri1aFHcK4CPh2Vh9UpwF0rJUSQekcQr8JTButAzEaAYc+GaXR0FPRh6O/vBx4GKpXKYDDgjGfYwyAWiyUSySRkUsiAkwH4GUDFJBCVRKfTR0ZGyGRyd3d3U1MT4mHYs9FFDnzIFJBKpUVFRVgsFgdZdnZ2ZmZmRkZGWlpaAmRxcXHA1QBewzUcBRHrWnR0tJ+/32uvv3ZT/2ZsbGxcXFxCQkJ8fHxycnJ6enpmZibwJwBOyM7OzsrKAkePiYlZ5ygbBAbNYWxpaXF2dt5ZZjAzMwNIAF6NjY1v375tamp69erVa9euwfyARqNTU1OnppA6KpoDgkwjCjxbAfHcbDWD6tVU9/axwle/X/LDXxR+5JGByk/VzstQByyVFWrXVKAaa1GtzaiuTtRAv87goM7IqJolwM/Y8sToKGpkBDU4iBoYQPV0ozraUc0N2jUV2qWEC3lZF/DpECQs5zTj07VzM3UKcDr5WbZNtTm0McH8/LPPFVljfyuAAMOejc/Y2BjowwCAAe7AAIABdGAQiUSAFmBggLEBMINIJBIKhZrlkkAaQ29vb3NzM+Jh2LPRRQ58yBSYnJwsLi5e8ewOf9mflZWVmZmJwWDSli0pKSk2NjbmWRYbG5uamhoZGfnmm2+amJhkZmZmaRgMJzAtwBNZWVnx8fERkK3pygDA8MykZ81hXFxcyM7ONjc336miSebm5jASQBVTjUH1JC8vLysrKwMDA0ARoI6qWCzWPBlkGlEAUWDjCnA4s7/7Q9XrR0pfe7X4jR8VHf0Gfy4Vg8rDaKv7IaReyEpRP+7j1MFL6lCivKwLBfgLhTnaRbkXivMuFOaqUxEIuAt5WPXSnAw1HmSnPd4qWwMScjKAP0EnN/NWVUkiub9fIl5UHsa6eRsfmgO0JgIMezZYVCoVVEkaGBgYHBwEwMBkMjkcjmYCg1gslkImgwwAw+TkpAQykP0MZzIwmUwqlTo0NNTX19fS0tLQ0LB+SJKmZ1+pYSqVCn4HBILfam6yZ9ohB0YU2GcKrAMM8EN8dnY2eMTH4XAAITKfZVgsFo/HJyUl/ed//qeJiUlubq7m3tafTklJAaiwDjBsJOkZKA0++CKRKDY21hKybcYjgWAkNGSgfKqXlxcWi83NzU1ISHB0dESj0SA2ycbGpqOjA/nNs89ueeR0DpIC/f2TP3yr9PUj6k7Pr71W/Or3i392tEDnUf6VoiztnAydAhwqH6udk6GNT4cKGaVBmcprveKgoqh4jLoQU06Gdk4GKjdTh5CtU4i7gE+/UoBzrK/OGBzonxDNba9I60ES99CcKwIMezDU4C8fjUbTzGEAGc9MJhN0YBAKhSBLQSKRSKVSmBNkMtlqJwMABtCTAfR+7u/vb21tfSYwzM/Pl5aW4nA4kUgEQ4JUKsVisUvQpx1wgkqlUigUcrlcoVCXVhMKhX19fXsgHHJIRIH9qsAGgWH9R/w1lwJg+MlPfoJGozcFDBtJet6UhwFoz+fzQ0NDrSDbMjNYWVmZmpqaQIZGo42MjAIDAxsbG4lEYllZmY+PD/A8GBoampubV1dXI7SwX2985LwOhgJtbeIjb5RrtHkufv3VyuvXOwVzU3UMWlJfl3N91Y1C/KXczAs4DCo/S7cQr1uI0ynE6ap/8LqFeJ1CvJorCNnaeWq0uJSHvUrAfVOcZ15V+rC9CUvub+ayJ+ZmlxB/wsG4I7ZylggwbEW1bW6zAhhWeBjWBAbgXpDJZFNTU7CrQQpVTJqYmNCsr0qj0UZGRgYGBlpbWxsbG9fxMCiVSoVCERoaamhoOD8/L5fLJRKJSqWSy+VCoVAB2fT0tFKppNFoOTk5CwsLCoW6v0pDQwMajd6mCMjmiAIvkgKbBQaQabAmIayYCQODsbHxpoAhOzs7MTEx4ikWDiVRbK0DGolEcnFxAUWTbGxsNogNmmuam5sDBwJ49fX1bWtrGxoaam9vDwkJgRfp6+tHREQgqQsv0icFuZY9UaC8XHDkjQpNYDjyRgUK1QafjEKpnFpYGBVPtHBYZdSRnCFyJrk/g9yXSe7HDZLyhgcJYyNFtLFyFqOOx+0SjVNlk5ML83Ny+QL0VADvB5l4gRVAgGHXBxd6Ll9aUiwp1V/Pq5RQtI9KpaLT6cDDMDAwMDQ0NDY2RqfT1/EwAFqYWrbJyUmpVAr3ZODz+aAhAwAGEokEgEEul6+6wsduA6VS/fQPOiXNzMwkJCQEBwdnZWWNjY35+/sPDg46OjrGxMRER0dnZ2dfv36dx+OBXbHZbAcHh1W7RWYgChxeBTYLDCuoYJ232wGG9PT0FZVYI5YtPDw8Kirqfx/9t/DlvUKhqK2tdXBw2FQCNAwMmsFIRkZGlpaWBQUFAwMDFRUVQUFBt27dMjY2RqPRBgYGnp6eLBbr8N5VyJUjCuyQAnn53FXAUP7Zmaa5uaUdOsKu70ahVHrU1lZRqeBInVyuU3V1wdDQrh8YOcCyAggwLCuxa/8rlAqlSqFUyRWqJYVCrlwCDRNVDAajvr6+p6dnNTCAHs8gJAmOR1omhcf/awKDSCSCayUxGIzR0VEymdzW1tbU1LQWMKggdFEDjEqlys3NffToESghv7i4+NVXX/X19d29e5fFYpmamk5MTKDR6Obm5pCQEFghJpNpb28Pv0UmEAUQBfYnMGCx2NjY2Ii1bDvAAPyQlZWVDg4Om0qAtrGxsbKygisjgdQFe3v75OTk+Ph4BwcH0N0ZjUYbGhqamprW1NRsgWeQuxFRAFFghQLZ2ZwVwPD6G+WnTjVOT6/+SnHFpvvlrUKpNCooeDssbGphoWBo6A9RUZ+mpf2bl1dSd/d+OcUX/TwQYNj1EZ6bneYIxFzhJEcwMTU7o8YFqGaAJjAMDw+DmqqaTRhA+4XJycmpqakZyDSZASQzSCSSiYkJAAygVhKTyQTA0N7e3tzcvCYwLMoXZTNzSqVcJpPhcPiwsLC0tLTAwECVSmVlZUUikdzc3AQCwZ07d6amptBodENDQ1BQEEhyUKlUHA7nzp07uy4ccgBEgYOjwNz8fHV1dUZGBvZJ06hptPbkOr4FsAiPxycmJv74xz82MjLabEhSdnZ2cnJyxFq2ZWCA85rm5+dxOJy1tfWmmAGmBfSy3bp1y8zMDFRJAsFI+vr6JiYmeDx+YWHh4NwCyJkiCuxfBdIwzJXAcKT845ONU1MHBhhUKtXE7Oyvw8ONCgoMCYQePl+lUvk1Nb3y4MEglIS5f9V/Uc4MAYZdH8mWHup7l0P/di3qzxdD8ip7oeOpiQEGBhKJNDQ0pAkMAoFgfHwcAAOLxQIriMXi6WUD5CCTyUBUElxclcvljoyMVFdXd3V1dXZ2Njc3a+YwQDFIytnZeZeQYlNPvHxpKTQkzNLKsre3d2Jiws7OLi0traCgoKur6+LFi4WFhZcuXaqsrDx37lxNTY2FhQWHwwFiFRUV6ejoCIVCKNpK7aZADFHgkCuwtLTU19dXXFxctGyFhYX5+fl5eXk5Txoej4drJeFwuOzsbIAYOTk5+GUDK8BVlTIzM93c3CIjI/F4fFZWFtgKTICCSzCLrMaPzMxM0JABJC1ELBsABjJ5KyFJ8FgLhcLQ0FALCwuADWuSAxyJZG1tbWlpeevWrWVS+PZ/QAvGxsYGBgYmJib3798vLy+fnp6GD4RMIAogCmxHgaRkxmpgOPFRg0x2kIBBpVJVUanfcXe/npsL1FhcWjqdmnoqJUWOpFJs5/7Y2LYIMGxMp22sRRrl/vly+J++iv+NdmhIah20pyeAgUwmPw0YxsfHQZVVKpU6Ojq6zAvTMDBMTk6CNAaQ98zj8SgUSkdHB4lE6urqamlp0QAG9UFprHF9l6z3b+b+6WtcFLZ5dm5GKFBjukqlmpqaotPpKpVKIpEARwebzeZwOAwGQyaTiUQikPS8tLTE5/NZLNbMzIwCMrA58ooocMgVWFxclMlkEqisGShIwOfzQQt2uoaBz/LospHJ5M7Ozrq6uuDg4IcPH4aFhcXHxycmJiYlJaWkpKRBhsFgQBnWjIyM9PR0DAaTmZkJswFMFyCRWjOdGmwFWj6HPmmPHj2KjIwcGRnZZswPlUr18/OzsLBYp24SAAnN1AW0hpmYmMCo4OnpWVJSwoe+O4Rdmof8vkIuH1Fg+wrEJ9APNDDQJBIchdLO4SwpFNZlZa/7+/OXv1AYFIm+7+sb1NKyfZWQPayvAAIM6+uzA0tn5+d0LJKOXop753LMVbuMxeUsZNjDoAkMbDabx+PBHgbQZkEikUxPT8tksjWBAUQljY+Pg+KqTCZzeHiYRCJ1dna2trbCwKBQKitaKGeM4t7VL/jEtu1j87p3LkVWtg2qS6Yql5TLkVKgcOqKywaPFGAR/HgB3Avw2xWbIG8RBRAFNq4Ai8W6devWl19+efv27bt373p6evr5+T169CgiIiI6OjouLi4JsoSEhNjY2KioqOjo6JiYmJCQkLS0NBqNRqfTBwcHSSRSX19fP2R9kAFPI5FIbG5urqmpIRAIsHMD9IRmMBjb/wj39va6u7uDoknrVExaHYwEApBAgzYvLy+ACts/n43LjqyJKHBIFIiLXwMYPvr4YHgYUnt7fxsR8Zf4+J8FB7ez2dOLi78NDzckEOCxi+zoeN3fnzE5Cc9BJnZDAQQYdkPVFftUphd0/QYV9uer8e/ohhRWP45KYjCZ9fUNvb29ZDJ5eHiYRqMxGAwYGEQikQT6qnJychLUR5qdnYUSGWYANoAcBvBFplgshoGBxWLRaLTBwUEYGNTVU5VKOkfkHVv14bWo47dKP7ZqOn4z9c7Dstjsxpm5RaVS7ZRc/++0ErKVFwayMVbMRd4iCiAKbFIBLpdrYmLyxeXLpqamTk5Ovr6+ISEhMTExSUlJaWlpGRkZUVFRqamp2dnZqampwGMQEBDg7OyclJQ0Pz8PHw18Th9/WjXfQDWU5+fnp6amwC+QqakpiUQCf6EA72ELEwqForS01NbWFi6atDo2ycLCwsTEBK1hwKuARqO9vLxKS0thr8IWTgDZBFEAUWB9BWJiaas9DB+fbNifOQxzy9+rqlSqPoHg12FhzUymOkRCIlmEQo/q6PR/8/IqGB4GVz0nl1fTaLOLi+uLgCzdpgIIMGxTwA1tPjs3Z+CU9VtU1NGvYk4bxI0xhSqVksFk1tXV9fT0kMnkkZGVwAByGECJJBCABGhhZuZbYIDznlcDw9DQUFdXV1tbm3xxUaFcWlDIVaqlsoaBD/UzT9o0v3MtIxJTr1KXeFXIl5bUsUqIIQogCuydAmw228TE5Nq1azY2Nu7u7g8fPoyMjExISEhLS8vKykpPTz9+/LilpWVhYWFubi4Oh0tNTY2MjHR3d09OTtYEhr26gunpaSwWq5kADTMDCFUyNTUFxVJBAJKhoSEajfb09CwuLhYIBOt/W7FXF4UcF1HghVEgOmYNYDj5yf5KehbNzICnEdvycjyZ3MJiSebmEru7jycmrhgIpUrlUFHx48BAJuJVWCHNbr5FgGE31YX2Df4WsrgTupYJv9OJ+OOlKJRFEo0zwaRTcbjsuvq6rs4uMpkMkp65XC7o8SwWi0F9pOnp6VnI5ufn5yADb4HDAYQqgUwGkPrM5XIZDMbw8DCR2N7Y2CRflKtUUL+F6v5j+piPLJr//HW2Y3Dh/MIC8DxAgUZI4vKu3wbIARAF1lFAExjc3Nw0gQELmZ+fX2xsLIFAyMvLw+FwGRkZiYmJ/v7+GAxmbm5unT2vs2hnH9Onp6cxGAxgBs1EZ2tra9CmDQQgGRoampiYeHh4FBcX8/l8zXN47BhZ54yRRYgCiAJbUiAqer8DA0kofDcmhgZ1jw1ubf2uv//p1FTu1FQdnf4DP79+gQC+breamoTu7tnFxfPp6dVQ4iW8CJnYVQUQYNhVeZ/YOYMnvn4n4ze6Eb+7GHHOJCklp7qoKL+woJBAyM/Pyy/ILyosIJSVldXX1zc0NDQ2Nra3t3d0dPT29pJIJDJkIyMjo6OjI5ANDw+TyeT+/v6+vr7u7m7QdaEOspKSEjwen5mRRSgsVCrl03Pz4Rm1H9xM/9utiveuYe9Hls/NzatUSyAS6YlTRN4gCiAK7IUCABiuX7++2sOAxWLxeHxBQQGouaQJDEFBQZmZmfvBwwA0EwgEcNEkmBlAZSQQgGRsbOzq6kogELhc7l7IjBwTUeCQKhAVRV0dkrSvPAwzi4t1dPrCkrqRnG15+Q8CAw2gLIV5ufyvcXFnMZil5TpIn6WmOlZWqrvBLM85pIP63C8bAYbnKrlsesYvrvr9L6N+pRN+9GKEqXtmbX07dWyYRh2lUsfGqDQajQaKqIwMD4+MjAwNDcGQAKZHRkaGIRsaGhpcNgqFQoKMQqEMDlKA9ff3tjXXNxAHb7jg/3wN96cb+efM0vOre6Fe0yqlOs0ZcSw819FHDoYo8DQF1gcGHA6Xn58P3AuawBAYGLivgEGlUvX397u7u4NCq3AdVeBVcHNzA6igWP4zr+leeJoyyHxEAUSB7SuwJjDskxwGqkTCWo4s8q6vf9jcPLO4WEOj/T9v71wKRaVSETmcl3189AmEAaEwtbf3l6Ghncg3Dtu/Jza/BwQYNq/ZVreAgn/U9NwxQLvtgX/3UtjbF0I+uB7pGpxf29rHYjKEXK6AzxMKBaIJ0YRYLJNNyqZkU9PTIHthYX5hYV79T/3f/Nzs7Mz0zPT0jLp6klQyKZqQjItE40KhgM/ncVijo2PZeXU3HNPevZb2+y8zPjZM9Y4u541LoZpIalSAvP9bvRJkO0QBRIEdVeCZwBAfH5+WlgacDHBI0j4EBpVK1dbe7uzsbGlpaWVldevWLUNDQ+BVYLPZMCHAEzuqIrIzRAFEgbUVeAow7HEOQzeP94/MzJ+FhLwZGOhWW6tQKhO7u//Ny6sL4gHHysr/DAriTU2pVKpGBuO9uLj/iYz8KCmpkkpd+yKRubusAAIMuyzwU3avVCryqztv2Ce8fyX8v84F/+nyIxPXjPSClj7y2LhQKJOKZFLxlGxycmpyanpqdmYGQIM6b2FO/TM9MzszNaNuxyCbmlKDhVQ2JZFJxQKBoK1rMDS58rJt+h8uxf/hctLntzGPUhtpLOFTTgSZjSiAKLD3CqwDDFlZWVgs9r333gPtFEHSM8hhCAwMzMjI2HIOww5etmb6wZJcXlJSYmVlZWJiYmtrm5qaSqfTNQkBrKw5ZwfPBNkVogCiwGoF1gSGPSyrurC05NvY+PPQ0DtVVd08XhSR+C/e3tkkkkql+hKPfy82dk4un15cfCcq6lpOzpxczpqcnF9aokuloErS6gtE5jwHBRBgeA4irziEUqGuWaRisWhNTbWVNe2PEkqMnDOOX4v+rXbAe5cffWmb4hNbWlLfPzjC4gtF4yKRUMiXSSVLiwuKpUWFHPpZnJcvzi7Oz8mkMjZf2ENh4EqJzqHFupYpf7r06DeosBPXo6y9s6NTSomd3dDhl5aQAKQV44C8RRTYNwqw2exbt25dv37d1tZ2dZUkLBZ79OhRbW3toqIiGBgSEhIePnyYkZGxf3IY4OrMEokkIyMjPj6eRCItLizsG5mRE0EUOKQK7CtgoEokZzCYv8bFtbBY8HhczMpCZWaqH40mJ9+CHA5qdyWb/X1f31+Fh1/PzUUyFmCt9moCAYY9UB5KHlAMj4w2NjaR+3tGR4ao1NF+8nBRZadvTJmJezbKNP7Tm1EfXQ8/ZxJ71TbZxCXd9gE2IKE8Ja8ttbA9MbctNKXufmS5nX/BdYf0M0axH92I+uRmNMoswcY3Lz67AZdfU1paQWdQKaT+DmL74uKCOgJJgVRP3YOxRg6JKLARBTYCDCgUav8DA3yxMzMz+8H1AZ8PMoEocJgVWBMYTny0N43byOPj/+bt7VJTozkiOpmZxgUFYE42ifTv3t4AJ6pptKC2NuHMjObKyPSeKIAAw57Irj7o0NBQQ0PDch+GESaTwedxxoU8qVjtUqDS2Z0kRnlDf255R0ZRe1x2S2RGY2RGY1RmY2RGQxS2MRbfnF5EzKvsqW8dJA8x6Wy+YFwslUgkYhGPyx4eVqdE9/f3NzQ0jI+Pw9/87dnVIgdGFEAUeLoC6wNDVlbW0aNHDxYwPP1akSWIAogCz1uBNYHh+ImGycm9aXYW1dHxL/fvAyRYWFpyqa7++aNHFOhZBUhzLTf395GRU4h/8nnfKesdDwGG9dTZpWUgeJdCoWgCA51OZzKZHA5HwOcLx0ViiXhmanJuVp2pMD8/q5AvKJcWlQq5CvpRLi0uyRcX5+fm52ZmZmSTU5PSSbFELFL3exYKeTwei8UaGxsjk8ktLS39/f1qB8NyZZJduihkt4gCiAJbVuBpwIDBYLIgO3r0qI6OTlFREVwlCYQkpaenI1/kb1l2ZENEgUOiwFrAUPa3v9VJJHsTMbikUPwdg3k3JqaFzT6dmvpBQkIPn685FuzJyUetrZotnzWXItN7ogACDHsgOwAGEomkCQw0Gu0xMAgEwnGhaEIkEUulUunUlGxmWjY9Oz0zNzs7Pzc7Nzc7PzczOzszMzM1PS2TTUknJyelateCSCwWqZFByOfzuVwunU4H/Z5ra2v50EcRyTXcg8FGDokosAEFNgIMKBSqsLBQExgCAgIQYNiAusgqiAKHXYE1geHP79aIRHsDDCqViiqRvBEQ8Iq/v2tNzezi3jg6DvttscnrR4Bhk4Lt3OoDAwONjY2gL9vw8DCNRmMwGGoPg0AgFApFIpFEIpFKpTKZbBqqrDo7Ows6Pc/NzWm2eZZK1cigjkUSiycmJtROBoEAdjIMDAy0trY2NjbKZDIkMGnnRg/ZE6LATiqwPjBkZ2e/++6758+fX+FhQIBhJ8cA2ReiwIurQEzM6k7Ppf/z2yqhcH4PLzqhu/uf3N01U5/38GSQQz9TAQQYninRbq3Q19cHgIFCoewIMEgkkomJCZG6qpJQIBBwOBwGgzE0NNTX19fS0tLa2ioWixFm2K3hRPaLKLANBdYEhsTERBCShMfj33///TNnz8DAkJ6enpCQ4O/vj8FgkJCkbQiPbIoocCgUCAoaXdXpueSHb5VVVu5lyXWlSoXKzPxtRIR0fi+55VDcATtxkQgw7ISKW9pHd3d3U1NTb2/vCmDg8/maHobJycmpqamZmZnZ2Vl1w7Zlm52dnZqagsKRpFKpVAwZAAZ1WNJyJgOVSh0cHOzt7W1ra2tpadHsnbSls0Y2QhRAFNh5BTYCDJ999hkCDDsvPbJHRIEXWoGRkWk7u4H/+u/y14+UvH6k+Mmfkp/9vPzGN92treovE/fEGFLp9319bcvK9uToyEE3pQACDJuSa8dWViqVHR0dTU1NPT09mhXPdAAAIABJREFUJBIJeBjgHIbx8fGJiQmJRDI5OSmTyWBgWICaPC9ANjc3Nz09LZVKJyATLds4ZMDJwOVyGQzG2NjY4OAgiUTq6elpbW0dHh6Wy9WNIJDGSTs2nMiOEAW2p8D6wIDD4a5evWpigoZzGNLT0+Pj4wMCAoqKisDHeXvHR7ZGFEAUeAEVIJNlv/t9zZE3ql4/UvokKsDkUHrkjeqf/6KivV2yV9dfODRUODy8V0dHjrtxBRBg2LhWO7mmQqFob29vbm7u7e0lk8kwMHC5XIFAoAkMmsywsLCwuLgol8sXFhbm5uampqYkEglIWtDkBCFkAoGAz+eDwKSxsbGRkZGhoSEymUwkEvv6+mZnZ3fyepB9IQogCmxDAQAMX3/9ta2trYeHR2BgYGRkZEJCAhySVABZfn4+SHqGgaG0tHRpaWkbR0Y2RRRAFHhhFejpnXzrR2Vr+RZgYCiGlpZVVe1lbNILOwAv1oUhwLA34ymXy9va2gAwUCiUkZERGo3GYrE4HI5QKATAIIVMAplUKp2enp6dnV1YWADAMD09DWiBx+NxuVweZIJlAx4GUC6JzWYzGAw6nU6j0cbGxoaGhjohAykNe3P9yFERBRAFNBR4JjDk5eURCAQEGDQ0QyYRBRAFnq2AnV0/5GHQJIQnpo+8UXHpcvv/3965B8Vx3fme2tp/tmpvtvhnpcr9I1shyW7uPm5cQfZ6s5tNyo6psh3bd5OYe+2K7fUbyVZsx481iW7FduwY+SbaxLKNLCWKZVnoiQALPyUEGiEe4jFCQiBgYIAZYGZ63owk9Dp3Z37op6Pu082gmWEY5tulQme6T5/H55zf6fPt8+gzZ/DeYW6Yee4DgiE7FeDcuXPNzc0tLS09PT0nT54cHBx0Op0sGDRNoylJtPGR3+8PBALhcJg0A22RFA6HvV7vxMSEy+VyJg66fXJycipx0AgDaYbxxDE2NjY6Ojo8PDwwMNDT09PW1jY+Po6JSdmpAYgVBCQCcwqG6urqvXv3GgXDxx9/jBEGCSScIAACVxEYGYl9/X/s/8tln5pNSfrSX33W2RW66h78AAEVAQgGFZXMnzt9+nRTU1N7e7uFYKA9Uml+EUkImp5ESxd8Pt/ExMTY6NjIyMjg4ACtnB4eHh4dHWXlQOKBBh9IWoyPj4+NjTmdTvqsW1tb2+Dg4DlsgZz5EkcMIGBBwCgYKisreUpSdXX1ww8//OijjxrXMHz00UdYw2ABFpdAAATWvjGwbPl+pWBYtnz/088cByIQSIYABEMylNLvJxqNNjY2trW10S5JPCWJ1zBo9A02kguJSUper3doaOjEiRMDAwNTnqkpj9flcrvd7vHxceeo89SpgROJg1ZE0HejXS4XDzjQzCW32+1yuWioYWRkhKYnHT9+PBaL0TLoixcvYswh/eWNEEHAksCcgmHlypVPPPHEvn379u7du2vXLl7DgBEGS664CAIgIHy+meIVjcuWGwcZPv3q1z4fHJwGIxBIhgAEQzKU0u8nEAgcPHiwvb3dKBhII/AiZvpJ32KjtQrxFcxDDo/HMzYysrfu497ewenpM6FwfF9Vj8fjcrkSYw6DDodjeHh4fHycNAPNUNLJBqfTOTQ0ZLfbOzs7eUnDpcSR/jwjRBAAARMCsmB46aWXjIuea2tr6+rqampqqqurSTBs2rTpjTfewAiDCVGcBgEQuEJg0++dxkGGZcv3v/xy/xVPcIGAJQEIBks8Gbvo9XobGhpowyJ50fPExAQLBlq4TMuYvV4vbXlEeyhNTU35fJ5tH7ateqPp7fcPhULBSOKIJg768LOmaR6PZyJxkGbgldAsG8bHx2lVQ29vb0dHx8TEBIYXMlbmCBgETAnIgoF2SZKnJO3Zs8coGDZu3Lh27Vpsq2rKFBdAAAQuE5ievvDd7x5etvwzaWLSp//zGw3uiTOXveB/EJiDAATDHIAydHliYqKhoaGjo+P48eMkGGjRMwsGHljweDzU0ee/iW2UtNGR0Rd/f/TZOvHG1hGvxxcJxz/iFolEwpcPkhD0ZTf5g26kQGhh9OTk5OykJqdzYGCgq6vL4XDwVxogHjJU+ggWBHQEZMFAIwyVlZXyl553J47a2lrdCMOnn36KRc86mPgJAiBgJFBd7V4uC4bl+3/7uyGjN5wBATMCEAxmZDJ7fmRkpKGhobOzM3nBQJOU6G8gEDje0/vEm/bVu8T6nVPeKXc4HCKlQDpBVg6XFUT8/1AoFAwGST/IAw4kGxwOB02R4iUNmaWA0EEABBIErAVDTU3Ngw8+eO+999bW1vIaho0bN1ZUVDQ0NFy8eBEUQQAEQMCawMzMxbv+V+uy5Z//5bKPli3/7MYbmwKBc9a34CoIyAQgGGQaC+ceGBhobGxMXjDQxxl8Ph990DkSDjUf6Xp8/cknd17aVO3WvBOhUCQUimsGo2DQnY9Gozz4QOLB5/NNTU3RaMPo6Gh/f//x48fD4fDC4UBMIJDfBGTBwFOSeIThww8/vOmmm2644QYWDB988MGmTZvWrl178OBBCIb8rjvIPQgkS+DgQd8X/3v8O27Lln+2ZctYsrfBHwgkCEAwZKci9PT0NDU1dXV10QjD0NDQ6Oioy+WiXZJoGhLNSiKpoBMM05HwR/tbyt51PVF1/oP64YDmC4WCOmGgG2RgIaFzkH4IhUJ+v9/r9ZJscDgcvb29Z87Mzm7E3KTs1BLEmjcErAVDXV3dzTfffOONN9bV1dEIAwkGjDDkTQVBRkEgDQQuXLj07w92Llve9L1bmmMxfKktDUjzKggIhiwU96VLlzo6Og4fPtzd3X3ixIn+/n5ZMExNTcmCgWci0dgCzSaKhoNbq5se2+xbveVs7efDgYAvlDjkEQYSDCwbdDqBf9I6af4bCoUCgYDX6x1IHJAKWagfiDL/CEAw5F+ZI8cgkAUC3d2hv/rygZqaiSzEjShznAAEQxYKkD7z3NzcTILh1KlTDoeDRhj4swm8VxJPQyLB4PP5/H5/NBx864OGx7eGn9o8/XnTcDDgC8YHGK6akkRDB9FofDE0Cwl2KAXD9OUjEol4vV6emATZkIVagijzicCcguF73/uecoTBZrPBPPOppiCvIJASgUuXxI4dLgwvpAQxX2+GYMhCydNnnltbW+12e29v78DAAH2hmaYk0f5FNMhAwwskFehjz5qmBQL+UMBfseGTx98OPPOW91BbfzgQSOiF2XXPRlVA8iAcDvO+qywneGwhGo1e1guz/4+Ojo6Nxac5okeShVqCKPOJwJyC4dZbb73++uv3Jo5du3Z98MEHtOi5tbUV5plPNQV5BQEQAIHsEIBgyAL3QCDQ0NDQ1tY2L8Hgv3wEgwGfZ2rN242rtsw89/ZUx7H+SDAcCsV3QKLBBDPB4PV6I5HIzMzM2bNnT58+LUsFcrNgiEajsVgsEAj09/efP38ePZIs1BJEmU8EXC7XqlWrHrj//ueee8646Lmuru72229fsWJFdXU1r2F49913X3/99ZaWFphnPtUU5BUEQAAEskMAgiEL3N1u94EDB9rb23WCYXx8fGJiQjnCcFksxP8PBYNul+u5d9qf3CN+vsnb2zccCQZDwUAwGEwIhlAkEo7FpmkyEo0tRKNRv197770/njh+4qP6jzb/YXN3V/eZ02d0moEFw/T0dCwWm56eHhoaikajWWCEKEEgnwi43e5Vq1bdbykYjCMMEAz5VEeQVxAAARDIJgEIhizQ7+/vP3jw4NGjR48dO3by5EmakjQ2NuZyuSYnJz0eD6975slINB8pEAj4/f5wOHKqf3D1O91P7hG/2Dg+MDgSjs9ICpJgIJ0wNDTkdrvPnIlLgkgkQn87Oo/+3zVr7vz+HXd8/7YH7ru/8WDTzMwMaQNZObBsiMViY2NjHo8nC4wQJQjkE4E5BcOtt976zW9+UzfC8Ktf/QojDPlUTZBXEAABEMgagWQFwyWTI2sJz3DEJtlN6fTFixfp/s7OzkOHDnV0dPT09Jw8eXJwcHBkZGR8fNztdpNg4BXP8sBCIBAfQwgGA5FIxN7TV/Zu3xNV4vXNztGx4fgAw+URhoBf+7C25rFHHn7llZcPHz5M+iEYDJ49e/b997c+9uhjP/9ZeXn586ufXPX4o4/6fL7JyUmn06lp2vT0dCQSYbVAQsLj8bhcLlrGQInPMHgEDwL5SMBaMNTW1t5zzz0lJSW6NQwQDPlYV5BnEAABEMgGgWQFg9xlpI4j/81GspOKk1O4SBwkGM6dO2ez2Q4fPkwfYaA9VXWCwefzeb1eGlXw+/0B6fAH/NPRyKGW44+/5/zJNrFuy9CEeywYDF8efAi3tx35/LP6D2trH/z3B7/xjW+89tprjsTxX2GuXLlq1aqVLzz/0x/94M7777v3jju+39PT43a7h4aGpqampqfjs5jkpc+xWEzTtJGRkXQBTKrY4AkE8o+AtWCorq7esmXLe++9V1tbW11dvWvXrq1bt7777ruvvfZac3Mz1jDkX31BjvOdwPnzIhAQPl/8H2YN53ttWKj8Lx3BkK5OLYVzMenDLF5jAOQzHA43NDS0tLR0dXXxRxicTieNMExNTem+0SYLhmAw4A/4Y9HwR009ZTu0J7dcqNze59d8sdiZcDji88W/xrBr17Z1v3n9/61dd/2Kf1yxYsXNN9/c3d3tThwPPPDAypVlP1n9xD13/+D+++696647T5w44fF4zEYYpqenQ6HQ8PDwuXPnzLI5r/MLVasRDwjkGAFrwbBnz57a2tq6ujr+0jMEQ44VMJILAmkl0N4u/uZvxD/8g7juOvH3fy++9S3x61+LSCStcSAwELiaQHYEw7x6mal7Nvbd03jmwnyOS5cuTUxM7N+/v7W1lQTDqVOnhoaGrAVDUDr8wUAsGtxW1/7I294n14ff/GNTzd7qHTt2NTQ0+v3+YDC4veqDnz6z+qVfvHTD9f943XXXPfTQQ319fZOTk+FwuLy8/JGHH1r9xMrHHnpg1crHVq1aqWkafdlNuYZheno6HA4PDw/HYrFLly7poKVeLtcWwtUVGL9AYCkQmFMw1NXV1dfXQzAshcJGHkAgZQKNjeLP/kx8/rno7xfHjonf/1589auipESEw1cFPTNz1c8kf5w7l6RHeMsvAmkQDNfW7UvXXbpe7Lx+JtPVP3+txznDMTMzc+HChb6+PnmLJPpqm9Pp5I8w0AgDz0ciGRBMHKFQKBgKTkfDG/cefXxz5Knfhd7a9PFXir78hf9WuHXr9kgk7Pf7Dx1qWrny8Wefffruu3/07W9/e+fOnS6XKxgMxmKx5ubmRx55+P/879K77rzjvvt+XF+/j1ZF87oF3ZQkEgxOpzMajTJYhkZnuBzZg4WDPWfCkV+Gi9wuLQLWgmHv3r2//e1vX3vtNV7DsHXr1g0bNrz66quYkrS0KgJyAwJJEWhqEn/xFyLxnaRZ/6OjYtkysXZt/OeFC+Ktt+LDDv/yL+KWW+K6Qgjx5JNi9+5Zz2vWiEceESQM2trEQw+JkyfFj38sPvxQ/PCH8VGLu+4So6OznvEfCBCBXBUMFr3S5C9x39foMJMJBhVw5cTMzMy5c+foKwdnpePM5ePs2bNnzpxpbW1tbGykLZLoq230mWe32017qsqCIZA4EmKBvsyW+NxCIPCbvd1l1RdXrQ983HDiUNOB/fsPeD2++IQlv39sbGzHjqqf//xnL7zwwsaNG9va2oaHhzVNC4fDsVisqanpmWeeef755/fV12/e/Mfx8fHTp09bCIZIJDI6OhoIBC5evGiklDxqpc9MyAZlmLB2EFjkBMwEw7Zt23bu3FlXV/foY4/edtttu3fvrqmp4TUMr7766uHDh7GGYZEXLpIHAmknQIJhePiqgFevFv/0T/Ezf/hDXDx88olwOOJTlZYtE8PD4qmn4mJAiPjMpeuvF8uXi5GR+M9f/ELcdltcHvz5n4sf/EC0t8eHLL7+dbF69VWB4wcI5Ixg0PU4ufOqO2/xk28xOszkwfnz568IgoRrxnCwNLisC86cThyxWIwdscuH3+9vaGiw2Wy8RRJ/5pm2SOI1DLyPajB4WSokPswWioQ0n/Zybe/KavHkm95Dzaei8a8uxEKhEG2j5PF4jh8/vm/fvm3btn3yySd2u93pdPp8vmAwGIlEOJGxWOz48eOBQIDVgnHRM+2bNDY25vV6Ey8t4uRkVkaSFy5cSL4IjD6V3f20nISpg8BiJmAtGPbs2bNjx47t27fX1NTs3bt3586dtIYBgmExlynSBgKZI6AUDOvWia98RczMiO9+V/zkJ7ORX7wY7/2vXy8OHhR/93fxqy0tcYXwwx+KnTvjfm65JT4c4fGIwkJRXT17109/Gj+PAwRkAjkjGKjXaOxiJnNG2a/V9X2pH8zywKALZk/o5AFJAp0w4J43ff2AFgmEQqFoNOpwOPbv39/S0tLZ2Sl/hGF0dJT3VPUlDk3TSADw95spnEg4Mjk5WV4z+PjuC0+96enoOBUM+ONTlRK6IhgMer3eoaGhzs7OI0eOdHZ29vf3U48/FArRBxk4eTQfiT/UICsHdkej0YmJiZMnT7rd7nA4TLOqWEcRNB3e8+fP685Yq4hkSpA2mGLlYLFnF/uZl0M2CbhBYOEJzCkYaNGzLBg2bNjwy1/+8tChQxhhWPjyQowgkF0CSsGwZk18GXQoJP76r8Wbb15J4He+E9cPwWBcOfT1xS8995zYtEmsXBkfbfja1+ILIVyu+JhDe/vsXS++CMFwBSBcRGApCwZjt5XOyO/IdW6jTmCFQA4zhcC9cOp/h8Ph0OUj/umExAfXgsFgV1fXgQMHaMWz/BGGsbExmpLk8XhIMPgTu6mSDEh8vzk8Kxgi4dHxiRfq3WXbLz37u5HeEwPBYOByVPFBBp/P53K5hoaG+vv7BwYGnE7nf/VFaAMlEgzyQgVe68wKQc4IuaemppqbmxsbG0l+uN3u6elpmnlFuEhl6Ujq9BiRT1IbzOltXmJgvp7JMPAXBBaSAAmGBxJfen7llVfWrVtXWVm5efNmmpJUXV1dV1e378N9NTU11dXVNMJQWVm5du3a7u5uCIaFLCnEBQKLgYBRMFy8KG64YXZlwt/+bXwmEh/f+pZ49tn4r9tui89W+tGPxL59oqND3H672LMnvs5hZiY+JUkWDP/xHxAMzA+OWQJLWTBwv1OnHHRdW35fTisQqBPMOoHm8LBOuDy3KEY9bPpqgTySQFKBRALpBE3TSAO43W6aj9TW1tbd3X3ixIm+vr6hoaGRkRGdYKD5SKwWpJGBSHQ60ucYf/rTwKr3xM/e7HMMOeILoS8fgUBA07SpqSm32z2aOGgttc/no/EKThj59F4+PB7PxMSEy+UaHx8fGxtzOp0jIyMOh2NoaOjkyZMtLS2tra29vb19fX12u72jo4M+I82Uzp49S9xoIce5c+eMkHWlIP/kkrJwsH/yM18ZkLx/gQMEFpyAtWCoqan59a9/XV5evnv3bp6S9M4776xfv97pdEIwLHhxIUIQyDIBEgyJr6rGU3L6tFizRnzhC6KzM/7z7rvFv/3bbAo1TXzxi2LbtvjPN98UN90k/vVf46sXYjHxz/8cX7Tw9NPxSyMjEAyzxPCfGYHFLhgsepDJXOJeJjuMHVkSDNzf5VXLSqmg0wk8nsBDCjye4Pf7SSp4vd6pxNHb20vzkY4ePWq323t7e/mrbWNjYxMTE/SZZ5/PR/ORaDLS1NTUwMDA4OCg1+uNRKPTsWhv/+CDv6y9b03Tz35Vc2qg35U4xsbGRkZGhoeHBwcH+/r6ent7e3p6jh07ZpeOrq6uzs7Ojo6O9vb2tra21tbWI0eONDc3H04cNputNXEcOnTIlvi03JEjR1pbW9va2jo6Oo4dO0YKZ2BgoK+vr6urKxqNso46c+YMiQemRzxp8EGnH2jwgUskXcupk5cE1j7NTAXnQSBzBKwFQ21t7TPPPHPnnXdu375dFgxvvfXW6OgoBEPmygUhg8DiJHDwoPiTPxH33BOfVvTAA2LFCvHlL19ZgdDWJr70pbgS2LBB3HprfElDKBTPR0+P+NM/jf88fz7+8+67RUGB2L8/7h4ejq9haG2dze6zz4rvfGfWjf9AgAgsdsGg69slIxJkP9wrZce1CYZY7MqQAk3jiSQOWScEEwe9uWep4PF4JicnJxJHW1vb1q1bDxw40NnZ2dvbe+rUqcHBwZGRkdHRUfmrbbzcmQKnbVW9Xq/T6UzEGZryabt31a59460NG7Y0Jw6bzdbU1HTkyBEasujr62tra9u/f/+BAwcOHjzY2Nhos9mo93/06NHOzs7u7u6enh4aMRgcHORRDhIttPCaF1GEw/HZUNFolCBEo1G/39/a2ur3+3ki0+nTp3mRtyweeOcoVg468aBc8MCFpdQSdFUuZZ1bV2fm+xNNAwgsPAGjYNiwYYM8JWnPnj20RRJNSXr//ffffvvt9evXQzAsfGEhRhDIOoHxcbFunXj5ZfHSS+KVV8TWrWJq6qpE9fSIF1+Mb6X6n/8ZX71Ax9mzorJSfPzx7M+WFvGb38x+KDoUEu+8IyYnZy8dOnRlD9bZU/gv7wnMQzDkPSsAAAEQAAEQAAEQAAEQAIG8IwDBkHdFjgyDAAiAAAiAAAiAAAiAQPIEIBiSZwWfIAACIAACIAACIAACIJB3BCAY8q7IkWEQAAEQAAEQAAEQAAEQSJ4ABEPyrOATBEAABEAABEAABEAABPKOAARD3hU5MgwCIAACIAACIAACIAACyROAYEieFXyCAAiAAAiAAAiAAAiAQN4RgGDIuyJHhkEABEAABEAABEAABEAgeQIQDMmzgk8QAAEQAAEQAAEQAAEQyDsCEAx5V+TIMAiAAAiAAAiAAAiAAAgkTwCCIXlW8AkCIAACIAACIAACIAACeUcAgiHvihwZBgEQAAEQAAEQAAEQAIHkCUAwJM8KPkEABEAABEAABEAABEAg7whAMORdkSPDIAACIAACIAACIAACIJA8AQiG5FnlgE9N00pLS0tKSux2eyrJraqqKikpKS8vTyUQ3AsCIJC7BNCY5G7ZIeUgsAgJ2O32kpKS0tJSTdNSSV55eXlJSUlVVVUqgeDeayAAwSBKS0sLEsc14LO+paKiIl0h2+12TmdBgWmpcYwVFRXWabO+Ssm2iMj6dlwFgTQSsNlsZWVlhYWFVC2Li4srKipSfOTokseGozt/DT81TZNTa7PZzAKh7KRoqmaBCyFKSkoKCgpKSkp0fpKMl5mkmEI0Jjr++JlFAg6Ho7y8vKioiBuT8vLyJdCYCCGS7CRkET61SAUFBRat4pzJs9lsVHbGlm3Oe+EhRQKmXc8Uw82J2+nRnrnnGT9xU6ThcDi4t0SpNQvQ4XAUFRUVFhamYpBCCEp5aWmpWURJnrfb7TabLcXhjiTjgrclSaCqqootVHYUFhamsV6ly1SFEMXFxXI6LSyRvKXYHbco9BQFAxoTC7a4lIsE6uvrdU9SssEl0Jgk30nIYsER/6KiohQVGrVslZWVKebFljgcDkeK4eTP7fkrGGw2m+7RnvZST1cvpLKyktq1NPaQ0p5ZZYBmXRalZ5wEAR2B+vp6qvnFxcX19fW2xFFVVUVP/cLCwhQfPBxdukzVbrdTgpN5mJHPRSsYGM4icaAxWSQFkaPJYNssLCysqqpyOBx2u33JNCa520nIYnXKdAucxaxlKOp8FAyVlZUsFQoLC3l0Mu2I09ULKS8vLygoKC4uTnsKMx0gnvGZJry0wyfbLC4u1gkDfvYn0y9PBlG6TJUVji7ByjRk+nFlZn2ZjleZ2dRPmmUn9ZARQj4QoPpTWFioe6PMjUm6pHtWGpPc7SRkse7laEuYTWJZjDtbUVPDQbN7HQ4H/0x7etLVcOTukzJ3U572yoAA50uAH+TKxW3prVrpMtV5hZPpx5UZokzHO9+CTtK/WXaSvB3e8pkAT3xXNiZlZWVpfCU3r0bAolDmFQ6sw4Kk2aUcbQnNsrMA5/NxhKE8cfD0HrK0VBb4appWUVHBoxbFxcX04tPa4GknIqqypF6MbRlf1TnMagY3i8qZ0w6Ho6ysjEdUioqKysrKdK9bKGRl6yNnxyzLnDBdgvmnMmF8FxwgwASsK7OyivK9Zg6zeivXbeO9yZgqNyNc1clhUeHJA73XtNlsvKuBhWEKIWjVJrc21HQoR1rMEMnxGjPLZ6z5ozFhUHAsfgL0Ar6wsDCNSV0kjYmuweGflFPZimnRJnUAZA7J2zKHWVpayqtBkt+FwrqZpTYwmWCVLZjc3OlyVFJSIrfDzIRZsUPGAreRQD4KBh0FftLrzif5U9M0+eHNNa+kpMTMPMxuoZcc8mQGDk3nMEsbW4JsHuTZbPFoQUGBUajItsdxcXbsdjurDjlh8pCufF52GxPG4cMBAskToBo4r51/zewuLabKzYhc2633AyGfFRUV9IJTd6NyIaa8arOoqKikpER+vuroKa1YCMHx6vzrfqIx0QHBz9wlYGYL15yjxdOY6NoN/klZYyuur6+XOyqccbOOgdnWKcrGirouypePHBFvpqJ8OWsWrLIZVLZgXMS8LoVRkIP7HsxE50GZMDn9cEMwzG4+eM11hTsKvNUjvXgoKCjgZ7munpHd0tIruqRpGtdy41oFtgRdOMafbAlsG+SHp1aXlpayVcu7sNXX18uhKWNkwVCYOGjdGO3mxi0RB06hKcORI4IbBK6BAD/kdPXcOqgFMFW2EeuU0FV6XJHykQ2Tm4KioiI5HE3TqEkpLi6WM87rHWXFnvq2qmhMZPhw5zQBsjWdgaSSo8XWmJg9atmKqekoKyuj3SMo7zztU25/HA4HjXYa13vITQ2/2eT9Y4xdFx1hs+aRxn8KCgq4EyWEYIVjTIayNIkAt5D19fWUQn7JomtOk391ostFPv+EYEhJMLA1Glsi7tPopAjE+M85AAALjUlEQVSflx/5VAU5NN0EA7O2wFhxOQRd4NQpUe5bzGYmh6aMka3duAiV49UNVijDkSOCGwTmRcBut3M91JmJdThcRTNqqpw268TQVX6/ZcwItxKyQfFJnSwXYvZjMroHtpn1KR+3xgQzMTQmRjg4k1sE5DpfVVUlz6i5hk+AsWksnsbEzNg5qcoRAxIGunaDSpb6DGVlZXJBU3fcOK7rcDiIsNxeyTeSW9k88r1GmPyKRLfDu1yaHAsRKCgo0KVZHtnQtZzKcDhAOIwEIBhSEgw8jsZqW0bMV+WT9DJeaaL8UlB31awtkIMlN7cO8jOehxd0wwi6W+SryhjZ2pWZVdqeMhxjsnEGBOYkwM8DGv6Wq+uc9woh2BiVtZevykFdg6myjcjhmLnJZJQyXgjBIw98OwWuaxzoqjJeM+tTmirHwg40JowCjlwnQHWevhBMbt1fYy/TIsvcXCyexsTM2NmKjV15TdMIgrItpcEEedUHdySUuSbtoevZ6xgqmykeXlAGy7fIVynZOoFBBJTNI0OQ+0UYYdCVTjI/IRhSEgxmVkroua7LJaGs6+yBb5HNwzoWvlcIoTQMDlP2KbspSfJrA2WM1uEo86UMR44abhBIkgDVJapmpBl0rb91ONZVUVm3lVWaY+FbZFPlk+zNwmEdPnVK5Ae2RVDKeM2ybB0vx4LGhFHAkesEuN2gTQLq6+vtdrumaTzvhebDJJlNM8ui25XGaG10fMs1NyZmSVJaMaWTL8mRMgG+ym/lKZHKHjm/xTdO++EA2Y9uzoVZynWJlFWNEqZFOJwX3SNDGY6cYLh1BCAYUhIMVOHkrrbMl1sBPslTBo1yn/woa7aFJXDIFreT9LewZGP4xjNm1s4JUNqeMhy+BQ4QuAYC9fX1VK+U6/XNAlwYUzWavFl65ny/ZRYU9XIqKipKS0tLEgevIErySaw0VWM6lW0RGhMjKJxZ/ASoziun5WiaRqN5SYpzttxMP/fNWgAlbbNHrdKKKQQOn5oR3V9uVbiTTVEUFhbqfNJPYqhrgnRJ5Rjl87y4Qj4pu43tlfEMz85QDtiaQVCGI0cNt44ABMMcgoGqlO4vm5B1hTOah1nF5VJRejBrC/gudlzb7cbwjWcgGBgyHIuBAPVc5Y8964yUfi6wqRpN3oLVfFsPskF6vtK9xcXFFk9rpRVzd0c3oG9MJxoTIxOcyVEC1rbGa3l1zQXdxX91V80syNgIKE1JJqn0YAxHvkXnNjN2Zch0L4fPGVQ6ONcUhdKPfFKXMPknxyifpHvNYCrbK+UtZgTMZl4oQ5YTBreRAARDGgSD2fRHo3mw9S7kCAMZEkYYjLUfZ3KXAJsSP8/khxa7dVczbapGk7cgrHzssX9jUDzZt7S0lPNF/o2eLV65WcfLCTAS5jDRmDAlOHKCAL0vNzN/Y1XnBkR2sNHRSbPQjMbI4c/ruW8MxwK1WXeZo+bEcyAsk/iMtYPe0Shf4VvfyFeVObKGqezW0y06jWFGAIKB+afugGCYQzDYVAfP+bOoo2av5JV1nQuSLYqj4Id0MoaqbB04TI5F56AkyQOsynxZh6PMlzIcXez4CQJKAtaVx1jVbaqD7cg6NGXdVlZpTirfwlGYmTzfonNYh2+c/EP+lcsKOTFyFGZZto6XQzASTiaDFDgaE8YIx2IgQCuCzISusarbVAdbupllUU6VxmhtdHwLR5GMrclgzZJkzBrfxZf4I7Z8SemgRCY/ccsYCGdTvmSWcvLDicQaBhlattwQDHMIBuuCsd4tgd8IyoFYb71i7CWkLhh4cwPZ5DhJymUVShtWWjuHo2wQU38nweHDkW8ElLbAELhWG9+csR/ZsTCmam0jcnr4zZnZe0qaesTygB+cyvwqmxqlFXO8uvdzurSZvZZj7GhMjMRwZtES4HprbT68wNc6I4uwMTEzdot2g3dJMm7rTJ9XoumOLCeYIZ+RKVVVVZF/+aTOrWweue2SxRLfyMMgctEoOxtmBMyaMiEEtbFztoScGDggGFISDGxCxjrHm6brlgHxeWPLxbatM2ALS9DVYA5BFzgtSFKOUVDgutcGyhiV1s4JUNow3aILnG+BAwQsCLCl6MyBbqEqWlBQoHzMGINdGFO1thFdqshklEu3+THJcxiUwp4CtNlsvLBBjkJpxSkKBt7vFY2JjBruxU+AHoJFRUW6FoM3+1dWaWW+FmFjYmbsZl0CyhcPvOiYmI1vEEN+iyHDMUuA7EfZPKb3OwzKQjSDQGlWZkdONtxMAIIhJcEghODNBPgjhXa7nUSz8inOt+Tul5659sgOpWDgPh/3e+Rb4AYBawJsXGVlZTabTdM0h8Mh75IkT32xDortTv6eaNpNVflENEsYmQy/5aJXaA6HgwPR7WDIPR5+2caelU0Nv7rTvT4gqkVFRRyOMoVmT1nuLclfh73mz8Yro0ZjosSCk6kQ4PpcVFRUVVXlSBz19fVkVgUFBTozsY6Lm6bMPfe5HbBOCV01669zrpW5czgc1HSYfTxe18AqbV/TNB5yUQ48cvrNcsQtFcO85i89z0swULILCwuVYyacbDiYAARDqoLBbrfz05pfGdJmz2wGjJscmqZxcyPfQrvLG7W+WVugC9Zi6E0IwX13XYzKF5zKGM2snZKhfMbzjnUcqbLZMmYEZ0BACGFhKcovelpDWwBTtbYRXfLIKMrLy5WtQVFRke4xxk9rtiYOga1bjoJf3ZE3Nj32TOflW2S3RVdDF4KcHuOrATQmMlW4s0igqqpK+bBWPgSt07nYGhOllVl3CSiDZhmhPoyxN2Jh+zp1YQRo0Tyy5JAbk4KCAmVvnvzopnWYEbCAwMO2HKkxzTgjE4BgSFUwULemrKyMX1Twp+bZtGTi7KY5f1xT+S72wA4LS2A/5LB4xgshHA6HnM6ioqKysjLlW0ZljBbWbjHPweFwlJaWcjPNvRZdyvETBMwIVFVVyVWI6u21VSR6GZY5U7W2EV0G5cdeRUUFy4bi4mL5TZt8l81mo6Ud9Cjl7ZLY8GXP9KRk/zKxqqoqjk53C//kMOUb+SoaE0YBRw4RcDgc5eXl3AJYPATnzNSiakyUj2yLvrKcO03T5PansLCwtLTUqPz5FqXtK1sJvoUc1s0jNW7cVbBoBuWWk6MwI2ANwWaz0Y0UJocGh5IABIMSS66etH7G52qukG4QAIEFJ4DGZMGRI0IQWMoErAXDUs75UskbBMNSKclEPvCMX1LFicyAQPYIoDHJHnvEDAJLkAAEQ64XKgRDrpfglfRrmsbTD4xTD6/4gwsEQAAELAmgMbHEg4sgAALzI+BwOGgmmG4vh/mFAt9ZJQDBkFX86Yuc10IUFBTMufYofdEiJBAAgaVGAI3JUitR5AcEskeAxyqpYbHeSSl7yUTMcxOAYJibUU74IFMsKirSbR2QE4lHIkEABBYPATQmi6cskBIQyHUCLBiKi4uhFnK6NCEYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBCAYcrr4kHgQAAEQAAEQAAEQAAEQyCwBCIbM8kXoIAACIAACIAACIAACIJDTBP4/eU86PD5hLswAAAAASUVORK5CYII=" + } + } }, { "cell_type": "code", From ec78ecb6cc676c9be421309cf637a1629fc35af7 Mon Sep 17 00:00:00 2001 From: Sonya <195730002+sonyyang-tw@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:08:56 +0800 Subject: [PATCH 115/180] docs(physim): remove Unitree training reference Remove the external Genesis Go2 training link from the parallel simulation notebook resources. Co-authored-by: Cursor <cursoragent@cursor.com> --- projects/PhySim/PhySim04_parallel_simulation.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/PhySim/PhySim04_parallel_simulation.ipynb b/projects/PhySim/PhySim04_parallel_simulation.ipynb index aab133b1..077b86b2 100644 --- a/projects/PhySim/PhySim04_parallel_simulation.ipynb +++ b/projects/PhySim/PhySim04_parallel_simulation.ipynb @@ -212,7 +212,6 @@ "If you’re interested in running more Genesis on AMD machines, here are some useful references:\n", "\n", "* **Genesis GitHub:** [https://github.com/Genesis-Embodied-AI/Genesis](https://github.com/Genesis-Embodied-AI/Genesis)\n", - "* **Train a Unitree Dog on Genesis:** [https://github.com/JingXunLin/Genesis_Go2](https://github.com/JingXunLin/Genesis_Go2)\n", "\n", "If you find aup learning cloud useful, please give us a star!\n", "\n", From 851c4fbb2ec0320fd84e6efb5dd4492dbf31ee90 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:33:18 +0800 Subject: [PATCH 116/180] feat(installer): add local access mode selection --- auplc_installer/cli.py | 87 +++++++++++++++++++++++++++--- auplc_installer/overlay.py | 44 ++++++++++++++- auplc_installer/state.py | 5 ++ auplc_installer/summary.py | 6 +++ auplc_installer/tui.py | 16 ++++++ tests/installer/test_local_auth.py | 71 ++++++++++++++++++++++++ 6 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 tests/installer/test_local_auth.py diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 390f2b32..f9ba5d61 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -38,7 +38,11 @@ pull_external_images, ) from auplc_installer.k3s import install_k3s_single_node, install_tools, remove_k3s -from auplc_installer.overlay import generate_values_overlay, try_load_courses_from_overlay +from auplc_installer.overlay import ( + generate_values_overlay, + try_load_access_settings_from_overlay, + try_load_courses_from_overlay, +) from auplc_installer.pack import pack_bundle from auplc_installer.progress import stage from auplc_installer.rocm import deploy_rocm_gpu_device_plugin @@ -155,6 +159,14 @@ <list> - comma-separated keys, e.g. cpu,gpu,Course-CV Env: AUPLC_COURSES + --access-mode=MODE + local - closed local accounts; installer creates an admin credential + personal - shared student session (legacy non-interactive default) + Env: AUPLC_ACCESS_MODE + --admin-username=NAME + Local-mode administrator username (default: admin). + Env: AUPLC_ADMIN_USERNAME + -y, --yes Assume yes to all prompts (for scripted/CI use). Env: AUPLC_YES=1 @@ -229,6 +241,8 @@ def _build_parser() -> argparse.ArgumentParser: p.add_argument("--mirror-pip", dest="mirror_pip", default=None) p.add_argument("--mirror-npm", dest="mirror_npm", default=None) p.add_argument("--courses", dest="courses", default=None) + p.add_argument("--access-mode", dest="access_mode", choices=("local", "personal"), default=None) + p.add_argument("--admin-username", dest="admin_username", default=None) p.add_argument("-y", "--yes", dest="assume_yes", action="store_true") p.add_argument("--dry-run", "--try-run", dest="dry_run", action="store_true") p.add_argument( @@ -272,6 +286,10 @@ def _apply_global_flags(state: InstallerState, args: argparse.Namespace) -> None state.mirror_npm = args.mirror_npm if args.courses is not None: state.courses = parse_selection_spec(args.courses) + if args.access_mode is not None: + state.access_mode = args.access_mode + if args.admin_username is not None: + state.admin_username = args.admin_username if args.assume_yes: state.assume_yes = True if args.verbose: @@ -357,6 +375,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: with stage("Provisioning GPU device access", idx=2, total=total): _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) paths = state.runtime_paths() + access_mode, admin_username = _resolve_access_settings(state) with stage("Generating values overlay (initial)", idx=3, total=total): # First pass: use local detection so image pulls / builds get the @@ -367,6 +386,8 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) @@ -432,17 +453,23 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) with stage("Deploying JupyterHub runtime (helm install + wait)", idx=9, total=total): - deploy_runtime(paths) + admin_password = deploy_runtime( + paths, + access_mode=access_mode, + admin_username=admin_username, + ) - _print_success_banner() + _print_success_banner(access_mode=access_mode, admin_username=admin_username, admin_password=admin_password) -def _print_success_banner() -> None: +def _print_success_banner(*, access_mode: str, admin_username: str, admin_password: str | None) -> None: """Show the post-install celebration / next-steps panel. The full "AUP Learning Cloud" figlet logo, a "ready" message, and the @@ -469,12 +496,23 @@ def _print_success_banner() -> None: log(" " + bold_green("You have successfully installed AUP Learning Cloud!")) log("") log(" " + bold("Open in your browser: ") + bold_cyan("http://localhost:30890")) - log(" " + dim("(auto-logged-in as 'student' — no login needed)")) + if access_mode == "local": + log(" " + dim(f"Sign in with local credentials for '{admin_username}'.")) + _print_created_admin_password(admin_password) + else: + log(" " + dim("Shared student session: no login needed.")) log("") log(" " + dim("kubectl is configured at $HOME/.kube/config; try ") + cyan("`kubectl get nodes`")) log("") +def _print_created_admin_password(admin_password: str | None) -> None: + if admin_password is not None: + from auplc_installer.colors import bold, bold_green + + log(" " + bold("Temporary admin password (shown once): ") + bold_green(admin_password)) + + def cmd_uninstall(state: InstallerState) -> None: ensure_sudo_session(assume_yes=state.assume_yes) keepalive = start_sudo_keepalive() @@ -608,15 +646,18 @@ def cmd_dev_deploy(state: InstallerState) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) + access_mode, admin_username = _resolve_access_settings(state) generate_values_overlay( state.gpu, image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) - deploy_runtime(paths, dev=True) + _print_created_admin_password(deploy_runtime(paths, dev=True, access_mode=access_mode, admin_username=admin_username)) def cmd_dev_upgrade(state: InstallerState) -> None: @@ -625,11 +666,15 @@ def cmd_dev_upgrade(state: InstallerState) -> None: paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) _preserve_courses_for_upgrade(state, paths.overlay_path) + _preserve_access_settings_for_upgrade(state, paths.overlay_path) + access_mode, admin_username = _resolve_access_settings(state) generate_values_overlay( state.gpu, image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) @@ -652,15 +697,18 @@ def cmd_rt_install(state: InstallerState) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) + access_mode, admin_username = _resolve_access_settings(state) generate_values_overlay( state.gpu, image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) - deploy_runtime(paths) + _print_created_admin_password(deploy_runtime(paths, access_mode=access_mode, admin_username=admin_username)) def cmd_rt_upgrade(state: InstallerState) -> None: @@ -669,11 +717,15 @@ def cmd_rt_upgrade(state: InstallerState) -> None: paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) _preserve_courses_for_upgrade(state, paths.overlay_path) + _preserve_access_settings_for_upgrade(state, paths.overlay_path) + access_mode, admin_username = _resolve_access_settings(state) generate_values_overlay( state.gpu, image_registry=state.image_registry, image_tag=state.image_tag, courses=state.courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) @@ -700,6 +752,23 @@ def _preserve_courses_for_upgrade(state: InstallerState, overlay_path: Path) -> log(f"Preserving previous course selection: {previous.description()}") +def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: Path) -> None: + if state.access_mode: + return + previous = try_load_access_settings_from_overlay(overlay_path) + if previous is None: + return + state.access_mode, state.admin_username = previous + log(f"Preserving previous access mode: {state.access_mode}") + + +def _resolve_access_settings(state: InstallerState) -> tuple[str, str]: + access_mode = state.access_mode or "personal" + if access_mode not in ("local", "personal"): + raise InstallerError("--access-mode must be local or personal") + return access_mode, state.admin_username or "admin" + + def cmd_rt_remove(state: InstallerState) -> None: remove_runtime() @@ -786,6 +855,8 @@ def main(argv: Sequence[str] | None = None) -> None: or tok.startswith("--mirror-pip=") or tok.startswith("--mirror-npm=") or tok.startswith("--courses=") + or tok.startswith("--access-mode=") + or tok.startswith("--admin-username=") or tok in ("-y", "--yes", "-v", "--verbose", "--version", "--dry-run", "--try-run") ): flags.append(tok) @@ -798,6 +869,8 @@ def main(argv: Sequence[str] | None = None) -> None: try: state = InstallerState.from_environment(script_dir=script_dir) _apply_global_flags(state, args) + if args.command not in (None, "tui") and not state.access_mode: + log("No --access-mode supplied; defaulting to personal shared student access.") _dispatch(args.command, list(args.rest), state, source_root=script_dir, dry_run=args.dry_run) except InstallerError as exc: log_error(str(exc)) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 5815f391..0734a94c 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -46,7 +46,9 @@ def emit_overlay( image_registry: str, image_tag: str, courses: CourseSelection, - offline_mode: bool, + access_mode: str = "personal", + admin_username: str = "admin", + offline_mode: bool = False, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" buf = StringIO() @@ -64,8 +66,17 @@ def emit_overlay( targets = " ".join(s.gpu_target for s in cfg.skus) buf.write(f"# Mixed gfx targets: {targets}\n") buf.write(f"# Env selection : {courses.description()}\n") + buf.write(f"# Access mode : {access_mode}\n") + buf.write(f"# Admin username: {admin_username}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") + auth_mode = "local" if access_mode == "local" else "auto-login" + buf.write(f" authMode: {auth_mode}\n") + buf.write(" adminUser:\n") + buf.write(f" enabled: {'true' if access_mode == 'local' else 'false'}\n") + buf.write(f' username: "{admin_username}"\n') + if access_mode == "local": + buf.write(' existingSecret: "jupyterhub-admin-credentials"\n') # --- accelerators --- any_accel_emitted = False @@ -163,7 +174,9 @@ def generate_values_overlay( image_registry: str, image_tag: str, courses: CourseSelection, - offline_mode: bool, + access_mode: str = "personal", + admin_username: str = "admin", + offline_mode: bool = False, overlay_path: Path, ) -> Path: """Render the overlay and write it to ``overlay_path``. Returns the path.""" @@ -174,6 +187,8 @@ def generate_values_overlay( image_registry=image_registry, image_tag=image_tag, courses=courses, + access_mode=access_mode, + admin_username=admin_username, offline_mode=offline_mode, ) overlay_path.write_text(text, encoding="utf-8") @@ -185,6 +200,8 @@ def generate_values_overlay( # ``rt upgrade`` (no ``--courses=`` flag) preserves whatever the user # originally installed with instead of silently expanding to "all". _COURSE_HEADER_RE = re.compile(r"^# (?:Env selection|Course selection)\s*:\s*(.+?)\s*$") +_ACCESS_MODE_HEADER_RE = re.compile(r"^# Access mode\s*:\s*(local|personal)\s*$") +_ADMIN_USERNAME_HEADER_RE = re.compile(r"^# Admin username\s*:\s*(.+?)\s*$") def try_load_courses_from_overlay(overlay_path: Path) -> CourseSelection | None: @@ -224,12 +241,35 @@ def try_load_courses_from_overlay(overlay_path: Path) -> CourseSelection | None: return None +def try_load_access_settings_from_overlay(overlay_path: Path) -> tuple[str, str] | None: + if not overlay_path.is_file(): + return None + try: + text = overlay_path.read_text(encoding="utf-8") + except OSError: + return None + access_mode = "" + admin_username = "" + for line in text.splitlines(): + mode_match = _ACCESS_MODE_HEADER_RE.match(line) + if mode_match: + access_mode = mode_match.group(1) + continue + username_match = _ADMIN_USERNAME_HEADER_RE.match(line) + if username_match: + admin_username = username_match.group(1) + if access_mode == "" or admin_username == "": + return None + return access_mode, admin_username + + # Re-exported so callers can import ``NONE_SENTINEL`` from a single module # without dipping into the lower-level catalog module. __all__ = [ "emit_overlay", "generate_values_overlay", "try_load_courses_from_overlay", + "try_load_access_settings_from_overlay", "GPU_RESOURCE_KEYS", "NONE_SENTINEL", ] diff --git a/auplc_installer/state.py b/auplc_installer/state.py index 5baf7ff4..899e5e8b 100644 --- a/auplc_installer/state.py +++ b/auplc_installer/state.py @@ -52,6 +52,9 @@ class InstallerState: # Course selection (drives image filtering + teams.mapping override) courses: CourseSelection = field(default_factory=CourseSelection.default) + access_mode: str = "" + admin_username: str = "" + # Non-interactive / scripted mode assume_yes: bool = False @@ -86,6 +89,8 @@ def from_environment(cls, *, script_dir: Path) -> InstallerState: mirror_npm=os.environ.get("MIRROR_NPM", ""), image_registry=os.environ.get("IMAGE_REGISTRY", DEFAULT_IMAGE_REGISTRY), image_tag=os.environ.get("IMAGE_TAG", DEFAULT_IMAGE_TAG), + access_mode=os.environ.get("AUPLC_ACCESS_MODE", ""), + admin_username=os.environ.get("AUPLC_ADMIN_USERNAME", ""), assume_yes=os.environ.get("AUPLC_YES", "0") == "1", verbose=os.environ.get("AUPLC_VERBOSE", "0") == "1", ) diff --git a/auplc_installer/summary.py b/auplc_installer/summary.py index 24df1cbf..2136cbc1 100644 --- a/auplc_installer/summary.py +++ b/auplc_installer/summary.py @@ -69,6 +69,9 @@ def format_configuration_summary(state: InstallerState, *, image_source_label: s lines.append(f" PyPI mirror : {state.mirror_pip or '(default)'}") lines.append(f" npm mirror : {state.mirror_npm or '(default)'}") lines.append(f" Environments : {state.courses.description()}") + lines.append(f" Access mode : {state.access_mode or 'personal'}") + if state.access_mode == "local": + lines.append(f" Admin username : {state.admin_username or 'admin'}") return "\n".join(lines) @@ -105,4 +108,7 @@ def row(key: str, value: str, *, accent: bool = False, faint: bool = False) -> s else: lines.append(row("npm mirror", "(default)", faint=True)) lines.append(row("Environments", state.courses.description(), accent=True)) + lines.append(row("Access mode", state.access_mode or "personal", accent=True)) + if state.access_mode == "local": + lines.append(row("Admin username", state.admin_username or "admin")) return "\n".join(lines) diff --git a/auplc_installer/tui.py b/auplc_installer/tui.py index 175cf0a8..f90a036e 100644 --- a/auplc_installer/tui.py +++ b/auplc_installer/tui.py @@ -630,6 +630,21 @@ def _flow_select_envs(state: InstallerState, *, allow_back: bool = False) -> boo return True +def _flow_select_access(state: InstallerState) -> None: + state.access_mode = _ask_select( + "Access mode", + ( + Choice("local", "local - sign in with managed local credentials (default)"), + Choice("personal", "personal - shared student session without a login"), + ), + default_value="local", + ) + if state.access_mode == "local": + state.admin_username = _ask_text("Administrator username", default=state.admin_username or "admin") + else: + state.admin_username = "" + + # Back-compat alias for any external callers. _flow_select_courses = _flow_select_envs @@ -672,6 +687,7 @@ def _flow_install(state: InstallerState) -> None: # Back from env selection in offline mode returns to GPU step. _flow_select_gpu(state) + _flow_select_access(state) log("\n" + format_configuration_summary_colored(state, image_source_label=image_source_label) + "\n") if not _ask_confirm("Proceed with installation?", default=True): raise _CancelledError diff --git a/tests/installer/test_local_auth.py b/tests/installer/test_local_auth.py new file mode 100644 index 00000000..5b0bf402 --- /dev/null +++ b/tests/installer/test_local_auth.py @@ -0,0 +1,71 @@ +import json +from pathlib import Path + +from auplc_installer.cli import _preserve_access_settings_for_upgrade +from auplc_installer.gpu import GpuConfig, append_product +from auplc_installer.overlay import generate_values_overlay, try_load_access_settings_from_overlay +from auplc_installer.state import InstallerState +from auplc_installer.tui import _flow_select_access + + +def test_overlay_emits_local_auth_and_round_trips_generated_headers(tmp_path: Path) -> None: + cfg = GpuConfig() + append_product(cfg, "AMD_Radeon_8060S_Graphics") + overlay = tmp_path / "values.local.yaml" + + generate_values_overlay( + cfg, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=InstallerState().courses, + access_mode="local", + admin_username="operator", + offline_mode=False, + overlay_path=overlay, + ) + + settings = try_load_access_settings_from_overlay(overlay) + rendered = json.loads(json.dumps(__import__("yaml").safe_load(overlay.read_text()))) + assert settings == ("local", "operator") + assert rendered["custom"]["authMode"] == "local" + assert rendered["custom"]["adminUser"] == { + "enabled": True, + "username": "operator", + "existingSecret": "jupyterhub-admin-credentials", + } + + +def test_bare_upgrade_restores_local_access_settings(tmp_path: Path) -> None: + cfg = GpuConfig() + append_product(cfg, "AMD_Radeon_8060S_Graphics") + overlay = tmp_path / "values.local.yaml" + generate_values_overlay( + cfg, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=InstallerState().courses, + access_mode="local", + admin_username="operator", + offline_mode=False, + overlay_path=overlay, + ) + + state = InstallerState() + _preserve_access_settings_for_upgrade(state, overlay) + + assert state.access_mode == "local" + assert state.admin_username == "operator" + + +def test_cli_defaults_to_personal_but_tui_defaults_to_local(monkeypatch) -> None: + state = InstallerState() + selections = iter(["local"]) + names = iter(["admin"]) + monkeypatch.setattr("auplc_installer.tui._ask_select", lambda *_args, **_kwargs: next(selections)) + monkeypatch.setattr("auplc_installer.tui._ask_text", lambda *_args, **_kwargs: next(names)) + + _flow_select_access(state) + + assert InstallerState().access_mode == "" + assert state.access_mode == "local" + assert state.admin_username == "admin" From 8ace2e1953994ae57333b3e008f1094dfbbd09d2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:33:32 +0800 Subject: [PATCH 117/180] feat(installer): create local admin credentials secret --- auplc_installer/helm.py | 46 ++++++++++++++++++++++++++-- tests/installer/test_admin_secret.py | 41 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/installer/test_admin_secret.py diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index 2a6799f8..49cb12af 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -10,10 +10,12 @@ from __future__ import annotations +import json +import secrets from dataclasses import dataclass from pathlib import Path -from auplc_installer.util import log, run_streaming +from auplc_installer.util import InstallerError, log, run, run_streaming DEV_VALUES_PATH = "runtime/values-dev.yaml" @@ -51,12 +53,51 @@ def _helm_install_args(paths: RuntimePaths, *, dev: bool = False) -> list[str]: return args -def deploy_runtime(paths: RuntimePaths, *, dev: bool = False) -> None: +def ensure_local_admin_secret(admin_username: str) -> str | None: + """Create the local admin credentials Secret, returning only a new password.""" + secret_name = "jupyterhub-admin-credentials" + existing = run( + ["kubectl", "get", "secret", secret_name, "--namespace", "jupyterhub"], + check=False, + ) + if existing.returncode == 0: + return None + + password = secrets.token_urlsafe(24) + payload = json.dumps( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": secret_name, "namespace": "jupyterhub"}, + "type": "Opaque", + "stringData": {"admin-username": admin_username, "admin-password": password}, + } + ) + created = run( + ["kubectl", "create", "--namespace", "jupyterhub", "--filename=-"], + check=False, + input_text=payload, + ) + if created.returncode == 0: + return password + if "AlreadyExists" in (created.stdout or ""): + return None + raise InstallerError("Failed to create local admin credentials Secret") + + +def deploy_runtime( + paths: RuntimePaths, + *, + dev: bool = False, + access_mode: str = "personal", + admin_username: str = "admin", +) -> str | None: """Initial Helm install of JupyterHub. Waits for hub/proxy/scheduler ready.""" msg = "Deploying AUP Learning Cloud Runtime" if dev: msg += " (dev mode)" log(msg + "...") + admin_password = ensure_local_admin_secret(admin_username) if access_mode == "local" else None cmd = [ "helm", "install", @@ -86,6 +127,7 @@ def deploy_runtime(paths: RuntimePaths, *, dev: bool = False) -> None: if dev: log("") log("Dev deployment ready. Admin UI: http://localhost:30890/hub/admin/users") + return admin_password def upgrade_runtime(paths: RuntimePaths, *, dev: bool = False) -> None: diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py new file mode 100644 index 00000000..133a3931 --- /dev/null +++ b/tests/installer/test_admin_secret.py @@ -0,0 +1,41 @@ +import json +import subprocess + +from auplc_installer.helm import ensure_local_admin_secret + + +def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(monkeypatch, capsys) -> None: + calls: list[tuple[list[str], str | None]] = [] + + def fake_run(command, *, check=True, input_text=None): + calls.append((command, input_text)) + return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + monkeypatch.setattr("auplc_installer.helm.secrets.token_urlsafe", lambda _length: "generated-password") + + password = ensure_local_admin_secret("operator") + + assert password == "generated-password" + assert calls[0] == ( + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + None, + ) + assert "generated-password" not in " ".join(calls[1][0]) + payload = json.loads(calls[1][1] or "") + assert payload["metadata"]["name"] == "jupyterhub-admin-credentials" + assert payload["stringData"] == {"admin-username": "operator", "admin-password": "generated-password"} + assert "generated-password" not in capsys.readouterr().out + + +def test_reuses_existing_local_admin_secret(monkeypatch) -> None: + calls: list[list[str]] = [] + + def fake_run(command, *, check=True, input_text=None): + calls.append(command) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + assert ensure_local_admin_secret("operator") is None + assert calls == [["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"]] From 030e3664c35dde9e99c0930e729cc690e890e84b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:33:47 +0800 Subject: [PATCH 118/180] feat(chart): configure local admin bootstrap --- runtime/chart/templates/hub/deployment.yaml | 16 +++++++-- runtime/chart/templates/hub/secret-admin.yaml | 4 +-- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 29 ++++++++++++++-- runtime/chart/values.yaml | 20 +++-------- runtime/values.yaml | 3 ++ tests/installer/test_chart_local_auth.py | 34 +++++++++++++++++++ 7 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 tests/installer/test_chart_local_auth.py diff --git a/runtime/chart/templates/hub/deployment.yaml b/runtime/chart/templates/hub/deployment.yaml index 1efc79a6..211828ed 100644 --- a/runtime/chart/templates/hub/deployment.yaml +++ b/runtime/chart/templates/hub/deployment.yaml @@ -215,8 +215,20 @@ spec: function on the user managed k8s Secret which is assumed to not be possible. */}} - name: {{ include "jupyterhub.hub.fullname" . }} - key: hub.config.ConfigurableHTTPProxy.auth_token + name: {{ include "jupyterhub.hub.fullname" . }} + key: hub.config.ConfigurableHTTPProxy.auth_token + {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} + - name: JUPYTERHUB_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} + key: admin-username + - name: JUPYTERHUB_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} + key: admin-password + {{- end }} {{- with .Values.hub.extraEnv }} {{- include "jupyterhub.extraEnv" . | nindent 12 }} {{- end }} diff --git a/runtime/chart/templates/hub/secret-admin.yaml b/runtime/chart/templates/hub/secret-admin.yaml index 4fcd7cc4..0e20e99f 100644 --- a/runtime/chart/templates/hub/secret-admin.yaml +++ b/runtime/chart/templates/hub/secret-admin.yaml @@ -19,7 +19,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */}} -{{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} +{{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled (not .Values.custom.adminUser.existingSecret) }} apiVersion: v1 kind: Secret metadata: @@ -32,6 +32,6 @@ metadata: "helm.sh/hook-weight": "-5" type: Opaque data: - api-token: {{ randAlphaNum 64 | b64enc | quote }} + admin-username: {{ .Values.custom.adminUser.username | b64enc | quote }} admin-password: {{ randAlphaNum 16 | b64enc | quote }} {{- end }} diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index ef7efffc..1f0f8b80 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 22ff5fec..bdaeddd5 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3173,13 +3173,14 @@ properties: properties: authMode: type: string - enum: [auto-login, dummy, github, multi] + enum: [auto-login, dummy, github, local, multi] description: | Authentication mode for the JupyterHub instance. - `auto-login`: No credentials required, auto-login as 'student' (for demos/single-node) - `dummy`: Accept any username/password (for testing) - `github`: GitHub App authentication only + - `local`: Closed local accounts managed by an administrator - `multi`: GitHub App + Local native accounts (recommended for production) adminUser: @@ -3187,14 +3188,17 @@ properties: additionalProperties: false description: | Auto-create admin user configuration. - When enabled, Helm will generate random credentials and store them in a Secret. + Bootstrap configuration for an administrator account. properties: enabled: type: boolean description: | Enable auto-admin creation on first install. Credentials will be stored in `jupyterhub-admin-credentials` secret. - + username: + type: string + existingSecret: + type: string notifications: type: object additionalProperties: false @@ -3783,6 +3787,25 @@ properties: enum: ["", IfNotPresent, Always, Never, "null"] description: Image pull policy. + allOf: + - if: + required: [authMode] + properties: + authMode: + const: local + then: + required: [adminUser] + properties: + adminUser: + required: [enabled, username, existingSecret] + properties: + enabled: + const: true + username: + minLength: 1 + existingSecret: + minLength: 1 + cull: type: object additionalProperties: false diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index a548691f..f60473b8 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -31,10 +31,11 @@ enabled: # custom can contain anything you want to pass to the hub pod, as all passed # Helm template values will be made available there. custom: - # Authentication mode: "auto-login" | "dummy" | "github" | "multi" + # Authentication mode: "auto-login" | "dummy" | "github" | "local" | "multi" # - auto-login: No credentials required, auto-login as 'student' (default, for single-node) # - dummy: Accept any username/password (for testing) # - github: GitHub App authentication + # - local: Closed local accounts managed by an administrator # - multi: GitHub App + Local accounts authMode: "auto-login" @@ -45,6 +46,8 @@ custom: # Auto-create admin user on first install (optional) adminUser: enabled: false + username: "admin" + existingSecret: "" # Accelerator configuration (GPU/NPU nodes) # Define these in runtime/values.yaml, not here @@ -162,20 +165,7 @@ hub: args: [] extraConfig: {} extraFiles: {} - extraEnv: - # Environment variables from secrets (for auto-admin feature) - JUPYTERHUB_API_TOKEN: - valueFrom: - secretKeyRef: - name: jupyterhub-admin-credentials - key: api-token - optional: true - JUPYTERHUB_ADMIN_PASSWORD: - valueFrom: - secretKeyRef: - name: jupyterhub-admin-credentials - key: admin-password - optional: true + extraEnv: {} extraContainers: [] extraVolumes: [] extraVolumeMounts: [] diff --git a/runtime/values.yaml b/runtime/values.yaml index 2a942e02..4ab78f9a 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -46,6 +46,7 @@ custom: # - auto-login: No credentials required, auto-login as 'student' (for single-node dev) # - dummy: Accept any username/password (for testing) # - github: GitHub App authentication + # - local: Closed local accounts managed by an administrator # - multi: GitHub App + Local accounts authMode: "auto-login" @@ -60,6 +61,8 @@ custom: # Auto-create admin user on first install adminUser: enabled: false + username: "admin" + existingSecret: "" # ============================================================================ # Notifications diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py new file mode 100644 index 00000000..9da80f24 --- /dev/null +++ b/tests/installer/test_chart_local_auth.py @@ -0,0 +1,34 @@ +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def test_local_chart_render_uses_existing_secret_only_for_hub_bootstrap() -> None: + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "--set", + "custom.authMode=local", + "--set", + "custom.adminUser.enabled=true", + "--set", + "custom.adminUser.username=operator", + "--set", + "custom.adminUser.existingSecret=jupyterhub-admin-credentials", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + assert "authMode: local" in result.stdout + assert "name: JUPYTERHUB_ADMIN_USERNAME" in result.stdout + assert "key: admin-username" in result.stdout + assert "name: JUPYTERHUB_ADMIN_PASSWORD" in result.stdout + assert "key: admin-password" in result.stdout + assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" not in result.stdout From 41e2333fd3a72cde224d6969bc354ced3f3058bb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:34:02 +0800 Subject: [PATCH 119/180] feat(hub): add closed local authenticator --- runtime/hub/core/authenticators/__init__.py | 6 ++- runtime/hub/core/authenticators/local.py | 14 ++++++ runtime/hub/core/config.py | 7 ++- runtime/hub/core/groups.py | 2 +- runtime/hub/core/handlers.py | 44 ++++--------------- runtime/hub/core/setup.py | 9 +++- runtime/hub/frontend/templates/login.html | 2 +- runtime/hub/tests/test_local_authenticator.py | 43 ++++++++++++++++++ 8 files changed, 84 insertions(+), 43 deletions(-) create mode 100644 runtime/hub/core/authenticators/local.py create mode 100644 runtime/hub/tests/test_local_authenticator.py diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index 7a491341..6cdca959 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -27,6 +27,7 @@ from core.authenticators.firstuse import CustomFirstUseAuthenticator from core.authenticators.github_app import GITHUB_USERNAME_PREFIX, CustomGitHubOAuthenticator from core.authenticators.jwt import RemoteLabAuthenticator +from core.authenticators.local import CustomLocalAuthenticator from core.authenticators.multi import CustomMultiAuthenticator LOCAL_ACCOUNT_PREFIX = "LocalAccount" @@ -37,7 +38,7 @@ def create_authenticator(auth_mode: str, **kwargs): Factory function to create the appropriate authenticator. Args: - auth_mode: Authentication mode ("auto-login", "dummy", "github", "multi") + auth_mode: Authentication mode ("auto-login", "dummy", "github", "local", "multi") **kwargs: Additional configuration options Returns: @@ -49,6 +50,8 @@ def create_authenticator(auth_mode: str, **kwargs): return "dummy" elif auth_mode == "github": return CustomGitHubOAuthenticator + elif auth_mode == "local": + return CustomLocalAuthenticator elif auth_mode == "multi": return CustomMultiAuthenticator else: @@ -61,6 +64,7 @@ def create_authenticator(auth_mode: str, **kwargs): "AutoLoginAuthenticator", "CustomGitHubOAuthenticator", "CustomFirstUseAuthenticator", + "CustomLocalAuthenticator", "CustomMultiAuthenticator", "create_authenticator", "LOCAL_ACCOUNT_PREFIX", diff --git a/runtime/hub/core/authenticators/local.py b/runtime/hub/core/authenticators/local.py new file mode 100644 index 00000000..ac3bf7f0 --- /dev/null +++ b/runtime/hub/core/authenticators/local.py @@ -0,0 +1,14 @@ +from core.authenticators.firstuse import CustomFirstUseAuthenticator + + +class CustomLocalAuthenticator(CustomFirstUseAuthenticator): + async def authenticate(self, _handler, data): + username = self.normalize_username(data.get("username", "")) + password = data.get("password", "") + if not username or not password or ":" in username: + return None + if not self._user_exists(username): + return None + if not self.check_password(username, password): + return None + return username diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 3925bbf0..5bc3f6f9 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -282,6 +282,7 @@ def __init__(self): self.single_node_mode: bool = False self.github_org_name: str = "" self.cluster_name: str = "" + self.admin_username: str = "admin" self.quota_enabled: bool = False # Parsed configuration @@ -320,6 +321,9 @@ def init(cls, config_path: str | Path) -> HubConfig: instance.auth_mode = raw_config.get("authMode", "auto-login") instance.github_org_name = raw_config.get("githubOrgName", "") instance.cluster_name = raw_config.get("clusterName", "") + admin_user = raw_config.get("adminUser", {}) + if isinstance(admin_user, dict): + instance.admin_username = admin_user.get("username", "admin") # Single-node mode: from config or auto-enable for auto-login single_node_mode = raw_config.get("singleNodeMode") @@ -345,8 +349,7 @@ def init(cls, config_path: str | Path) -> HubConfig: if instance._config.quota.enabled is not None: instance.quota_enabled = instance._config.quota.enabled else: - # Disable quota for auto-login and dummy modes by default - instance.quota_enabled = instance.auth_mode not in ("auto-login", "dummy") + instance.quota_enabled = instance.auth_mode not in ("auto-login", "dummy", "local") instance._config.quota.enabled = instance.quota_enabled cls._initialized = True diff --git a/runtime/hub/core/groups.py b/runtime/hub/core/groups.py index 30d7b085..2dc1d832 100644 --- a/runtime/hub/core/groups.py +++ b/runtime/hub/core/groups.py @@ -709,7 +709,7 @@ def resolve_resources_for_user( """Resolve the resources visible to a user for UI and spawn flows.""" username = user.name.strip() - if auth_mode in ["auto-login", "dummy"]: + if auth_mode in ["auto-login", "dummy", "local"]: return all_resources available_resources = get_resources_for_user(user, team_resource_mapping) diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index d1a93dee..5cd38dd2 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -217,12 +217,8 @@ async def get(self): if ":" in username: username = username.split(":", 1)[1] - needs_change = False - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - needs_change = authenticator.needs_password_change(username) - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) + needs_change = firstuse_auth.needs_password_change(username) if firstuse_auth else False self.set_header("Content-Type", "application/json") self.finish(json.dumps({"needs_password_change": needs_change})) @@ -243,12 +239,8 @@ async def get(self): if ":" in username: username = username.split(":", 1)[1] - is_forced = False - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - is_forced = authenticator.needs_password_change(username) - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) + is_forced = firstuse_auth.needs_password_change(username) if firstuse_auth else False html = await self.render_template( "change-password.html", password_changed=password_changed, forced_change=is_forced or forced @@ -288,12 +280,7 @@ def _render_error(msg: str): self.set_status(400) return self.finish(html) - firstuse_auth = None - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - firstuse_auth = authenticator - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: html = await _render_error("Password change not available") @@ -380,12 +367,7 @@ async def post(self): + f"admin/reset-password?user={target_user}&error=Cannot+reset+password+for+GitHub+users" ) - firstuse_auth = None - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - firstuse_auth = authenticator - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: return self.redirect(self.hub.base_url + "admin/reset-password?error=Password+reset+not+available") @@ -461,12 +443,7 @@ async def post(self): self.set_header("Content-Type", "application/json") return self.finish(json.dumps({"error": "Cannot set password for GitHub users"})) - firstuse_auth = None - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - firstuse_auth = authenticator - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: self.set_status(500) @@ -552,12 +529,7 @@ async def post(self): json.dumps({"error": f"Cannot set password for GitHub user: {entry['username']}"}) ) - firstuse_auth = None - if isinstance(self.authenticator, MultiAuthenticator): - for authenticator in self.authenticator._authenticators: - if isinstance(authenticator, CustomFirstUseAuthenticator): - firstuse_auth = authenticator - break + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: self.set_status(500) diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index df5e2b6c..826c341c 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -175,7 +175,7 @@ async def auth_state_hook(spawner, auth_state): # Set authenticator based on mode c.JupyterHub.authenticator_class = create_authenticator(config.auth_mode) - if config.auth_mode == "auto-login": + if config.auth_mode in ("auto-login", "local"): c.Authenticator.allow_all = True elif config.auth_mode == "multi": c.MultiAuthenticator.authenticators = [ @@ -349,7 +349,12 @@ async def delete(self, group_name): # ========================================================================= admin_password = os.environ.get("JUPYTERHUB_ADMIN_PASSWORD", "") - admin_username = "admin" + admin_username = os.environ.get("JUPYTERHUB_ADMIN_USERNAME", "admin") + + if config.auth_mode == "local" and not admin_password: + raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_PASSWORD") + if config.auth_mode == "local" and not os.environ.get("JUPYTERHUB_ADMIN_USERNAME"): + raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_USERNAME") if admin_password: c.Authenticator.admin_users = {admin_username} diff --git a/runtime/hub/frontend/templates/login.html b/runtime/hub/frontend/templates/login.html index 50377583..7a820507 100755 --- a/runtime/hub/frontend/templates/login.html +++ b/runtime/hub/frontend/templates/login.html @@ -154,7 +154,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L <p class="login-error font-medium mb-4 text-center">{{ login_error }}</p> {% endif %} - {% if authenticator_mode == 'dummy' %} + {% if authenticator_mode in ['dummy', 'local'] %} <!-- Dummy Authenticator: Simple login form --> <form action="{{ base_url }}login?next={{ next | urlencode }}" method="post" role="form" class="space-y-6"> <input type="hidden" name="_xsrf" value="{{ xsrf }}" /> diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py new file mode 100644 index 00000000..e6ff1b01 --- /dev/null +++ b/runtime/hub/tests/test_local_authenticator.py @@ -0,0 +1,43 @@ +import asyncio +import importlib.util +import sys +import types +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LOCAL_AUTHENTICATOR = ROOT / "core" / "authenticators" / "local.py" + + +class FakeFirstUseAuthenticator: + def normalize_username(self, username): + return username.lower() + + def _user_exists(self, username): + return username == "existing" + + def check_password(self, username, password): + return username == "existing" and password == "correct-password" + + +def test_local_authenticator_rejects_first_use_and_accepts_existing_password() -> None: + core = types.ModuleType("core") + authenticators = types.ModuleType("core.authenticators") + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator + sys.modules.update( + { + "core": core, + "core.authenticators": authenticators, + "core.authenticators.firstuse": firstuse, + } + ) + spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + authenticator = module.CustomLocalAuthenticator() + + assert asyncio.run(authenticator.authenticate(None, {"username": "EXISTING", "password": "correct-password"})) == "existing" + assert asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "wrong-password"})) is None + assert asyncio.run(authenticator.authenticate(None, {"username": "new", "password": "valid-password"})) is None From ca22d9ca1db0c7d37012492a150dea11d1815f21 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:34:12 +0800 Subject: [PATCH 120/180] docs: document single-node access modes --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 566443c1..a6947ffb 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,18 @@ cd aup-learning-cloud ./auplc-installer install ``` +### Single-Node Access + +The interactive installer defaults to `local` access, which creates a closed local administrator account. Choose `personal` for the shared student session used by earlier single-node installs. + +For scripted installs, `personal` remains the compatibility default. Select local access explicitly when credentials are required: + +```bash +./auplc-installer install --access-mode=local --admin-username=admin +``` + +The installer generates the administrator password only when it creates `jupyterhub-admin-credentials` and displays it once after a successful deployment. Re-running against an existing Secret reuses the credentials without rotating or redisplaying them. Local users are created and assigned passwords through the Admin UI. + A successful install looks like this: ```text From 27504c968b468348e628ea6f36654d66aa32cf80 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:47:06 +0800 Subject: [PATCH 121/180] fix(installer): prepare namespace for local credentials --- auplc_installer/helm.py | 11 +++++ tests/installer/test_admin_secret.py | 65 +++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index 49cb12af..05c1e303 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -53,9 +53,20 @@ def _helm_install_args(paths: RuntimePaths, *, dev: bool = False) -> list[str]: return args +def _ensure_namespace() -> None: + existing = run(["kubectl", "get", "namespace", "jupyterhub"], check=False) + if existing.returncode == 0: + return + created = run(["kubectl", "create", "namespace", "jupyterhub"], check=False) + if created.returncode == 0 or "AlreadyExists" in (created.stdout or ""): + return + raise InstallerError("Failed to create jupyterhub namespace") + + def ensure_local_admin_secret(admin_username: str) -> str | None: """Create the local admin credentials Secret, returning only a new password.""" secret_name = "jupyterhub-admin-credentials" + _ensure_namespace() existing = run( ["kubectl", "get", "secret", secret_name, "--namespace", "jupyterhub"], check=False, diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py index 133a3931..0f345186 100644 --- a/tests/installer/test_admin_secret.py +++ b/tests/installer/test_admin_secret.py @@ -1,7 +1,11 @@ import json import subprocess +from pathlib import Path -from auplc_installer.helm import ensure_local_admin_secret +import pytest + +from auplc_installer.helm import RuntimePaths, deploy_runtime, ensure_local_admin_secret +from auplc_installer.util import InstallerError def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(monkeypatch, capsys) -> None: @@ -9,7 +13,7 @@ def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(mo def fake_run(command, *, check=True, input_text=None): calls.append((command, input_text)) - return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") + return subprocess.CompletedProcess(command, 1 if len(calls) in (1, 3) else 0, "") monkeypatch.setattr("auplc_installer.helm.run", fake_run) monkeypatch.setattr("auplc_installer.helm.secrets.token_urlsafe", lambda _length: "generated-password") @@ -17,12 +21,14 @@ def fake_run(command, *, check=True, input_text=None): password = ensure_local_admin_secret("operator") assert password == "generated-password" - assert calls[0] == ( + assert calls[0] == (["kubectl", "get", "namespace", "jupyterhub"], None) + assert calls[1] == (["kubectl", "create", "namespace", "jupyterhub"], None) + assert calls[2] == ( ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], None, ) - assert "generated-password" not in " ".join(calls[1][0]) - payload = json.loads(calls[1][1] or "") + assert "generated-password" not in " ".join(calls[3][0]) + payload = json.loads(calls[3][1] or "") assert payload["metadata"]["name"] == "jupyterhub-admin-credentials" assert payload["stringData"] == {"admin-username": "operator", "admin-password": "generated-password"} assert "generated-password" not in capsys.readouterr().out @@ -38,4 +44,51 @@ def fake_run(command, *, check=True, input_text=None): monkeypatch.setattr("auplc_installer.helm.run", fake_run) assert ensure_local_admin_secret("operator") is None - assert calls == [["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"]] + assert calls == [ + ["kubectl", "get", "namespace", "jupyterhub"], + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + ] + + +def test_deploy_orders_namespace_secret_and_helm_without_printing_new_password(monkeypatch, capsys) -> None: + calls: list[tuple[str, list[str], str | None]] = [] + + def fake_run(command, *, check=True, input_text=None): + calls.append(("run", command, input_text)) + return subprocess.CompletedProcess(command, 1 if len(calls) in (1, 3) else 0, "") + + def failing_stream(command, **_kwargs): + calls.append(("stream", command, None)) + raise InstallerError("helm failed") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + monkeypatch.setattr("auplc_installer.helm.run_streaming", failing_stream) + monkeypatch.setattr("auplc_installer.helm.secrets.token_urlsafe", lambda _length: "generated-password") + + with pytest.raises(InstallerError, match="helm failed"): + deploy_runtime( + RuntimePaths(Path("chart"), Path("values.yaml"), Path("values.local.yaml")), + access_mode="local", + admin_username="operator", + ) + + assert [command for _, command, _ in calls] == [ + ["kubectl", "get", "namespace", "jupyterhub"], + ["kubectl", "create", "namespace", "jupyterhub"], + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + ["kubectl", "create", "--namespace", "jupyterhub", "--filename=-"], + [ + "helm", + "install", + "jupyterhub", + "chart", + "--namespace", + "jupyterhub", + "--create-namespace", + "-f", + "values.yaml", + "-f", + "values.local.yaml", + ], + ] + assert "generated-password" not in capsys.readouterr().out From e657a0071bccace2f609d60e8d2a848521fc044c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:47:18 +0800 Subject: [PATCH 122/180] fix(hub): exclude administrators from reset targets --- runtime/hub/core/handlers.py | 2 +- runtime/hub/tests/test_onboarding_handlers.py | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index 5cd38dd2..041602f7 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -327,7 +327,7 @@ async def get(self): from jupyterhub.orm import User for user in self.db.query(User).all(): - if not user.name.startswith(GITHUB_USERNAME_PREFIX) and user.name != "admin": + if not user.name.startswith(GITHUB_USERNAME_PREFIX) and not user.admin: native_users.append(user.name) html = await self.render_template( diff --git a/runtime/hub/tests/test_onboarding_handlers.py b/runtime/hub/tests/test_onboarding_handlers.py index 13eacb9b..e833bb56 100644 --- a/runtime/hub/tests/test_onboarding_handlers.py +++ b/runtime/hub/tests/test_onboarding_handlers.py @@ -15,11 +15,17 @@ jupyterhub_module = types.ModuleType("jupyterhub") apihandlers_module = types.ModuleType("jupyterhub.apihandlers") handlers_module = types.ModuleType("jupyterhub.handlers") + orm_module = types.ModuleType("jupyterhub.orm") apihandlers_module.APIHandler = type("APIHandler", (), {}) handlers_module.BaseHandler = type("BaseHandler", (), {}) + orm_module.User = type("User", (), {}) + scopes_module = types.ModuleType("jupyterhub.scopes") + scopes_module.needs_scope = lambda _scope: lambda handler: handler sys.modules["jupyterhub"] = jupyterhub_module sys.modules["jupyterhub.apihandlers"] = apihandlers_module sys.modules["jupyterhub.handlers"] = handlers_module + sys.modules["jupyterhub.orm"] = orm_module + sys.modules["jupyterhub.scopes"] = scopes_module if "multiauthenticator" not in sys.modules: multiauthenticator_module = types.ModuleType("multiauthenticator") @@ -35,6 +41,7 @@ auth_module = types.ModuleType("core.authenticators") auth_module.__path__ = [str(AUTHENTICATORS)] auth_module.CustomFirstUseAuthenticator = type("CustomFirstUseAuthenticator", (), {}) + auth_module.GITHUB_USERNAME_PREFIX = "github:" sys.modules["core.authenticators"] = auth_module if "sqlalchemy" not in sys.modules: @@ -136,11 +143,13 @@ def load_module(name: str, path: Path): UserOnboardingState = models.UserOnboardingState DismissMyOnboardingHandler = handlers.DismissMyOnboardingHandler GetMyOnboardingHandler = handlers.GetMyOnboardingHandler +AdminResetPasswordHandler = handlers.AdminResetPasswordHandler class DummyUser: - def __init__(self, name: str): + def __init__(self, name: str, admin: bool = False): self.name = name + self.admin = admin class FakeQuery: @@ -157,6 +166,9 @@ def filter_by(self, **kwargs): def first(self): return self._filtered[0] if self._filtered else None + def all(self): + return self._filtered + def one_or_none(self): return self.first() @@ -217,6 +229,32 @@ def make_handler(handler_cls, username: str): return handler, captured +def test_admin_reset_listing_excludes_all_administrators() -> None: + handler = object.__new__(AdminResetPasswordHandler) + handler.current_user = DummyUser("operator", admin=True) + handler.db = FakeDb( + [ + DummyUser("operator", admin=True), + DummyUser("admin", admin=True), + DummyUser("learner"), + DummyUser("github:member"), + ] + ) + handler.get_argument = lambda _name, default="": default + rendered = {} + + async def render_template(_name, **kwargs): + rendered.update(kwargs) + return "html" + + handler.render_template = render_template + handler.finish = lambda _html: None + + asyncio.run(handler.get()) + + assert rendered["native_users"] == ["learner"] + + def test_get_my_onboarding_returns_visible_when_no_state_exists(monkeypatch): monkeypatch.setattr(database, "session_scope", fake_session_scope(FakeDb())) handler, captured = make_handler(GetMyOnboardingHandler, "alice") From 9a6fc632f7ecbbf4cbfa9b3e8a914f6e88f86b20 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:52:41 +0800 Subject: [PATCH 123/180] style: format local authentication changes --- auplc_installer/cli.py | 4 +++- runtime/hub/tests/test_local_authenticator.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f9ba5d61..29f206ec 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -657,7 +657,9 @@ def cmd_dev_deploy(state: InstallerState) -> None: offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) - _print_created_admin_password(deploy_runtime(paths, dev=True, access_mode=access_mode, admin_username=admin_username)) + _print_created_admin_password( + deploy_runtime(paths, dev=True, access_mode=access_mode, admin_username=admin_username) + ) def cmd_dev_upgrade(state: InstallerState) -> None: diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py index e6ff1b01..f6a4ecb7 100644 --- a/runtime/hub/tests/test_local_authenticator.py +++ b/runtime/hub/tests/test_local_authenticator.py @@ -38,6 +38,9 @@ def test_local_authenticator_rejects_first_use_and_accepts_existing_password() - spec.loader.exec_module(module) authenticator = module.CustomLocalAuthenticator() - assert asyncio.run(authenticator.authenticate(None, {"username": "EXISTING", "password": "correct-password"})) == "existing" + assert ( + asyncio.run(authenticator.authenticate(None, {"username": "EXISTING", "password": "correct-password"})) + == "existing" + ) assert asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "wrong-password"})) is None assert asyncio.run(authenticator.authenticate(None, {"username": "new", "password": "valid-password"})) is None From d5abf57292e223064c2b7fdd5d29aeeb7029cb86 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:15:50 +0800 Subject: [PATCH 124/180] fix(installer): harden local credential lifecycle --- auplc_installer/auth.py | 19 ++++ auplc_installer/cli.py | 25 ++++-- auplc_installer/helm.py | 46 +++++++++- auplc_installer/overlay.py | 4 + tests/installer/test_admin_secret.py | 127 +++++++++++++++++++++++++-- tests/installer/test_local_auth.py | 49 ++++++++++- 6 files changed, 253 insertions(+), 17 deletions(-) create mode 100644 auplc_installer/auth.py diff --git a/auplc_installer/auth.py b/auplc_installer/auth.py new file mode 100644 index 00000000..ba6359a3 --- /dev/null +++ b/auplc_installer/auth.py @@ -0,0 +1,19 @@ +"""Local administrator username validation.""" + +from __future__ import annotations + +import re + +from auplc_installer.util import InstallerError + +LOCAL_USERNAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") + + +def validate_local_admin_username(username: str) -> str: + """Return a canonical local administrator username or raise an actionable error.""" + if not LOCAL_USERNAME_PATTERN.fullmatch(username): + raise InstallerError( + "Local administrator username must use lowercase ASCII letters, digits, '.', '_' or '-', " + "start with a letter or digit, and be at most 64 characters." + ) + return username diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 29f206ec..9cf6510f 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -11,6 +11,7 @@ import argparse import contextlib +import re import sys import time from collections.abc import Sequence @@ -18,6 +19,7 @@ from typing import NoReturn from auplc_installer import __version__ +from auplc_installer.auth import validate_local_admin_username from auplc_installer.catalog import parse_selection_spec from auplc_installer.gpu import ( detect_and_configure_gpu, @@ -507,10 +509,14 @@ def _print_success_banner(*, access_mode: str, admin_username: str, admin_passwo def _print_created_admin_password(admin_password: str | None) -> None: - if admin_password is not None: + if admin_password is not None and sys.stdout.isatty(): from auplc_installer.colors import bold, bold_green log(" " + bold("Temporary admin password (shown once): ") + bold_green(admin_password)) + elif admin_password is not None: + log(" Retrieve credentials safely: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo") + else: + log(" Existing credentials were preserved. Retrieve the password: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo") def cmd_uninstall(state: InstallerState) -> None: @@ -680,7 +686,7 @@ def cmd_dev_upgrade(state: InstallerState) -> None: offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) - upgrade_runtime(paths, dev=True) + upgrade_runtime(paths, dev=True, access_mode=access_mode, admin_username=admin_username) def cmd_dev_reinstall(state: InstallerState) -> None: @@ -731,7 +737,7 @@ def cmd_rt_upgrade(state: InstallerState) -> None: offline_mode=state.offline_mode, overlay_path=paths.overlay_path, ) - upgrade_runtime(paths) + upgrade_runtime(paths, access_mode=access_mode, admin_username=admin_username) def _preserve_courses_for_upgrade(state: InstallerState, overlay_path: Path) -> None: @@ -755,9 +761,15 @@ def _preserve_courses_for_upgrade(state: InstallerState, overlay_path: Path) -> def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: Path) -> None: + previous = try_load_access_settings_from_overlay(overlay_path) + if previous is None and overlay_path.is_file(): + text = overlay_path.read_text(encoding="utf-8") + if re.search(r"^\s*authMode:\s*(github|multi|dummy)\s*$", text, re.MULTILINE): + raise InstallerError("Existing overlay uses an advanced authMode; use operator-managed Helm values instead of installer upgrade") if state.access_mode: + if state.access_mode == "local" and not state.admin_username and previous and previous[0] == "local": + state.admin_username = previous[1] return - previous = try_load_access_settings_from_overlay(overlay_path) if previous is None: return state.access_mode, state.admin_username = previous @@ -768,7 +780,10 @@ def _resolve_access_settings(state: InstallerState) -> tuple[str, str]: access_mode = state.access_mode or "personal" if access_mode not in ("local", "personal"): raise InstallerError("--access-mode must be local or personal") - return access_mode, state.admin_username or "admin" + admin_username = state.admin_username or "admin" + if access_mode == "local": + admin_username = validate_local_admin_username(admin_username) + return access_mode, admin_username def cmd_rt_remove(state: InstallerState) -> None: diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index 05c1e303..97c1670c 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -15,6 +15,7 @@ from dataclasses import dataclass from pathlib import Path +from auplc_installer.auth import validate_local_admin_username from auplc_installer.util import InstallerError, log, run, run_streaming DEV_VALUES_PATH = "runtime/values-dev.yaml" @@ -66,22 +67,50 @@ def _ensure_namespace() -> None: def ensure_local_admin_secret(admin_username: str) -> str | None: """Create the local admin credentials Secret, returning only a new password.""" secret_name = "jupyterhub-admin-credentials" + admin_username = validate_local_admin_username(admin_username) _ensure_namespace() existing = run( - ["kubectl", "get", "secret", secret_name, "--namespace", "jupyterhub"], + ["kubectl", "get", "secret", secret_name, "--namespace", "jupyterhub", "-o", "json"], check=False, ) if existing.returncode == 0: + try: + data = json.loads(existing.stdout).get("data", {}) + except json.JSONDecodeError as exc: + raise InstallerError("Unable to inspect existing local admin credentials Secret") from exc + missing = {"admin-password", "api-token"}.difference(data) + if missing: + raise InstallerError(f"Existing local admin credentials Secret is missing {', '.join(sorted(missing))}") + encoded_username = data.get("admin-username") + if encoded_username is None: + run( + [ + "kubectl", "patch", "secret", secret_name, "--namespace", "jupyterhub", "--type", "merge", "--patch", + json.dumps({"stringData": {"admin-username": admin_username}}, separators=(",", ":")), + ] + ) + return None + import base64 + + try: + stored_username = base64.b64decode(encoded_username).decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise InstallerError("Existing local admin credentials Secret has an invalid admin-username") from exc + if stored_username != admin_username: + raise InstallerError("Existing local admin credentials Secret belongs to a different administrator username") return None + if "NotFound" not in (existing.stdout or ""): + raise InstallerError("Unable to inspect local admin credentials Secret; verify Kubernetes access and RBAC") password = secrets.token_urlsafe(24) + api_token = secrets.token_urlsafe(32) payload = json.dumps( { "apiVersion": "v1", "kind": "Secret", "metadata": {"name": secret_name, "namespace": "jupyterhub"}, "type": "Opaque", - "stringData": {"admin-username": admin_username, "admin-password": password}, + "stringData": {"admin-username": admin_username, "admin-password": password, "api-token": api_token}, } ) created = run( @@ -92,7 +121,7 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: if created.returncode == 0: return password if "AlreadyExists" in (created.stdout or ""): - return None + return ensure_local_admin_secret(admin_username) raise InstallerError("Failed to create local admin credentials Secret") @@ -141,8 +170,16 @@ def deploy_runtime( return admin_password -def upgrade_runtime(paths: RuntimePaths, *, dev: bool = False) -> None: +def upgrade_runtime( + paths: RuntimePaths, + *, + dev: bool = False, + access_mode: str = "personal", + admin_username: str = "admin", +) -> None: """Helm upgrade. Used after values changes.""" + if access_mode == "local": + ensure_local_admin_secret(admin_username) cmd = [ "helm", "upgrade", @@ -154,6 +191,7 @@ def upgrade_runtime(paths: RuntimePaths, *, dev: bool = False) -> None: *_helm_install_args(paths, dev=dev), ] run_streaming(cmd) + run_streaming(["kubectl", "rollout", "status", "deployment/hub", "--namespace", "jupyterhub", "--timeout=600s"]) def remove_runtime() -> None: diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 0734a94c..2abdf9d3 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -15,6 +15,7 @@ from io import StringIO from pathlib import Path +from auplc_installer.auth import validate_local_admin_username from auplc_installer.catalog import ( BASE_TEAM_MAPPING, NONE_SENTINEL, @@ -71,7 +72,10 @@ def emit_overlay( buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") auth_mode = "local" if access_mode == "local" else "auto-login" + if access_mode == "local": + admin_username = validate_local_admin_username(admin_username) buf.write(f" authMode: {auth_mode}\n") + buf.write(" singleNodeMode: true\n") buf.write(" adminUser:\n") buf.write(f" enabled: {'true' if access_mode == 'local' else 'false'}\n") buf.write(f' username: "{admin_username}"\n') diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py index 0f345186..be4b8db8 100644 --- a/tests/installer/test_admin_secret.py +++ b/tests/installer/test_admin_secret.py @@ -4,7 +4,7 @@ import pytest -from auplc_installer.helm import RuntimePaths, deploy_runtime, ensure_local_admin_secret +from auplc_installer.helm import RuntimePaths, deploy_runtime, ensure_local_admin_secret, upgrade_runtime from auplc_installer.util import InstallerError @@ -13,7 +13,9 @@ def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(mo def fake_run(command, *, check=True, input_text=None): calls.append((command, input_text)) - return subprocess.CompletedProcess(command, 1 if len(calls) in (1, 3) else 0, "") + if len(calls) == 3: + return subprocess.CompletedProcess(command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found') + return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") monkeypatch.setattr("auplc_installer.helm.run", fake_run) monkeypatch.setattr("auplc_installer.helm.secrets.token_urlsafe", lambda _length: "generated-password") @@ -24,13 +26,17 @@ def fake_run(command, *, check=True, input_text=None): assert calls[0] == (["kubectl", "get", "namespace", "jupyterhub"], None) assert calls[1] == (["kubectl", "create", "namespace", "jupyterhub"], None) assert calls[2] == ( - ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub", "-o", "json"], None, ) assert "generated-password" not in " ".join(calls[3][0]) payload = json.loads(calls[3][1] or "") assert payload["metadata"]["name"] == "jupyterhub-admin-credentials" - assert payload["stringData"] == {"admin-username": "operator", "admin-password": "generated-password"} + assert payload["stringData"] == { + "admin-username": "operator", + "admin-password": "generated-password", + "api-token": "generated-password", + } assert "generated-password" not in capsys.readouterr().out @@ -39,6 +45,20 @@ def test_reuses_existing_local_admin_secret(monkeypatch) -> None: def fake_run(command, *, check=True, input_text=None): calls.append(command) + if command[2] == "secret": + return subprocess.CompletedProcess( + command, + 0, + json.dumps( + { + "data": { + "admin-username": "b3BlcmF0b3I=", + "admin-password": "cGFzc3dvcmQ=", + "api-token": "dG9rZW4=", + } + } + ), + ) return subprocess.CompletedProcess(command, 0, "") monkeypatch.setattr("auplc_installer.helm.run", fake_run) @@ -46,7 +66,7 @@ def fake_run(command, *, check=True, input_text=None): assert ensure_local_admin_secret("operator") is None assert calls == [ ["kubectl", "get", "namespace", "jupyterhub"], - ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub", "-o", "json"], ] @@ -55,7 +75,9 @@ def test_deploy_orders_namespace_secret_and_helm_without_printing_new_password(m def fake_run(command, *, check=True, input_text=None): calls.append(("run", command, input_text)) - return subprocess.CompletedProcess(command, 1 if len(calls) in (1, 3) else 0, "") + if len(calls) == 3: + return subprocess.CompletedProcess(command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found') + return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") def failing_stream(command, **_kwargs): calls.append(("stream", command, None)) @@ -75,7 +97,7 @@ def failing_stream(command, **_kwargs): assert [command for _, command, _ in calls] == [ ["kubectl", "get", "namespace", "jupyterhub"], ["kubectl", "create", "namespace", "jupyterhub"], - ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub"], + ["kubectl", "get", "secret", "jupyterhub-admin-credentials", "--namespace", "jupyterhub", "-o", "json"], ["kubectl", "create", "--namespace", "jupyterhub", "--filename=-"], [ "helm", @@ -92,3 +114,94 @@ def failing_stream(command, **_kwargs): ], ] assert "generated-password" not in capsys.readouterr().out + + +def test_existing_legacy_secret_is_patched_without_rotating_credentials(monkeypatch) -> None: + calls: list[tuple[list[str], str | None]] = [] + + def fake_run(command, *, check=True, input_text=None): + calls.append((command, input_text)) + if command[2] == "secret": + return subprocess.CompletedProcess( + command, + 0, + json.dumps({"data": {"admin-password": "cGFzc3dvcmQ=", "api-token": "dG9rZW4="}}), + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + assert ensure_local_admin_secret("operator") is None + assert calls[-1][0] == [ + "kubectl", + "patch", + "secret", + "jupyterhub-admin-credentials", + "--namespace", + "jupyterhub", + "--type", + "merge", + "--patch", + '{"stringData":{"admin-username":"operator"}}', + ] + + +def test_existing_secret_requires_complete_matching_contract(monkeypatch) -> None: + def fake_run(command, *, check=True, input_text=None): + if command[2] == "secret": + return subprocess.CompletedProcess( + command, + 0, + json.dumps( + {"data": {"admin-username": "b3RoZXI=", "admin-password": "cGFzc3dvcmQ="}} + ), + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + with pytest.raises(InstallerError, match="api-token"): + ensure_local_admin_secret("operator") + + +def test_secret_lookup_fails_closed_for_non_not_found_errors(monkeypatch) -> None: + def fake_run(command, *, check=True, input_text=None): + if command[2] == "secret": + return subprocess.CompletedProcess(command, 1, "Error from server (Forbidden): secrets is forbidden") + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + with pytest.raises(InstallerError, match="Unable to inspect"): + ensure_local_admin_secret("operator") + + +def test_local_upgrade_ensures_secret_and_waits_for_hub(monkeypatch) -> None: + calls: list[list[str]] = [] + + def fake_run(command, *, check=True, input_text=None): + calls.append(command) + if command[2] == "secret": + return subprocess.CompletedProcess( + command, + 0, + json.dumps( + { + "data": { + "admin-username": "b3BlcmF0b3I=", + "admin-password": "cGFzc3dvcmQ=", + "api-token": "dG9rZW4=", + } + } + ), + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + monkeypatch.setattr("auplc_installer.helm.run_streaming", lambda command, **_kwargs: calls.append(command)) + + upgrade_runtime(RuntimePaths(Path("chart"), Path("values.yaml"), Path("values.local.yaml")), access_mode="local", admin_username="operator") + + assert calls[1][:3] == ["kubectl", "get", "secret"] + assert any(command[:2] == ["helm", "upgrade"] for command in calls) + assert ["kubectl", "rollout", "status", "deployment/hub", "--namespace", "jupyterhub", "--timeout=600s"] in calls diff --git a/tests/installer/test_local_auth.py b/tests/installer/test_local_auth.py index 5b0bf402..28c60d4b 100644 --- a/tests/installer/test_local_auth.py +++ b/tests/installer/test_local_auth.py @@ -1,7 +1,9 @@ import json from pathlib import Path -from auplc_installer.cli import _preserve_access_settings_for_upgrade +import pytest + +from auplc_installer.cli import _preserve_access_settings_for_upgrade, _resolve_access_settings from auplc_installer.gpu import GpuConfig, append_product from auplc_installer.overlay import generate_values_overlay, try_load_access_settings_from_overlay from auplc_installer.state import InstallerState @@ -69,3 +71,48 @@ def test_cli_defaults_to_personal_but_tui_defaults_to_local(monkeypatch) -> None assert InstallerState().access_mode == "" assert state.access_mode == "local" assert state.admin_username == "admin" + + +@pytest.mark.parametrize("username", ["Admin", "admin:name", 'admin"name', "admin\nname", "-admin"]) +def test_local_admin_username_rejects_unsafe_values(username: str) -> None: + state = InstallerState(access_mode="local", admin_username=username) + + with pytest.raises(Exception, match="lowercase ASCII"): + _resolve_access_settings(state) + + +def test_explicit_local_upgrade_without_username_preserves_previous_username(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text("# Access mode : local\n# Admin username: operator\n", encoding="utf-8") + state = InstallerState(access_mode="local") + + _preserve_access_settings_for_upgrade(state, overlay) + + assert _resolve_access_settings(state) == ("local", "operator") + + +def test_upgrade_rejects_unmanaged_advanced_auth_overlay(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text("custom:\n authMode: github\n", encoding="utf-8") + + with pytest.raises(Exception, match="operator-managed Helm values"): + _preserve_access_settings_for_upgrade(InstallerState(), overlay) + + +def test_local_overlay_retains_single_node_runtime_behavior(tmp_path: Path) -> None: + cfg = GpuConfig() + append_product(cfg, "AMD_Radeon_8060S_Graphics") + overlay = tmp_path / "values.local.yaml" + + generate_values_overlay( + cfg, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=InstallerState().courses, + access_mode="local", + admin_username="operator", + overlay_path=overlay, + ) + + rendered = __import__("yaml").safe_load(overlay.read_text()) + assert rendered["custom"]["singleNodeMode"] is True From 99b5aae72ed4bab0a16beea98db05a584118a0c0 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:16:02 +0800 Subject: [PATCH 125/180] fix(chart): restore local admin token wiring --- runtime/chart/templates/NOTES.txt | 2 +- runtime/chart/templates/hub/deployment.yaml | 5 ++++ runtime/chart/templates/hub/secret-admin.yaml | 1 + runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 1 + tests/installer/test_chart_local_auth.py | 28 +++++++++++++++++++ 6 files changed, 37 insertions(+), 2 deletions(-) diff --git a/runtime/chart/templates/NOTES.txt b/runtime/chart/templates/NOTES.txt index b9554fae..42f09ffb 100644 --- a/runtime/chart/templates/NOTES.txt +++ b/runtime/chart/templates/NOTES.txt @@ -42,7 +42,7 @@ SOFTWARE. {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} ### Admin Credentials (auto-generated) - Admin username: admin + Admin username: {{ .Values.custom.adminUser.username }} Get admin password: kubectl -n {{ .Release.Namespace }} get secret jupyterhub-admin-credentials -o go-template='{{"{{index .data \"admin-password\" | base64decode}}"}}' diff --git a/runtime/chart/templates/hub/deployment.yaml b/runtime/chart/templates/hub/deployment.yaml index 211828ed..74b0b385 100644 --- a/runtime/chart/templates/hub/deployment.yaml +++ b/runtime/chart/templates/hub/deployment.yaml @@ -228,6 +228,11 @@ spec: secretKeyRef: name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} key: admin-password + - name: JUPYTERHUB_API_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} + key: api-token {{- end }} {{- with .Values.hub.extraEnv }} {{- include "jupyterhub.extraEnv" . | nindent 12 }} diff --git a/runtime/chart/templates/hub/secret-admin.yaml b/runtime/chart/templates/hub/secret-admin.yaml index 0e20e99f..5cd8e1b4 100644 --- a/runtime/chart/templates/hub/secret-admin.yaml +++ b/runtime/chart/templates/hub/secret-admin.yaml @@ -34,4 +34,5 @@ type: Opaque data: admin-username: {{ .Values.custom.adminUser.username | b64enc | quote }} admin-password: {{ randAlphaNum 16 | b64enc | quote }} + api-token: {{ randAlphaNum 32 | b64enc | quote }} {{- end }} diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 1f0f8b80..0d46c668 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index bdaeddd5..f1a211b9 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3197,6 +3197,7 @@ properties: Credentials will be stored in `jupyterhub-admin-credentials` secret. username: type: string + pattern: "^[a-z0-9][a-z0-9._-]{0,63}$" existingSecret: type: string notifications: diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index 9da80f24..ff5af7b2 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -31,4 +31,32 @@ def test_local_chart_render_uses_existing_secret_only_for_hub_bootstrap() -> Non assert "key: admin-username" in result.stdout assert "name: JUPYTERHUB_ADMIN_PASSWORD" in result.stdout assert "key: admin-password" in result.stdout + assert "name: JUPYTERHUB_API_TOKEN" in result.stdout + assert "key: api-token" in result.stdout assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" not in result.stdout + + +def test_local_chart_schema_rejects_uppercase_admin_username() -> None: + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "--set", + "custom.authMode=local", + "--set", + "custom.adminUser.enabled=true", + "--set", + "custom.adminUser.username=Operator", + "--set", + "custom.adminUser.existingSecret=jupyterhub-admin-credentials", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "does not match pattern" in result.stderr From 2fd3519f57525ed83cc3890ba1633362440c2d2e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:16:17 +0800 Subject: [PATCH 126/180] fix(hub): fail closed for local admin bootstrap --- runtime/hub/core/authenticators/__init__.py | 3 +- runtime/hub/core/authenticators/local.py | 8 ++- runtime/hub/core/config.py | 2 +- runtime/hub/core/setup.py | 11 +++- .../tests/test_config_resource_metadata.py | 11 ++++ runtime/hub/tests/test_local_authenticator.py | 56 ++++++++++++++++++- 6 files changed, 82 insertions(+), 9 deletions(-) diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index 6cdca959..b1f55049 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -55,8 +55,7 @@ def create_authenticator(auth_mode: str, **kwargs): elif auth_mode == "multi": return CustomMultiAuthenticator else: - print(f"[WARN] Unknown auth mode: {auth_mode}, falling back to dummy") - return "dummy" + raise ValueError(f"Unknown authentication mode: {auth_mode}") __all__ = [ diff --git a/runtime/hub/core/authenticators/local.py b/runtime/hub/core/authenticators/local.py index ac3bf7f0..dcad0689 100644 --- a/runtime/hub/core/authenticators/local.py +++ b/runtime/hub/core/authenticators/local.py @@ -1,11 +1,15 @@ +import re + from core.authenticators.firstuse import CustomFirstUseAuthenticator +LOCAL_USERNAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") + class CustomLocalAuthenticator(CustomFirstUseAuthenticator): async def authenticate(self, _handler, data): - username = self.normalize_username(data.get("username", "")) + username = data.get("username", "") password = data.get("password", "") - if not username or not password or ":" in username: + if not LOCAL_USERNAME_PATTERN.fullmatch(username) or not password: return None if not self._user_exists(username): return None diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 5bc3f6f9..25d21617 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -330,7 +330,7 @@ def init(cls, config_path: str | Path) -> HubConfig: if single_node_mode is not None: instance.single_node_mode = single_node_mode else: - instance.single_node_mode = instance.auth_mode == "auto-login" + instance.single_node_mode = instance.auth_mode in ("auto-login", "local") # Parse structured configuration instance._config = ParsedConfig.from_dicts( diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index 826c341c..6200d956 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -357,9 +357,6 @@ async def delete(self, group_name): raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_USERNAME") if admin_password: - c.Authenticator.admin_users = {admin_username} - print(f"[SETUP] Admin user configured: {admin_username}") - try: from core.authenticators.models import UserPassword from core.database import session_scope @@ -367,6 +364,10 @@ async def delete(self, group_name): with session_scope() as session: user_pw = session.query(UserPassword).filter_by(username=admin_username).first() if user_pw: + if config.auth_mode == "local" and not bcrypt.checkpw( + admin_password.encode(), user_pw.password_hash + ): + raise RuntimeError("Local administrator password does not match the credentials Secret") print(f"[SETUP] Admin '{admin_username}' password already set") else: password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) @@ -378,7 +379,11 @@ async def delete(self, group_name): session.add(user_pw) print(f"[SETUP] Admin '{admin_username}' password set automatically") except Exception as e: + if config.auth_mode == "local": + raise RuntimeError("Failed to bootstrap local administrator credentials") from e print(f"[SETUP] Warning: Failed to set admin password: {e}") + c.Authenticator.admin_users = {admin_username} + print(f"[SETUP] Admin user configured: {admin_username}") # ========================================================================= # Template Vars diff --git a/runtime/hub/tests/test_config_resource_metadata.py b/runtime/hub/tests/test_config_resource_metadata.py index 61dac30b..64eb3de8 100644 --- a/runtime/hub/tests/test_config_resource_metadata.py +++ b/runtime/hub/tests/test_config_resource_metadata.py @@ -96,3 +96,14 @@ def test_code_server_extra_trusted_domains_parse_from_config(): ) assert parsed_config.codeServer.extraTrustedDomains == ["docs.example.edu", "git.example.edu"] + + +def test_local_auth_mode_defaults_to_single_node_runtime_behavior(tmp_path: Path): + config_path = tmp_path / "hub-config.yaml" + config_path.write_text("authMode: local\n", encoding="utf-8") + config.HubConfig._instance = None + config.HubConfig._initialized = False + + hub_config = config.HubConfig.init(config_path) + + assert hub_config.single_node_mode is True diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py index f6a4ecb7..48a6b11c 100644 --- a/runtime/hub/tests/test_local_authenticator.py +++ b/runtime/hub/tests/test_local_authenticator.py @@ -4,8 +4,11 @@ import types from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] LOCAL_AUTHENTICATOR = ROOT / "core" / "authenticators" / "local.py" +AUTHENTICATORS = ROOT / "core" / "authenticators" / "__init__.py" class FakeFirstUseAuthenticator: @@ -39,8 +42,59 @@ def test_local_authenticator_rejects_first_use_and_accepts_existing_password() - authenticator = module.CustomLocalAuthenticator() assert ( - asyncio.run(authenticator.authenticate(None, {"username": "EXISTING", "password": "correct-password"})) + asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "correct-password"})) == "existing" ) assert asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "wrong-password"})) is None assert asyncio.run(authenticator.authenticate(None, {"username": "new", "password": "valid-password"})) is None + + +def test_local_authenticator_rejects_noncanonical_usernames() -> None: + core = types.ModuleType("core") + authenticators = types.ModuleType("core.authenticators") + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator + sys.modules.update( + { + "core": core, + "core.authenticators": authenticators, + "core.authenticators.firstuse": firstuse, + } + ) + spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + authenticator = module.CustomLocalAuthenticator() + + for username in ("EXISTING", "existing:admin", 'existing"', "existing\n"): + assert asyncio.run(authenticator.authenticate(None, {"username": username, "password": "correct-password"})) is None + + +def test_authenticator_factory_rejects_unknown_mode() -> None: + core = types.ModuleType("core") + authenticators = types.ModuleType("core.authenticators") + sys.modules.update({"core": core, "core.authenticators": authenticators}) + for name, attribute in ( + ("auto_login", "AutoLoginAuthenticator"), + ("firstuse", "CustomFirstUseAuthenticator"), + ("github_app", "GITHUB_USERNAME_PREFIX"), + ("jwt", "RemoteLabAuthenticator"), + ("local", "CustomLocalAuthenticator"), + ("multi", "CustomMultiAuthenticator"), + ): + module = types.ModuleType(f"core.authenticators.{name}") + setattr(module, attribute, type(attribute, (), {}) if attribute != "GITHUB_USERNAME_PREFIX" else "github:") + if name == "github_app": + module.CustomGitHubOAuthenticator = type("CustomGitHubOAuthenticator", (), {}) + sys.modules[module.__name__] = module + + spec = importlib.util.spec_from_file_location("core.authenticators", AUTHENTICATORS) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + with pytest.raises(ValueError, match="Unknown authentication mode"): + module.create_authenticator("unexpected") From d700de8a5fa837b69967302176bb4c27258cabb0 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:16:30 +0800 Subject: [PATCH 127/180] docs: clarify local credential recovery --- README.md | 7 +++++-- .../reference.md | 8 +++++--- .../reference.md | 15 +++++++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a6947ffb..f391fb64 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,9 @@ For scripted installs, `personal` remains the compatibility default. Select loca ./auplc-installer install --access-mode=local --admin-username=admin ``` -The installer generates the administrator password only when it creates `jupyterhub-admin-credentials` and displays it once after a successful deployment. Re-running against an existing Secret reuses the credentials without rotating or redisplaying them. Local users are created and assigned passwords through the Admin UI. +The installer generates the administrator password and API token only when it creates `jupyterhub-admin-credentials`. It displays the password once after a successful interactive deployment. Re-running against an existing Secret reuses the credentials without rotating them; recover credentials with `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo`. Local users are created and assigned passwords through the Admin UI. + +Local mode is an installer MVP for `http://localhost:30890` on a trusted single-node host. It does not configure TLS or restrict the K3s NodePort from LAN reachability; do not treat local credentials as a network exposure control. A successful install looks like this: @@ -119,7 +121,8 @@ This operation needs root privileges. Requesting sudo password... You have successfully installed AUP Learning Cloud! Open in your browser: http://localhost:30890 - (auto-logged-in as 'student' — no login needed) + Sign in with the selected local administrator credentials. + (Use `--access-mode=personal` for the compatibility shared student session.) kubectl is configured at $HOME/.kube/config; try `kubectl get nodes` ``` diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md index 8c659dab..638d287a 100644 --- a/skills/configure-aup-learning-cloud-auth/reference.md +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -17,13 +17,15 @@ source of truth; verify keys against them. ```yaml custom: - authMode: "auto-login" # auto-login | dummy | github | multi + authMode: "auto-login" # auto-login | dummy | github | local | multi ``` - `auto-login` — shared, no credentials. Quota auto-disables unless explicitly enabled. Checked-in single-node default. - `dummy` — accepts any username/password. Testing only. - `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. +- `local` — closed, administrator-managed local accounts. Requires an existing + credentials Secret with `admin-username`, `admin-password`, and `api-token`. - `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends in `/hub/github/oauth_callback`. @@ -35,8 +37,8 @@ custom: enabled: true ``` -The chart creates the `jupyterhub-admin-credentials` secret and bootstraps the -`admin` user. Retrieve: +The chart or installer creates `jupyterhub-admin-credentials` and bootstraps the +configured administrator. Retrieve: ```bash kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 656aac83..1585dd67 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -81,16 +81,18 @@ sudo apt install python3-questionary python3-prompt-toolkit ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) Open in your browser: http://localhost:30890 - (auto-logged-in as 'student' — no login needed) + Sign in with the selected local administrator credentials. ``` ## Default deployment facts -The checked-in defaults describe a local deployment: NodePort **30890**, -`local-path` storage, ingress **disabled**, prePuller **disabled**, and -`custom.authMode: auto-login`. To change auth, courses, or accelerators, layer -a values overlay (see configure-aup-learning-cloud-courses) and -`./auplc-installer rt upgrade`. +The checked-in chart defaults use `custom.authMode: auto-login`, while the +interactive installer defaults to `local` and creates `jupyterhub-admin-credentials` +with `admin-username`, `admin-password`, and `api-token`. Scripted installs keep +the `personal` compatibility default unless `--access-mode=local` is supplied. +The single-node NodePort is not a TLS or LAN exposure boundary; use local mode only +on a trusted host/network. To change generated installer values, run +`./auplc-installer rt upgrade`; it preserves and validates an existing local Secret. ## Offline / air-gapped (pack) @@ -124,6 +126,7 @@ installation verifies and installs it from the bundle. | `localhost:30890` refused | Proxy not up or NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | | `docker` permission denied | User not in docker group | re-run `usermod -aG docker $USER` then re-login / `newgrp docker` | | Need to re-apply values only | Changed the overlay, not images | `./auplc-installer rt upgrade` (don't reinstall k3s) | +| Local administrator password unavailable | Existing Secret is intentionally preserved | `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo` | ## Out of scope From 5922fb5395c89d052061e818cb2bc6fb1b70a1e6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:19:19 +0800 Subject: [PATCH 128/180] style: format local auth hardening --- auplc_installer/cli.py | 12 +++++++++--- auplc_installer/helm.py | 14 ++++++++++++-- runtime/hub/tests/test_local_authenticator.py | 5 ++++- tests/installer/test_admin_secret.py | 18 ++++++++++++------ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 9cf6510f..61eee6ad 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -514,9 +514,13 @@ def _print_created_admin_password(admin_password: str | None) -> None: log(" " + bold("Temporary admin password (shown once): ") + bold_green(admin_password)) elif admin_password is not None: - log(" Retrieve credentials safely: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo") + log( + " Retrieve credentials safely: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo" + ) else: - log(" Existing credentials were preserved. Retrieve the password: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo") + log( + " Existing credentials were preserved. Retrieve the password: kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo" + ) def cmd_uninstall(state: InstallerState) -> None: @@ -765,7 +769,9 @@ def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: P if previous is None and overlay_path.is_file(): text = overlay_path.read_text(encoding="utf-8") if re.search(r"^\s*authMode:\s*(github|multi|dummy)\s*$", text, re.MULTILINE): - raise InstallerError("Existing overlay uses an advanced authMode; use operator-managed Helm values instead of installer upgrade") + raise InstallerError( + "Existing overlay uses an advanced authMode; use operator-managed Helm values instead of installer upgrade" + ) if state.access_mode: if state.access_mode == "local" and not state.admin_username and previous and previous[0] == "local": state.admin_username = previous[1] diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index 97c1670c..42e93dfe 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -85,7 +85,15 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: if encoded_username is None: run( [ - "kubectl", "patch", "secret", secret_name, "--namespace", "jupyterhub", "--type", "merge", "--patch", + "kubectl", + "patch", + "secret", + secret_name, + "--namespace", + "jupyterhub", + "--type", + "merge", + "--patch", json.dumps({"stringData": {"admin-username": admin_username}}, separators=(",", ":")), ] ) @@ -97,7 +105,9 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: except (ValueError, UnicodeDecodeError) as exc: raise InstallerError("Existing local admin credentials Secret has an invalid admin-username") from exc if stored_username != admin_username: - raise InstallerError("Existing local admin credentials Secret belongs to a different administrator username") + raise InstallerError( + "Existing local admin credentials Secret belongs to a different administrator username" + ) return None if "NotFound" not in (existing.stdout or ""): raise InstallerError("Unable to inspect local admin credentials Secret; verify Kubernetes access and RBAC") diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py index 48a6b11c..9be6ab04 100644 --- a/runtime/hub/tests/test_local_authenticator.py +++ b/runtime/hub/tests/test_local_authenticator.py @@ -69,7 +69,10 @@ def test_local_authenticator_rejects_noncanonical_usernames() -> None: authenticator = module.CustomLocalAuthenticator() for username in ("EXISTING", "existing:admin", 'existing"', "existing\n"): - assert asyncio.run(authenticator.authenticate(None, {"username": username, "password": "correct-password"})) is None + assert ( + asyncio.run(authenticator.authenticate(None, {"username": username, "password": "correct-password"})) + is None + ) def test_authenticator_factory_rejects_unknown_mode() -> None: diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py index be4b8db8..9e45f26c 100644 --- a/tests/installer/test_admin_secret.py +++ b/tests/installer/test_admin_secret.py @@ -14,7 +14,9 @@ def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(mo def fake_run(command, *, check=True, input_text=None): calls.append((command, input_text)) if len(calls) == 3: - return subprocess.CompletedProcess(command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found') + return subprocess.CompletedProcess( + command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found' + ) return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") monkeypatch.setattr("auplc_installer.helm.run", fake_run) @@ -76,7 +78,9 @@ def test_deploy_orders_namespace_secret_and_helm_without_printing_new_password(m def fake_run(command, *, check=True, input_text=None): calls.append(("run", command, input_text)) if len(calls) == 3: - return subprocess.CompletedProcess(command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found') + return subprocess.CompletedProcess( + command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found' + ) return subprocess.CompletedProcess(command, 1 if len(calls) == 1 else 0, "") def failing_stream(command, **_kwargs): @@ -152,9 +156,7 @@ def fake_run(command, *, check=True, input_text=None): return subprocess.CompletedProcess( command, 0, - json.dumps( - {"data": {"admin-username": "b3RoZXI=", "admin-password": "cGFzc3dvcmQ="}} - ), + json.dumps({"data": {"admin-username": "b3RoZXI=", "admin-password": "cGFzc3dvcmQ="}}), ) return subprocess.CompletedProcess(command, 0, "") @@ -200,7 +202,11 @@ def fake_run(command, *, check=True, input_text=None): monkeypatch.setattr("auplc_installer.helm.run", fake_run) monkeypatch.setattr("auplc_installer.helm.run_streaming", lambda command, **_kwargs: calls.append(command)) - upgrade_runtime(RuntimePaths(Path("chart"), Path("values.yaml"), Path("values.local.yaml")), access_mode="local", admin_username="operator") + upgrade_runtime( + RuntimePaths(Path("chart"), Path("values.yaml"), Path("values.local.yaml")), + access_mode="local", + admin_username="operator", + ) assert calls[1][:3] == ["kubectl", "get", "secret"] assert any(command[:2] == ["helm", "upgrade"] for command in calls) From 2aacc5c534cd056313c1c58b5e38486a573e5307 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:52:32 +0800 Subject: [PATCH 129/180] fix(installer): harden local auth lifecycle --- auplc_installer/cli.py | 13 +++- auplc_installer/helm.py | 61 +++++++++++---- auplc_installer/tui.py | 4 + .../SKILL.md | 18 +++-- .../reference.md | 25 +++++- .../SKILL.md | 21 ++++- .../reference.md | 3 + tests/installer/test_admin_secret.py | 27 ++++++- tests/installer/test_cli_install_options.py | 13 ++++ tests/installer/test_local_auth.py | 78 ++++++++++++++++++- 10 files changed, 230 insertions(+), 33 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 61eee6ad..9a8b2428 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -315,6 +315,7 @@ def _install_pull_and_label( def cmd_install_plan(state: InstallerState, *, legacy_pull: bool = False) -> None: """Print the install Configuration summary without side effects.""" + _resolve_access_settings(state) _, label = _install_pull_and_label(state, legacy_pull=legacy_pull) sys.stdout.write(format_configuration_summary(state, image_source_label=label) + "\n") @@ -694,6 +695,7 @@ def cmd_dev_upgrade(state: InstallerState) -> None: def cmd_dev_reinstall(state: InstallerState) -> None: + _preserve_access_settings_for_upgrade(state, state.runtime_paths().overlay_path) _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() @@ -765,13 +767,17 @@ def _preserve_courses_for_upgrade(state: InstallerState, overlay_path: Path) -> def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: Path) -> None: - previous = try_load_access_settings_from_overlay(overlay_path) - if previous is None and overlay_path.is_file(): + if overlay_path.is_file(): text = overlay_path.read_text(encoding="utf-8") - if re.search(r"^\s*authMode:\s*(github|multi|dummy)\s*$", text, re.MULTILINE): + if re.search( + r"^\s*authMode\s*:\s*(?:\"(?:github|multi|dummy)\"|'(?:github|multi|dummy)'|github|multi|dummy)\s*(?:#.*)?$", + text, + re.MULTILINE, + ): raise InstallerError( "Existing overlay uses an advanced authMode; use operator-managed Helm values instead of installer upgrade" ) + previous = try_load_access_settings_from_overlay(overlay_path) if state.access_mode: if state.access_mode == "local" and not state.admin_username and previous and previous[0] == "local": state.admin_username = previous[1] @@ -797,6 +803,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: + _preserve_access_settings_for_upgrade(state, state.runtime_paths().overlay_path) _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index 42e93dfe..d3fb22ea 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -10,7 +10,9 @@ from __future__ import annotations +import base64 import json +import re import secrets from dataclasses import dataclass from pathlib import Path @@ -19,6 +21,7 @@ from auplc_installer.util import InstallerError, log, run, run_streaming DEV_VALUES_PATH = "runtime/values-dev.yaml" +_KUBECTL_ERROR_CATEGORY_RE = re.compile(r"\(([^()]+)\):") @dataclass @@ -64,6 +67,41 @@ def _ensure_namespace() -> None: raise InstallerError("Failed to create jupyterhub namespace") +def _decode_secret_value(data: dict[str, object], key: str) -> str: + encoded_value = data.get(key) + if not isinstance(encoded_value, str) or not encoded_value: + raise InstallerError(f"Existing local admin credentials Secret has an invalid {key}") + try: + value = base64.b64decode(encoded_value, validate=True).decode("utf-8") + except (UnicodeDecodeError, ValueError) as exc: + raise InstallerError(f"Existing local admin credentials Secret has an invalid {key}") from exc + if not value: + raise InstallerError(f"Existing local admin credentials Secret has an invalid {key}") + return value + + +def _parse_existing_local_admin_secret(secret_json: str) -> tuple[str | None, str, str]: + try: + payload = json.loads(secret_json) + except json.JSONDecodeError as exc: + raise InstallerError("Unable to inspect existing local admin credentials Secret") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise InstallerError("Existing local admin credentials Secret has an invalid data object") + data = payload["data"] + password = _decode_secret_value(data, "admin-password") + api_token = _decode_secret_value(data, "api-token") + if "admin-username" not in data: + return None, password, api_token + return _decode_secret_value(data, "admin-username"), password, api_token + + +def _kubectl_error_category(output: str | None) -> str: + if not output: + return "unknown kubectl error" + match = _KUBECTL_ERROR_CATEGORY_RE.search(output) + return match.group(1) if match else "unknown kubectl error" + + def ensure_local_admin_secret(admin_username: str) -> str | None: """Create the local admin credentials Secret, returning only a new password.""" secret_name = "jupyterhub-admin-credentials" @@ -74,15 +112,8 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: check=False, ) if existing.returncode == 0: - try: - data = json.loads(existing.stdout).get("data", {}) - except json.JSONDecodeError as exc: - raise InstallerError("Unable to inspect existing local admin credentials Secret") from exc - missing = {"admin-password", "api-token"}.difference(data) - if missing: - raise InstallerError(f"Existing local admin credentials Secret is missing {', '.join(sorted(missing))}") - encoded_username = data.get("admin-username") - if encoded_username is None: + stored_username, _, _ = _parse_existing_local_admin_secret(existing.stdout) + if stored_username is None: run( [ "kubectl", @@ -98,19 +129,17 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: ] ) return None - import base64 - - try: - stored_username = base64.b64decode(encoded_username).decode("utf-8") - except (ValueError, UnicodeDecodeError) as exc: - raise InstallerError("Existing local admin credentials Secret has an invalid admin-username") from exc + validate_local_admin_username(stored_username) if stored_username != admin_username: raise InstallerError( "Existing local admin credentials Secret belongs to a different administrator username" ) return None if "NotFound" not in (existing.stdout or ""): - raise InstallerError("Unable to inspect local admin credentials Secret; verify Kubernetes access and RBAC") + category = _kubectl_error_category(existing.stdout) + raise InstallerError( + f"Unable to inspect local admin credentials Secret ({category}); verify Kubernetes access and RBAC" + ) password = secrets.token_urlsafe(24) api_token = secrets.token_urlsafe(32) diff --git a/auplc_installer/tui.py b/auplc_installer/tui.py index f90a036e..f53aa6a1 100644 --- a/auplc_installer/tui.py +++ b/auplc_installer/tui.py @@ -791,6 +791,7 @@ def _flow_dev(state: InstallerState) -> None: if sub == "deploy": while True: if _flow_select_envs(state, allow_back=True): + _flow_select_access(state) cmd_dev_deploy(state) return break @@ -800,6 +801,7 @@ def _flow_dev(state: InstallerState) -> None: raise _CancelledError while True: if _flow_select_envs(state, allow_back=True): + _flow_select_access(state) cmd_dev_reinstall(state) return break @@ -836,6 +838,7 @@ def _flow_rt(state: InstallerState) -> None: if sub == "install": while True: if _flow_select_envs(state, allow_back=True): + _flow_select_access(state) cmd_rt_install(state) return break @@ -848,6 +851,7 @@ def _flow_rt(state: InstallerState) -> None: raise _CancelledError while True: if _flow_select_envs(state, allow_back=True): + _flow_select_access(state) cmd_rt_reinstall(state) return break diff --git a/skills/configure-aup-learning-cloud-auth/SKILL.md b/skills/configure-aup-learning-cloud-auth/SKILL.md index 12003da8..16cfd8a0 100644 --- a/skills/configure-aup-learning-cloud-auth/SKILL.md +++ b/skills/configure-aup-learning-cloud-auth/SKILL.md @@ -2,7 +2,7 @@ name: configure-aup-learning-cloud-auth description: >- Group: Maintain AUP Learning Cloud. Configures authentication for AUP Learning - Cloud: auth modes (auto-login/dummy/github/multi), GitHub App / OAuth, GitHub + Cloud: auth modes (auto-login/dummy/github/local/multi), GitHub App / OAuth, GitHub team-to-group sync, native local accounts, password policy and forced first-login change, and admin bootstrap. Use when the user wants to set or switch custom.authMode, enable GitHub login, create or migrate a GitHub @@ -43,6 +43,7 @@ GitHub App walkthrough, value blocks, and troubleshooting are in | `auto-login` | Local demo / single dev box | No credentials; quota auto-disabled unless forced. The checked-in default. | | `dummy` | Throwaway testing only | Accepts any user/password; not for real use; its login can 404 in normal setups. | | `github` | Org-backed SSO | GitHub App only; team membership syncs into Hub groups. | +| `local` | Closed single-node or standalone local auth | Administrator-managed local accounts only; no GitHub setup. | | `multi` | GitHub + local accounts | Combined login page; native accounts for users without GitHub. | `custom.authMode` is the single switch. Confirm the target mode with the user @@ -68,13 +69,15 @@ login blip). teams are intersected with `custom.teams.mapping`. Mapping *which resource* a group sees stays in the configure-courses skill — this skill only makes the groups exist. -5. **Native accounts (multi).** The first-use authenticator has +5. **Native accounts (local/multi).** The first-use authenticator has `create_users = False`, so accounts must be created by an admin before login (see manage-users skill). Password policy: ≥8 chars with upper, lower, digit, and special; users can be forced to change on first login. -6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` to have - the chart mint the `jupyterhub-admin-credentials` secret and the `admin` - user. +6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` with a + canonical `custom.adminUser.username`. Without `existingSecret`, the chart + creates `jupyterhub-admin-credentials`; with `existingSecret`, it uses the + named external Secret and never rotates it. The single-node installer creates + and validates its lifecycle Secret before Helm runs. 7. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay>` must succeed. 8. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: @@ -84,6 +87,11 @@ login blip). lands in the right groups, and (if bootstrapped) the admin can log in. Read the secret with the commands in [reference.md](reference.md). +If a Helm install/upgrade fails, inspect `helm status jupyterhub -n jupyterhub` +before retrying. On a single-node install, `./auplc-installer rt upgrade` or +`./auplc-installer rt reinstall` preserves `jupyterhub-admin-credentials`; do +not delete that Secret unless intentionally resetting credentials. + ## Safety - **Secrets never go in tracked files.** `client_secret`, the App private key, diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md index 638d287a..276b4f98 100644 --- a/skills/configure-aup-learning-cloud-auth/reference.md +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -24,8 +24,11 @@ custom: enabled. Checked-in single-node default. - `dummy` — accepts any username/password. Testing only. - `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. -- `local` — closed, administrator-managed local accounts. Requires an existing - credentials Secret with `admin-username`, `admin-password`, and `api-token`. +- `local` — closed, administrator-managed local accounts. It requires + `custom.adminUser.enabled: true` and a canonical username. The chart can + create credentials or use an external Secret with `admin-password` and an + optional `api-token`; the username always comes from + `custom.adminUser.username`. - `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends in `/hub/github/oauth_callback`. @@ -37,8 +40,12 @@ custom: enabled: true ``` -The chart or installer creates `jupyterhub-admin-credentials` and bootstraps the -configured administrator. Retrieve: +Without `existingSecret`, the chart creates `jupyterhub-admin-credentials` and +bootstraps the configured administrator. The installer creates and validates the +same Secret for its local lifecycle. With `existingSecret`, the external Secret +is never created or rotated by the chart and must contain `admin-password`; an +`api-token` is optional for legacy two-key Secrets. Retrieve chart-created +credentials: ```bash kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ @@ -143,6 +150,16 @@ kubectl rollout status -n jupyterhub deploy/hub kubectl logs -n jupyterhub deployment/hub | grep -i -E 'admin|github|oauth' ``` +If Helm reports a failed release, inspect it before retrying: + +```bash +helm status jupyterhub -n jupyterhub +``` + +On a single-node host, `rt upgrade` and `rt reinstall` retain the installer +Secret. Do not delete it unless intentionally resetting the local administrator +credentials. + ## Troubleshooting | Symptom | Likely cause | First checks | diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md index 575141db..081f11c0 100644 --- a/skills/install-aup-learning-cloud-single-node/SKILL.md +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -50,6 +50,10 @@ table, offline flow, and troubleshooting are in **[reference.md](reference.md)** (local from `dockerfiles/`). For a quick demo prefer `pull`. 4. **Online or offline**: a normal machine with internet, or an air-gapped one that needs a `pack` bundle (see reference). +5. **Access mode**: interactive installs default to `local` and prompt for a + canonical administrator username. Use `personal` only for the compatibility + shared student session; scripted installs remain `personal` unless passed + `--access-mode=local --admin-username=<name>`. ## Phase 2 — Verify the environment @@ -84,9 +88,16 @@ kubectl get nodes # the node is Ready kubectl get pods -n jupyterhub # hub + proxy Running, no CrashLoop/ImagePull ``` -Open `http://localhost:30890` — the default values auto-log-in as `student` -(NodePort 30890, `local-path` storage, ingress disabled). Spawn a CPU notebook, -then a GPU notebook, and confirm the GPU pod schedules. +Open `http://localhost:30890` — local interactive installs display a login form; +sign in with the configured administrator credentials. Scripted `personal` +installs retain the compatibility shared student session. The NodePort is 30890, +storage is `local-path`, and ingress is disabled. Spawn a CPU notebook, then a +GPU notebook, and confirm the GPU pod schedules. + +If Helm fails, inspect `helm status jupyterhub -n jupyterhub` before retrying. +For a Hub-only retry, use `./auplc-installer rt upgrade` or +`./auplc-installer rt reinstall`; both retain `jupyterhub-admin-credentials`. +Do not delete the Secret unless intentionally resetting local credentials. ## Safety @@ -100,6 +111,10 @@ Stop and get explicit confirmation before: Never commit changes to the checkout. The installer writes a local values overlay (e.g. `values.local.yaml`); do not commit it. +Local mode remains localhost-oriented MVP guidance only. It does not configure +TLS or restrict NodePort LAN reachability, so credentials are not a network +authorization boundary. + ## Reference Flag-by-flag table, the offline `pack`/air-gapped flow, `dev`/`rt` diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 1585dd67..9d1a4050 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -93,6 +93,9 @@ the `personal` compatibility default unless `--access-mode=local` is supplied. The single-node NodePort is not a TLS or LAN exposure boundary; use local mode only on a trusted host/network. To change generated installer values, run `./auplc-installer rt upgrade`; it preserves and validates an existing local Secret. +If a release fails, run `helm status jupyterhub -n jupyterhub` before retrying. +`rt upgrade` and `rt reinstall` retain `jupyterhub-admin-credentials`; do not +delete it unless intentionally resetting local credentials. ## Offline / air-gapped (pack) diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py index 9e45f26c..ae229890 100644 --- a/tests/installer/test_admin_secret.py +++ b/tests/installer/test_admin_secret.py @@ -174,10 +174,35 @@ def fake_run(command, *, check=True, input_text=None): monkeypatch.setattr("auplc_installer.helm.run", fake_run) - with pytest.raises(InstallerError, match="Unable to inspect"): + with pytest.raises(InstallerError, match=r"Unable to inspect.*\(Forbidden\)"): ensure_local_admin_secret("operator") +@pytest.mark.parametrize( + "secret_data", + [ + [], + {"data": []}, + {"data": {"admin-username": "b3BlcmF0b3I=", "admin-password": "!!!", "api-token": "dG9rZW4="}}, + {"data": {"admin-username": "b3BlcmF0b3I=", "admin-password": "", "api-token": "dG9rZW4="}}, + {"data": {"admin-username": "b3BlcmF0b3I=", "admin-password": "cGFzc3dvcmQ=", "api-token": ""}}, + ], +) +def test_existing_secret_rejects_invalid_json_data_without_exposing_values(monkeypatch, secret_data) -> None: + def fake_run(command, *, check=True, input_text=None): + if command[2] == "secret": + return subprocess.CompletedProcess(command, 0, json.dumps(secret_data)) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + with pytest.raises(InstallerError) as exc_info: + ensure_local_admin_secret("operator") + + assert "!!!" not in str(exc_info.value) + assert "cGFzc3dvcmQ=" not in str(exc_info.value) + + def test_local_upgrade_ensures_secret_and_waits_for_hub(monkeypatch) -> None: calls: list[list[str]] = [] diff --git a/tests/installer/test_cli_install_options.py b/tests/installer/test_cli_install_options.py index 5960d6a9..4d7a0c8c 100644 --- a/tests/installer/test_cli_install_options.py +++ b/tests/installer/test_cli_install_options.py @@ -173,6 +173,19 @@ def test_install_dry_run_defaults_to_pull() -> None: assert " Image source : pull" in out +@pytest.mark.parametrize("username", ["Admin", "admin:name", 'admin"name']) +@patch("auplc_installer.cli._resolve_source_root") +@patch("auplc_installer.cli.InstallerState.from_environment") +def test_main_install_dry_run_rejects_unsafe_local_admin_username(mock_from_env, mock_root, username: str) -> None: + mock_root.return_value = Path("/repo") + mock_from_env.return_value = InstallerState() + + with pytest.raises(SystemExit) as exc_info: + main(["install", "--dry-run", "--access-mode=local", f"--admin-username={username}"]) + + assert exc_info.value.code == 1 + + def test_help_flag_prints_usage() -> None: buf = io.StringIO() with redirect_stdout(buf): diff --git a/tests/installer/test_local_auth.py b/tests/installer/test_local_auth.py index 28c60d4b..72770dbb 100644 --- a/tests/installer/test_local_auth.py +++ b/tests/installer/test_local_auth.py @@ -3,8 +3,15 @@ import pytest -from auplc_installer.cli import _preserve_access_settings_for_upgrade, _resolve_access_settings +from auplc_installer import tui +from auplc_installer.cli import ( + _preserve_access_settings_for_upgrade, + _resolve_access_settings, + cmd_dev_reinstall, + cmd_rt_reinstall, +) from auplc_installer.gpu import GpuConfig, append_product +from auplc_installer.helm import RuntimePaths from auplc_installer.overlay import generate_values_overlay, try_load_access_settings_from_overlay from auplc_installer.state import InstallerState from auplc_installer.tui import _flow_select_access @@ -99,6 +106,75 @@ def test_upgrade_rejects_unmanaged_advanced_auth_overlay(tmp_path: Path) -> None _preserve_access_settings_for_upgrade(InstallerState(), overlay) +@pytest.mark.parametrize( + "auth_mode", + ['"github"', "'multi' # operator-managed", '"dummy"'], +) +def test_upgrade_rejects_quoted_or_commented_advanced_auth_overlay(tmp_path: Path, auth_mode: str) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text( + f"# Access mode : local\n# Admin username: operator\ncustom:\n authMode: {auth_mode}\n", + encoding="utf-8", + ) + + with pytest.raises(Exception, match="operator-managed Helm values"): + _preserve_access_settings_for_upgrade(InstallerState(), overlay) + + +@pytest.mark.parametrize( + ("menu", "action", "command"), + [ + ("dev", "deploy", "cmd_dev_deploy"), + ("dev", "reinstall", "cmd_dev_reinstall"), + ("rt", "install", "cmd_rt_install"), + ("rt", "reinstall", "cmd_rt_reinstall"), + ], +) +def test_tui_runtime_deploy_and_reinstall_prompt_for_access_mode( + monkeypatch, menu: str, action: str, command: str +) -> None: + selected_access = [] + monkeypatch.setattr("auplc_installer.tui._ask_select", lambda *_args, **_kwargs: action) + monkeypatch.setattr("auplc_installer.tui._ask_confirm", lambda *_args, **_kwargs: True) + monkeypatch.setattr("auplc_installer.tui._flow_select_envs", lambda *_args, **_kwargs: True) + monkeypatch.setattr("auplc_installer.tui._flow_select_access", lambda state: selected_access.append(state)) + monkeypatch.setattr(f"auplc_installer.cli.{command}", lambda _state: None) + + if menu == "dev": + tui._flow_dev(InstallerState()) + else: + tui._flow_rt(InstallerState()) + + assert len(selected_access) == 1 + + +@pytest.mark.parametrize( + ("reinstall", "install"), + [ + (cmd_dev_reinstall, "auplc_installer.cli.cmd_dev_deploy"), + (cmd_rt_reinstall, "auplc_installer.cli.cmd_rt_install"), + ], +) +def test_reinstall_preserves_local_access_before_removing_release( + monkeypatch, tmp_path: Path, reinstall, install: str +) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text("# Access mode : local\n# Admin username: operator\n", encoding="utf-8") + state = InstallerState() + monkeypatch.setattr(state, "runtime_paths", lambda: RuntimePaths(Path("chart"), Path("values"), overlay)) + observed = [] + monkeypatch.setattr("auplc_installer.cli._provision_gpu_access_for_local_hardware", lambda **_kwargs: None) + monkeypatch.setattr("auplc_installer.cli.remove_runtime", lambda: observed.append("removed")) + monkeypatch.setattr("auplc_installer.cli.time.sleep", lambda _seconds: None) + monkeypatch.setattr( + install, lambda current_state: observed.append((current_state.access_mode, current_state.admin_username)) + ) + + reinstall(state) + + assert observed == ["removed", ("local", "operator")] + + def test_local_overlay_retains_single_node_runtime_behavior(tmp_path: Path) -> None: cfg = GpuConfig() append_product(cfg, "AMD_Radeon_8060S_Graphics") From 2b3c362389f01886ae99bc90e0a3415b14caf1de Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:52:49 +0800 Subject: [PATCH 130/180] fix(chart): support legacy local admin secrets --- runtime/chart/templates/NOTES.txt | 18 ++++++- runtime/chart/templates/hub/deployment.yaml | 8 ++-- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 14 +++++- tests/installer/test_chart_local_auth.py | 53 ++++++++++++++++++++- 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/runtime/chart/templates/NOTES.txt b/runtime/chart/templates/NOTES.txt index 42f09ffb..90137e8a 100644 --- a/runtime/chart/templates/NOTES.txt +++ b/runtime/chart/templates/NOTES.txt @@ -40,7 +40,20 @@ SOFTWARE. - Hub image: Custom AUP Learning Cloud Hub (based on Z2JH {{ .Chart.Version }}) {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} -### Admin Credentials (auto-generated) +{{- $admin_secret := .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} +{{- if .Values.custom.adminUser.existingSecret }} +### Admin Credentials (external Secret) + + Administrator username: {{ .Values.custom.adminUser.username }} + Credential Secret: {{ $admin_secret }} + + Get admin password: + kubectl -n {{ .Release.Namespace }} get secret {{ $admin_secret }} -o go-template='{{"{{index .data \"admin-password\" | base64decode}}"}}' + + This chart does not create or rotate the external Secret. If it includes an + `api-token` key, retrieve it from {{ $admin_secret }} for scripts. +{{- else }} +### Admin Credentials (chart-created Secret) Admin username: {{ .Values.custom.adminUser.username }} @@ -50,6 +63,7 @@ SOFTWARE. Get API token (for scripts): export JUPYTERHUB_TOKEN=$(kubectl -n {{ .Release.Namespace }} get secret jupyterhub-admin-credentials -o go-template='{{"{{index .data \"api-token\" | base64decode}}"}}') +{{- end }} {{- end }} ### Followup links @@ -104,7 +118,7 @@ SOFTWARE. The k8s Service {{ $proxy_service }} is exposed via NodePorts. That means that all the k8s cluster's nodes are exposing the k8s Service via those ports. - {{- if and .Values.custom .Values.custom.authMode (eq .Values.custom.authMode "auto-login") }} + {{- if and .Values.custom .Values.custom.authMode (or (eq .Values.custom.authMode "auto-login") (eq .Values.custom.authMode "local")) }} Single-node mode detected. To get your node's IP address, run: diff --git a/runtime/chart/templates/hub/deployment.yaml b/runtime/chart/templates/hub/deployment.yaml index 74b0b385..a776a359 100644 --- a/runtime/chart/templates/hub/deployment.yaml +++ b/runtime/chart/templates/hub/deployment.yaml @@ -219,10 +219,7 @@ spec: key: hub.config.ConfigurableHTTPProxy.auth_token {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} - name: JUPYTERHUB_ADMIN_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} - key: admin-username + value: {{ .Values.custom.adminUser.username | quote }} - name: JUPYTERHUB_ADMIN_PASSWORD valueFrom: secretKeyRef: @@ -233,6 +230,9 @@ spec: secretKeyRef: name: {{ .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} key: api-token + {{- if .Values.custom.adminUser.existingSecret }} + optional: true + {{- end }} {{- end }} {{- with .Values.hub.extraEnv }} {{- include "jupyterhub.extraEnv" . | nindent 12 }} diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 0d46c668..00efce32 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username"],"properties":{"enabled":{"const":true},"username":{"minLength":1}}}}}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["existingSecret"],"properties":{"existingSecret":{"minLength":1}}}}},"then":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index f1a211b9..b9c34361 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3798,14 +3798,26 @@ properties: required: [adminUser] properties: adminUser: - required: [enabled, username, existingSecret] + required: [enabled, username] properties: enabled: const: true username: minLength: 1 + + - if: + required: [adminUser] + properties: + adminUser: + required: [existingSecret] + properties: existingSecret: minLength: 1 + then: + required: [authMode] + properties: + authMode: + const: local cull: type: object diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index ff5af7b2..87edd8ef 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -28,11 +28,12 @@ def test_local_chart_render_uses_existing_secret_only_for_hub_bootstrap() -> Non assert "authMode: local" in result.stdout assert "name: JUPYTERHUB_ADMIN_USERNAME" in result.stdout - assert "key: admin-username" in result.stdout + assert 'value: "operator"' in result.stdout assert "name: JUPYTERHUB_ADMIN_PASSWORD" in result.stdout assert "key: admin-password" in result.stdout assert "name: JUPYTERHUB_API_TOKEN" in result.stdout assert "key: api-token" in result.stdout + assert "optional: true" in result.stdout assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" not in result.stdout @@ -60,3 +61,53 @@ def test_local_chart_schema_rejects_uppercase_admin_username() -> None: assert result.returncode != 0 assert "does not match pattern" in result.stderr + + +def test_local_chart_render_allows_chart_managed_credentials() -> None: + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "--set", + "custom.authMode=local", + "--set", + "custom.adminUser.enabled=true", + "--set", + "custom.adminUser.username=operator", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" in result.stdout + assert 'name: JUPYTERHUB_ADMIN_USERNAME\n value: "operator"' in result.stdout + + +def test_chart_schema_rejects_existing_secret_outside_local_mode() -> None: + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "--set", + "custom.authMode=auto-login", + "--set", + "custom.adminUser.enabled=true", + "--set", + "custom.adminUser.username=operator", + "--set", + "custom.adminUser.existingSecret=legacy-admin-credentials", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "authMode" in result.stderr From 93753d36231df185fc706f0699d54fa0927e7361 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:53:10 +0800 Subject: [PATCH 131/180] fix(hub): preserve local administrator credentials --- runtime/hub/core/authenticators/local.py | 5 +- runtime/hub/core/handlers.py | 4 +- runtime/hub/core/setup.py | 75 +++++++------ runtime/hub/tests/test_local_authenticator.py | 102 ++++++++++++++++++ runtime/hub/tests/test_onboarding_handlers.py | 56 ++++++++++ 5 files changed, 204 insertions(+), 38 deletions(-) diff --git a/runtime/hub/core/authenticators/local.py b/runtime/hub/core/authenticators/local.py index dcad0689..e51336c4 100644 --- a/runtime/hub/core/authenticators/local.py +++ b/runtime/hub/core/authenticators/local.py @@ -6,10 +6,13 @@ class CustomLocalAuthenticator(CustomFirstUseAuthenticator): + def validate_username(self, username): + return bool(LOCAL_USERNAME_PATTERN.fullmatch(username)) + async def authenticate(self, _handler, data): username = data.get("username", "") password = data.get("password", "") - if not LOCAL_USERNAME_PATTERN.fullmatch(username) or not password: + if not self.validate_username(username) or not password: return None if not self._user_exists(username): return None diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index 041602f7..d4e073ee 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -689,8 +689,8 @@ async def post(self): results["failed"] += 1 results["results"].append(result) continue - if not self.authenticator.validate_username(username): - result["error"] = f"Invalid username: {username}" + if not self.authenticator.validate_username(requested_username): + result["error"] = f"Invalid username: {requested_username}" results["failed"] += 1 results["results"].append(result) continue diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index 6200d956..e89a3bb6 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -49,6 +49,32 @@ pass +def _bootstrap_admin_password(admin_username: str, admin_password: str) -> None: + from core.authenticators.models import UserPassword + from core.database import session_scope + + with session_scope() as session: + user_pw = session.query(UserPassword).filter_by(username=admin_username).first() + if user_pw: + print(f"[SETUP] Admin '{admin_username}' password already set") + return + password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) + session.add( + UserPassword( + username=admin_username, + password_hash=password_hash, + force_change=False, + ) + ) + print(f"[SETUP] Admin '{admin_username}' password set automatically") + + +def _configure_api_token(c: Any, api_token: str | None, admin_username: str) -> None: + if api_token: + c.JupyterHub.api_tokens = {api_token: admin_username} + print(f"[SETUP] API token loaded for administrator '{admin_username}'") + + def setup_hub(c: Any) -> None: """ Set up JupyterHub with business logic from core. @@ -328,14 +354,24 @@ async def delete(self, group_name): except Exception as e: print(f"[QUOTA] Warning: Failed to run quota migration: {e}") + # ========================================================================= + # Auto-Create Admin User + # ========================================================================= + + admin_password = os.environ.get("JUPYTERHUB_ADMIN_PASSWORD", "") + admin_username = os.environ.get("JUPYTERHUB_ADMIN_USERNAME", "admin") + + if config.auth_mode == "local" and not admin_password: + raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_PASSWORD") + if config.auth_mode == "local" and not os.environ.get("JUPYTERHUB_ADMIN_USERNAME"): + raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_USERNAME") + # ========================================================================= # API Token # ========================================================================= api_token = os.environ.get("JUPYTERHUB_API_TOKEN") - if api_token: - c.JupyterHub.api_tokens = {api_token: "admin"} - print("[SETUP] API token loaded for admin user") + _configure_api_token(c, api_token, admin_username) # ========================================================================= # Template Paths @@ -344,40 +380,9 @@ async def delete(self, group_name): template_path = os.environ.get("JUPYTERHUB_TEMPLATE_PATH", "/tmp/custom_templates") c.JupyterHub.template_paths = [template_path] - # ========================================================================= - # Auto-Create Admin User - # ========================================================================= - - admin_password = os.environ.get("JUPYTERHUB_ADMIN_PASSWORD", "") - admin_username = os.environ.get("JUPYTERHUB_ADMIN_USERNAME", "admin") - - if config.auth_mode == "local" and not admin_password: - raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_PASSWORD") - if config.auth_mode == "local" and not os.environ.get("JUPYTERHUB_ADMIN_USERNAME"): - raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_USERNAME") - if admin_password: try: - from core.authenticators.models import UserPassword - from core.database import session_scope - - with session_scope() as session: - user_pw = session.query(UserPassword).filter_by(username=admin_username).first() - if user_pw: - if config.auth_mode == "local" and not bcrypt.checkpw( - admin_password.encode(), user_pw.password_hash - ): - raise RuntimeError("Local administrator password does not match the credentials Secret") - print(f"[SETUP] Admin '{admin_username}' password already set") - else: - password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) - user_pw = UserPassword( - username=admin_username, - password_hash=password_hash, - force_change=False, - ) - session.add(user_pw) - print(f"[SETUP] Admin '{admin_username}' password set automatically") + _bootstrap_admin_password(admin_username, admin_password) except Exception as e: if config.auth_mode == "local": raise RuntimeError("Failed to bootstrap local administrator credentials") from e diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py index 9be6ab04..0266e3fa 100644 --- a/runtime/hub/tests/test_local_authenticator.py +++ b/runtime/hub/tests/test_local_authenticator.py @@ -2,6 +2,7 @@ import importlib.util import sys import types +from contextlib import contextmanager from pathlib import Path import pytest @@ -9,6 +10,7 @@ ROOT = Path(__file__).resolve().parents[1] LOCAL_AUTHENTICATOR = ROOT / "core" / "authenticators" / "local.py" AUTHENTICATORS = ROOT / "core" / "authenticators" / "__init__.py" +SETUP = ROOT / "core" / "setup.py" class FakeFirstUseAuthenticator: @@ -75,6 +77,106 @@ def test_local_authenticator_rejects_noncanonical_usernames() -> None: ) +def test_local_authenticator_validate_username_matches_login_policy() -> None: + core = types.ModuleType("core") + authenticators = types.ModuleType("core.authenticators") + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator + sys.modules.update( + { + "core": core, + "core.authenticators": authenticators, + "core.authenticators.firstuse": firstuse, + } + ) + spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + authenticator = module.CustomLocalAuthenticator() + + assert authenticator.validate_username("existing") + for username in ("EXISTING", "existing:admin", 'existing"', "existing\n", "a" * 65): + assert not authenticator.validate_username(username) + + +def test_bootstrap_admin_password_preserves_a_password_changed_after_first_start(monkeypatch) -> None: + bcrypt = types.ModuleType("bcrypt") + bcrypt.gensalt = lambda: b"salt" + bcrypt.hashpw = lambda password, _salt: b"hash:" + password + bcrypt.checkpw = lambda password, password_hash: password_hash == b"hash:" + password + + class FakeUserPassword: + def __init__(self, username, password_hash, force_change): + self.username = username + self.password_hash = password_hash + self.force_change = force_change + + class FakeQuery: + def __init__(self, rows): + self.rows = rows + self.username = "" + + def filter_by(self, *, username): + self.username = username + return self + + def first(self): + return next((row for row in self.rows if row.username == self.username), None) + + class FakeSession: + def __init__(self): + self.rows = [] + + def query(self, _model): + return FakeQuery(self.rows) + + def add(self, row): + self.rows.append(row) + + session = FakeSession() + models = types.ModuleType("core.authenticators.models") + models.UserPassword = FakeUserPassword + database = types.ModuleType("core.database") + + @contextmanager + def session_scope(): + yield session + + database.session_scope = session_scope + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + monkeypatch.setitem(sys.modules, "core.authenticators.models", models) + monkeypatch.setitem(sys.modules, "core.database", database) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + module._bootstrap_admin_password("operator", "InitialPassword1!") + session.rows[0].password_hash = bcrypt.hashpw(b"ChangedPassword1!", bcrypt.gensalt()) + module._bootstrap_admin_password("operator", "InitialPassword1!") + + assert bcrypt.checkpw(b"ChangedPassword1!", session.rows[0].password_hash) + assert not bcrypt.checkpw(b"InitialPassword1!", session.rows[0].password_hash) + + +def test_api_token_is_assigned_to_the_configured_administrator(monkeypatch) -> None: + bcrypt = types.ModuleType("bcrypt") + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + config = types.SimpleNamespace(JupyterHub=types.SimpleNamespace()) + + module._configure_api_token(config, "token", "operator") + + assert config.JupyterHub.api_tokens == {"token": "operator"} + + def test_authenticator_factory_rejects_unknown_mode() -> None: core = types.ModuleType("core") authenticators = types.ModuleType("core.authenticators") diff --git a/runtime/hub/tests/test_onboarding_handlers.py b/runtime/hub/tests/test_onboarding_handlers.py index e833bb56..96415a02 100644 --- a/runtime/hub/tests/test_onboarding_handlers.py +++ b/runtime/hub/tests/test_onboarding_handlers.py @@ -27,6 +27,20 @@ sys.modules["jupyterhub.orm"] = orm_module sys.modules["jupyterhub.scopes"] = scopes_module +if "jupyterhub.roles" not in sys.modules: + roles_module = types.ModuleType("jupyterhub.roles") + roles_module.assign_default_roles = lambda *_args, **_kwargs: None + sys.modules["jupyterhub.roles"] = roles_module + +if "jupyterhub.utils" not in sys.modules: + utils_module = types.ModuleType("jupyterhub.utils") + + async def maybe_future(value): + return value + + utils_module.maybe_future = maybe_future + sys.modules["jupyterhub.utils"] = utils_module + if "multiauthenticator" not in sys.modules: multiauthenticator_module = types.ModuleType("multiauthenticator") multiauthenticator_module.MultiAuthenticator = type("MultiAuthenticator", (), {}) @@ -144,6 +158,7 @@ def load_module(name: str, path: Path): DismissMyOnboardingHandler = handlers.DismissMyOnboardingHandler GetMyOnboardingHandler = handlers.GetMyOnboardingHandler AdminResetPasswordHandler = handlers.AdminResetPasswordHandler +AdminAPIProvisionUsersHandler = handlers.AdminAPIProvisionUsersHandler class DummyUser: @@ -255,6 +270,47 @@ async def render_template(_name, **kwargs): assert rendered["native_users"] == ["learner"] +def test_admin_provisioning_rejects_username_that_local_login_would_reject(monkeypatch) -> None: + class FakeAuthenticator: + def __init__(self): + self.validated_usernames = [] + + def validate_username(self, username): + self.validated_usernames.append(username) + return username == username.lower() and ":" not in username + + class FakeFirstUseAuthenticator: + def normalize_username(self, username): + return username.lower() + + def _check_password_strength(self, _password): + return None + + def set_password(self, *_args, **_kwargs): + raise AssertionError("invalid username must not set a password") + + authenticator = FakeAuthenticator() + handler = object.__new__(AdminAPIProvisionUsersHandler) + handler.current_user = DummyUser("operator", admin=True) + handler.authenticator = authenticator + handler.request = types.SimpleNamespace( + body=json.dumps({"users": [{"username": "Admin", "password": "Password1!"}]}).encode("utf-8") + ) + handler.find_user = lambda _username: None + captured = {} + handler.set_header = lambda key, value: captured.setdefault("headers", {}).__setitem__(key, value) + handler.finish = lambda payload: captured.setdefault("body", payload) + handler.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) + monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) + + asyncio.run(handler.post()) + + payload = json.loads(captured["body"]) + assert authenticator.validated_usernames == ["Admin"] + assert payload["failed"] == 1 + assert payload["results"][0]["error"] == "Invalid username: Admin" + + def test_get_my_onboarding_returns_visible_when_no_state_exists(monkeypatch): monkeypatch.setattr(database, "session_scope", fake_session_scope(FakeDb())) handler, captured = make_handler(GetMyOnboardingHandler, "alice") From 2820825deab7a1a664f675f0a915b639bf57cb26 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:58:02 +0800 Subject: [PATCH 132/180] fix(installer): capture local credential inspections --- auplc_installer/helm.py | 9 ++-- auplc_installer/util.py | 5 +- tests/installer/test_admin_secret.py | 70 ++++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/auplc_installer/helm.py b/auplc_installer/helm.py index d3fb22ea..f735f3b9 100644 --- a/auplc_installer/helm.py +++ b/auplc_installer/helm.py @@ -58,7 +58,7 @@ def _helm_install_args(paths: RuntimePaths, *, dev: bool = False) -> list[str]: def _ensure_namespace() -> None: - existing = run(["kubectl", "get", "namespace", "jupyterhub"], check=False) + existing = _run_kubectl_inspection(["kubectl", "get", "namespace", "jupyterhub"]) if existing.returncode == 0: return created = run(["kubectl", "create", "namespace", "jupyterhub"], check=False) @@ -67,6 +67,10 @@ def _ensure_namespace() -> None: raise InstallerError("Failed to create jupyterhub namespace") +def _run_kubectl_inspection(command: list[str]): + return run(command, check=False, capture_output=True) + + def _decode_secret_value(data: dict[str, object], key: str) -> str: encoded_value = data.get(key) if not isinstance(encoded_value, str) or not encoded_value: @@ -107,9 +111,8 @@ def ensure_local_admin_secret(admin_username: str) -> str | None: secret_name = "jupyterhub-admin-credentials" admin_username = validate_local_admin_username(admin_username) _ensure_namespace() - existing = run( + existing = _run_kubectl_inspection( ["kubectl", "get", "secret", secret_name, "--namespace", "jupyterhub", "-o", "json"], - check=False, ) if existing.returncode == 0: stored_username, _, _ = _parse_existing_local_admin_secret(existing.stdout) diff --git a/auplc_installer/util.py b/auplc_installer/util.py index 752d2eff..dc7ddafd 100644 --- a/auplc_installer/util.py +++ b/auplc_installer/util.py @@ -63,6 +63,7 @@ def run( env: Mapping[str, str] | None = None, cwd: str | Path | None = None, input_text: str | None = None, + capture_output: bool = False, ) -> subprocess.CompletedProcess[str]: """Run a command synchronously, optionally with sudo, raise on failure. @@ -83,7 +84,7 @@ def run( """ full = _build_cmd(cmd, sudo=sudo) - if _VERBOSE or input_text is not None: + if _VERBOSE or input_text is not None or capture_output: # Verbose path or stdin-feeding path: subprocess.run is fine # (Popen with stdin pipes complicates feeding ``input_text``). popen_kwargs: dict[str, object] = { @@ -93,7 +94,7 @@ def run( "text": True, "input": input_text, } - if not _VERBOSE: + if capture_output or not _VERBOSE: # Quiet but with input_text: still capture for failure dump. popen_kwargs["stdout"] = subprocess.PIPE popen_kwargs["stderr"] = subprocess.STDOUT diff --git a/tests/installer/test_admin_secret.py b/tests/installer/test_admin_secret.py index ae229890..822a4a56 100644 --- a/tests/installer/test_admin_secret.py +++ b/tests/installer/test_admin_secret.py @@ -11,7 +11,7 @@ def test_creates_local_admin_secret_through_stdin_without_leaking_credentials(monkeypatch, capsys) -> None: calls: list[tuple[list[str], str | None]] = [] - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): calls.append((command, input_text)) if len(calls) == 3: return subprocess.CompletedProcess( @@ -45,7 +45,7 @@ def fake_run(command, *, check=True, input_text=None): def test_reuses_existing_local_admin_secret(monkeypatch) -> None: calls: list[list[str]] = [] - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): calls.append(command) if command[2] == "secret": return subprocess.CompletedProcess( @@ -75,7 +75,7 @@ def fake_run(command, *, check=True, input_text=None): def test_deploy_orders_namespace_secret_and_helm_without_printing_new_password(monkeypatch, capsys) -> None: calls: list[tuple[str, list[str], str | None]] = [] - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): calls.append(("run", command, input_text)) if len(calls) == 3: return subprocess.CompletedProcess( @@ -123,7 +123,7 @@ def failing_stream(command, **_kwargs): def test_existing_legacy_secret_is_patched_without_rotating_credentials(monkeypatch) -> None: calls: list[tuple[list[str], str | None]] = [] - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): calls.append((command, input_text)) if command[2] == "secret": return subprocess.CompletedProcess( @@ -151,7 +151,7 @@ def fake_run(command, *, check=True, input_text=None): def test_existing_secret_requires_complete_matching_contract(monkeypatch) -> None: - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): if command[2] == "secret": return subprocess.CompletedProcess( command, @@ -167,7 +167,7 @@ def fake_run(command, *, check=True, input_text=None): def test_secret_lookup_fails_closed_for_non_not_found_errors(monkeypatch) -> None: - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): if command[2] == "secret": return subprocess.CompletedProcess(command, 1, "Error from server (Forbidden): secrets is forbidden") return subprocess.CompletedProcess(command, 0, "") @@ -189,7 +189,7 @@ def fake_run(command, *, check=True, input_text=None): ], ) def test_existing_secret_rejects_invalid_json_data_without_exposing_values(monkeypatch, secret_data) -> None: - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): if command[2] == "secret": return subprocess.CompletedProcess(command, 0, json.dumps(secret_data)) return subprocess.CompletedProcess(command, 0, "") @@ -206,7 +206,7 @@ def fake_run(command, *, check=True, input_text=None): def test_local_upgrade_ensures_secret_and_waits_for_hub(monkeypatch) -> None: calls: list[list[str]] = [] - def fake_run(command, *, check=True, input_text=None): + def fake_run(command, *, check=True, input_text=None, capture_output=False): calls.append(command) if command[2] == "secret": return subprocess.CompletedProcess( @@ -236,3 +236,57 @@ def fake_run(command, *, check=True, input_text=None): assert calls[1][:3] == ["kubectl", "get", "secret"] assert any(command[:2] == ["helm", "upgrade"] for command in calls) assert ["kubectl", "rollout", "status", "deployment/hub", "--namespace", "jupyterhub", "--timeout=600s"] in calls + + +def test_verbose_first_install_captures_secret_inspection_without_printing_credentials(monkeypatch, capsys) -> None: + calls: list[tuple[list[str], bool]] = [] + + def fake_run(command, *, check=True, input_text=None, capture_output=False): + calls.append((command, capture_output)) + if command[1:3] == ["get", "secret"]: + assert capture_output + return subprocess.CompletedProcess( + command, 1, 'Error from server (NotFound): secrets "jupyterhub-admin-credentials" not found' + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + monkeypatch.setattr("auplc_installer.util._VERBOSE", True) + monkeypatch.setattr("auplc_installer.helm.secrets.token_urlsafe", lambda _length: "generated-password") + + assert ensure_local_admin_secret("operator") == "generated-password" + assert calls[0][1] + assert calls[1][1] + assert calls[2][1] is False + assert "generated-password" not in capsys.readouterr().out + + +def test_verbose_reuse_captures_secret_inspection_without_printing_credentials(monkeypatch, capsys) -> None: + calls: list[tuple[list[str], bool]] = [] + + def fake_run(command, *, check=True, input_text=None, capture_output=False): + calls.append((command, capture_output)) + if command[1:3] == ["get", "secret"]: + assert capture_output + return subprocess.CompletedProcess( + command, + 0, + json.dumps( + { + "data": { + "admin-username": "b3BlcmF0b3I=", + "admin-password": "c2VjcmV0LXBhc3N3b3Jk", + "api-token": "dG9rZW4=", + } + } + ), + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + monkeypatch.setattr("auplc_installer.util._VERBOSE", True) + + assert ensure_local_admin_secret("operator") is None + assert calls[0][1] + assert calls[1][1] + assert "secret-password" not in capsys.readouterr().out From 9ad5171010f889900bb4b520755cd681e223ca46 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:58:28 +0800 Subject: [PATCH 133/180] fix(hub): protect secret-managed local admin credentials --- README.md | 2 +- runtime/hub/core/authenticators/local.py | 11 +++ runtime/hub/core/handlers.py | 35 +++++++++ runtime/hub/core/setup.py | 8 +- runtime/hub/tests/test_local_authenticator.py | 72 +++++++++++++++++- runtime/hub/tests/test_onboarding_handlers.py | 76 +++++++++++++++++++ 6 files changed, 198 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f391fb64..78826c07 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ For scripted installs, `personal` remains the compatibility default. Select loca ./auplc-installer install --access-mode=local --admin-username=admin ``` -The installer generates the administrator password and API token only when it creates `jupyterhub-admin-credentials`. It displays the password once after a successful interactive deployment. Re-running against an existing Secret reuses the credentials without rotating them; recover credentials with `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo`. Local users are created and assigned passwords through the Admin UI. +The installer generates the administrator password and API token only when it creates `jupyterhub-admin-credentials`. It displays the password once after a successful interactive deployment. Re-running against an existing Secret reuses the credentials without rotating them; recover credentials with `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo`. The configured bootstrap administrator remains Secret-managed and cannot change or reset its password through the UI. Other local users are created and assigned passwords through the Admin UI. Local mode is an installer MVP for `http://localhost:30890` on a trusted single-node host. It does not configure TLS or restrict the K3s NodePort from LAN reachability; do not treat local credentials as a network exposure control. diff --git a/runtime/hub/core/authenticators/local.py b/runtime/hub/core/authenticators/local.py index e51336c4..16ef9540 100644 --- a/runtime/hub/core/authenticators/local.py +++ b/runtime/hub/core/authenticators/local.py @@ -9,6 +9,17 @@ class CustomLocalAuthenticator(CustomFirstUseAuthenticator): def validate_username(self, username): return bool(LOCAL_USERNAME_PATTERN.fullmatch(username)) + def _user_exists(self, username): + db = getattr(self, "db", None) or getattr(getattr(self, "parent", None), "db", None) + if db is None: + return False + try: + from jupyterhub.orm import User + + return db.query(User).filter_by(name=username).first() is not None + except Exception: + return False + async def authenticate(self, _handler, data): username = data.get("username", "") password = data.get("password", "") diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index d4e073ee..f3e91f49 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -31,6 +31,7 @@ import asyncio import json +import os from datetime import datetime, timezone from typing import Any from urllib.parse import urlencode, urlparse, urlunparse @@ -81,6 +82,10 @@ MAX_NATIVE_PASSWORD_BYTES = 72 +def _is_secret_managed_bootstrap_admin(username: str) -> bool: + return _handler_config["auth_mode"] == "local" and username == os.environ.get("JUPYTERHUB_ADMIN_USERNAME") + + def _serialize_dismissed_at(value: datetime | None) -> str | None: """Serialize onboarding dismissal timestamps for API responses.""" if value is None: @@ -280,6 +285,13 @@ def _render_error(msg: str): self.set_status(400) return self.finish(html) + if _is_secret_managed_bootstrap_admin(username): + html = await _render_error( + "The configured local administrator password is managed by the Kubernetes Secret" + ) + self.set_status(403) + return self.finish(html) + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -367,6 +379,12 @@ async def post(self): + f"admin/reset-password?user={target_user}&error=Cannot+reset+password+for+GitHub+users" ) + if _is_secret_managed_bootstrap_admin(username): + return self.redirect( + self.hub.base_url + + f"admin/reset-password?user={target_user}&error=Configured+local+administrator+password+is+managed+by+the+Kubernetes+Secret" + ) + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -443,6 +461,15 @@ async def post(self): self.set_header("Content-Type", "application/json") return self.finish(json.dumps({"error": "Cannot set password for GitHub users"})) + if _is_secret_managed_bootstrap_admin(username): + self.set_status(403) + self.set_header("Content-Type", "application/json") + return self.finish( + json.dumps( + {"error": "The configured local administrator password is managed by the Kubernetes Secret"} + ) + ) + firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -528,6 +555,14 @@ async def post(self): return self.finish( json.dumps({"error": f"Cannot set password for GitHub user: {entry['username']}"}) ) + if _is_secret_managed_bootstrap_admin(entry["username"]): + self.set_status(403) + self.set_header("Content-Type", "application/json") + return self.finish( + json.dumps( + {"error": "The configured local administrator password is managed by the Kubernetes Secret"} + ) + ) firstuse_auth = _find_firstuse_authenticator(self.authenticator) diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index e89a3bb6..d3e2298f 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -49,13 +49,17 @@ pass -def _bootstrap_admin_password(admin_username: str, admin_password: str) -> None: +def _bootstrap_admin_password(admin_username: str, admin_password: str, *, require_match: bool = False) -> None: from core.authenticators.models import UserPassword from core.database import session_scope with session_scope() as session: user_pw = session.query(UserPassword).filter_by(username=admin_username).first() if user_pw: + if require_match and not bcrypt.checkpw(admin_password.encode(), user_pw.password_hash): + raise RuntimeError( + "Existing administrator password hash does not match the configured credentials Secret" + ) print(f"[SETUP] Admin '{admin_username}' password already set") return password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) @@ -382,7 +386,7 @@ async def delete(self, group_name): if admin_password: try: - _bootstrap_admin_password(admin_username, admin_password) + _bootstrap_admin_password(admin_username, admin_password, require_match=config.auth_mode == "local") except Exception as e: if config.auth_mode == "local": raise RuntimeError("Failed to bootstrap local administrator credentials") from e diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py index 0266e3fa..836c9568 100644 --- a/runtime/hub/tests/test_local_authenticator.py +++ b/runtime/hub/tests/test_local_authenticator.py @@ -29,11 +29,16 @@ def test_local_authenticator_rejects_first_use_and_accepts_existing_password() - authenticators = types.ModuleType("core.authenticators") firstuse = types.ModuleType("core.authenticators.firstuse") firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator + jupyterhub = types.ModuleType("jupyterhub") + orm = types.ModuleType("jupyterhub.orm") + orm.User = type("User", (), {}) sys.modules.update( { "core": core, "core.authenticators": authenticators, "core.authenticators.firstuse": firstuse, + "jupyterhub": jupyterhub, + "jupyterhub.orm": orm, } ) spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) @@ -43,6 +48,19 @@ def test_local_authenticator_rejects_first_use_and_accepts_existing_password() - spec.loader.exec_module(module) authenticator = module.CustomLocalAuthenticator() + class ExistingUserQuery: + def filter_by(self, **_kwargs): + return self + + def first(self): + return object() + + class ExistingUserDb: + def query(self, _model): + return ExistingUserQuery() + + authenticator.db = ExistingUserDb() + assert ( asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "correct-password"})) == "existing" @@ -101,7 +119,54 @@ def test_local_authenticator_validate_username_matches_login_policy() -> None: assert not authenticator.validate_username(username) -def test_bootstrap_admin_password_preserves_a_password_changed_after_first_start(monkeypatch) -> None: +def test_local_authenticator_fails_closed_without_a_working_hub_database() -> None: + class DatabaseAgnosticFirstUseAuthenticator: + def _user_exists(self, _username): + return True + + def check_password(self, _username, _password): + return True + + core = types.ModuleType("core") + authenticators = types.ModuleType("core.authenticators") + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = DatabaseAgnosticFirstUseAuthenticator + jupyterhub = types.ModuleType("jupyterhub") + orm = types.ModuleType("jupyterhub.orm") + orm.User = type("User", (), {}) + sys.modules.update( + { + "core": core, + "core.authenticators": authenticators, + "core.authenticators.firstuse": firstuse, + "jupyterhub": jupyterhub, + "jupyterhub.orm": orm, + } + ) + spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + unavailable_db = module.CustomLocalAuthenticator() + unavailable_db.db = None + unavailable_db.parent = types.SimpleNamespace(db=None) + + class FailingDb: + def query(self, _model): + raise RuntimeError("database unavailable") + + failing_db = module.CustomLocalAuthenticator() + failing_db.db = FailingDb() + + assert ( + asyncio.run(unavailable_db.authenticate(None, {"username": "existing", "password": "correct-password"})) is None + ) + assert asyncio.run(failing_db.authenticate(None, {"username": "existing", "password": "correct-password"})) is None + + +def test_bootstrap_admin_password_rejects_secret_mismatch_for_existing_hash(monkeypatch) -> None: bcrypt = types.ModuleType("bcrypt") bcrypt.gensalt = lambda: b"salt" bcrypt.hashpw = lambda password, _salt: b"hash:" + password @@ -154,10 +219,11 @@ def session_scope(): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - module._bootstrap_admin_password("operator", "InitialPassword1!") + module._bootstrap_admin_password("operator", "InitialPassword1!", require_match=True) session.rows[0].password_hash = bcrypt.hashpw(b"ChangedPassword1!", bcrypt.gensalt()) - module._bootstrap_admin_password("operator", "InitialPassword1!") + with pytest.raises(RuntimeError, match="does not match"): + module._bootstrap_admin_password("operator", "InitialPassword1!", require_match=True) assert bcrypt.checkpw(b"ChangedPassword1!", session.rows[0].password_hash) assert not bcrypt.checkpw(b"InitialPassword1!", session.rows[0].password_hash) diff --git a/runtime/hub/tests/test_onboarding_handlers.py b/runtime/hub/tests/test_onboarding_handlers.py index 96415a02..27dedea6 100644 --- a/runtime/hub/tests/test_onboarding_handlers.py +++ b/runtime/hub/tests/test_onboarding_handlers.py @@ -158,6 +158,8 @@ def load_module(name: str, path: Path): DismissMyOnboardingHandler = handlers.DismissMyOnboardingHandler GetMyOnboardingHandler = handlers.GetMyOnboardingHandler AdminResetPasswordHandler = handlers.AdminResetPasswordHandler +AdminAPISetPasswordHandler = handlers.AdminAPISetPasswordHandler +ChangePasswordHandler = handlers.ChangePasswordHandler AdminAPIProvisionUsersHandler = handlers.AdminAPIProvisionUsersHandler @@ -311,6 +313,80 @@ def set_password(self, *_args, **_kwargs): assert payload["results"][0]["error"] == "Invalid username: Admin" +def test_password_management_rejects_the_secret_managed_local_administrator(monkeypatch) -> None: + class FakeFirstUseAuthenticator: + async def authenticate(self, *_args): + raise AssertionError("bootstrap administrator password must not be authenticated for a change") + + def set_password(self, *_args, **_kwargs): + raise AssertionError("bootstrap administrator password must not be changed") + + monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") + monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") + monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) + + change = object.__new__(ChangePasswordHandler) + change.current_user = DummyUser("operator") + change.authenticator = object() + change.hub = types.SimpleNamespace(base_url="/hub/") + change.get_body_argument = lambda name, default=None: { + "current_password": "OldPassword1!", + "new_password": "NewPassword1!", + "confirm_password": "NewPassword1!", + }.get(name, default) + change.set_status = lambda status: setattr(change, "status", status) + change.finish = lambda payload: setattr(change, "body", payload) + + async def render_template(_name, **kwargs): + return kwargs["error_message"] + + change.render_template = render_template + asyncio.run(change.post()) + + assert change.status == 403 + assert "managed by the Kubernetes Secret" in change.body + + admin_api = object.__new__(AdminAPISetPasswordHandler) + admin_api.current_user = DummyUser("manager", admin=True) + admin_api.authenticator = object() + admin_api.request = types.SimpleNamespace( + body=json.dumps({"username": "operator", "password": "NewPassword1!"}).encode("utf-8") + ) + admin_api.set_status = lambda status: setattr(admin_api, "status", status) + admin_api.set_header = lambda *_args: None + admin_api.finish = lambda payload: setattr(admin_api, "body", payload) + admin_api.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) + asyncio.run(admin_api.post()) + + assert admin_api.status == 403 + assert "managed by the Kubernetes Secret" in json.loads(admin_api.body)["error"] + + +def test_admin_password_management_keeps_other_local_users_changeable(monkeypatch) -> None: + class FakeFirstUseAuthenticator: + def set_password(self, username, password, force_change=True): + assert (username, password, force_change) == ("learner", "NewPassword1!", True) + return "Password set for learner (force change on next login)" + + monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") + monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") + monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) + + handler = object.__new__(AdminAPISetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.request = types.SimpleNamespace( + body=json.dumps({"username": "learner", "password": "NewPassword1!"}).encode("utf-8") + ) + handler.set_header = lambda *_args: None + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) + + asyncio.run(handler.post()) + + assert json.loads(handler.body)["message"].startswith("Password set for learner") + + def test_get_my_onboarding_returns_visible_when_no_state_exists(monkeypatch): monkeypatch.setattr(database, "session_scope", fake_session_scope(FakeDb())) handler, captured = make_handler(GetMyOnboardingHandler, "alice") From 988895d52bf0804364071ddec15446561e586a9d Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:58:59 +0800 Subject: [PATCH 134/180] fix(chart): require local admin credential Secret --- runtime/chart/templates/NOTES.txt | 9 ++++--- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 4 ++- .../SKILL.md | 15 ++++++----- .../reference.md | 21 ++++++++------- tests/installer/test_chart_local_auth.py | 27 +++++++++++++++++-- 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/runtime/chart/templates/NOTES.txt b/runtime/chart/templates/NOTES.txt index 90137e8a..e36739f8 100644 --- a/runtime/chart/templates/NOTES.txt +++ b/runtime/chart/templates/NOTES.txt @@ -42,7 +42,7 @@ SOFTWARE. {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} {{- $admin_secret := .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} {{- if .Values.custom.adminUser.existingSecret }} -### Admin Credentials (external Secret) +### Admin Credentials (local mode external Secret) Administrator username: {{ .Values.custom.adminUser.username }} Credential Secret: {{ $admin_secret }} @@ -50,10 +50,11 @@ SOFTWARE. Get admin password: kubectl -n {{ .Release.Namespace }} get secret {{ $admin_secret }} -o go-template='{{"{{index .data \"admin-password\" | base64decode}}"}}' - This chart does not create or rotate the external Secret. If it includes an - `api-token` key, retrieve it from {{ $admin_secret }} for scripts. + Local mode requires `custom.adminUser.existingSecret`; this chart does not + create or rotate that Secret. If it includes an `api-token` key, retrieve it + from {{ $admin_secret }} for scripts. {{- else }} -### Admin Credentials (chart-created Secret) +### Admin Credentials (chart-created Secret for non-local auth) Admin username: {{ .Values.custom.adminUser.username }} diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 00efce32..96e3e9fc 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username"],"properties":{"enabled":{"const":true},"username":{"minLength":1}}}}}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["existingSecret"],"properties":{"existingSecret":{"minLength":1}}}}},"then":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["existingSecret"],"properties":{"existingSecret":{"minLength":1}}}}},"then":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index b9c34361..e98a501c 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3798,12 +3798,14 @@ properties: required: [adminUser] properties: adminUser: - required: [enabled, username] + required: [enabled, username, existingSecret] properties: enabled: const: true username: minLength: 1 + existingSecret: + minLength: 1 - if: required: [adminUser] diff --git a/skills/configure-aup-learning-cloud-auth/SKILL.md b/skills/configure-aup-learning-cloud-auth/SKILL.md index 16cfd8a0..b7b22025 100644 --- a/skills/configure-aup-learning-cloud-auth/SKILL.md +++ b/skills/configure-aup-learning-cloud-auth/SKILL.md @@ -74,15 +74,18 @@ login blip). (see manage-users skill). Password policy: ≥8 chars with upper, lower, digit, and special; users can be forced to change on first login. 6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` with a - canonical `custom.adminUser.username`. Without `existingSecret`, the chart - creates `jupyterhub-admin-credentials`; with `existingSecret`, it uses the - named external Secret and never rotates it. The single-node installer creates - and validates its lifecycle Secret before Helm runs. + canonical `custom.adminUser.username`. Direct Helm local mode requires a + nonempty `custom.adminUser.existingSecret`; create that Secret before Helm + runs. The single-node installer creates and validates its lifecycle Secret + before Helm runs. Chart-managed credentials remain available for non-local + authentication modes. 7. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay>` must succeed. -8. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: +8. **Apply.** Single-node: `./auplc-installer rt upgrade`. For a direct Helm + local deployment, first create the configured existing Secret, then run `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f - runtime/values.yaml -f <overlay>`. + runtime/values.yaml -f <overlay>`. Multi/manual non-local deployments can + use chart-managed credentials when no existing Secret is configured. 9. **Verify.** Load the Hub: the expected login page appears, a GitHub user lands in the right groups, and (if bootstrapped) the admin can log in. Read the secret with the commands in [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md index 276b4f98..46549820 100644 --- a/skills/configure-aup-learning-cloud-auth/reference.md +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -25,10 +25,10 @@ custom: - `dummy` — accepts any username/password. Testing only. - `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. - `local` — closed, administrator-managed local accounts. It requires - `custom.adminUser.enabled: true` and a canonical username. The chart can - create credentials or use an external Secret with `admin-password` and an - optional `api-token`; the username always comes from - `custom.adminUser.username`. + `custom.adminUser.enabled: true`, a canonical username, and a nonempty + `custom.adminUser.existingSecret` containing `admin-password`. The username + always comes from `custom.adminUser.username`; `api-token` remains optional + for direct Helm startup. - `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends in `/hub/github/oauth_callback`. @@ -40,12 +40,13 @@ custom: enabled: true ``` -Without `existingSecret`, the chart creates `jupyterhub-admin-credentials` and -bootstraps the configured administrator. The installer creates and validates the -same Secret for its local lifecycle. With `existingSecret`, the external Secret -is never created or rotated by the chart and must contain `admin-password`; an -`api-token` is optional for legacy two-key Secrets. Retrieve chart-created -credentials: +Direct Helm local mode requires `existingSecret`; create the external Secret +before Helm runs. The installer creates and validates +`jupyterhub-admin-credentials` for its local lifecycle. With `existingSecret`, +the external Secret is never created or rotated by the chart and must contain +`admin-password`; an `api-token` is optional for direct Helm startup. +Chart-managed credentials remain available for non-local authentication modes. +Retrieve chart-created credentials: ```bash kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index 87edd8ef..f82fbf07 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -63,7 +63,7 @@ def test_local_chart_schema_rejects_uppercase_admin_username() -> None: assert "does not match pattern" in result.stderr -def test_local_chart_render_allows_chart_managed_credentials() -> None: +def test_local_chart_schema_requires_a_nonempty_existing_secret() -> None: result = subprocess.run( [ "helm", @@ -78,13 +78,36 @@ def test_local_chart_render_allows_chart_managed_credentials() -> None: "custom.adminUser.username=operator", ], cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "existingSecret" in result.stderr + + +def test_nonlocal_chart_render_retains_chart_managed_credentials() -> None: + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "--set", + "custom.authMode=multi", + "--set", + "custom.adminUser.enabled=true", + "--set", + "custom.adminUser.username=operator", + ], + cwd=ROOT, check=True, capture_output=True, text=True, ) assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" in result.stdout - assert 'name: JUPYTERHUB_ADMIN_USERNAME\n value: "operator"' in result.stdout def test_chart_schema_rejects_existing_secret_outside_local_mode() -> None: From 35146eb7010e0afd7ffaa4598788c2e4fd4893c9 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:18:52 +0800 Subject: [PATCH 135/180] docs(skills): support custom admin secret names --- skills/manage-aup-learning-cloud-users/SKILL.md | 7 ++++--- skills/manage-aup-learning-cloud-users/reference.md | 5 ++++- .../scripts/hub-api-env.sh | 12 +++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/skills/manage-aup-learning-cloud-users/SKILL.md b/skills/manage-aup-learning-cloud-users/SKILL.md index 6de97b3d..0ca2cd0c 100644 --- a/skills/manage-aup-learning-cloud-users/SKILL.md +++ b/skills/manage-aup-learning-cloud-users/SKILL.md @@ -42,7 +42,8 @@ in **[reference.md](reference.md)**. scripts. - `manage_users.py` requires `JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN` for every subcommand. The bundled `scripts/hub-api-env.sh` derives both from the - `jupyterhub-admin-credentials` secret and checks reachability. + admin credentials Secret and checks reachability. Set `HUB_ADMIN_SECRET` when + `custom.adminUser.existingSecret` uses a non-default name. - Quota subcommands use the Hub admin API. `kubectl` is only needed to bootstrap an API token from `jupyterhub-admin-credentials` or inspect scheduled quota refresh CronJobs. @@ -80,8 +81,8 @@ current admin are protected from deletion. ``` (Or export `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` yourself — see reference.) - Use `HUB_URL="https://hub.example.com"` and `HUB_NAMESPACE=<namespace>` when - the Hub is not the default local NodePort in namespace `jupyterhub`. + Use `HUB_URL="https://hub.example.com"`, `HUB_NAMESPACE=<namespace>`, and + `HUB_ADMIN_SECRET=<secret-name>` when the deployment uses non-default values. 3. **Generate a roster template**: ```bash diff --git a/skills/manage-aup-learning-cloud-users/reference.md b/skills/manage-aup-learning-cloud-users/reference.md index ea12f92f..b43afe7c 100644 --- a/skills/manage-aup-learning-cloud-users/reference.md +++ b/skills/manage-aup-learning-cloud-users/reference.md @@ -28,7 +28,8 @@ admin-credentials secret (requires `custom.adminUser.enabled`): ```bash export JUPYTERHUB_URL="http://localhost:30890" -export JUPYTERHUB_TOKEN=$(kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ +export HUB_ADMIN_SECRET="jupyterhub-admin-credentials" +export JUPYTERHUB_TOKEN=$(kubectl -n jupyterhub get secret "$HUB_ADMIN_SECRET" \ -o jsonpath='{.data.api-token}' | base64 -d) ``` @@ -39,6 +40,8 @@ it (don't execute) so the exports land in your shell: source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh # override the URL if not localhost:30890: HUB_URL="https://hub.example.com" source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +# also set HUB_ADMIN_SECRET when custom.adminUser.existingSecret is non-default: +HUB_ADMIN_SECRET="external-admin" source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh ``` CLI **quota** commands call the Hub admin API, so they need a valid API token diff --git a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh index 49b8f3a7..31e2c544 100644 --- a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +++ b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh @@ -8,19 +8,21 @@ # HUB_URL="https://hub.example.com" source scripts/hub-api-env.sh # # Environment inputs (all optional): -# HUB_URL Hub base URL (default: http://localhost:30890) -# HUB_NAMESPACE Kubernetes namespace (default: jupyterhub) +# HUB_URL Hub base URL (default: http://localhost:30890) +# HUB_NAMESPACE Kubernetes namespace (default: jupyterhub) +# HUB_ADMIN_SECRET Admin Secret name (default: jupyterhub-admin-credentials) # # Exports on success: JUPYTERHUB_URL, JUPYTERHUB_TOKEN _auplc_ns="${HUB_NAMESPACE:-jupyterhub}" _auplc_url="${HUB_URL:-http://localhost:30890}" +_auplc_secret="${HUB_ADMIN_SECRET:-jupyterhub-admin-credentials}" -_auplc_token="$(kubectl -n "$_auplc_ns" get secret jupyterhub-admin-credentials \ +_auplc_token="$(kubectl -n "$_auplc_ns" get secret "$_auplc_secret" \ -o jsonpath='{.data.api-token}' 2>/dev/null | base64 -d 2>/dev/null)" if [ -z "$_auplc_token" ]; then - echo "hub-api-env: could not read api-token from secret 'jupyterhub-admin-credentials'" >&2 + echo "hub-api-env: could not read api-token from secret '$_auplc_secret'" >&2 echo " - is custom.adminUser.enabled: true and the Hub deployed?" >&2 echo " - is your kube context/namespace ('$_auplc_ns') correct?" >&2 # This file is meant to be sourced; `return` exits the caller's shell. The @@ -45,4 +47,4 @@ fi echo "hub-api-env: exported JUPYTERHUB_URL=$JUPYTERHUB_URL and JUPYTERHUB_TOKEN (hidden)" -unset _auplc_ns _auplc_url _auplc_token _auplc_code +unset _auplc_ns _auplc_url _auplc_secret _auplc_token _auplc_code From ffe6e963d939fcd23277174336d9d49a9f18940d Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:41:25 +0800 Subject: [PATCH 136/180] fix(installer): keep personal access as TUI default --- README.md | 4 +-- auplc_installer/tui.py | 6 ++-- .../SKILL.md | 5 ++- .../reference.md | 8 ++--- tests/installer/test_local_auth.py | 31 +++++++++++++++---- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 78826c07..69681fb8 100644 --- a/README.md +++ b/README.md @@ -86,9 +86,9 @@ cd aup-learning-cloud ### Single-Node Access -The interactive installer defaults to `local` access, which creates a closed local administrator account. Choose `personal` for the shared student session used by earlier single-node installs. +The interactive installer defaults to `personal` access, preserving the shared student session used by earlier single-node installs. Choose `local` when individual managed accounts are required. -For scripted installs, `personal` remains the compatibility default. Select local access explicitly when credentials are required: +Both interactive and scripted installs keep `personal` as the compatibility default. Select local access explicitly when credentials are required: ```bash ./auplc-installer install --access-mode=local --admin-username=admin diff --git a/auplc_installer/tui.py b/auplc_installer/tui.py index f53aa6a1..b1a4cf01 100644 --- a/auplc_installer/tui.py +++ b/auplc_installer/tui.py @@ -634,10 +634,10 @@ def _flow_select_access(state: InstallerState) -> None: state.access_mode = _ask_select( "Access mode", ( - Choice("local", "local - sign in with managed local credentials (default)"), - Choice("personal", "personal - shared student session without a login"), + Choice("personal", "personal - shared student session without a login (default)"), + Choice("local", "local - sign in with managed local credentials"), ), - default_value="local", + default_value="personal", ) if state.access_mode == "local": state.admin_username = _ask_text("Administrator username", default=state.admin_username or "admin") diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md index 081f11c0..1b224ee2 100644 --- a/skills/install-aup-learning-cloud-single-node/SKILL.md +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -50,9 +50,8 @@ table, offline flow, and troubleshooting are in **[reference.md](reference.md)** (local from `dockerfiles/`). For a quick demo prefer `pull`. 4. **Online or offline**: a normal machine with internet, or an air-gapped one that needs a `pack` bundle (see reference). -5. **Access mode**: interactive installs default to `local` and prompt for a - canonical administrator username. Use `personal` only for the compatibility - shared student session; scripted installs remain `personal` unless passed +5. **Access mode**: interactive and scripted installs default to the `personal` + shared student session. Select local managed accounts explicitly with `--access-mode=local --admin-username=<name>`. ## Phase 2 — Verify the environment diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 9d1a4050..a6e03079 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -86,10 +86,10 @@ sudo apt install python3-questionary python3-prompt-toolkit ## Default deployment facts -The checked-in chart defaults use `custom.authMode: auto-login`, while the -interactive installer defaults to `local` and creates `jupyterhub-admin-credentials` -with `admin-username`, `admin-password`, and `api-token`. Scripted installs keep -the `personal` compatibility default unless `--access-mode=local` is supplied. +The checked-in chart and interactive installer default to `personal` access via +`custom.authMode: auto-login`. Selecting `--access-mode=local` creates +`jupyterhub-admin-credentials` with `admin-username`, `admin-password`, and +`api-token`. The single-node NodePort is not a TLS or LAN exposure boundary; use local mode only on a trusted host/network. To change generated installer values, run `./auplc-installer rt upgrade`; it preserves and validates an existing local Secret. diff --git a/tests/installer/test_local_auth.py b/tests/installer/test_local_auth.py index 72770dbb..05588bda 100644 --- a/tests/installer/test_local_auth.py +++ b/tests/installer/test_local_auth.py @@ -66,18 +66,37 @@ def test_bare_upgrade_restores_local_access_settings(tmp_path: Path) -> None: assert state.admin_username == "operator" -def test_cli_defaults_to_personal_but_tui_defaults_to_local(monkeypatch) -> None: +def test_cli_and_tui_default_to_personal(monkeypatch) -> None: state = InstallerState() - selections = iter(["local"]) - names = iter(["admin"]) - monkeypatch.setattr("auplc_installer.tui._ask_select", lambda *_args, **_kwargs: next(selections)) - monkeypatch.setattr("auplc_installer.tui._ask_text", lambda *_args, **_kwargs: next(names)) + selected_defaults = [] + + def select_default(*_args, **kwargs): + selected_defaults.append(kwargs["default_value"]) + return kwargs["default_value"] + + monkeypatch.setattr("auplc_installer.tui._ask_select", select_default) + monkeypatch.setattr( + "auplc_installer.tui._ask_text", + lambda *_args, **_kwargs: pytest.fail("personal mode must not prompt for an administrator"), + ) _flow_select_access(state) assert InstallerState().access_mode == "" + assert selected_defaults == ["personal"] + assert state.access_mode == "personal" + assert state.admin_username == "" + + +def test_tui_local_mode_remains_selectable(monkeypatch) -> None: + state = InstallerState() + monkeypatch.setattr("auplc_installer.tui._ask_select", lambda *_args, **_kwargs: "local") + monkeypatch.setattr("auplc_installer.tui._ask_text", lambda *_args, **_kwargs: "operator") + + _flow_select_access(state) + assert state.access_mode == "local" - assert state.admin_username == "admin" + assert state.admin_username == "operator" @pytest.mark.parametrize("username", ["Admin", "admin:name", 'admin"name', "admin\nname", "-admin"]) From c8facdee0bc33bb5917e1d1d8de31dc0daa69c6e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:06:28 +0800 Subject: [PATCH 137/180] refactor(auth): add composable provider configuration --- runtime/hub/core/config.py | 149 ++++++++++-- .../tests/test_config_resource_metadata.py | 216 +++++++++++++++++- 2 files changed, 344 insertions(+), 21 deletions(-) diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 25d21617..cf521e33 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -39,11 +39,13 @@ from __future__ import annotations +import warnings +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal import yaml -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator # ============================================================================= # YAML Configuration Models @@ -257,6 +259,108 @@ def from_dicts( return cls.model_validate(raw_config) +LegacyAuthMode = Literal["auto-login", "dummy", "github", "local", "multi"] + + +@dataclass(frozen=True, slots=True) +class AuthCapabilities: + """Enabled authentication providers normalized from canonical or legacy configuration.""" + + auto_login: bool + dummy: bool + native: bool + github: bool + + @property + def effective_mode(self) -> LegacyAuthMode: + """Project capabilities onto the temporary legacy mode consumed downstream.""" + + match self: + case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False): + return "auto-login" + case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False): + return "dummy" + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False): + return "local" + case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True): + return "github" + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True): + return "multi" + case _: + raise AuthConfigurationError("auth must enable one exclusive provider or native + github") + + +@dataclass(frozen=True, slots=True) +class AuthConfigurationError(ValueError): + """Raised when the Hub authentication provider configuration is invalid.""" + + detail: str + + def __str__(self) -> str: + return f"Invalid authentication configuration: {self.detail}" + + +class CanonicalAuthConfig(BaseModel): + """Strict raw-YAML model for the public canonical authentication flags.""" + + autoLogin: bool = False + dummy: bool = False + native: bool = False + github: bool = False + + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + + def capabilities(self) -> AuthCapabilities: + """Return the immutable provider capability contract.""" + + return AuthCapabilities( + auto_login=self.autoLogin, + dummy=self.dummy, + native=self.native, + github=self.github, + ) + + +def _legacy_auth_capabilities(mode: str) -> AuthCapabilities: + """Parse a one-release legacy authMode value into canonical capabilities.""" + + match mode: + case "auto-login": + return AuthCapabilities(True, False, False, False) + case "dummy": + return AuthCapabilities(False, True, False, False) + case "github": + return AuthCapabilities(False, False, False, True) + case "local": + return AuthCapabilities(False, False, True, False) + case "multi": + return AuthCapabilities(False, False, True, True) + case _: + raise AuthConfigurationError("authMode must be one of auto-login, dummy, github, local, or multi") + + +def _parse_auth_capabilities(raw_config: dict[str, Any]) -> tuple[AuthCapabilities, bool]: + """Parse explicit configuration form presence before defaulted models erase it.""" + + canonical_present = "auth" in raw_config + legacy_present = "authMode" in raw_config + if canonical_present and legacy_present: + raise AuthConfigurationError("cannot specify both authMode and auth") + if canonical_present: + try: + capabilities = CanonicalAuthConfig.model_validate(raw_config["auth"]).capabilities() + except ValidationError as error: + raise AuthConfigurationError(f"auth must be a strict provider mapping: {error}") from error + try: + _ = capabilities.effective_mode + except AuthConfigurationError as error: + raise AuthConfigurationError("auth must enable one exclusive provider or native + github") from error + return capabilities, False + if legacy_present and raw_config["authMode"] is not None: + return _legacy_auth_capabilities(raw_config["authMode"]), True + return AuthCapabilities(True, False, False, False), False + + # ============================================================================= # Hub Configuration Singleton # ============================================================================= @@ -279,6 +383,7 @@ class HubConfig: def __init__(self): # Runtime settings self.auth_mode: str = "auto-login" + self._auth: AuthCapabilities = AuthCapabilities(True, False, False, False) self.single_node_mode: bool = False self.github_org_name: str = "" self.cluster_name: str = "" @@ -302,10 +407,7 @@ def init(cls, config_path: str | Path) -> HubConfig: Returns: The initialized HubConfig instance """ - if cls._instance is None: - cls._instance = cls() - - instance = cls._instance + instance = cls() config_path = Path(config_path) # Load configuration from YAML file @@ -313,24 +415,36 @@ def init(cls, config_path: str | Path) -> HubConfig: raise FileNotFoundError(f"Configuration file not found: {config_path}") with open(config_path, encoding="utf-8") as f: - raw_config = yaml.safe_load(f) or {} + raw_config = yaml.safe_load(f) + if raw_config is None: + raw_config = {} + if not isinstance(raw_config, dict): + raise AuthConfigurationError("Hub configuration must be a YAML mapping") print(f"[CONFIG] Loaded configuration from {config_path}") # Extract runtime settings - instance.auth_mode = raw_config.get("authMode", "auto-login") + instance._auth, legacy_auth = _parse_auth_capabilities(raw_config) + instance.auth_mode = instance._auth.effective_mode + if legacy_auth: + warnings.warn( + "authMode is deprecated; configure authentication with auth provider flags instead", + DeprecationWarning, + stacklevel=2, + ) instance.github_org_name = raw_config.get("githubOrgName", "") instance.cluster_name = raw_config.get("clusterName", "") admin_user = raw_config.get("adminUser", {}) if isinstance(admin_user, dict): instance.admin_username = admin_user.get("username", "admin") - # Single-node mode: from config or auto-enable for auto-login - single_node_mode = raw_config.get("singleNodeMode") - if single_node_mode is not None: - instance.single_node_mode = single_node_mode - else: + # Canonical providers use neutral policy defaults; legacy input retains historical defaults. + if "singleNodeMode" in raw_config: + instance.single_node_mode = raw_config["singleNodeMode"] + elif legacy_auth: instance.single_node_mode = instance.auth_mode in ("auto-login", "local") + else: + instance.single_node_mode = False # Parse structured configuration instance._config = ParsedConfig.from_dicts( @@ -345,13 +459,14 @@ def init(cls, config_path: str | Path) -> HubConfig: notifications=raw_config.get("notifications"), ) - # Quota enabled: from config or auto-detect based on auth_mode + # Canonical providers use neutral policy defaults; legacy input retains historical defaults. if instance._config.quota.enabled is not None: instance.quota_enabled = instance._config.quota.enabled else: - instance.quota_enabled = instance.auth_mode not in ("auto-login", "dummy", "local") + instance.quota_enabled = instance.auth_mode not in ("auto-login", "dummy", "local") if legacy_auth else True instance._config.quota.enabled = instance.quota_enabled + cls._instance = instance cls._initialized = True # Log configuration @@ -394,6 +509,12 @@ def platform_display_name(self) -> str: return f"{base} {self.cluster_name}" return base + @property + def auth(self) -> AuthCapabilities: + """Get typed authentication provider capabilities for new consumers.""" + + return self._auth + @property def resources(self) -> ResourcesConfig: """Get resources configuration.""" diff --git a/runtime/hub/tests/test_config_resource_metadata.py b/runtime/hub/tests/test_config_resource_metadata.py index 64eb3de8..a5688036 100644 --- a/runtime/hub/tests/test_config_resource_metadata.py +++ b/runtime/hub/tests/test_config_resource_metadata.py @@ -20,6 +20,7 @@ import importlib.util import sys import types +import warnings from pathlib import Path import pytest @@ -47,6 +48,55 @@ def load_module(name: str, path: Path): ParsedConfig = config.ParsedConfig ResourceMetadata = config.ResourceMetadata +ProviderFlags = tuple[bool, bool, bool, bool] +AUTH_FLAG_NAMES = ("autoLogin", "dummy", "native", "github") +VALID_CANONICAL_AUTH = ( + ((True, False, False, False), "auto-login"), + ((False, True, False, False), "dummy"), + ((False, False, True, False), "local"), + ((False, False, False, True), "github"), + ((False, False, True, True), "multi"), +) +INVALID_CANONICAL_AUTH = ( + (False, False, False, False), + (True, True, False, False), + (True, False, True, False), + (True, False, False, True), + (False, True, True, False), + (False, True, False, True), + (True, True, True, False), + (True, True, False, True), + (True, False, True, True), + (False, True, True, True), + (True, True, True, True), +) + + +def write_hub_config(tmp_path: Path, contents: str) -> Path: + config_path = tmp_path / "hub-config.yaml" + config_path.write_text(contents, encoding="utf-8") + return config_path + + +def canonical_auth_yaml(flags: ProviderFlags) -> str: + lines = ["auth:"] + lines.extend(f" {name}: {str(enabled).lower()}" for name, enabled in zip(AUTH_FLAG_NAMES, flags)) + return "\n".join(lines) + "\n" + + +def assert_auth_configuration_rejected(tmp_path: Path, contents: str, expected_message: str) -> None: + with pytest.raises(ValueError) as raised: + config.HubConfig.init(write_hub_config(tmp_path, contents)) + + assert raised.value.__class__ is config.AuthConfigurationError + assert expected_message in str(raised.value) + + +@pytest.fixture(autouse=True) +def restore_hub_config_singleton(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(config.HubConfig, "_instance", None) + monkeypatch.setattr(config.HubConfig, "_initialized", False) + def test_resource_metadata_default_path_omitted_or_null_stays_none(): assert ResourceMetadata().defaultPath is None @@ -98,12 +148,164 @@ def test_code_server_extra_trusted_domains_parse_from_config(): assert parsed_config.codeServer.extraTrustedDomains == ["docs.example.edu", "git.example.edu"] -def test_local_auth_mode_defaults_to_single_node_runtime_behavior(tmp_path: Path): - config_path = tmp_path / "hub-config.yaml" - config_path.write_text("authMode: local\n", encoding="utf-8") - config.HubConfig._instance = None - config.HubConfig._initialized = False +def test_legacy_github_mode_preserves_existing_runtime_defaults(tmp_path: Path): + hub_config = config.HubConfig.init(write_hub_config(tmp_path, "authMode: github\n")) + + assert hub_config.auth_mode == "github" + assert hub_config.single_node_mode is False + assert hub_config.quota_enabled is True + + +def test_absent_auth_forms_preserve_existing_auto_login_compatibility(tmp_path: Path): + hub_config = config.HubConfig.init(write_hub_config(tmp_path, "resources: {}\n")) + + assert hub_config.auth_mode == "auto-login" + + +@pytest.mark.parametrize(("flags", "expected_mode"), VALID_CANONICAL_AUTH) +def test_canonical_auth_flags_normalize_to_capabilities_and_neutral_policy( + tmp_path: Path, flags: ProviderFlags, expected_mode: str +): + hub_config = config.HubConfig.init(write_hub_config(tmp_path, canonical_auth_yaml(flags))) + + assert hub_config.auth_mode == expected_mode + assert (hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github) == flags + assert hub_config.single_node_mode is False + assert hub_config.quota_enabled is True + + +@pytest.mark.parametrize("flags", INVALID_CANONICAL_AUTH) +def test_canonical_auth_rejects_each_invalid_boolean_combination(tmp_path: Path, flags: ProviderFlags): + assert_auth_configuration_rejected(tmp_path, canonical_auth_yaml(flags), "native + github") + + +@pytest.mark.parametrize( + ("legacy_mode", "expected_flags", "expected_single_node", "expected_quota"), + [ + ("auto-login", (True, False, False, False), True, False), + ("dummy", (False, True, False, False), False, False), + ("github", (False, False, False, True), False, True), + ("local", (False, False, True, False), True, False), + ("multi", (False, False, True, True), False, True), + ], +) +def test_explicit_legacy_modes_map_to_capabilities_and_preserve_policy_defaults( + tmp_path: Path, legacy_mode: str, expected_flags: ProviderFlags, expected_single_node: bool, expected_quota: bool +): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config.HubConfig.init(write_hub_config(tmp_path, f"authMode: {legacy_mode}\n")) + + assert hub_config.auth_mode == legacy_mode + assert ( + hub_config.auth.auto_login, + hub_config.auth.dummy, + hub_config.auth.native, + hub_config.auth.github, + ) == expected_flags + assert hub_config.single_node_mode is expected_single_node + assert hub_config.quota_enabled is expected_quota + + +def test_legacy_auth_emits_one_actionable_deprecation_warning_per_initialization(tmp_path: Path): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hub_config = config.HubConfig.init(write_hub_config(tmp_path, "authMode: local\n")) + _ = hub_config.auth + _ = hub_config.auth + + legacy_warnings = [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] + assert len(legacy_warnings) == 1 + assert "authMode" in str(legacy_warnings[0].message) + assert "auth" in str(legacy_warnings[0].message) + + +def test_absent_auth_forms_use_compatibility_auto_login_with_neutral_defaults(tmp_path: Path): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hub_config = config.HubConfig.init(write_hub_config(tmp_path, "resources: {}\n")) + + assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] + assert hub_config.auth_mode == "auto-login" + assert (hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github) == ( + True, + False, + False, + False, + ) + assert hub_config.single_node_mode is False + assert hub_config.quota_enabled is True + + +def test_mixed_legacy_and_canonical_auth_forms_are_rejected(tmp_path: Path): + assert_auth_configuration_rejected(tmp_path, "authMode: local\nauth: {}\n", "both authMode and auth") + + +@pytest.mark.parametrize( + "contents", + [ + "auth: []\n", + 'auth:\n autoLogin: "true"\n', + "auth:\n native: 1\n", + "auth:\n autoLogin: true\n ldap: false\n", + ], +) +def test_malformed_canonical_auth_is_rejected_before_hub_setup(tmp_path: Path, contents: str): + assert_auth_configuration_rejected(tmp_path, contents, "auth") + + +@pytest.mark.parametrize( + ("contents", "expected_single_node", "expected_quota"), + [ + ("auth:\n native: true\nsingleNodeMode: true\nquota:\n enabled: false\n", True, False), + ("authMode: local\nsingleNodeMode: false\nquota:\n enabled: true\n", False, True), + ], +) +def test_explicit_runtime_policy_values_override_auth_compatibility_defaults( + tmp_path: Path, contents: str, expected_single_node: bool, expected_quota: bool +): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config.HubConfig.init(write_hub_config(tmp_path, contents)) + + assert hub_config.single_node_mode is expected_single_node + assert hub_config.quota_enabled is expected_quota + + +def test_hub_config_singleton_is_reset_before_each_case(): + assert config.HubConfig.is_initialized() is False + with pytest.raises(RuntimeError): + config.HubConfig.get() + + +@pytest.mark.parametrize("contents", ["[]\n", "false\n", "0\n", '""\n']) +def test_falsey_non_mapping_yaml_roots_are_rejected(tmp_path: Path, contents: str): + assert_auth_configuration_rejected(tmp_path, contents, "YAML mapping") + + +def test_null_legacy_mode_is_absent_compatibility_without_warning(tmp_path: Path): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hub_config = config.HubConfig.init(write_hub_config(tmp_path, "authMode: null\n")) + + assert hub_config.auth_mode == "auto-login" + assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] + + +def test_null_legacy_mode_with_canonical_auth_is_rejected(tmp_path: Path): + assert_auth_configuration_rejected(tmp_path, "authMode: null\nauth:\n native: true\n", "both authMode and auth") + - hub_config = config.HubConfig.init(config_path) +def test_failed_initializations_preserve_or_recover_singleton_state(tmp_path: Path): + with pytest.raises(ValidationError, match="Input should be a valid dictionary"): + config.HubConfig.init(write_hub_config(tmp_path, "quota: invalid\n")) + assert config.HubConfig._instance is None + assert config.HubConfig._initialized is False + with pytest.raises(RuntimeError): + config.HubConfig.get() - assert hub_config.single_node_mode is True + valid = config.HubConfig.init(write_hub_config(tmp_path, "auth:\n native: true\n")) + with pytest.raises(ValidationError, match="Input should be a valid dictionary"): + config.HubConfig.init(write_hub_config(tmp_path, "auth:\n github: true\nquota: invalid\n")) + assert config.HubConfig.get() is valid + assert valid.auth.native is True From 773a38a4d86bb93dfc93f72906b531f27335faf1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:11:28 +0800 Subject: [PATCH 138/180] refactor(chart): validate composable auth providers --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 126 +++++++-- runtime/chart/values.yaml | 15 +- tests/installer/test_chart_local_auth.py | 333 +++++++++++++++++------ 4 files changed, 351 insertions(+), 125 deletions(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 96e3e9fc..79bb90be 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","local","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"if":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}},"then":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled","username","existingSecret"],"properties":{"enabled":{"const":true},"username":{"minLength":1},"existingSecret":{"minLength":1}}}}}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["existingSecret"],"properties":{"existingSecret":{"minLength":1}}}}},"then":{"required":["authMode"],"properties":{"authMode":{"const":"local"}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":["string","null"],"enum":[null,"auto-login","dummy","github","local","multi"]},"auth":{"type":"object","additionalProperties":false,"properties":{"autoLogin":{"type":"boolean"},"dummy":{"type":"boolean"},"native":{"type":"boolean"},"github":{"type":"boolean"}},"oneOf":[{"required":["autoLogin"],"properties":{"autoLogin":{"const":true},"dummy":{"const":false},"native":{"const":false},"github":{"const":false}}},{"required":["dummy"],"properties":{"autoLogin":{"const":false},"dummy":{"const":true},"native":{"const":false},"github":{"const":false}}},{"required":["native"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":false}}},{"required":["github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":false},"github":{"const":true}}},{"required":["native","github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":true}}}]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"not":{"required":["authMode","auth"]}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"oneOf":[{"required":["auth"],"properties":{"auth":{"required":["native"],"properties":{"native":{"const":true}}}}},{"required":["authMode"],"properties":{"authMode":{"enum":["local","multi"]}}}],"properties":{"adminUser":{"required":["username"],"properties":{"username":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index e98a501c..04026014 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3172,29 +3172,99 @@ properties: accelerators, resources, teams, quota management, and API service configuration. properties: authMode: - type: string - enum: [auto-login, dummy, github, local, multi] + type: [string, "null"] + enum: [null, auto-login, dummy, github, local, multi] description: | - Authentication mode for the JupyterHub instance. + Deprecated authentication mode retained for one release. Do not combine + this field with `custom.auth`. - `auto-login`: No credentials required, auto-login as 'student' (for demos/single-node) - `dummy`: Accept any username/password (for testing) - `github`: GitHub App authentication only - - `local`: Closed local accounts managed by an administrator - - `multi`: GitHub App + Local native accounts (recommended for production) + - `local`: Native accounts without GitHub authentication + - `multi`: GitHub App + native accounts (recommended for production) + + auth: + type: object + additionalProperties: false + description: | + Composable authentication providers. Omitted providers are disabled. + Exactly one of auto-login, dummy, native, GitHub, or native plus GitHub + must be enabled whenever this object is present. + properties: + autoLogin: + type: boolean + dummy: + type: boolean + native: + type: boolean + github: + type: boolean + oneOf: + - required: [autoLogin] + properties: + autoLogin: + const: true + dummy: + const: false + native: + const: false + github: + const: false + - required: [dummy] + properties: + autoLogin: + const: false + dummy: + const: true + native: + const: false + github: + const: false + - required: [native] + properties: + autoLogin: + const: false + dummy: + const: false + native: + const: true + github: + const: false + - required: [github] + properties: + autoLogin: + const: false + dummy: + const: false + native: + const: false + github: + const: true + - required: [native, github] + properties: + autoLogin: + const: false + dummy: + const: false + native: + const: true + github: + const: true adminUser: type: object additionalProperties: false description: | - Auto-create admin user configuration. - Bootstrap configuration for an administrator account. + Bootstrap configuration for an administrator account in native + authentication modes. Leave `existingSecret` empty for chart-generated + credentials, or provide an external Secret with `admin-password` and + optional `api-token` keys. properties: enabled: type: boolean description: | - Enable auto-admin creation on first install. - Credentials will be stored in `jupyterhub-admin-credentials` secret. + Enable administrator bootstrap on first install. username: type: string pattern: "^[a-z0-9][a-z0-9._-]{0,63}$" @@ -3789,37 +3859,35 @@ properties: description: Image pull policy. allOf: + - not: + required: [authMode, auth] - if: - required: [authMode] - properties: - authMode: - const: local - then: required: [adminUser] properties: adminUser: - required: [enabled, username, existingSecret] + required: [enabled] properties: enabled: const: true - username: - minLength: 1 - existingSecret: - minLength: 1 - - - if: - required: [adminUser] + then: + oneOf: + - required: [auth] + properties: + auth: + required: [native] + properties: + native: + const: true + - required: [authMode] + properties: + authMode: + enum: [local, multi] properties: adminUser: - required: [existingSecret] + required: [username] properties: - existingSecret: + username: minLength: 1 - then: - required: [authMode] - properties: - authMode: - const: local cull: type: object diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index f60473b8..65826e9a 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -31,19 +31,22 @@ enabled: # custom can contain anything you want to pass to the hub pod, as all passed # Helm template values will be made available there. custom: - # Authentication mode: "auto-login" | "dummy" | "github" | "local" | "multi" + # Authentication defaults to compatibility auto-login when neither the + # canonical custom.auth object nor deprecated custom.authMode is provided. + # Deprecated authMode values retained for one release: + # "auto-login" | "dummy" | "github" | "local" | "multi" # - auto-login: No credentials required, auto-login as 'student' (default, for single-node) # - dummy: Accept any username/password (for testing) # - github: GitHub App authentication - # - local: Closed local accounts managed by an administrator - # - multi: GitHub App + Local accounts - authMode: "auto-login" - + # - local: Native accounts without GitHub authentication + # - multi: GitHub App + native accounts # Cluster display name (optional). Appended to "AUP Learning Cloud" in the UI. # Example: "City/University" → "AUP Learning Cloud City/University" clusterName: "" - # Auto-create admin user on first install (optional) + # Bootstrap the admin user for native authentication modes (optional). + # Leave existingSecret empty for the chart-generated Secret, or provide an + # external Secret with admin-password and optional api-token keys. adminUser: enabled: false username: "admin" diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index f82fbf07..22445c16 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -1,81 +1,159 @@ +import importlib.util +import itertools +import json import subprocess +import sys +import warnings +from collections.abc import Mapping from pathlib import Path +import pytest +import yaml + +from scripts.generate_values_schema import remove_descriptions + ROOT = Path(__file__).resolve().parents[2] +CHART = "runtime/chart" +AUTH_KEYS = ("autoLogin", "dummy", "native", "github") +LEGACY_MODES = ("auto-login", "dummy", "github", "local", "multi") +VALID_COMBINATIONS = { + (True, False, False, False), + (False, True, False, False), + (False, False, True, False), + (False, False, False, True), + (False, False, True, True), +} +ALL_COMBINATIONS = tuple(itertools.product((False, True), repeat=len(AUTH_KEYS))) +INVALID_COMBINATIONS = tuple(case for case in ALL_COMBINATIONS if case not in VALID_COMBINATIONS) -def test_local_chart_render_uses_existing_secret_only_for_hub_bootstrap() -> None: - result = subprocess.run( - [ - "helm", - "template", - "jupyterhub", - "runtime/chart", - "--set", - "custom.authMode=local", - "--set", - "custom.adminUser.enabled=true", - "--set", - "custom.adminUser.username=operator", - "--set", - "custom.adminUser.existingSecret=jupyterhub-admin-credentials", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, +def render(*settings: str, string_settings: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: + command = ["helm", "template", "jupyterhub", CHART] + for setting in settings: + command.extend(("--set", setting)) + for setting in string_settings: + command.extend(("--set-string", setting)) + return subprocess.run(command, cwd=ROOT, check=False, capture_output=True, text=True) + + +def auth_settings(combination: tuple[bool, bool, bool, bool]) -> tuple[str, ...]: + return tuple( + f"custom.auth.{key}={str(enabled).lower()}" for key, enabled in zip(AUTH_KEYS, combination, strict=True) ) - assert "authMode: local" in result.stdout - assert "name: JUPYTERHUB_ADMIN_USERNAME" in result.stdout - assert 'value: "operator"' in result.stdout - assert "name: JUPYTERHUB_ADMIN_PASSWORD" in result.stdout - assert "key: admin-password" in result.stdout - assert "name: JUPYTERHUB_API_TOKEN" in result.stdout - assert "key: api-token" in result.stdout - assert "optional: true" in result.stdout - assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" not in result.stdout +def rendered_documents(output: str) -> list[Mapping[str, object]]: + return [document for document in yaml.safe_load_all(output) if isinstance(document, dict)] + + +def document_by_kind(documents: list[Mapping[str, object]], kind: str) -> Mapping[str, object]: + return next(document for document in documents if document.get("kind") == kind) -def test_local_chart_schema_rejects_uppercase_admin_username() -> None: + +@pytest.mark.parametrize("combination", sorted(VALID_COMBINATIONS)) +def test_chart_accepts_canonical_auth_truth_table( + combination: tuple[bool, bool, bool, bool], +) -> None: + result = render(*auth_settings(combination)) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("combination", INVALID_COMBINATIONS) +def test_chart_rejects_invalid_canonical_auth_combinations( + combination: tuple[bool, bool, bool, bool], +) -> None: + result = render(*auth_settings(combination)) + + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr + + +@pytest.mark.parametrize("auth_mode", LEGACY_MODES) +def test_chart_accepts_legacy_auth_modes(auth_mode: str) -> None: + result = render(f"custom.authMode={auth_mode}") + + assert result.returncode == 0, result.stderr + + +def test_chart_accepts_absent_auth_forms_without_injecting_a_default() -> None: + result = render() + + assert result.returncode == 0, result.stderr + config_map = document_by_kind(rendered_documents(result.stdout), "ConfigMap") + custom = yaml.safe_load(config_map["data"]["hub-config.yaml"]) + assert "auth" not in custom + assert "authMode" not in custom + + +def test_null_legacy_mode_renders_as_compatibility_absent_without_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: result = subprocess.run( - [ - "helm", - "template", - "jupyterhub", - "runtime/chart", - "--set", - "custom.authMode=local", - "--set", - "custom.adminUser.enabled=true", - "--set", - "custom.adminUser.username=Operator", - "--set", - "custom.adminUser.existingSecret=jupyterhub-admin-credentials", - ], + ["helm", "template", "jupyterhub", CHART, "--set-json", "custom.authMode=null"], cwd=ROOT, check=False, capture_output=True, text=True, ) + assert result.returncode == 0, result.stderr + config_map = document_by_kind(rendered_documents(result.stdout), "ConfigMap") + rendered_config = config_map["data"]["hub-config.yaml"] + custom = yaml.safe_load(rendered_config) + assert "authMode" in custom + assert custom["authMode"] is None + assert "auth" not in custom + + config_path = tmp_path / "hub-config.yaml" + config_path.write_text(rendered_config, encoding="utf-8") + spec = importlib.util.spec_from_file_location("chart_contract_config", ROOT / "runtime/hub/core/config.py") + assert spec is not None + assert spec.loader is not None + config_module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, config_module) + spec.loader.exec_module(config_module) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hub_config = config_module.HubConfig.init(config_path) + + assert hub_config.auth_mode == "auto-login" + assert ( + hub_config.auth.auto_login, + hub_config.auth.dummy, + hub_config.auth.native, + hub_config.auth.github, + ) == (True, False, False, False) + assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] + + +def test_chart_accepts_minimal_native_set_override() -> None: + result = render("custom.auth.native=true") + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("auth_mode", LEGACY_MODES) +@pytest.mark.parametrize("auth_key", AUTH_KEYS) +def test_chart_rejects_mixed_legacy_and_canonical_auth(auth_mode: str, auth_key: str) -> None: + result = render(f"custom.authMode={auth_mode}", f"custom.auth.{auth_key}=true") + assert result.returncode != 0 - assert "does not match pattern" in result.stderr + assert "values don't meet the specifications" in result.stderr -def test_local_chart_schema_requires_a_nonempty_existing_secret() -> None: +@pytest.mark.parametrize("auth_key", AUTH_KEYS) +def test_chart_rejects_null_legacy_mode_with_canonical_auth(auth_key: str) -> None: result = subprocess.run( [ "helm", "template", "jupyterhub", - "runtime/chart", - "--set", - "custom.authMode=local", - "--set", - "custom.adminUser.enabled=true", + CHART, + "--set-json", + "custom.authMode=null", "--set", - "custom.adminUser.username=operator", + f"custom.auth.{auth_key}=true", ], cwd=ROOT, check=False, @@ -84,53 +162,130 @@ def test_local_chart_schema_requires_a_nonempty_existing_secret() -> None: ) assert result.returncode != 0 - assert "existingSecret" in result.stderr + assert "at '/custom': 'not' failed" in result.stderr -def test_nonlocal_chart_render_retains_chart_managed_credentials() -> None: +def test_chart_rejects_empty_canonical_auth_object() -> None: result = subprocess.run( - [ - "helm", - "template", - "jupyterhub", - "runtime/chart", - "--set", - "custom.authMode=multi", - "--set", - "custom.adminUser.enabled=true", - "--set", - "custom.adminUser.username=operator", - ], + ["helm", "template", "jupyterhub", CHART, "--set-json", "custom.auth={}"], cwd=ROOT, - check=True, + check=False, capture_output=True, text=True, ) - assert "kind: Secret\nmetadata:\n name: jupyterhub-admin-credentials" in result.stdout + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr -def test_chart_schema_rejects_existing_secret_outside_local_mode() -> None: - result = subprocess.run( - [ - "helm", - "template", - "jupyterhub", - "runtime/chart", - "--set", - "custom.authMode=auto-login", - "--set", - "custom.adminUser.enabled=true", - "--set", - "custom.adminUser.username=operator", - "--set", - "custom.adminUser.existingSecret=legacy-admin-credentials", - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, +@pytest.mark.parametrize("auth_key", AUTH_KEYS) +def test_chart_rejects_non_boolean_canonical_auth_values(auth_key: str) -> None: + result = render(string_settings=(f"custom.auth.{auth_key}=true",)) + + assert result.returncode != 0 + assert f"at '/custom/auth/{auth_key}': got string, want boolean" in result.stderr + + +def test_chart_rejects_unknown_canonical_auth_key() -> None: + result = render("custom.auth.native=true", "custom.auth.password=true") + + assert result.returncode != 0 + assert "additional properties 'password' not allowed" in result.stderr + + +@pytest.mark.parametrize( + "provider_settings", + [ + ("custom.auth.autoLogin=true",), + ("custom.auth.dummy=true",), + ("custom.auth.github=true",), + (), + ("custom.authMode=auto-login",), + ("custom.authMode=dummy",), + ("custom.authMode=github",), + ], +) +def test_chart_rejects_admin_bootstrap_without_native( + provider_settings: tuple[str, ...], +) -> None: + result = render( + *provider_settings, + "custom.adminUser.enabled=true", + "custom.adminUser.username=operator", + ) + + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr + + +@pytest.mark.parametrize( + "provider_settings", + [ + ("custom.auth.native=true",), + ("custom.auth.native=true", "custom.auth.github=true"), + ("custom.authMode=local",), + ("custom.authMode=multi",), + ], +) +@pytest.mark.parametrize("existing_secret", ["", "external-admin-credentials"]) +def test_native_admin_bootstrap_renders_generated_or_external_secret( + provider_settings: tuple[str, ...], existing_secret: str +) -> None: + settings = [ + *provider_settings, + "custom.adminUser.enabled=true", + "custom.adminUser.username=operator", + ] + if existing_secret: + settings.append(f"custom.adminUser.existingSecret={existing_secret}") + + result = render(*settings) + + assert result.returncode == 0, result.stderr + documents = rendered_documents(result.stdout) + deployment = document_by_kind(documents, "Deployment") + container = deployment["spec"]["template"]["spec"]["containers"][0] + environment = {entry["name"]: entry for entry in container["env"]} + selected_secret = existing_secret or "jupyterhub-admin-credentials" + assert environment["JUPYTERHUB_ADMIN_USERNAME"]["value"] == "operator" + assert environment["JUPYTERHUB_ADMIN_PASSWORD"]["valueFrom"]["secretKeyRef"] == { + "name": selected_secret, + "key": "admin-password", + } + expected_token_ref = {"name": selected_secret, "key": "api-token"} + if existing_secret: + expected_token_ref["optional"] = True + assert environment["JUPYTERHUB_API_TOKEN"]["valueFrom"]["secretKeyRef"] == expected_token_ref + admin_secrets = [ + document + for document in documents + if document.get("kind") == "Secret" + and document.get("metadata", {}).get("name") == "jupyterhub-admin-credentials" + ] + assert bool(admin_secrets) is not bool(existing_secret) + if admin_secrets: + assert set(admin_secrets[0]["data"]) == {"admin-username", "admin-password", "api-token"} + + +@pytest.mark.parametrize( + "provider_settings", + [("custom.auth.native=true",), ("custom.authMode=local",), ("custom.authMode=multi",)], +) +def test_native_admin_bootstrap_rejects_uppercase_username( + provider_settings: tuple[str, ...], +) -> None: + result = render( + *provider_settings, + "custom.adminUser.enabled=true", + "custom.adminUser.username=Operator", ) assert result.returncode != 0 - assert "authMode" in result.stderr + assert "does not match pattern" in result.stderr + + +def test_generated_values_schema_matches_yaml_source() -> None: + yaml_schema = yaml.safe_load((ROOT / "runtime/chart/values.schema.yaml").read_text()) + json_schema = json.loads((ROOT / "runtime/chart/values.schema.json").read_text()) + + assert json_schema == remove_descriptions(yaml_schema) From 085076469030d43eac3616df7aa52c560ac990d7 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:46:40 +0800 Subject: [PATCH 139/180] refactor(hub): compose native and GitHub authentication --- runtime/hub/core/authenticators/__init__.py | 49 +++--- runtime/hub/core/authenticators/local.py | 32 ---- .../hub/tests/test_authenticator_factory.py | 153 ++++++++++++++++++ 3 files changed, 178 insertions(+), 56 deletions(-) delete mode 100644 runtime/hub/core/authenticators/local.py create mode 100644 runtime/hub/tests/test_authenticator_factory.py diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index b1f55049..92799af6 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -27,35 +27,37 @@ from core.authenticators.firstuse import CustomFirstUseAuthenticator from core.authenticators.github_app import GITHUB_USERNAME_PREFIX, CustomGitHubOAuthenticator from core.authenticators.jwt import RemoteLabAuthenticator -from core.authenticators.local import CustomLocalAuthenticator from core.authenticators.multi import CustomMultiAuthenticator +from core.config import AuthCapabilities, AuthConfigurationError, LegacyAuthMode LOCAL_ACCOUNT_PREFIX = "LocalAccount" -def create_authenticator(auth_mode: str, **kwargs): - """ - Factory function to create the appropriate authenticator. +def create_authenticator(auth: AuthCapabilities | LegacyAuthMode) -> type | str: + """Select the JupyterHub authenticator class for validated capabilities.""" - Args: - auth_mode: Authentication mode ("auto-login", "dummy", "github", "local", "multi") - **kwargs: Additional configuration options - - Returns: - Authenticator class (not instance) - """ - if auth_mode == "auto-login": - return AutoLoginAuthenticator - elif auth_mode == "dummy": - return "dummy" - elif auth_mode == "github": - return CustomGitHubOAuthenticator - elif auth_mode == "local": - return CustomLocalAuthenticator - elif auth_mode == "multi": - return CustomMultiAuthenticator - else: - raise ValueError(f"Unknown authentication mode: {auth_mode}") + match auth: + case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False) | "auto-login": + return AutoLoginAuthenticator + case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False) | "dummy": + return "dummy" + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False) | "local": + return CustomFirstUseAuthenticator + case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True) | "github": + return CustomGitHubOAuthenticator + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True) | "multi": + return CustomMultiAuthenticator + case AuthCapabilities(): + raise AuthConfigurationError("auth must enable one exclusive provider or native + github") + # Todo 13: remove the effective-mode compatibility boundary after Todo 6 consumes config.auth. + case str(): + raise ValueError(f"Unknown authentication mode: {auth}") + case bool(): + raise AuthConfigurationError("authentication capabilities cannot be boolean values") + case unsupported: + raise AuthConfigurationError( + f"authentication capabilities must be AuthCapabilities or a supported effective mode, got {type(unsupported).__name__}" + ) __all__ = [ @@ -63,7 +65,6 @@ def create_authenticator(auth_mode: str, **kwargs): "AutoLoginAuthenticator", "CustomGitHubOAuthenticator", "CustomFirstUseAuthenticator", - "CustomLocalAuthenticator", "CustomMultiAuthenticator", "create_authenticator", "LOCAL_ACCOUNT_PREFIX", diff --git a/runtime/hub/core/authenticators/local.py b/runtime/hub/core/authenticators/local.py deleted file mode 100644 index 16ef9540..00000000 --- a/runtime/hub/core/authenticators/local.py +++ /dev/null @@ -1,32 +0,0 @@ -import re - -from core.authenticators.firstuse import CustomFirstUseAuthenticator - -LOCAL_USERNAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") - - -class CustomLocalAuthenticator(CustomFirstUseAuthenticator): - def validate_username(self, username): - return bool(LOCAL_USERNAME_PATTERN.fullmatch(username)) - - def _user_exists(self, username): - db = getattr(self, "db", None) or getattr(getattr(self, "parent", None), "db", None) - if db is None: - return False - try: - from jupyterhub.orm import User - - return db.query(User).filter_by(name=username).first() is not None - except Exception: - return False - - async def authenticate(self, _handler, data): - username = data.get("username", "") - password = data.get("password", "") - if not self.validate_username(username) or not password: - return None - if not self._user_exists(username): - return None - if not self.check_password(username, password): - return None - return username diff --git a/runtime/hub/tests/test_authenticator_factory.py b/runtime/hub/tests/test_authenticator_factory.py new file mode 100644 index 00000000..5048b61a --- /dev/null +++ b/runtime/hub/tests/test_authenticator_factory.py @@ -0,0 +1,153 @@ +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +AUTHENTICATORS = ROOT / "core" / "authenticators" / "__init__.py" +CONFIG = ROOT / "core" / "config.py" + + +def _install_core_packages(module_patch: pytest.MonkeyPatch) -> types.ModuleType: + core = types.ModuleType("core") + core.__path__ = [str(ROOT / "core")] + authenticators = types.ModuleType("core.authenticators") + authenticators.__path__ = [str(ROOT / "core" / "authenticators")] + module_patch.setitem(sys.modules, "core", core) + module_patch.setitem(sys.modules, "core.authenticators", authenticators) + core.authenticators = authenticators + return core + + +@contextmanager +def _loaded_factory(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[types.ModuleType, types.ModuleType]]: + with monkeypatch.context() as module_patch: + core = _install_core_packages(module_patch) + config_spec = importlib.util.spec_from_file_location("core.config", CONFIG) + assert config_spec is not None and config_spec.loader is not None + config = importlib.util.module_from_spec(config_spec) + module_patch.setitem(sys.modules, "core.config", config) + core.config = config + config_spec.loader.exec_module(config) + + auto_login = types.ModuleType("core.authenticators.auto_login") + auto_login.AutoLoginAuthenticator = type("AutoLoginAuthenticator", (), {}) + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = type("CustomFirstUseAuthenticator", (), {"prefix": ""}) + github_app = types.ModuleType("core.authenticators.github_app") + github_app.CustomGitHubOAuthenticator = type("CustomGitHubOAuthenticator", (), {"prefix": "github:"}) + github_app.GITHUB_USERNAME_PREFIX = "github:" + jwt = types.ModuleType("core.authenticators.jwt") + jwt.RemoteLabAuthenticator = type("RemoteLabAuthenticator", (), {}) + multi = types.ModuleType("core.authenticators.multi") + multi.CustomMultiAuthenticator = type("CustomMultiAuthenticator", (), {}) + for fake_module in (auto_login, firstuse, github_app, jwt, multi): + module_patch.setitem(sys.modules, fake_module.__name__, fake_module) + + spec = importlib.util.spec_from_file_location("core.authenticators", AUTHENTICATORS) + assert spec is not None and spec.loader is not None + authenticator_factory = importlib.util.module_from_spec(spec) + module_patch.setitem(sys.modules, "core.authenticators", authenticator_factory) + core.authenticators = authenticator_factory + spec.loader.exec_module(authenticator_factory) + yield authenticator_factory, config + + +@pytest.mark.parametrize( + ("mode", "expected_name"), + [ + ("auto-login", "AutoLoginAuthenticator"), + ("dummy", "dummy"), + ("local", "CustomFirstUseAuthenticator"), + ("github", "CustomGitHubOAuthenticator"), + ("multi", "CustomMultiAuthenticator"), + ], +) +def test_factory_preserves_legacy_projection_and_prefix_contract( + monkeypatch: pytest.MonkeyPatch, mode: str, expected_name: str +) -> None: + with _loaded_factory(monkeypatch) as (factory, _config): + selected = factory.create_authenticator(mode) + + assert selected == "dummy" if expected_name == "dummy" else selected.__name__ == expected_name + assert factory.GITHUB_USERNAME_PREFIX == "github:" + assert factory.CustomGitHubOAuthenticator.prefix == "github:" + assert factory.CustomFirstUseAuthenticator.prefix == "" + assert "CustomLocalAuthenticator" not in factory.__all__ + + +@pytest.mark.parametrize( + ("capabilities", "expected_name"), + [ + ((True, False, False, False), "AutoLoginAuthenticator"), + ((False, True, False, False), "dummy"), + ((False, False, True, False), "CustomFirstUseAuthenticator"), + ((False, False, False, True), "CustomGitHubOAuthenticator"), + ((False, False, True, True), "CustomMultiAuthenticator"), + ], +) +def test_factory_selects_authenticator_for_canonical_capabilities( + monkeypatch: pytest.MonkeyPatch, capabilities: tuple[bool, bool, bool, bool], expected_name: str +) -> None: + with _loaded_factory(monkeypatch) as (factory, config): + selected = factory.create_authenticator(config.AuthCapabilities(*capabilities)) + + assert selected == "dummy" if expected_name == "dummy" else selected.__name__ == expected_name + + +@pytest.mark.parametrize( + "capabilities", + [ + (False, False, False, False), + (True, False, True, False), + (False, True, False, True), + (True, True, False, False), + ], +) +def test_factory_rejects_invalid_capabilities_before_authenticator_construction( + monkeypatch: pytest.MonkeyPatch, capabilities: tuple[bool, bool, bool, bool] +) -> None: + with _loaded_factory(monkeypatch) as (factory, config), pytest.raises(config.AuthConfigurationError): + factory.create_authenticator(config.AuthCapabilities(*capabilities)) + + +@pytest.mark.parametrize("malformed_auth", (None, 1, True, (), object())) +def test_factory_rejects_malformed_runtime_inputs(monkeypatch: pytest.MonkeyPatch, malformed_auth) -> None: + with _loaded_factory(monkeypatch) as (factory, config), pytest.raises(config.AuthConfigurationError): + factory.create_authenticator(malformed_auth) + + +def test_factory_module_cleanup_survives_a_forced_test_failure(monkeypatch: pytest.MonkeyPatch) -> None: + module_names = ( + "core", + "core.config", + "core.authenticators", + "core.authenticators.auto_login", + "core.authenticators.firstuse", + "core.authenticators.github_app", + "core.authenticators.jwt", + "core.authenticators.multi", + ) + missing = object() + original_modules = {name: sys.modules.get(name, missing) for name in module_names} + + with pytest.raises(AssertionError, match="forced cleanup probe"), _loaded_factory(monkeypatch): + raise AssertionError("forced cleanup probe") + + for name, original_module in original_modules.items(): + if original_module is missing: + assert name not in sys.modules + else: + assert sys.modules[name] is original_module + + +def test_authenticator_factory_rejects_unknown_mode(monkeypatch: pytest.MonkeyPatch) -> None: + with ( + _loaded_factory(monkeypatch) as (factory, _config), + pytest.raises(ValueError, match="Unknown authentication mode"), + ): + factory.create_authenticator("unexpected") From c1998cc1e1aa03b1cc64baf2e7736d282e6011fb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:36:56 +0800 Subject: [PATCH 140/180] fix(auth): fail closed for unknown native users --- runtime/hub/core/authenticators/firstuse.py | 17 +- runtime/hub/tests/test_local_authenticator.py | 271 ----------------- .../hub/tests/test_native_authenticator.py | 281 ++++++++++++++++++ 3 files changed, 288 insertions(+), 281 deletions(-) delete mode 100644 runtime/hub/tests/test_local_authenticator.py create mode 100644 runtime/hub/tests/test_native_authenticator.py diff --git a/runtime/hub/core/authenticators/firstuse.py b/runtime/hub/core/authenticators/firstuse.py index 0ac3765c..685b82b4 100644 --- a/runtime/hub/core/authenticators/firstuse.py +++ b/runtime/hub/core/authenticators/firstuse.py @@ -58,19 +58,16 @@ def normalize_username(self, username): def _user_exists(self, username): """Check if user exists in JupyterHub database.""" - if self.db is None: - if hasattr(self, "parent") and self.parent: - db = self.parent.db - if db is None: - return True - else: - return True - else: - db = self.db + db = getattr(self, "db", None) + if db is None: + db = getattr(getattr(self, "parent", None), "db", None) + if db is None: + self.log.warning("Native authentication denied because Hub database is unavailable") + return False from jupyterhub.orm import User - return db.query(User).filter_by(name=username).first() is not None + return bool(db.query(User).filter_by(name=username).first()) def _get_user_password(self, username: str) -> UserPassword | None: """Get user password record from database.""" diff --git a/runtime/hub/tests/test_local_authenticator.py b/runtime/hub/tests/test_local_authenticator.py deleted file mode 100644 index 836c9568..00000000 --- a/runtime/hub/tests/test_local_authenticator.py +++ /dev/null @@ -1,271 +0,0 @@ -import asyncio -import importlib.util -import sys -import types -from contextlib import contextmanager -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -LOCAL_AUTHENTICATOR = ROOT / "core" / "authenticators" / "local.py" -AUTHENTICATORS = ROOT / "core" / "authenticators" / "__init__.py" -SETUP = ROOT / "core" / "setup.py" - - -class FakeFirstUseAuthenticator: - def normalize_username(self, username): - return username.lower() - - def _user_exists(self, username): - return username == "existing" - - def check_password(self, username, password): - return username == "existing" and password == "correct-password" - - -def test_local_authenticator_rejects_first_use_and_accepts_existing_password() -> None: - core = types.ModuleType("core") - authenticators = types.ModuleType("core.authenticators") - firstuse = types.ModuleType("core.authenticators.firstuse") - firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator - jupyterhub = types.ModuleType("jupyterhub") - orm = types.ModuleType("jupyterhub.orm") - orm.User = type("User", (), {}) - sys.modules.update( - { - "core": core, - "core.authenticators": authenticators, - "core.authenticators.firstuse": firstuse, - "jupyterhub": jupyterhub, - "jupyterhub.orm": orm, - } - ) - spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - authenticator = module.CustomLocalAuthenticator() - - class ExistingUserQuery: - def filter_by(self, **_kwargs): - return self - - def first(self): - return object() - - class ExistingUserDb: - def query(self, _model): - return ExistingUserQuery() - - authenticator.db = ExistingUserDb() - - assert ( - asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "correct-password"})) - == "existing" - ) - assert asyncio.run(authenticator.authenticate(None, {"username": "existing", "password": "wrong-password"})) is None - assert asyncio.run(authenticator.authenticate(None, {"username": "new", "password": "valid-password"})) is None - - -def test_local_authenticator_rejects_noncanonical_usernames() -> None: - core = types.ModuleType("core") - authenticators = types.ModuleType("core.authenticators") - firstuse = types.ModuleType("core.authenticators.firstuse") - firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator - sys.modules.update( - { - "core": core, - "core.authenticators": authenticators, - "core.authenticators.firstuse": firstuse, - } - ) - spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - authenticator = module.CustomLocalAuthenticator() - - for username in ("EXISTING", "existing:admin", 'existing"', "existing\n"): - assert ( - asyncio.run(authenticator.authenticate(None, {"username": username, "password": "correct-password"})) - is None - ) - - -def test_local_authenticator_validate_username_matches_login_policy() -> None: - core = types.ModuleType("core") - authenticators = types.ModuleType("core.authenticators") - firstuse = types.ModuleType("core.authenticators.firstuse") - firstuse.CustomFirstUseAuthenticator = FakeFirstUseAuthenticator - sys.modules.update( - { - "core": core, - "core.authenticators": authenticators, - "core.authenticators.firstuse": firstuse, - } - ) - spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - authenticator = module.CustomLocalAuthenticator() - - assert authenticator.validate_username("existing") - for username in ("EXISTING", "existing:admin", 'existing"', "existing\n", "a" * 65): - assert not authenticator.validate_username(username) - - -def test_local_authenticator_fails_closed_without_a_working_hub_database() -> None: - class DatabaseAgnosticFirstUseAuthenticator: - def _user_exists(self, _username): - return True - - def check_password(self, _username, _password): - return True - - core = types.ModuleType("core") - authenticators = types.ModuleType("core.authenticators") - firstuse = types.ModuleType("core.authenticators.firstuse") - firstuse.CustomFirstUseAuthenticator = DatabaseAgnosticFirstUseAuthenticator - jupyterhub = types.ModuleType("jupyterhub") - orm = types.ModuleType("jupyterhub.orm") - orm.User = type("User", (), {}) - sys.modules.update( - { - "core": core, - "core.authenticators": authenticators, - "core.authenticators.firstuse": firstuse, - "jupyterhub": jupyterhub, - "jupyterhub.orm": orm, - } - ) - spec = importlib.util.spec_from_file_location("core.authenticators.local", LOCAL_AUTHENTICATOR) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - unavailable_db = module.CustomLocalAuthenticator() - unavailable_db.db = None - unavailable_db.parent = types.SimpleNamespace(db=None) - - class FailingDb: - def query(self, _model): - raise RuntimeError("database unavailable") - - failing_db = module.CustomLocalAuthenticator() - failing_db.db = FailingDb() - - assert ( - asyncio.run(unavailable_db.authenticate(None, {"username": "existing", "password": "correct-password"})) is None - ) - assert asyncio.run(failing_db.authenticate(None, {"username": "existing", "password": "correct-password"})) is None - - -def test_bootstrap_admin_password_rejects_secret_mismatch_for_existing_hash(monkeypatch) -> None: - bcrypt = types.ModuleType("bcrypt") - bcrypt.gensalt = lambda: b"salt" - bcrypt.hashpw = lambda password, _salt: b"hash:" + password - bcrypt.checkpw = lambda password, password_hash: password_hash == b"hash:" + password - - class FakeUserPassword: - def __init__(self, username, password_hash, force_change): - self.username = username - self.password_hash = password_hash - self.force_change = force_change - - class FakeQuery: - def __init__(self, rows): - self.rows = rows - self.username = "" - - def filter_by(self, *, username): - self.username = username - return self - - def first(self): - return next((row for row in self.rows if row.username == self.username), None) - - class FakeSession: - def __init__(self): - self.rows = [] - - def query(self, _model): - return FakeQuery(self.rows) - - def add(self, row): - self.rows.append(row) - - session = FakeSession() - models = types.ModuleType("core.authenticators.models") - models.UserPassword = FakeUserPassword - database = types.ModuleType("core.database") - - @contextmanager - def session_scope(): - yield session - - database.session_scope = session_scope - monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) - monkeypatch.setitem(sys.modules, "core.authenticators.models", models) - monkeypatch.setitem(sys.modules, "core.database", database) - spec = importlib.util.spec_from_file_location("core.setup", SETUP) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - module._bootstrap_admin_password("operator", "InitialPassword1!", require_match=True) - session.rows[0].password_hash = bcrypt.hashpw(b"ChangedPassword1!", bcrypt.gensalt()) - - with pytest.raises(RuntimeError, match="does not match"): - module._bootstrap_admin_password("operator", "InitialPassword1!", require_match=True) - assert bcrypt.checkpw(b"ChangedPassword1!", session.rows[0].password_hash) - assert not bcrypt.checkpw(b"InitialPassword1!", session.rows[0].password_hash) - - -def test_api_token_is_assigned_to_the_configured_administrator(monkeypatch) -> None: - bcrypt = types.ModuleType("bcrypt") - monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) - spec = importlib.util.spec_from_file_location("core.setup", SETUP) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - config = types.SimpleNamespace(JupyterHub=types.SimpleNamespace()) - - module._configure_api_token(config, "token", "operator") - - assert config.JupyterHub.api_tokens == {"token": "operator"} - - -def test_authenticator_factory_rejects_unknown_mode() -> None: - core = types.ModuleType("core") - authenticators = types.ModuleType("core.authenticators") - sys.modules.update({"core": core, "core.authenticators": authenticators}) - for name, attribute in ( - ("auto_login", "AutoLoginAuthenticator"), - ("firstuse", "CustomFirstUseAuthenticator"), - ("github_app", "GITHUB_USERNAME_PREFIX"), - ("jwt", "RemoteLabAuthenticator"), - ("local", "CustomLocalAuthenticator"), - ("multi", "CustomMultiAuthenticator"), - ): - module = types.ModuleType(f"core.authenticators.{name}") - setattr(module, attribute, type(attribute, (), {}) if attribute != "GITHUB_USERNAME_PREFIX" else "github:") - if name == "github_app": - module.CustomGitHubOAuthenticator = type("CustomGitHubOAuthenticator", (), {}) - sys.modules[module.__name__] = module - - spec = importlib.util.spec_from_file_location("core.authenticators", AUTHENTICATORS) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - with pytest.raises(ValueError, match="Unknown authentication mode"): - module.create_authenticator("unexpected") diff --git a/runtime/hub/tests/test_native_authenticator.py b/runtime/hub/tests/test_native_authenticator.py new file mode 100644 index 00000000..b9515540 --- /dev/null +++ b/runtime/hub/tests/test_native_authenticator.py @@ -0,0 +1,281 @@ +import asyncio +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +FIRSTUSE = ROOT / "core" / "authenticators" / "firstuse.py" + + +class _FakeLog: + def __init__(self) -> None: + self.warnings = [] + + def warning(self, *args) -> None: + self.warnings.append(args) + + def info(self, *_args) -> None: + pass + + +class _HubUserQuery: + def __init__(self, result, queried_names) -> None: + self._result = result + self._queried_names = queried_names + + def filter_by(self, *, name): + self._queried_names.append(name) + return self + + def first(self): + return self._result + + +class _HubDatabase: + def __init__(self, result) -> None: + self._result = result + self.queried_names = [] + + def query(self, _model): + return _HubUserQuery(self._result, self.queried_names) + + +class _RaisingHubDatabase: + def query(self, _model): + raise RuntimeError("database unavailable") + + +class _UnexpectedHubDatabase: + def query(self, _model): + raise AssertionError("unexpected Hub database query") + + +def _install_core_packages(module_patch: pytest.MonkeyPatch) -> None: + core = types.ModuleType("core") + core.__path__ = [str(ROOT / "core")] + authenticators = types.ModuleType("core.authenticators") + authenticators.__path__ = [str(ROOT / "core" / "authenticators")] + core.authenticators = authenticators + module_patch.setitem(sys.modules, "core", core) + module_patch.setitem(sys.modules, "core.authenticators", authenticators) + + +@contextmanager +def _loaded_firstuse_authenticator(monkeypatch: pytest.MonkeyPatch) -> Iterator[type]: + with monkeypatch.context() as module_patch: + _install_core_packages(module_patch) + bcrypt = types.ModuleType("bcrypt") + bcrypt.gensalt = lambda: b"salt" + bcrypt.hashpw = lambda password, _salt: b"hash:" + password + bcrypt.checkpw = lambda password, password_hash: password_hash == b"hash:" + password + + class FakeFirstUseAuthenticator: + def __init__(self) -> None: + self.log = _FakeLog() + + firstuseauthenticator = types.ModuleType("firstuseauthenticator") + firstuseauthenticator.FirstUseAuthenticator = FakeFirstUseAuthenticator + models = types.ModuleType("core.authenticators.models") + models.UserPassword = type("UserPassword", (), {}) + database = types.ModuleType("core.database") + database.get_session = lambda: None + database.session_scope = lambda: None + jupyterhub = types.ModuleType("jupyterhub") + orm = types.ModuleType("jupyterhub.orm") + orm.User = type("User", (), {}) + jupyterhub.orm = orm + for fake_module in (bcrypt, firstuseauthenticator, models, database, jupyterhub, orm): + module_patch.setitem(sys.modules, fake_module.__name__, fake_module) + + spec = importlib.util.spec_from_file_location("core.authenticators.firstuse", FIRSTUSE) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + module_patch.setitem(sys.modules, "core.authenticators.firstuse", module) + spec.loader.exec_module(module) + yield module.CustomFirstUseAuthenticator + + +def test_firstuse_module_cleanup_survives_a_forced_test_failure(monkeypatch: pytest.MonkeyPatch) -> None: + module_names = ( + "core", + "core.authenticators", + "core.authenticators.firstuse", + "core.authenticators.models", + "core.database", + "bcrypt", + "firstuseauthenticator", + "jupyterhub", + "jupyterhub.orm", + ) + missing = object() + original_modules = {name: sys.modules.get(name, missing) for name in module_names} + + with pytest.raises(AssertionError, match="forced cleanup probe"), _loaded_firstuse_authenticator(monkeypatch): + raise AssertionError("forced cleanup probe") + + for name, original_module in original_modules.items(): + if original_module is missing: + assert name not in sys.modules + else: + assert sys.modules[name] is original_module + + +def test_precreated_user_sets_password_after_one_normalized_lookup(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + hub_db = _HubDatabase(object()) + authenticator.db = hub_db + calls = [] + authenticator.normalize_username = lambda username: calls.append(("normalize", username)) or "learner" + authenticator.user_has_password = lambda username: calls.append(("has_password", username)) or False + authenticator._validate_password = lambda password: calls.append(("validate", password)) or True + authenticator.set_password = lambda username, password, force_change: calls.append( + ("set_password", username, password, force_change) + ) + + authenticated = asyncio.run(authenticator.authenticate(None, {"username": "LEARNER", "password": "Password1!"})) + + assert authenticated == "learner" + assert authenticator.create_users is False + assert hub_db.queried_names == ["learner"] + assert calls == [ + ("normalize", "LEARNER"), + ("has_password", "learner"), + ("validate", "Password1!"), + ("set_password", "learner", "Password1!", False), + ] + + +@pytest.mark.parametrize( + ("submitted_password", "expected_result"), + [("Password1!", "learner"), ("wrong-password", None)], +) +def test_existing_user_authentication_checks_normalized_username( + monkeypatch: pytest.MonkeyPatch, submitted_password: str, expected_result: str | None +) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + hub_db = _HubDatabase(object()) + authenticator.db = hub_db + calls = [] + authenticator.normalize_username = lambda username: calls.append(("normalize", username)) or "learner" + authenticator.user_has_password = lambda username: calls.append(("has_password", username)) or True + authenticator.check_password = lambda username, password: ( + calls.append(("check_password", username, password)) or (password == "Password1!") + ) + + authenticated = asyncio.run( + authenticator.authenticate(None, {"username": "LEARNER", "password": submitted_password}) + ) + + assert authenticated == expected_result + assert hub_db.queried_names == ["learner"] + assert calls == [ + ("normalize", "LEARNER"), + ("has_password", "learner"), + ("check_password", "learner", submitted_password), + ] + + +def test_missing_child_and_parent_database_rejects_without_password_side_effect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + authenticator.db = None + authenticator.parent = types.SimpleNamespace(db=None) + authenticator.user_has_password = lambda _username: False + authenticator._validate_password = lambda _password: True + password_changes = [] + authenticator.set_password = lambda *args: password_changes.append(args) + + authenticated = asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "Password1!"})) + + assert authenticated is None + assert password_changes == [] + assert authenticator.log.warnings + + +def test_missing_parent_database_rejects_without_password_side_effect(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + authenticator.db = None + authenticator.parent = types.SimpleNamespace() + password_changes = [] + authenticator.set_password = lambda *args: password_changes.append(args) + + authenticated = asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "Password1!"})) + + assert authenticated is None + assert password_changes == [] + assert authenticator.log.warnings + + +@pytest.mark.parametrize("query_result", [None, False], ids=["none", "falsey"]) +def test_unknown_user_query_result_rejects_without_password_side_effect( + monkeypatch: pytest.MonkeyPatch, query_result +) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + authenticator.db = _HubDatabase(query_result) + password_changes = [] + authenticator.set_password = lambda *args: password_changes.append(args) + + authenticated = asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "Password1!"})) + + assert authenticated is None + assert password_changes == [] + + +def test_database_query_error_propagates_without_password_side_effect(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + authenticator.db = _RaisingHubDatabase() + password_changes = [] + authenticator.set_password = lambda *args: password_changes.append(args) + + with pytest.raises(RuntimeError, match="database unavailable"): + asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "Password1!"})) + + assert password_changes == [] + + +def test_parent_database_fallback_supports_multiauth_child(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + parent_db = _HubDatabase(object()) + authenticator.db = None + authenticator.parent = types.SimpleNamespace(db=parent_db) + + assert authenticator._user_exists("learner") is True + assert parent_db.queried_names == ["learner"] + + +def test_child_database_takes_precedence_over_parent_database(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + child_db = _HubDatabase(object()) + authenticator.db = child_db + authenticator.parent = types.SimpleNamespace(db=_UnexpectedHubDatabase()) + + assert authenticator._user_exists("learner") is True + assert child_db.queried_names == ["learner"] + + +def test_weak_first_use_password_rejects_without_password_storage(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_firstuse_authenticator(monkeypatch) as authenticator_type: + authenticator = authenticator_type() + authenticator.db = _HubDatabase(object()) + authenticator.user_has_password = lambda _username: False + password_changes = [] + authenticator.set_password = lambda *args: password_changes.append(args) + + authenticated = asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "weak"})) + + assert authenticated is None + assert password_changes == [] From f59d0d3d3f5fe452301b6a5f524b66296391799b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:12:43 +0800 Subject: [PATCH 141/180] refactor(hub): gate provider setup by capabilities --- runtime/hub/core/setup.py | 36 ++- runtime/hub/tests/test_auth_provider_setup.py | 296 ++++++++++++++++++ 2 files changed, 319 insertions(+), 13 deletions(-) create mode 100644 runtime/hub/tests/test_auth_provider_setup.py diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index d3e2298f..8ac6cb40 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -108,11 +108,18 @@ def setup_hub(c: Any) -> None: # Get the initialized config singleton config = HubConfig.get() - github_app_id = z2jh.get_config("hub.config.GitHubOAuthenticator.app_id", "") - github_app_installation_id = z2jh.get_config("hub.config.GitHubOAuthenticator.installation_id", "") - github_app_private_key = z2jh.get_config("hub.config.GitHubOAuthenticator.private_key", "") - github_app_private_key_file = z2jh.get_config("hub.config.GitHubOAuthenticator.private_key_file", "") - github_team_sync_ttl_seconds = z2jh.get_config("hub.config.GitHubOAuthenticator.team_sync_ttl_seconds", 3600) + auth = config.auth + github_app_id = "" + github_app_installation_id = "" + github_app_private_key = "" + github_app_private_key_file = "" + github_team_sync_ttl_seconds = 3600 + if auth.github: + github_app_id = z2jh.get_config("hub.config.GitHubOAuthenticator.app_id", "") + github_app_installation_id = z2jh.get_config("hub.config.GitHubOAuthenticator.installation_id", "") + github_app_private_key = z2jh.get_config("hub.config.GitHubOAuthenticator.private_key", "") + github_app_private_key_file = z2jh.get_config("hub.config.GitHubOAuthenticator.private_key_file", "") + github_team_sync_ttl_seconds = z2jh.get_config("hub.config.GitHubOAuthenticator.team_sync_ttl_seconds", 3600) # ========================================================================= # Configure Spawner @@ -139,7 +146,11 @@ def _start_metrics_updater(): # Ensure system-managed groups exist at startup (before any user logs in). # Note: load_groups does NOT set properties on existing groups, so the # source=system backfill is handled lazily in the admin groups API handler. - c.JupyterHub.load_groups = {"native-users": [], "github-users": []} + c.JupyterHub.load_groups = {} + if auth.native: + c.JupyterHub.load_groups["native-users"] = [] + if auth.github: + c.JupyterHub.load_groups["github-users"] = [] # ========================================================================= # Configure Authenticator @@ -152,7 +163,7 @@ async def auth_state_hook(spawner, auth_state): if auth_state is None: spawner.github_access_token = None # Still assign native users to their default group - if not spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): + if auth.native and not spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): try: from core.groups import assign_user_to_group @@ -162,7 +173,7 @@ async def auth_state_hook(spawner, auth_state): return spawner.github_access_token = auth_state.get("access_token") - if spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): + if auth.github and spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): try: from core.groups import sync_github_teams_for_user @@ -191,7 +202,7 @@ async def auth_state_hook(spawner, auth_state): assign_user_to_group(spawner.user, "github-users", spawner.user.db) except Exception as e: print(f"[GROUPS] Warning: Failed to assign github-users group for {spawner.user.name}: {e}") - elif not spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): + elif auth.native and not spawner.user.name.startswith(GITHUB_USERNAME_PREFIX): # Native user with auth_state but no GitHub teams try: from core.groups import assign_user_to_group @@ -202,12 +213,11 @@ async def auth_state_hook(spawner, auth_state): c.Spawner.auth_state_hook = auth_state_hook - # Set authenticator based on mode - c.JupyterHub.authenticator_class = create_authenticator(config.auth_mode) + c.JupyterHub.authenticator_class = create_authenticator(auth) - if config.auth_mode in ("auto-login", "local"): + if auth.auto_login or (auth.native and not auth.github): c.Authenticator.allow_all = True - elif config.auth_mode == "multi": + if auth.native and auth.github: c.MultiAuthenticator.authenticators = [ { "authenticator_class": CustomGitHubOAuthenticator, diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py new file mode 100644 index 00000000..f5f99eca --- /dev/null +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -0,0 +1,296 @@ +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import anyio +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SETUP = ROOT / "core" / "setup.py" +CONFIG = ROOT / "core" / "config.py" +GITHUB_SETTINGS = { + "hub.config.GitHubOAuthenticator.app_id": "app-id", + "hub.config.GitHubOAuthenticator.installation_id": "installation-id", + "hub.config.GitHubOAuthenticator.private_key": "private-key", + "hub.config.GitHubOAuthenticator.private_key_file": "private-key-file", + "hub.config.GitHubOAuthenticator.team_sync_ttl_seconds": 123, +} +MODULE_NAMES = tuple( + ( + "bcrypt|core|core.z2jh|core.config|core.authenticators|core.database|core.handlers|core.metrics_updater|" + "core.spawner|core.groups|jupyterhub|jupyterhub.apihandlers|jupyterhub.apihandlers.groups|tornado|" + "tornado.web|core.setup" + ) + .replace("|", "\n") + .splitlines() +) +_module = types.ModuleType + + +def _config_for(auth: object) -> types.SimpleNamespace: + return types.SimpleNamespace( + auth=auth, + auth_mode=auth.effective_mode, + accelerators={}, + build_quota_rates=lambda: {}, + quota_enabled=True, + quota=types.SimpleNamespace(minimumToStart=0, defaultQuota=0), + teams=types.SimpleNamespace(mapping={"learners": ["cpu"]}), + github_org_name="example-org", + platform_display_name="AUP Learning Cloud", + cluster_name="", + ) + + +@contextmanager +def _loaded_setup( + monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool], *, fail_setup: bool = False +) -> Iterator[types.SimpleNamespace]: + with monkeypatch.context() as module_patch: + for variable in ("JUPYTERHUB_ADMIN_PASSWORD", "JUPYTERHUB_ADMIN_USERNAME", "JUPYTERHUB_API_TOKEN"): + monkeypatch.delenv(variable, raising=False) + module_patch.setattr( + importlib.import_module("asyncio"), + "get_event_loop", + lambda: types.SimpleNamespace(call_later=lambda *_args: None), + ) + bcrypt = _module("bcrypt") + module_patch.setitem(sys.modules, "bcrypt", bcrypt) + core = _module("core") + core.__path__ = [str(ROOT / "core")] + module_patch.setitem(sys.modules, "core", core) + + config_spec = importlib.util.spec_from_file_location("core.config", CONFIG) + assert config_spec is not None and config_spec.loader is not None + config_module = importlib.util.module_from_spec(config_spec) + module_patch.setitem(sys.modules, "core.config", config_module) + core.config = config_module + config_spec.loader.exec_module(config_module) + auth = config_module.AuthCapabilities(*providers) + config = _config_for(auth) + config_module.HubConfig._instance, config_module.HubConfig._initialized = config, True + + settings_reads: list[str] = [] + z2jh = _module("core.z2jh") + + def get_config(key: str, default: object = None) -> object: + settings_reads.append(key) + if fail_setup and key == "hub.db.type": + raise RuntimeError("forced setup failure") + if key.startswith("hub.config.GitHubOAuthenticator") and not auth.github: + raise AssertionError(f"GitHub settings accessed for disabled provider: {key}") + return GITHUB_SETTINGS.get(key, default) + + z2jh.get_config = get_config + core.z2jh = z2jh + module_patch.setitem(sys.modules, "core.z2jh", z2jh) + + authenticator_types = { + "auto": type("AutoLoginAuthenticator", (), {}), + "github": type("CustomGitHubOAuthenticator", (), {}), + "native": type("CustomFirstUseAuthenticator", (), {}), + "multi": type("CustomMultiAuthenticator", (), {}), + } + factory_inputs: list[object] = [] + authenticators = _module("core.authenticators") + authenticators.GITHUB_USERNAME_PREFIX = "github:" + authenticators.CustomGitHubOAuthenticator = authenticator_types["github"] + authenticators.CustomFirstUseAuthenticator = authenticator_types["native"] + + def create_authenticator(_input: object) -> type | str: + factory_inputs.append(_input) + if auth.auto_login: + return authenticator_types["auto"] + if auth.dummy: + return "dummy" + if auth.native and auth.github: + return authenticator_types["multi"] + if auth.github: + return authenticator_types["github"] + return authenticator_types["native"] + + authenticators.create_authenticator = create_authenticator + core.authenticators = authenticators + module_patch.setitem(sys.modules, "core.authenticators", authenticators) + + database = _module("core.database") + database.init_database = database.create_all_tables = lambda *_args: None + module_patch.setitem(sys.modules, "core.database", database) + handlers = _module("core.handlers") + handlers.configure_handlers, handlers.get_handlers = lambda **_kwargs: None, lambda: [] + module_patch.setitem(sys.modules, "core.handlers", handlers) + metrics = _module("core.metrics_updater") + metrics.start_metrics_updater = lambda: None + module_patch.setitem(sys.modules, "core.metrics_updater", metrics) + spawner = _module("core.spawner") + spawner.RemoteLabKubeSpawner = type("RemoteLabKubeSpawner", (), {"configure_from_config": lambda _config: None}) + module_patch.setitem(sys.modules, "core.spawner", spawner) + + group_assignments: list[tuple[str, str]] = [] + team_syncs: list[tuple[object, ...]] = [] + groups = _module("core.groups") + groups.assign_user_to_group = lambda user, group, _db: group_assignments.append((user.name, group)) + + async def sync_github_teams_for_user(*args: object, **kwargs: object) -> bool: + team_syncs.append((*args, kwargs)) + return True + + groups.sync_github_teams_for_user = sync_github_teams_for_user + groups.is_readonly_group, groups.is_undeletable_group = lambda _group: False, lambda _group: False + module_patch.setitem(sys.modules, "core.groups", groups) + + jupyterhub = _module("jupyterhub") + apihandlers = _module("jupyterhub.apihandlers") + apihandlers.default_handlers = [] + api_groups = _module("jupyterhub.apihandlers.groups") + api_groups.GroupAPIHandler = type("GroupAPIHandler", (), {}) + api_groups.GroupUsersAPIHandler = type("GroupUsersAPIHandler", (), {}) + jupyterhub.apihandlers = apihandlers + apihandlers.groups = api_groups + module_patch.setitem(sys.modules, "jupyterhub", jupyterhub) + module_patch.setitem(sys.modules, "jupyterhub.apihandlers", apihandlers) + module_patch.setitem(sys.modules, "jupyterhub.apihandlers.groups", api_groups) + tornado = _module("tornado") + web = _module("tornado.web") + web.HTTPError = RuntimeError + tornado.web = web + module_patch.setitem(sys.modules, "tornado", tornado) + module_patch.setitem(sys.modules, "tornado.web", web) + + setup_spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert setup_spec is not None and setup_spec.loader is not None + setup_module = importlib.util.module_from_spec(setup_spec) + module_patch.setitem(sys.modules, "core.setup", setup_module) + setup_spec.loader.exec_module(setup_module) + if auth.native: + monkeypatch.setenv("JUPYTERHUB_ADMIN_PASSWORD", "Password1!") + monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "admin") + setup_module._bootstrap_admin_password = lambda *_args, **_kwargs: None + hub = types.SimpleNamespace(template_vars={}, extra_handlers=[]) + c = types.SimpleNamespace( + JupyterHub=hub, + Authenticator=types.SimpleNamespace(), + Spawner=types.SimpleNamespace(), + MultiAuthenticator=types.SimpleNamespace(), + ) + yield types.SimpleNamespace( + auth=auth, + c=c, + factory_inputs=factory_inputs, + group_assignments=group_assignments, + team_syncs=team_syncs, + authenticator_types=authenticator_types, + settings_reads=settings_reads, + setup=setup_module, + ) + + +@pytest.mark.parametrize( + ("providers", "expected_groups"), + [ + ((True, False, False, False), {}), + ((False, True, False, False), {}), + ((False, False, True, False), {"native-users": []}), + ((False, False, False, True), {"github-users": []}), + ((False, False, True, True), {"native-users": [], "github-users": []}), + ], +) +def test_setup_passes_typed_capabilities_and_creates_only_enabled_groups( + monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool], expected_groups: dict[str, list[object]] +) -> None: + with _loaded_setup(monkeypatch, providers) as state: + state.setup.setup_hub(state.c) + + assert state.factory_inputs == [state.auth] + assert state.c.JupyterHub.load_groups == expected_groups + + +@pytest.mark.parametrize("providers", ((False, False, False, True), (False, False, True, True))) +def test_setup_loads_github_settings_for_each_github_capability( + monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool] +) -> None: + with _loaded_setup(monkeypatch, providers) as state: + state.setup.setup_hub(state.c) + + assert set(GITHUB_SETTINGS).issubset(state.settings_reads) + + +def test_native_only_setup_never_reads_github_settings(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, True, False)) as state: + state.setup.setup_hub(state.c) + + assert not any(key.startswith("hub.config.GitHubOAuthenticator") for key in state.settings_reads) + + +@pytest.mark.parametrize("providers", ((False, False, False, True), (False, False, True, True))) +def test_github_prefixed_users_sync_teams_for_each_github_capability( + monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool] +) -> None: + with _loaded_setup(monkeypatch, providers) as state: + state.setup.setup_hub(state.c) + github_user = types.SimpleNamespace(name="github:octo", db=object()) + spawner = types.SimpleNamespace(user=github_user) + + anyio.run(state.c.Spawner.auth_state_hook, spawner, {"access_token": "token"}) + + assert spawner.github_access_token == "token" + assert len(state.team_syncs) == 1 + assert state.group_assignments == [("github:octo", "github-users")] + + +def test_native_user_retains_native_group_without_github_sync(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, True, True)) as state: + state.setup.setup_hub(state.c) + native_user = types.SimpleNamespace(name="learner", db=object()) + spawner = types.SimpleNamespace(user=native_user) + + anyio.run(state.c.Spawner.auth_state_hook, spawner, None) + + assert spawner.github_access_token is None + assert state.team_syncs == [] + assert state.group_assignments == [("learner", "native-users")] + + +def test_github_only_preserves_direct_callback_path(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True)) as state: + state.setup.setup_hub(state.c) + + assert state.c.JupyterHub.authenticator_class is state.authenticator_types["github"] + assert not hasattr(state.c.MultiAuthenticator, "authenticators") + + +def test_composed_auth_preserves_prefixed_github_and_unprefixed_native_callbacks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with _loaded_setup(monkeypatch, (False, False, True, True)) as state: + state.setup.setup_hub(state.c) + + assert state.c.JupyterHub.authenticator_class is state.authenticator_types["multi"] + assert state.c.MultiAuthenticator.authenticators == [ + {"authenticator_class": state.authenticator_types["github"], "url_prefix": "/github"}, + { + "authenticator_class": state.authenticator_types["native"], + "url_prefix": "/native", + "config": {"prefix": "", "allow_all": True}, + }, + ] + + +def test_setup_module_cleanup_survives_a_forced_setup_failure(monkeypatch: pytest.MonkeyPatch) -> None: + missing = object() + original_modules = {name: sys.modules.get(name, missing) for name in MODULE_NAMES} + + with ( + pytest.raises(RuntimeError, match="forced setup failure"), + _loaded_setup(monkeypatch, (False, False, False, True), fail_setup=True) as state, + ): + state.setup.setup_hub(state.c) + + for name, original_module in original_modules.items(): + if original_module is missing: + assert name not in sys.modules + else: + assert sys.modules[name] is original_module From e8120660522f1262dde646f6e2350260c4b994ef Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:27:40 +0800 Subject: [PATCH 142/180] fix(hub): make admin credentials bootstrap-only --- runtime/hub/core/setup.py | 53 ++++--- runtime/hub/tests/test_admin_bootstrap.py | 134 ++++++++++++++++++ .../tests/test_admin_bootstrap_failures.py | 102 +++++++++++++ .../hub/tests/test_setup_admin_bootstrap.py | 104 ++++++++++++++ 4 files changed, 364 insertions(+), 29 deletions(-) create mode 100644 runtime/hub/tests/test_admin_bootstrap.py create mode 100644 runtime/hub/tests/test_admin_bootstrap_failures.py create mode 100644 runtime/hub/tests/test_setup_admin_bootstrap.py diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index 8ac6cb40..d55b5e94 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -49,28 +49,28 @@ pass -def _bootstrap_admin_password(admin_username: str, admin_password: str, *, require_match: bool = False) -> None: +def _bootstrap_admin_password(admin_username: str, admin_password: str) -> None: from core.authenticators.models import UserPassword from core.database import session_scope + created = False with session_scope() as session: user_pw = session.query(UserPassword).filter_by(username=admin_username).first() - if user_pw: - if require_match and not bcrypt.checkpw(admin_password.encode(), user_pw.password_hash): - raise RuntimeError( - "Existing administrator password hash does not match the configured credentials Secret" + if user_pw is None: + password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) + session.add( + UserPassword( + username=admin_username, + password_hash=password_hash, + force_change=False, ) - print(f"[SETUP] Admin '{admin_username}' password already set") - return - password_hash = bcrypt.hashpw(admin_password.encode(), bcrypt.gensalt()) - session.add( - UserPassword( - username=admin_username, - password_hash=password_hash, - force_change=False, ) - ) + created = True + + if created: print(f"[SETUP] Admin '{admin_username}' password set automatically") + else: + print(f"[SETUP] Admin '{admin_username}' password already set") def _configure_api_token(c: Any, api_token: str | None, admin_username: str) -> None: @@ -243,6 +243,7 @@ async def auth_state_hook(spawner, auth_state): team_resource_mapping=dict(config.teams.mapping), github_org=config.github_org_name, auth_mode=config.auth_mode, + access_policy=config.resources.effective_access_policy, platform_name=config.platform_display_name, ) @@ -374,18 +375,10 @@ async def delete(self, group_name): admin_password = os.environ.get("JUPYTERHUB_ADMIN_PASSWORD", "") admin_username = os.environ.get("JUPYTERHUB_ADMIN_USERNAME", "admin") - - if config.auth_mode == "local" and not admin_password: - raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_PASSWORD") - if config.auth_mode == "local" and not os.environ.get("JUPYTERHUB_ADMIN_USERNAME"): - raise RuntimeError("Local authentication requires JUPYTERHUB_ADMIN_USERNAME") - - # ========================================================================= - # API Token - # ========================================================================= - api_token = os.environ.get("JUPYTERHUB_API_TOKEN") - _configure_api_token(c, api_token, admin_username) + + if admin_password and not auth.native: + raise RuntimeError("Administrator password bootstrap requires native authentication") # ========================================================================= # Template Paths @@ -396,11 +389,13 @@ async def delete(self, group_name): if admin_password: try: - _bootstrap_admin_password(admin_username, admin_password, require_match=config.auth_mode == "local") + _bootstrap_admin_password(admin_username, admin_password) except Exception as e: - if config.auth_mode == "local": - raise RuntimeError("Failed to bootstrap local administrator credentials") from e - print(f"[SETUP] Warning: Failed to set admin password: {e}") + raise RuntimeError("Failed to bootstrap administrator credentials") from e + + _configure_api_token(c, api_token, admin_username) + + if admin_password: c.Authenticator.admin_users = {admin_username} print(f"[SETUP] Admin user configured: {admin_username}") diff --git a/runtime/hub/tests/test_admin_bootstrap.py b/runtime/hub/tests/test_admin_bootstrap.py new file mode 100644 index 00000000..094c0bfd --- /dev/null +++ b/runtime/hub/tests/test_admin_bootstrap.py @@ -0,0 +1,134 @@ +import importlib.util +import inspect +import sys +import types +from contextlib import contextmanager +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SETUP = ROOT / "core" / "setup.py" + + +def test_bootstrap_admin_password_preserves_existing_hash(monkeypatch) -> None: + bcrypt = types.ModuleType("bcrypt") + bcrypt.gensalt = lambda: b"salt" + bcrypt.hashpw = lambda password, _salt: b"hash:" + password + bcrypt.checkpw = lambda password, password_hash: password_hash == b"hash:" + password + + class FakeUserPassword: + def __init__(self, username, password_hash, force_change): + self.username = username + self.password_hash = password_hash + self.force_change = force_change + + class FakeQuery: + def __init__(self, rows): + self.rows = rows + self.username = "" + + def filter_by(self, *, username): + self.username = username + return self + + def first(self): + return next((row for row in self.rows if row.username == self.username), None) + + class FakeSession: + def __init__(self): + self.rows = [] + + def query(self, _model): + return FakeQuery(self.rows) + + def add(self, row): + self.rows.append(row) + + session = FakeSession() + models = types.ModuleType("core.authenticators.models") + models.UserPassword = FakeUserPassword + database = types.ModuleType("core.database") + + @contextmanager + def session_scope(): + yield session + + database.session_scope = session_scope + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + monkeypatch.setitem(sys.modules, "core.authenticators.models", models) + monkeypatch.setitem(sys.modules, "core.database", database) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + module._bootstrap_admin_password("operator", "InitialPassword1!") + session.rows[0].password_hash = bcrypt.hashpw(b"ChangedPassword1!", bcrypt.gensalt()) + module._bootstrap_admin_password("operator", "InitialPassword1!") + + assert "require_match" not in inspect.signature(module._bootstrap_admin_password).parameters + assert bcrypt.checkpw(b"ChangedPassword1!", session.rows[0].password_hash) + assert not bcrypt.checkpw(b"InitialPassword1!", session.rows[0].password_hash) + + +def test_bootstrap_admin_password_does_not_compare_same_secret_on_restart(monkeypatch) -> None: + bcrypt = types.ModuleType("bcrypt") + bcrypt.gensalt = lambda: b"salt" + bcrypt.hashpw = lambda password, _salt: b"hash:" + password + bcrypt.checkpw = lambda *_args: (_ for _ in ()).throw(AssertionError("bootstrap must not compare password hashes")) + + class FakeUserPassword: + def __init__(self, username, password_hash, force_change): + self.username = username + self.password_hash = password_hash + self.force_change = force_change + + class FakeQuery: + def __init__(self, rows): + self.rows = rows + + def filter_by(self, *, username): + self.username = username + return self + + def first(self): + return next((row for row in self.rows if row.username == self.username), None) + + session = types.SimpleNamespace(rows=[]) + session.query = lambda _model: FakeQuery(session.rows) + session.add = session.rows.append + models = types.ModuleType("core.authenticators.models") + models.UserPassword = FakeUserPassword + database = types.ModuleType("core.database") + + @contextmanager + def session_scope(): + yield session + + database.session_scope = session_scope + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + monkeypatch.setitem(sys.modules, "core.authenticators.models", models) + monkeypatch.setitem(sys.modules, "core.database", database) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + module._bootstrap_admin_password("operator", "InitialPassword1!") + module._bootstrap_admin_password("operator", "InitialPassword1!") + + assert len(session.rows) == 1 + assert session.rows[0].password_hash == b"hash:InitialPassword1!" + + +def test_api_token_is_assigned_to_the_configured_administrator(monkeypatch) -> None: + bcrypt = types.ModuleType("bcrypt") + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + config = types.SimpleNamespace(JupyterHub=types.SimpleNamespace()) + + module._configure_api_token(config, "token", "operator") + + assert config.JupyterHub.api_tokens == {"token": "operator"} diff --git a/runtime/hub/tests/test_admin_bootstrap_failures.py b/runtime/hub/tests/test_admin_bootstrap_failures.py new file mode 100644 index 00000000..c80e1a59 --- /dev/null +++ b/runtime/hub/tests/test_admin_bootstrap_failures.py @@ -0,0 +1,102 @@ +import importlib.util +import sys +import types +from contextlib import contextmanager +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SETUP = ROOT / "core" / "setup.py" + + +@pytest.mark.parametrize("failure", ("query", "hash", "add", "commit")) +def test_bootstrap_failure_never_prints_password_success(monkeypatch: pytest.MonkeyPatch, capsys, failure: str) -> None: + class FakeUserPassword: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + + class FakeQuery: + def filter_by(self, **_kwargs): + return self + + def first(self): + if failure == "query": + raise OSError("query failed") + return None + + class FakeSession: + def query(self, _model): + return FakeQuery() + + def add(self, _row) -> None: + if failure == "add": + raise OSError("add failed") + + @contextmanager + def session_scope(): + yield FakeSession() + if failure == "commit": + raise OSError("commit failed") + + bcrypt = types.ModuleType("bcrypt") + bcrypt.gensalt = lambda: b"salt" + bcrypt.hashpw = lambda _password, _salt: ( + (_ for _ in ()).throw(OSError("hash failed")) if failure == "hash" else b"hash" + ) + models = types.ModuleType("core.authenticators.models") + models.UserPassword = FakeUserPassword + database = types.ModuleType("core.database") + database.session_scope = session_scope + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + monkeypatch.setitem(sys.modules, "core.authenticators.models", models) + monkeypatch.setitem(sys.modules, "core.database", database) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + with pytest.raises(OSError): + module._bootstrap_admin_password("operator", "Password1!") + + assert "Admin 'operator' password" not in capsys.readouterr().out + + +def test_existing_password_commit_failure_never_prints_success(monkeypatch: pytest.MonkeyPatch, capsys) -> None: + class FakeUserPassword: + username = "operator" + + class FakeQuery: + def filter_by(self, **_kwargs): + return self + + def first(self): + return FakeUserPassword() + + class FakeSession: + def query(self, _model): + return FakeQuery() + + @contextmanager + def session_scope(): + yield FakeSession() + raise OSError("commit failed") + + bcrypt = types.ModuleType("bcrypt") + bcrypt.hashpw = lambda *_args: (_ for _ in ()).throw(AssertionError("existing rows must not hash")) + models = types.ModuleType("core.authenticators.models") + models.UserPassword = FakeUserPassword + database = types.ModuleType("core.database") + database.session_scope = session_scope + monkeypatch.setitem(sys.modules, "bcrypt", bcrypt) + monkeypatch.setitem(sys.modules, "core.authenticators.models", models) + monkeypatch.setitem(sys.modules, "core.database", database) + spec = importlib.util.spec_from_file_location("core.setup", SETUP) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + with pytest.raises(OSError, match="commit failed"): + module._bootstrap_admin_password("operator", "Password1!") + + assert "Admin 'operator' password" not in capsys.readouterr().out diff --git a/runtime/hub/tests/test_setup_admin_bootstrap.py b/runtime/hub/tests/test_setup_admin_bootstrap.py new file mode 100644 index 00000000..de0f89aa --- /dev/null +++ b/runtime/hub/tests/test_setup_admin_bootstrap.py @@ -0,0 +1,104 @@ +import pytest +from test_auth_provider_setup import _loaded_setup + + +def test_native_setup_does_not_require_optional_admin_bootstrap(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, True, False)) as state: + monkeypatch.delenv("JUPYTERHUB_ADMIN_PASSWORD", raising=False) + monkeypatch.delenv("JUPYTERHUB_ADMIN_USERNAME", raising=False) + + state.setup.setup_hub(state.c) + + assert not hasattr(state.c.Authenticator, "admin_users") + + +def test_enabled_bootstrap_failure_aborts_before_administrator_registration(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, True, False)) as state: + + def fail_bootstrap(_username: str, _password: str) -> None: + raise OSError("database unavailable") + + state.setup._bootstrap_admin_password = fail_bootstrap + + with pytest.raises(RuntimeError, match="Failed to bootstrap administrator credentials") as error: + state.setup.setup_hub(state.c) + + assert isinstance(error.value.__cause__, OSError) + assert not hasattr(state.c.Authenticator, "admin_users") + + +def test_github_only_rejects_stale_password_before_bootstrap_or_token_configuration( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True)) as state: + calls: list[tuple[str, str]] = [] + monkeypatch.setenv("JUPYTERHUB_ADMIN_PASSWORD", "Password1!") + monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") + monkeypatch.setenv("JUPYTERHUB_API_TOKEN", "token-value") + state.setup._bootstrap_admin_password = lambda username, password: calls.append((username, password)) + + with pytest.raises(RuntimeError, match="requires native authentication"): + state.setup.setup_hub(state.c) + + assert calls == [] + assert not hasattr(state.c.JupyterHub, "api_tokens") + assert not hasattr(state.c.Authenticator, "admin_users") + output = capsys.readouterr().out + assert "API token loaded" not in output + assert "Admin user configured" not in output + + +def test_token_only_remains_available_without_native_bootstrap(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True)) as state: + monkeypatch.setenv("JUPYTERHUB_API_TOKEN", "token-value") + + state.setup.setup_hub(state.c) + + assert state.c.JupyterHub.api_tokens == {"token-value": "admin"} + assert not hasattr(state.c.Authenticator, "admin_users") + + +def test_bootstrap_failure_preserves_existing_token_and_admin_state( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + with _loaded_setup(monkeypatch, (False, False, True, False)) as state: + state.c.JupyterHub.api_tokens = {"existing-token": "existing-admin"} + state.c.Authenticator.admin_users = {"existing-admin"} + monkeypatch.setenv("JUPYTERHUB_API_TOKEN", "token-value") + + def fail_bootstrap(_username: str, _password: str) -> None: + raise OSError("database unavailable") + + state.setup._bootstrap_admin_password = fail_bootstrap + + with pytest.raises(RuntimeError, match="Failed to bootstrap administrator credentials"): + state.setup.setup_hub(state.c) + + assert state.c.JupyterHub.api_tokens == {"existing-token": "existing-admin"} + assert state.c.Authenticator.admin_users == {"existing-admin"} + output = capsys.readouterr().out + assert "API token loaded" not in output + assert "Admin user configured" not in output + + +def test_token_failure_follows_successful_bootstrap_without_final_admin_state( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + with _loaded_setup(monkeypatch, (False, False, True, False)) as state: + calls: list[tuple[str, str]] = [] + monkeypatch.setenv("JUPYTERHUB_API_TOKEN", "token-value") + state.setup._bootstrap_admin_password = lambda username, password: calls.append((username, password)) + + def fail_token(_config, _token: str, _username: str) -> None: + raise OSError("token storage unavailable") + + state.setup._configure_api_token = fail_token + + with pytest.raises(OSError, match="token storage unavailable"): + state.setup.setup_hub(state.c) + + assert calls == [("admin", "Password1!")] + assert not hasattr(state.c.Authenticator, "admin_users") + output = capsys.readouterr().out + assert "API token loaded" not in output + assert "Admin user configured" not in output From e2c53a611184bd56a641e1f2a3044f419fd1bc11 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:28:55 +0800 Subject: [PATCH 143/180] fix(hub): preserve native password and onboarding semantics --- runtime/hub/core/handlers.py | 46 +- .../hub/tests/onboarding_handlers_support.py | 165 +++++++ runtime/hub/tests/test_onboarding_handlers.py | 432 ++---------------- runtime/hub/tests/test_password_handlers.py | 228 +++++++++ 4 files changed, 434 insertions(+), 437 deletions(-) create mode 100644 runtime/hub/tests/onboarding_handlers_support.py create mode 100644 runtime/hub/tests/test_password_handlers.py diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index f3e91f49..c47360ca 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -31,9 +31,8 @@ import asyncio import json -import os from datetime import datetime, timezone -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlparse, urlunparse from jupyterhub.apihandlers import APIHandler @@ -63,6 +62,9 @@ StatsUserHandler, ) +if TYPE_CHECKING: + from core.config import ResourceAccessPolicy + # ============================================================================= # Module-level configuration (set via configure_handlers) # ============================================================================= @@ -75,6 +77,7 @@ "default_quota": 0, "team_resource_mapping": {}, "auth_mode": "auto-login", + "access_policy": "group-mapped", "platform_name": "AUP Learning Cloud", } @@ -82,10 +85,6 @@ MAX_NATIVE_PASSWORD_BYTES = 72 -def _is_secret_managed_bootstrap_admin(username: str) -> bool: - return _handler_config["auth_mode"] == "local" and username == os.environ.get("JUPYTERHUB_ADMIN_USERNAME") - - def _serialize_dismissed_at(value: datetime | None) -> str | None: """Serialize onboarding dismissal timestamps for API responses.""" if value is None: @@ -142,6 +141,7 @@ def configure_handlers( team_resource_mapping: dict[str, list[str]] | None = None, github_org: str = "", auth_mode: str = "auto-login", + access_policy: ResourceAccessPolicy = "group-mapped", platform_name: str = "AUP Learning Cloud", ) -> None: """Configure handler module with runtime settings.""" @@ -156,6 +156,7 @@ def configure_handlers( _handler_config["team_resource_mapping"] = team_resource_mapping _handler_config["github_org"] = github_org _handler_config["auth_mode"] = auth_mode + _handler_config["access_policy"] = access_policy _handler_config["platform_name"] = platform_name @@ -285,13 +286,6 @@ def _render_error(msg: str): self.set_status(400) return self.finish(html) - if _is_secret_managed_bootstrap_admin(username): - html = await _render_error( - "The configured local administrator password is managed by the Kubernetes Secret" - ) - self.set_status(403) - return self.finish(html) - firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -379,12 +373,6 @@ async def post(self): + f"admin/reset-password?user={target_user}&error=Cannot+reset+password+for+GitHub+users" ) - if _is_secret_managed_bootstrap_admin(username): - return self.redirect( - self.hub.base_url - + f"admin/reset-password?user={target_user}&error=Configured+local+administrator+password+is+managed+by+the+Kubernetes+Secret" - ) - firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -461,15 +449,6 @@ async def post(self): self.set_header("Content-Type", "application/json") return self.finish(json.dumps({"error": "Cannot set password for GitHub users"})) - if _is_secret_managed_bootstrap_admin(username): - self.set_status(403) - self.set_header("Content-Type", "application/json") - return self.finish( - json.dumps( - {"error": "The configured local administrator password is managed by the Kubernetes Secret"} - ) - ) - firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -555,15 +534,6 @@ async def post(self): return self.finish( json.dumps({"error": f"Cannot set password for GitHub user: {entry['username']}"}) ) - if _is_secret_managed_bootstrap_admin(entry["username"]): - self.set_status(403) - self.set_header("Content-Type", "application/json") - return self.finish( - json.dumps( - {"error": "The configured local administrator password is managed by the Kubernetes Secret"} - ) - ) - firstuse_auth = _find_firstuse_authenticator(self.authenticator) if not firstuse_auth: @@ -1173,7 +1143,7 @@ async def get(self): resolve_resources_for_user( self.current_user, _handler_config.get("team_resource_mapping", {}), - _handler_config.get("auth_mode", "auto-login"), + _handler_config["access_policy"], list(config.resources.images.keys()), ) ) diff --git a/runtime/hub/tests/onboarding_handlers_support.py b/runtime/hub/tests/onboarding_handlers_support.py new file mode 100644 index 00000000..cc4c118b --- /dev/null +++ b/runtime/hub/tests/onboarding_handlers_support.py @@ -0,0 +1,165 @@ +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" +AUTHENTICATORS = CORE / "authenticators" + + +class DummyUser: + def __init__(self, name: str, admin: bool = False) -> None: + self.name = name + self.admin = admin + + +class FakeQuery: + def __init__(self, rows: list[object]) -> None: + self._rows = rows + self._filtered = rows + + def filter_by(self, **kwargs: object) -> "FakeQuery": + self._filtered = [ + row for row in self._rows if all(getattr(row, key, None) == value for key, value in kwargs.items()) + ] + return self + + def first(self) -> object | None: + return self._filtered[0] if self._filtered else None + + def all(self) -> list[object]: + return self._filtered + + +class FakeDb: + def __init__(self, rows: list[object] | None = None) -> None: + self.rows = rows or [] + self.commits = 0 + + def query(self, _model: object) -> FakeQuery: + return FakeQuery(self.rows) + + def add(self, row: object) -> None: + self.rows.append(row) + + def commit(self) -> None: + self.commits += 1 + + +@contextmanager +def fake_session_scope(db: FakeDb) -> Iterator[FakeDb]: + yield db + for row in db.rows: + if hasattr(row, "detached"): + row.detached = True + db.commit() + + +def make_handler(handler_cls: type, username: str) -> tuple[object, dict[str, object]]: + handler = object.__new__(handler_cls) + handler.current_user = DummyUser(username) + captured: dict[str, object] = {} + handler.set_header = lambda key, value: captured.setdefault("headers", {}).__setitem__(key, value) + handler.finish = lambda payload: captured.setdefault("body", payload) + return handler, captured + + +@contextmanager +def load_handlers(monkeypatch: pytest.MonkeyPatch) -> Iterator[types.SimpleNamespace]: + with monkeypatch.context() as module_patch: + core = types.ModuleType("core") + core.__path__ = [str(CORE)] + module_patch.setitem(sys.modules, "core", core) + authenticators = types.ModuleType("core.authenticators") + authenticators.__path__ = [str(AUTHENTICATORS)] + native_authenticator = type("CustomFirstUseAuthenticator", (), {}) + authenticators.CustomFirstUseAuthenticator = native_authenticator + authenticators.GITHUB_USERNAME_PREFIX = "github:" + module_patch.setitem(sys.modules, "core.authenticators", authenticators) + + database = types.ModuleType("core.database") + database.Base = type("Base", (), {"__init__": lambda self, **kwargs: self.__dict__.update(kwargs)}) + database.session_scope = lambda: (_ for _ in ()).throw(AssertionError("session_scope must be patched")) + module_patch.setitem(sys.modules, "core.database", database) + + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.Boolean = sqlalchemy.DateTime = sqlalchemy.Integer = sqlalchemy.LargeBinary = sqlalchemy.String = ( + lambda *_args: None + ) + sqlalchemy.func = types.SimpleNamespace(now=lambda: None) + sqlalchemy_orm = types.ModuleType("sqlalchemy.orm") + sqlalchemy_orm.Mapped = type("Mapped", (), {"__class_getitem__": classmethod(lambda cls, _item: cls)}) + sqlalchemy_orm.mapped_column = lambda *_args, **_kwargs: None + module_patch.setitem(sys.modules, "sqlalchemy", sqlalchemy) + module_patch.setitem(sys.modules, "sqlalchemy.orm", sqlalchemy_orm) + + jupyterhub = types.ModuleType("jupyterhub") + apihandlers = types.ModuleType("jupyterhub.apihandlers") + handlers = types.ModuleType("jupyterhub.handlers") + orm = types.ModuleType("jupyterhub.orm") + roles = types.ModuleType("jupyterhub.roles") + scopes = types.ModuleType("jupyterhub.scopes") + utils = types.ModuleType("jupyterhub.utils") + apihandlers.APIHandler = type("APIHandler", (), {}) + handlers.BaseHandler = type("BaseHandler", (), {}) + orm.User = type("User", (), {}) + roles.assign_default_roles = lambda *_args, **_kwargs: None + scopes.needs_scope = lambda _scope: lambda handler: handler + + async def maybe_future(value): + return value + + utils.maybe_future = maybe_future + module_patch.setitem(sys.modules, "jupyterhub", jupyterhub) + module_patch.setitem(sys.modules, "jupyterhub.apihandlers", apihandlers) + module_patch.setitem(sys.modules, "jupyterhub.handlers", handlers) + module_patch.setitem(sys.modules, "jupyterhub.orm", orm) + module_patch.setitem(sys.modules, "jupyterhub.roles", roles) + module_patch.setitem(sys.modules, "jupyterhub.scopes", scopes) + module_patch.setitem(sys.modules, "jupyterhub.utils", utils) + + multi = types.ModuleType("multiauthenticator") + multi_authenticator = type("MultiAuthenticator", (), {}) + multi.MultiAuthenticator = multi_authenticator + module_patch.setitem(sys.modules, "multiauthenticator", multi) + quota = types.ModuleType("core.quota") + quota.BatchQuotaRequest = quota.QuotaAction = quota.QuotaModifyRequest = quota.QuotaRefreshRequest = type( + "Quota", (), {} + ) + quota.get_quota_manager = lambda: None + module_patch.setitem(sys.modules, "core.quota", quota) + stats = types.ModuleType("core.stats_handlers") + for name in ( + "StatsActiveSSEHandler", + "StatsDistributionHandler", + "StatsHourlyHandler", + "StatsMyUsageHandler", + "StatsOverviewHandler", + "StatsUsageHandler", + "StatsUserHandler", + ): + setattr(stats, name, type(name, (), {})) + module_patch.setitem(sys.modules, "core.stats_handlers", stats) + + def load(name: str, path: Path) -> types.ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + module_patch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + return module + + models = load("core.authenticators.models", AUTHENTICATORS / "models.py") + handler_module = load("core.handlers", CORE / "handlers.py") + yield types.SimpleNamespace( + database=database, + handlers=handler_module, + models=models, + multi_authenticator=multi_authenticator, + native_authenticator=native_authenticator, + ) diff --git a/runtime/hub/tests/test_onboarding_handlers.py b/runtime/hub/tests/test_onboarding_handlers.py index 27dedea6..f727cfb7 100644 --- a/runtime/hub/tests/test_onboarding_handlers.py +++ b/runtime/hub/tests/test_onboarding_handlers.py @@ -1,395 +1,20 @@ import asyncio -import importlib.util import json -import sys -import types -from contextlib import contextmanager from datetime import datetime, timezone -from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] -CORE = ROOT / "core" -AUTHENTICATORS = CORE / "authenticators" +import pytest +from onboarding_handlers_support import FakeDb, fake_session_scope, load_handlers, make_handler -if "jupyterhub.apihandlers" not in sys.modules: - jupyterhub_module = types.ModuleType("jupyterhub") - apihandlers_module = types.ModuleType("jupyterhub.apihandlers") - handlers_module = types.ModuleType("jupyterhub.handlers") - orm_module = types.ModuleType("jupyterhub.orm") - apihandlers_module.APIHandler = type("APIHandler", (), {}) - handlers_module.BaseHandler = type("BaseHandler", (), {}) - orm_module.User = type("User", (), {}) - scopes_module = types.ModuleType("jupyterhub.scopes") - scopes_module.needs_scope = lambda _scope: lambda handler: handler - sys.modules["jupyterhub"] = jupyterhub_module - sys.modules["jupyterhub.apihandlers"] = apihandlers_module - sys.modules["jupyterhub.handlers"] = handlers_module - sys.modules["jupyterhub.orm"] = orm_module - sys.modules["jupyterhub.scopes"] = scopes_module -if "jupyterhub.roles" not in sys.modules: - roles_module = types.ModuleType("jupyterhub.roles") - roles_module.assign_default_roles = lambda *_args, **_kwargs: None - sys.modules["jupyterhub.roles"] = roles_module +@pytest.fixture +def loaded_handlers(monkeypatch: pytest.MonkeyPatch): + with load_handlers(monkeypatch) as state: + yield state -if "jupyterhub.utils" not in sys.modules: - utils_module = types.ModuleType("jupyterhub.utils") - async def maybe_future(value): - return value - - utils_module.maybe_future = maybe_future - sys.modules["jupyterhub.utils"] = utils_module - -if "multiauthenticator" not in sys.modules: - multiauthenticator_module = types.ModuleType("multiauthenticator") - multiauthenticator_module.MultiAuthenticator = type("MultiAuthenticator", (), {}) - sys.modules["multiauthenticator"] = multiauthenticator_module - -if "core" not in sys.modules: - core_module = types.ModuleType("core") - core_module.__path__ = [str(CORE)] - sys.modules["core"] = core_module - -if "core.authenticators" not in sys.modules: - auth_module = types.ModuleType("core.authenticators") - auth_module.__path__ = [str(AUTHENTICATORS)] - auth_module.CustomFirstUseAuthenticator = type("CustomFirstUseAuthenticator", (), {}) - auth_module.GITHUB_USERNAME_PREFIX = "github:" - sys.modules["core.authenticators"] = auth_module - -if "sqlalchemy" not in sys.modules: - sqlalchemy_module = types.ModuleType("sqlalchemy") - - class _SQLAType: - def __init__(self, *args, **kwargs): - pass - - class _Func: - @staticmethod - def now(): - return None - - sqlalchemy_module.Boolean = _SQLAType - sqlalchemy_module.DateTime = _SQLAType - sqlalchemy_module.Integer = _SQLAType - sqlalchemy_module.LargeBinary = _SQLAType - sqlalchemy_module.String = _SQLAType - sqlalchemy_module.func = _Func() - sys.modules["sqlalchemy"] = sqlalchemy_module - -if "sqlalchemy.orm" in sys.modules: - sqlalchemy_orm_module = sys.modules["sqlalchemy.orm"] -else: - sqlalchemy_orm_module = types.ModuleType("sqlalchemy.orm") - sys.modules["sqlalchemy.orm"] = sqlalchemy_orm_module - - -class Mapped: - def __class_getitem__(cls, _item): - return cls - - -def mapped_column(*args, **kwargs): - return None - - -if not hasattr(sqlalchemy_orm_module, "Mapped"): - sqlalchemy_orm_module.Mapped = Mapped -if not hasattr(sqlalchemy_orm_module, "mapped_column"): - sqlalchemy_orm_module.mapped_column = mapped_column -if not hasattr(sqlalchemy_orm_module, "Session"): - sqlalchemy_orm_module.Session = type("Session", (), {}) - -if "core.database" not in sys.modules: - database_module = types.ModuleType("core.database") - - class Base: - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) - - @contextmanager - def session_scope(): - raise AssertionError("session_scope must be patched in onboarding tests") - - database_module.Base = Base - database_module.session_scope = session_scope - sys.modules["core.database"] = database_module - -if "core.quota" not in sys.modules: - quota_module = types.ModuleType("core.quota") - quota_module.BatchQuotaRequest = type("BatchQuotaRequest", (), {}) - quota_module.QuotaAction = type("QuotaAction", (), {}) - quota_module.QuotaModifyRequest = type("QuotaModifyRequest", (), {}) - quota_module.QuotaRefreshRequest = type("QuotaRefreshRequest", (), {}) - quota_module.get_quota_manager = lambda: None - sys.modules["core.quota"] = quota_module - -if "core.stats_handlers" not in sys.modules: - stats_module = types.ModuleType("core.stats_handlers") - for name in [ - "StatsActiveSSEHandler", - "StatsDistributionHandler", - "StatsHourlyHandler", - "StatsMyUsageHandler", - "StatsOverviewHandler", - "StatsUsageHandler", - "StatsUserHandler", - ]: - setattr(stats_module, name, type(name, (), {})) - sys.modules["core.stats_handlers"] = stats_module - - -def load_module(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -models = load_module("core.authenticators.models", AUTHENTICATORS / "models.py") -handlers = load_module("core.handlers", CORE / "handlers.py") -database = sys.modules["core.database"] - -UserOnboardingState = models.UserOnboardingState -DismissMyOnboardingHandler = handlers.DismissMyOnboardingHandler -GetMyOnboardingHandler = handlers.GetMyOnboardingHandler -AdminResetPasswordHandler = handlers.AdminResetPasswordHandler -AdminAPISetPasswordHandler = handlers.AdminAPISetPasswordHandler -ChangePasswordHandler = handlers.ChangePasswordHandler -AdminAPIProvisionUsersHandler = handlers.AdminAPIProvisionUsersHandler - - -class DummyUser: - def __init__(self, name: str, admin: bool = False): - self.name = name - self.admin = admin - - -class FakeQuery: - def __init__(self, rows): - self._rows = rows - self._filtered = rows - - def filter_by(self, **kwargs): - self._filtered = [ - row for row in self._rows if all(getattr(row, key, None) == value for key, value in kwargs.items()) - ] - return self - - def first(self): - return self._filtered[0] if self._filtered else None - - def all(self): - return self._filtered - - def one_or_none(self): - return self.first() - - -class FakeDb: - def __init__(self, rows=None): - self.rows = rows or [] - self.commits = 0 - - def query(self, _model): - return FakeQuery(self.rows) - - def add(self, obj): - self.rows.append(obj) - - def flush(self): - pass - - def commit(self): - self.commits += 1 - - -class DetachedAwareState: - def __init__(self, username: str, dismissed_at): - self.username = username - self._dismissed_at = dismissed_at - self.detached = False - - @property - def dismissed_at(self): - if self.detached: - raise RuntimeError("detached instance access") - return self._dismissed_at - - @dismissed_at.setter - def dismissed_at(self, value): - self._dismissed_at = value - - -def fake_session_scope(db): - @contextmanager - def _scope(): - yield db - for row in getattr(db, "rows", []): - if hasattr(row, "detached"): - row.detached = True - db.commit() - - return _scope - - -def make_handler(handler_cls, username: str): - handler = object.__new__(handler_cls) - handler.current_user = DummyUser(username) - captured = {} - handler.set_header = lambda key, value: captured.setdefault("headers", {}).__setitem__(key, value) - handler.finish = lambda payload: captured.setdefault("body", payload) - return handler, captured - - -def test_admin_reset_listing_excludes_all_administrators() -> None: - handler = object.__new__(AdminResetPasswordHandler) - handler.current_user = DummyUser("operator", admin=True) - handler.db = FakeDb( - [ - DummyUser("operator", admin=True), - DummyUser("admin", admin=True), - DummyUser("learner"), - DummyUser("github:member"), - ] - ) - handler.get_argument = lambda _name, default="": default - rendered = {} - - async def render_template(_name, **kwargs): - rendered.update(kwargs) - return "html" - - handler.render_template = render_template - handler.finish = lambda _html: None - - asyncio.run(handler.get()) - - assert rendered["native_users"] == ["learner"] - - -def test_admin_provisioning_rejects_username_that_local_login_would_reject(monkeypatch) -> None: - class FakeAuthenticator: - def __init__(self): - self.validated_usernames = [] - - def validate_username(self, username): - self.validated_usernames.append(username) - return username == username.lower() and ":" not in username - - class FakeFirstUseAuthenticator: - def normalize_username(self, username): - return username.lower() - - def _check_password_strength(self, _password): - return None - - def set_password(self, *_args, **_kwargs): - raise AssertionError("invalid username must not set a password") - - authenticator = FakeAuthenticator() - handler = object.__new__(AdminAPIProvisionUsersHandler) - handler.current_user = DummyUser("operator", admin=True) - handler.authenticator = authenticator - handler.request = types.SimpleNamespace( - body=json.dumps({"users": [{"username": "Admin", "password": "Password1!"}]}).encode("utf-8") - ) - handler.find_user = lambda _username: None - captured = {} - handler.set_header = lambda key, value: captured.setdefault("headers", {}).__setitem__(key, value) - handler.finish = lambda payload: captured.setdefault("body", payload) - handler.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) - monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) - - asyncio.run(handler.post()) - - payload = json.loads(captured["body"]) - assert authenticator.validated_usernames == ["Admin"] - assert payload["failed"] == 1 - assert payload["results"][0]["error"] == "Invalid username: Admin" - - -def test_password_management_rejects_the_secret_managed_local_administrator(monkeypatch) -> None: - class FakeFirstUseAuthenticator: - async def authenticate(self, *_args): - raise AssertionError("bootstrap administrator password must not be authenticated for a change") - - def set_password(self, *_args, **_kwargs): - raise AssertionError("bootstrap administrator password must not be changed") - - monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") - monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") - monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) - - change = object.__new__(ChangePasswordHandler) - change.current_user = DummyUser("operator") - change.authenticator = object() - change.hub = types.SimpleNamespace(base_url="/hub/") - change.get_body_argument = lambda name, default=None: { - "current_password": "OldPassword1!", - "new_password": "NewPassword1!", - "confirm_password": "NewPassword1!", - }.get(name, default) - change.set_status = lambda status: setattr(change, "status", status) - change.finish = lambda payload: setattr(change, "body", payload) - - async def render_template(_name, **kwargs): - return kwargs["error_message"] - - change.render_template = render_template - asyncio.run(change.post()) - - assert change.status == 403 - assert "managed by the Kubernetes Secret" in change.body - - admin_api = object.__new__(AdminAPISetPasswordHandler) - admin_api.current_user = DummyUser("manager", admin=True) - admin_api.authenticator = object() - admin_api.request = types.SimpleNamespace( - body=json.dumps({"username": "operator", "password": "NewPassword1!"}).encode("utf-8") - ) - admin_api.set_status = lambda status: setattr(admin_api, "status", status) - admin_api.set_header = lambda *_args: None - admin_api.finish = lambda payload: setattr(admin_api, "body", payload) - admin_api.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) - asyncio.run(admin_api.post()) - - assert admin_api.status == 403 - assert "managed by the Kubernetes Secret" in json.loads(admin_api.body)["error"] - - -def test_admin_password_management_keeps_other_local_users_changeable(monkeypatch) -> None: - class FakeFirstUseAuthenticator: - def set_password(self, username, password, force_change=True): - assert (username, password, force_change) == ("learner", "NewPassword1!", True) - return "Password set for learner (force change on next login)" - - monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") - monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") - monkeypatch.setattr(handlers, "_find_firstuse_authenticator", lambda _authenticator: FakeFirstUseAuthenticator()) - - handler = object.__new__(AdminAPISetPasswordHandler) - handler.current_user = DummyUser("manager", admin=True) - handler.authenticator = object() - handler.request = types.SimpleNamespace( - body=json.dumps({"username": "learner", "password": "NewPassword1!"}).encode("utf-8") - ) - handler.set_header = lambda *_args: None - handler.finish = lambda payload: setattr(handler, "body", payload) - handler.log = types.SimpleNamespace(error=lambda *_args, **_kwargs: None) - - asyncio.run(handler.post()) - - assert json.loads(handler.body)["message"].startswith("Password set for learner") - - -def test_get_my_onboarding_returns_visible_when_no_state_exists(monkeypatch): - monkeypatch.setattr(database, "session_scope", fake_session_scope(FakeDb())) - handler, captured = make_handler(GetMyOnboardingHandler, "alice") +def test_get_my_onboarding_returns_visible_when_no_state_exists(loaded_handlers, monkeypatch) -> None: + monkeypatch.setattr(loaded_handlers.database, "session_scope", lambda: fake_session_scope(FakeDb())) + handler, captured = make_handler(loaded_handlers.handlers.GetMyOnboardingHandler, "alice") asyncio.run(handler.get()) @@ -397,37 +22,46 @@ def test_get_my_onboarding_returns_visible_when_no_state_exists(monkeypatch): assert json.loads(captured["body"]) == {"should_show": True, "dismissed_at": None} -def test_get_my_onboarding_returns_hidden_when_current_user_already_dismissed(monkeypatch): - dismissed_at = datetime(2026, 4, 22, 12, 30, 0, tzinfo=timezone.utc) - db = FakeDb([DetachedAwareState(username="alice", dismissed_at=dismissed_at)]) - monkeypatch.setattr(database, "session_scope", fake_session_scope(db)) - handler, captured = make_handler(GetMyOnboardingHandler, "alice") +def test_get_my_onboarding_returns_hidden_when_current_user_already_dismissed(loaded_handlers, monkeypatch) -> None: + class DetachedAwareState: + def __init__(self, username: str, dismissed_at: datetime) -> None: + self.username = username + self._dismissed_at = dismissed_at + self.detached = False + + @property + def dismissed_at(self) -> datetime: + if self.detached: + raise RuntimeError("detached instance access") + return self._dismissed_at + + dismissed_at = datetime(2026, 4, 22, 12, 30, tzinfo=timezone.utc) + state = DetachedAwareState(username="alice", dismissed_at=dismissed_at) + monkeypatch.setattr(loaded_handlers.database, "session_scope", lambda: fake_session_scope(FakeDb([state]))) + handler, captured = make_handler(loaded_handlers.handlers.GetMyOnboardingHandler, "alice") asyncio.run(handler.get()) assert captured["headers"]["Content-Type"] == "application/json" - assert json.loads(captured["body"]) == { - "should_show": False, - "dismissed_at": dismissed_at.isoformat(), - } + assert json.loads(captured["body"]) == {"should_show": False, "dismissed_at": dismissed_at.isoformat()} -def test_dismiss_my_onboarding_persists_dismissal_for_current_user(monkeypatch): - existing_state = UserOnboardingState( +def test_dismiss_my_onboarding_persists_dismissal_for_current_user(loaded_handlers, monkeypatch) -> None: + existing_state = loaded_handlers.models.UserOnboardingState( username="bob", - dismissed_at=datetime(2026, 4, 21, 8, 0, 0, tzinfo=timezone.utc), + dismissed_at=datetime(2026, 4, 21, 8, tzinfo=timezone.utc), ) db = FakeDb([existing_state]) - monkeypatch.setattr(database, "session_scope", fake_session_scope(db)) - handler, captured = make_handler(DismissMyOnboardingHandler, "alice") + monkeypatch.setattr(loaded_handlers.database, "session_scope", lambda: fake_session_scope(db)) + handler, captured = make_handler(loaded_handlers.handlers.DismissMyOnboardingHandler, "alice") asyncio.run(handler.post()) payload = json.loads(captured["body"]) + dismissed_at = datetime.fromisoformat(payload["dismissed_at"]) assert captured["headers"]["Content-Type"] == "application/json" assert payload["should_show"] is False assert payload["dismissed_at"] is not None - dismissed_at = datetime.fromisoformat(payload["dismissed_at"]) assert dismissed_at.tzinfo == timezone.utc assert db.commits == 1 assert len(db.rows) == 2 diff --git a/runtime/hub/tests/test_password_handlers.py b/runtime/hub/tests/test_password_handlers.py new file mode 100644 index 00000000..4e8611bc --- /dev/null +++ b/runtime/hub/tests/test_password_handlers.py @@ -0,0 +1,228 @@ +import asyncio +import json +from types import SimpleNamespace + +import pytest +from onboarding_handlers_support import DummyUser, FakeDb, load_handlers + + +@pytest.fixture +def loaded_handlers(monkeypatch: pytest.MonkeyPatch): + with load_handlers(monkeypatch) as state: + yield state + + +class PasswordAuthenticator: + def __init__(self) -> None: + self.changes: list[tuple[object, ...]] = [] + + async def authenticate(self, _handler, data): + return data["username"] + + def set_password(self, username, password, force_change=True): + self.changes.append((username, password, force_change)) + return f"Password set for {username}" + + def mark_force_password_change(self, username, force): + self.changes.append(("mark", username, force)) + + def clear_force_password_change(self, username): + self.changes.append(("clear", username)) + + def batch_set_passwords(self, users, force_change=True): + self.changes.extend((entry["username"], entry["password"], force_change) for entry in users) + return {"success": len(users), "failed": 0, "results": []} + + +def configure_local_bootstrap(monkeypatch, handlers) -> None: + monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") + monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") + + +def test_bootstrap_admin_can_change_own_password(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.ChangePasswordHandler) + handler.current_user = DummyUser("operator") + handler.authenticator = object() + handler.hub = SimpleNamespace(base_url="/hub/") + handler.get_body_argument = lambda name, default=None: { + "current_password": "OldPassword1!", + "new_password": "NewPassword1!", + "confirm_password": "NewPassword1!", + }.get(name, default) + handler.set_status = lambda status: setattr(handler, "status", status) + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.redirect = lambda url: setattr(handler, "redirect_url", url) + handler.render_template = lambda _name, **kwargs: kwargs["error_message"] + + asyncio.run(handler.post()) + + assert handler.redirect_url == "/hub/auth/change-password?password_changed=1" + assert authenticator.changes == [("operator", "NewPassword1!", False)] + + +def test_admin_can_reset_bootstrap_administrator(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.AdminResetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.hub = SimpleNamespace(base_url="/hub/") + handler.get_body_argument = lambda name, default=None: { + "target_user": "operator", + "new_password": "NewPassword1!", + "confirm_password": "NewPassword1!", + "force_change": "off", + }.get(name, default) + handler.redirect = lambda url: setattr(handler, "redirect_url", url) + + asyncio.run(handler.post()) + + assert handler.redirect_url == "/hub/admin/reset-password?success=1&user=operator" + assert authenticator.changes == [("operator", "NewPassword1!", False), ("clear", "operator")] + + +def test_admin_reset_listing_excludes_administrators_and_github_users(loaded_handlers) -> None: + handler = object.__new__(loaded_handlers.handlers.AdminResetPasswordHandler) + handler.current_user = DummyUser("operator", admin=True) + handler.db = FakeDb( + [ + DummyUser("operator", admin=True), + DummyUser("admin", admin=True), + DummyUser("learner"), + DummyUser("github:octo"), + ] + ) + handler.get_argument = lambda _name, default="": default + rendered = {} + + async def render_template(_name, **kwargs): + rendered.update(kwargs) + return "html" + + handler.render_template = render_template + handler.finish = lambda _html: None + + asyncio.run(handler.get()) + + assert rendered["native_users"] == ["learner"] + + +def test_admin_api_can_set_bootstrap_administrator_password(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.request = SimpleNamespace(body=b'{"username":"operator","password":"NewPassword1!"}') + handler.set_header = lambda *_args: None + handler.set_status = lambda status: setattr(handler, "status", status) + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = SimpleNamespace(error=lambda *_args, **_kwargs: None) + + asyncio.run(handler.post()) + + assert json.loads(handler.body) == {"message": "Password set for operator"} + assert authenticator.changes == [("operator", "NewPassword1!", True)] + + +def test_admin_api_keeps_other_local_users_changeable(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.request = SimpleNamespace(body=b'{"username":"learner","password":"NewPassword1!"}') + handler.set_header = lambda *_args: None + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = SimpleNamespace(error=lambda *_args, **_kwargs: None) + + asyncio.run(handler.post()) + + assert json.loads(handler.body) == {"message": "Password set for learner"} + assert authenticator.changes == [("learner", "NewPassword1!", True)] + + +def test_admin_api_batch_can_set_bootstrap_administrator_password(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.AdminAPIBatchSetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.request = SimpleNamespace(body=b'{"users":[{"username":"operator","password":"NewPassword1!"}]}') + handler.set_header = lambda *_args: None + handler.set_status = lambda status: setattr(handler, "status", status) + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = SimpleNamespace(error=lambda *_args, **_kwargs: None) + + asyncio.run(handler.post()) + + assert json.loads(handler.body)["success"] == 1 + assert authenticator.changes == [("operator", "NewPassword1!", True)] + + +def test_github_users_remain_blocked_from_native_password_changes(loaded_handlers, monkeypatch) -> None: + authenticator = PasswordAuthenticator() + configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) + handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) + handler.current_user = DummyUser("manager", admin=True) + handler.authenticator = object() + handler.request = SimpleNamespace(body=b'{"username":"github:octo","password":"NewPassword1!"}') + handler.set_header = lambda *_args: None + handler.set_status = lambda status: setattr(handler, "status", status) + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = SimpleNamespace(error=lambda *_args, **_kwargs: None) + + asyncio.run(handler.post()) + + assert handler.status == 400 + assert json.loads(handler.body) == {"error": "Cannot set password for GitHub users"} + assert authenticator.changes == [] + + +def test_admin_provisioning_rejects_username_that_local_login_would_reject(loaded_handlers, monkeypatch) -> None: + class LoginAuthenticator: + def validate_username(self, username): + return username == username.lower() and ":" not in username + + class NativeAuthenticator: + def normalize_username(self, username): + return username.lower() + + def _check_password_strength(self, _password): + return None + + def set_password(self, *_args, **_kwargs): + raise AssertionError("invalid username must not set a password") + + handler = object.__new__(loaded_handlers.handlers.AdminAPIProvisionUsersHandler) + handler.current_user = DummyUser("operator", admin=True) + handler.authenticator = LoginAuthenticator() + handler.request = SimpleNamespace(body=b'{"users":[{"username":"Admin","password":"Password1!"}]}') + handler.find_user = lambda _username: None + handler.set_header = lambda *_args: None + handler.finish = lambda payload: setattr(handler, "body", payload) + handler.log = SimpleNamespace(error=lambda *_args, **_kwargs: None) + monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: NativeAuthenticator()) + + asyncio.run(handler.post()) + + payload = json.loads(handler.body) + assert payload["failed"] == 1 + assert payload["results"][0]["error"] == "Invalid username: Admin" + + +def test_password_handlers_find_native_authenticator_directly_and_in_composition(loaded_handlers) -> None: + native = loaded_handlers.native_authenticator() + composed = loaded_handlers.multi_authenticator() + composed._authenticators = [native] + + assert loaded_handlers.handlers._find_firstuse_authenticator(native) is native + assert loaded_handlers.handlers._find_firstuse_authenticator(composed) is native From c4aa1f57c90b9f8534afc209833a7e1963b22532 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:37:58 +0800 Subject: [PATCH 144/180] refactor(config): add explicit resource access policy --- runtime/hub/core/config.py | 38 +++++- .../hub/tests/test_access_policy_config.py | 121 ++++++++++++++++++ 2 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 runtime/hub/tests/test_access_policy_config.py diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index cf521e33..e8e234a6 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -42,10 +42,10 @@ import warnings from dataclasses import dataclass from pathlib import Path -from typing import Any, Literal +from typing import Any, Final, Literal, assert_never import yaml -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator # ============================================================================= # YAML Configuration Models @@ -149,6 +149,10 @@ def validate_default_path(cls, value: str | None) -> str | None: model_config = {"extra": "allow"} +ResourceAccessPolicy = Literal["all", "group-mapped"] +DEFAULT_RESOURCE_ACCESS_POLICY: Final[ResourceAccessPolicy] = "group-mapped" + + class ResourcesConfig(BaseModel): """Resources configuration (images, requirements, and metadata).""" @@ -156,6 +160,18 @@ class ResourcesConfig(BaseModel): requirements: dict[str, ResourceRequirements] = Field(default_factory=dict) metadata: dict[str, ResourceMetadata] = Field(default_factory=dict) groupOrder: list[str] = Field(default_factory=list) + accessPolicy: ResourceAccessPolicy | None = None + + @model_validator(mode="before") + @classmethod + def reject_explicit_null_access_policy(cls, value: Any) -> Any: + if isinstance(value, dict) and "accessPolicy" in value and value["accessPolicy"] is None: + raise ValueError("accessPolicy must be all or group-mapped") + return value + + @property + def effective_access_policy(self) -> ResourceAccessPolicy: + return self.accessPolicy or DEFAULT_RESOURCE_ACCESS_POLICY model_config = {"extra": "allow"} @@ -361,6 +377,16 @@ def _parse_auth_capabilities(raw_config: dict[str, Any]) -> tuple[AuthCapabiliti return AuthCapabilities(True, False, False, False), False +def _legacy_resource_access_policy(mode: LegacyAuthMode) -> ResourceAccessPolicy: + match mode: + case "auto-login" | "dummy" | "local": + return "all" + case "github" | "multi": + return "group-mapped" + case unreachable: + assert_never(unreachable) + + # ============================================================================= # Hub Configuration Singleton # ============================================================================= @@ -425,7 +451,8 @@ def init(cls, config_path: str | Path) -> HubConfig: # Extract runtime settings instance._auth, legacy_auth = _parse_auth_capabilities(raw_config) - instance.auth_mode = instance._auth.effective_mode + effective_mode = instance._auth.effective_mode + instance.auth_mode = effective_mode if legacy_auth: warnings.warn( "authMode is deprecated; configure authentication with auth provider flags instead", @@ -459,6 +486,11 @@ def init(cls, config_path: str | Path) -> HubConfig: notifications=raw_config.get("notifications"), ) + if instance._config.resources.accessPolicy is None: + instance._config.resources.accessPolicy = ( + _legacy_resource_access_policy(effective_mode) if legacy_auth else DEFAULT_RESOURCE_ACCESS_POLICY + ) + # Canonical providers use neutral policy defaults; legacy input retains historical defaults. if instance._config.quota.enabled is not None: instance.quota_enabled = instance._config.quota.enabled diff --git a/runtime/hub/tests/test_access_policy_config.py b/runtime/hub/tests/test_access_policy_config.py new file mode 100644 index 00000000..c26fc186 --- /dev/null +++ b/runtime/hub/tests/test_access_policy_config.py @@ -0,0 +1,121 @@ +import importlib.util +import sys +import types +import warnings +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "core" / "config.py" +AUTH_FLAGS = ("autoLogin", "dummy", "native", "github") +CANONICAL_PROVIDERS = ( + (True, False, False, False), + (False, True, False, False), + (False, False, True, False), + (False, False, False, True), + (False, False, True, True), +) +LEGACY_POLICIES = { + "auto-login": "all", + "dummy": "all", + "local": "all", + "github": "group-mapped", + "multi": "group-mapped", +} + + +@pytest.fixture +def config_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + spec = importlib.util.spec_from_file_location("task7_access_policy_config", CONFIG) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module + + +def write_config(tmp_path: Path, data: dict[str, object]) -> Path: + path = tmp_path / "hub-config.yaml" + path.write_text(yaml.safe_dump(data), encoding="utf-8") + return path + + +def canonical_auth(flags: tuple[bool, bool, bool, bool]) -> dict[str, dict[str, bool]]: + return {"auth": dict(zip(AUTH_FLAGS, flags, strict=True))} + + +@pytest.mark.parametrize("providers", CANONICAL_PROVIDERS) +def test_canonical_providers_default_to_group_mapped_access( + config_module: types.ModuleType, tmp_path: Path, providers: tuple[bool, bool, bool, bool] +) -> None: + hub_config = config_module.HubConfig.init(write_config(tmp_path, canonical_auth(providers))) + + assert hub_config.resources.effective_access_policy == "group-mapped" + + +def test_absent_auth_forms_default_to_group_mapped_access(config_module: types.ModuleType, tmp_path: Path) -> None: + hub_config = config_module.HubConfig.init(write_config(tmp_path, {"resources": {}})) + + assert hub_config.resources.effective_access_policy == "group-mapped" + + +@pytest.mark.parametrize(("auth_mode", "expected_policy"), LEGACY_POLICIES.items()) +def test_legacy_modes_preserve_implicit_resource_access_policy( + config_module: types.ModuleType, tmp_path: Path, auth_mode: str, expected_policy: str +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config_module.HubConfig.init(write_config(tmp_path, {"authMode": auth_mode})) + + assert hub_config.resources.effective_access_policy == expected_policy + + +@pytest.mark.parametrize("providers", CANONICAL_PROVIDERS) +@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) +def test_explicit_policy_is_independent_of_canonical_provider_and_runtime_policy( + config_module: types.ModuleType, + tmp_path: Path, + providers: tuple[bool, bool, bool, bool], + access_policy: str, +) -> None: + raw_config: dict[str, object] = canonical_auth(providers) + raw_config.update( + { + "singleNodeMode": True, + "quota": {"enabled": False}, + "resources": {"accessPolicy": access_policy}, + } + ) + + hub_config = config_module.HubConfig.init(write_config(tmp_path, raw_config)) + + assert hub_config.resources.effective_access_policy == access_policy + assert hub_config.single_node_mode is True + assert hub_config.quota_enabled is False + + +@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) +@pytest.mark.parametrize("auth_mode", tuple(LEGACY_POLICIES)) +def test_explicit_policy_overrides_legacy_implicit_policy( + config_module: types.ModuleType, tmp_path: Path, auth_mode: str, access_policy: str +) -> None: + raw_config = {"authMode": auth_mode, "resources": {"accessPolicy": access_policy}} + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config_module.HubConfig.init(write_config(tmp_path, raw_config)) + + assert hub_config.resources.effective_access_policy == access_policy + + +@pytest.mark.parametrize("invalid_policy", ("unknown", None, True, 1, ["all"])) +def test_direct_parser_rejects_invalid_explicit_access_policy( + config_module: types.ModuleType, tmp_path: Path, invalid_policy: object +) -> None: + raw_config = {"resources": {"accessPolicy": invalid_policy}} + + with pytest.raises(ValidationError): + config_module.HubConfig.init(write_config(tmp_path, raw_config)) From 6b00c40b7bf96173c709a16ff953fb04ebe3ce42 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:38:24 +0800 Subject: [PATCH 145/180] refactor(runtime): decouple resource resolution from auth --- runtime/hub/core/groups.py | 29 +++--- runtime/hub/tests/groups_test_support.py | 89 +++++++++++++++++++ runtime/hub/tests/test_groups.py | 87 +++--------------- .../hub/tests/test_groups_module_isolation.py | 23 +++++ 4 files changed, 139 insertions(+), 89 deletions(-) create mode 100644 runtime/hub/tests/groups_test_support.py create mode 100644 runtime/hub/tests/test_groups_module_isolation.py diff --git a/runtime/hub/core/groups.py b/runtime/hub/core/groups.py index 2dc1d832..2c09fdbe 100644 --- a/runtime/hub/core/groups.py +++ b/runtime/hub/core/groups.py @@ -32,6 +32,7 @@ import logging import time from contextlib import suppress +from typing import TYPE_CHECKING, assert_never import aiohttp import jwt @@ -41,6 +42,9 @@ from core.authenticators.github_app import GITHUB_USERNAME_PREFIX +if TYPE_CHECKING: + from core.config import ResourceAccessPolicy + log = logging.getLogger("jupyterhub.groups") GITHUB_TEAM_SOURCE = "github-team" @@ -703,23 +707,24 @@ def get_resources_for_user( def resolve_resources_for_user( user: JupyterHubUser, team_resource_mapping: dict[str, list[str]], - auth_mode: str, + access_policy: ResourceAccessPolicy, all_resources: list[str], ) -> list[str]: """Resolve the resources visible to a user for UI and spawn flows.""" username = user.name.strip() - if auth_mode in ["auto-login", "dummy", "local"]: - return all_resources - - available_resources = get_resources_for_user(user, team_resource_mapping) - if available_resources: - return available_resources - - if not username.startswith(GITHUB_USERNAME_PREFIX): - return team_resource_mapping.get("native-users", team_resource_mapping.get("official", [])) - - return ["none"] + match access_policy: + case "all": + return all_resources + case "group-mapped": + available_resources = get_resources_for_user(user, team_resource_mapping) + if available_resources: + return available_resources + if not username.startswith(GITHUB_USERNAME_PREFIX): + return team_resource_mapping.get("native-users", team_resource_mapping.get("official", [])) + return ["none"] + case unreachable: + assert_never(unreachable) def is_readonly_group(group: ORMGroup) -> bool: diff --git a/runtime/hub/tests/groups_test_support.py b/runtime/hub/tests/groups_test_support.py new file mode 100644 index 00000000..ecb8d32e --- /dev/null +++ b/runtime/hub/tests/groups_test_support.py @@ -0,0 +1,89 @@ +import importlib.util +import sys +import types +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" +MODULE_NAMES = ( + "aiohttp", + "jupyterhub", + "jupyterhub.orm", + "jupyterhub.user", + "sqlalchemy", + "sqlalchemy.orm", + "core", + "core.authenticators", + "core.authenticators.github_app", + "core.groups", +) +MISSING = object() + + +def load_groups_module() -> types.ModuleType: + original_modules = {name: sys.modules.get(name, MISSING) for name in MODULE_NAMES} + try: + aiohttp_module = types.ModuleType("aiohttp") + aiohttp_module.ClientSession = object + jupyterhub_module = types.ModuleType("jupyterhub") + jupyterhub_module.__path__ = [] + orm_module = types.ModuleType("jupyterhub.orm") + orm_module.Group = type("Group", (), {}) + user_module = types.ModuleType("jupyterhub.user") + user_module.User = type("User", (), {}) + jupyterhub_module.orm, jupyterhub_module.user = orm_module, user_module + sqlalchemy_module = types.ModuleType("sqlalchemy") + sqlalchemy_module.__path__ = [] + sa_orm_module = types.ModuleType("sqlalchemy.orm") + sa_orm_module.Session = type("Session", (), {}) + sqlalchemy_module.orm = sa_orm_module + core_module = types.ModuleType("core") + core_module.__path__ = [str(CORE)] + authenticators_module = types.ModuleType("core.authenticators") + authenticators_module.__path__ = [str(CORE / "authenticators")] + github_app_module = types.ModuleType("core.authenticators.github_app") + github_app_module.GITHUB_USERNAME_PREFIX = "github:" + authenticators_module.github_app = github_app_module + core_module.authenticators = authenticators_module + sys.modules.update( + { + "aiohttp": aiohttp_module, + "jupyterhub": jupyterhub_module, + "jupyterhub.orm": orm_module, + "jupyterhub.user": user_module, + "sqlalchemy": sqlalchemy_module, + "sqlalchemy.orm": sa_orm_module, + "core": core_module, + "core.authenticators": authenticators_module, + "core.authenticators.github_app": github_app_module, + } + ) + spec = importlib.util.spec_from_file_location("core.groups", CORE / "groups.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["core.groups"] = module + spec.loader.exec_module(module) + return module + finally: + for name, original_module in original_modules.items(): + if original_module is MISSING: + sys.modules.pop(name, None) + else: + sys.modules[name] = original_module + + +class DummyGroup: + def __init__(self, name: str, source: str = "github-team") -> None: + self.name = name + self.properties = {"source": source} + + +class DummyOrmUser: + def __init__(self, groups: list[DummyGroup]) -> None: + self.groups = groups + + +class DummyUser: + def __init__(self, groups: list[DummyGroup], name: str = "github:test") -> None: + self.name = name + self.orm_user = DummyOrmUser(groups) diff --git a/runtime/hub/tests/test_groups.py b/runtime/hub/tests/test_groups.py index 3276b4dd..3a1d4ac8 100644 --- a/runtime/hub/tests/test_groups.py +++ b/runtime/hub/tests/test_groups.py @@ -18,60 +18,10 @@ # SOFTWARE. import asyncio -import importlib.util -import sys -import types -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -CORE = ROOT / "core" - -if "aiohttp" not in sys.modules: - aiohttp_module = types.ModuleType("aiohttp") - aiohttp_module.ClientSession = object - sys.modules["aiohttp"] = aiohttp_module - -if "jupyterhub.orm" not in sys.modules: - orm_module = types.ModuleType("jupyterhub.orm") - orm_module.Group = type("Group", (), {}) - sys.modules["jupyterhub.orm"] = orm_module - -if "jupyterhub.user" not in sys.modules: - user_module = types.ModuleType("jupyterhub.user") - user_module.User = type("User", (), {}) - sys.modules["jupyterhub.user"] = user_module - -if "sqlalchemy.orm" not in sys.modules: - sa_orm_module = types.ModuleType("sqlalchemy.orm") - sa_orm_module.Session = type("Session", (), {}) - sys.modules["sqlalchemy.orm"] = sa_orm_module - -if "core" not in sys.modules: - core_module = types.ModuleType("core") - core_module.__path__ = [str(CORE)] - sys.modules["core"] = core_module - -if "core.authenticators" not in sys.modules: - authenticators_module = types.ModuleType("core.authenticators") - authenticators_module.__path__ = [str(CORE / "authenticators")] - sys.modules["core.authenticators"] = authenticators_module - -if "core.authenticators.github_app" not in sys.modules: - github_app_module = types.ModuleType("core.authenticators.github_app") - github_app_module.GITHUB_USERNAME_PREFIX = "github:" - sys.modules["core.authenticators.github_app"] = github_app_module - - -def load_module(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -groups = load_module("core.groups", CORE / "groups.py") + +from groups_test_support import DummyGroup, DummyUser, load_groups_module + +groups = load_groups_module() resolve_resources_for_user = groups.resolve_resources_for_user fetch_github_team_members = groups.fetch_github_team_members get_github_app_installation_token = groups.get_github_app_installation_token @@ -79,23 +29,6 @@ def load_module(name: str, path: Path): sync_user_github_teams = groups.sync_user_github_teams -class DummyGroup: - def __init__(self, name, source="github-team"): - self.name = name - self.properties = {"source": source} - - -class DummyOrmUser: - def __init__(self, groups): - self.groups = groups - - -class DummyUser: - def __init__(self, groups, name="github:test"): - self.name = name - self.orm_user = DummyOrmUser(groups) - - class DummyQuery: def filter_by(self, **kwargs): return self @@ -309,7 +242,7 @@ def test_resolve_resources_for_user_uses_group_mapping(): resources = resolve_resources_for_user( user, {"team-a": ["cpu", "course-a"], "team-b": ["course-a", "course-b"]}, - "multi", + "group-mapped", ["cpu", "gpu", "code-cpu", "course-a", "course-b"], ) @@ -317,13 +250,13 @@ def test_resolve_resources_for_user_uses_group_mapping(): assert resources.count("course-a") == 1 -def test_resolve_resources_for_user_falls_back_for_native_users(): +def test_resolve_resources_for_group_mapped_native_user_uses_native_users_mapping(): user = DummyUser([], name="native-user") resources = resolve_resources_for_user( user, {"official": ["cpu"], "native-users": ["code-cpu"]}, - "multi", + "group-mapped", ["cpu", "gpu", "code-cpu"], ) @@ -333,14 +266,14 @@ def test_resolve_resources_for_user_falls_back_for_native_users(): def test_resolve_resources_for_user_denies_unmapped_github_users(): user = DummyUser([]) - resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "multi", ["cpu", "gpu"]) + resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "group-mapped", ["cpu", "gpu"]) assert resources == ["none"] -def test_resolve_resources_for_user_uses_all_resources_for_auto_login(): +def test_resolve_resources_for_user_uses_all_resources_for_all_policy(): user = DummyUser([], name="demo-user") - resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "auto-login", ["cpu", "gpu", "code-cpu"]) + resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "all", ["cpu", "gpu", "code-cpu"]) assert resources == ["cpu", "gpu", "code-cpu"] diff --git a/runtime/hub/tests/test_groups_module_isolation.py b/runtime/hub/tests/test_groups_module_isolation.py new file mode 100644 index 00000000..576b2d1a --- /dev/null +++ b/runtime/hub/tests/test_groups_module_isolation.py @@ -0,0 +1,23 @@ +import subprocess +import sys +from itertools import permutations +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +GROUPS = "runtime/hub/tests/test_groups.py" +ONBOARDING = "runtime/hub/tests/test_onboarding_handlers.py" + + +@pytest.mark.parametrize("test_order", permutations((GROUPS, ONBOARDING))) +def test_groups_collection_does_not_contaminate_onboarding(test_order: tuple[str, str]) -> None: + result = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--collect-only", "-q", *test_order], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr From c4c5ed8b0b1a9c16dab01c0d9da6d658017ec4b2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:38:52 +0800 Subject: [PATCH 146/180] refactor(runtime): propagate resource access policy --- runtime/hub/core/handlers.py | 9 +- runtime/hub/core/spawner/kubernetes.py | 6 +- runtime/hub/tests/provider_setup_support.py | 25 +++ runtime/hub/tests/test_auth_provider_setup.py | 52 +++---- .../hub/tests/test_resource_access_runtime.py | 143 ++++++++++++++++++ 5 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 runtime/hub/tests/provider_setup_support.py create mode 100644 runtime/hub/tests/test_resource_access_runtime.py diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index c47360ca..4d7605db 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -145,15 +145,12 @@ def configure_handlers( platform_name: str = "AUP Learning Cloud", ) -> None: """Configure handler module with runtime settings.""" - if accelerator_options is not None: - _handler_config["accelerator_options"] = accelerator_options - if quota_rates is not None: - _handler_config["quota_rates"] = quota_rates + _handler_config["accelerator_options"] = accelerator_options or {} + _handler_config["quota_rates"] = quota_rates or {} _handler_config["quota_enabled"] = quota_enabled _handler_config["minimum_quota_to_start"] = minimum_quota_to_start _handler_config["default_quota"] = default_quota - if team_resource_mapping is not None: - _handler_config["team_resource_mapping"] = team_resource_mapping + _handler_config["team_resource_mapping"] = team_resource_mapping or {} _handler_config["github_org"] = github_org _handler_config["auth_mode"] = auth_mode _handler_config["access_policy"] = access_policy diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index 7cd093c8..dba21dae 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -50,7 +50,7 @@ ) if TYPE_CHECKING: - from core.config import HubConfig + from core.config import HubConfig, ResourceAccessPolicy # NPU Security Config @@ -91,6 +91,7 @@ class RemoteLabKubeSpawner(KubeSpawner): # Runtime settings (set by jupyterhub_config.py) github_org_name: str = "" auth_mode: str = "auto-login" + access_policy: ResourceAccessPolicy = "group-mapped" single_node_mode: bool = False quota_enabled: bool | None = False @@ -132,6 +133,7 @@ def configure_from_config(cls, config: HubConfig) -> None: # Basic spawner settings cls.auth_mode = config.auth_mode + cls.access_policy = config.resources.effective_access_policy cls.single_node_mode = config.single_node_mode cls.github_org_name = config.github_org_name @@ -189,7 +191,7 @@ def _resolve_user_resources(self) -> list[str]: available_resources = resolve_resources_for_user( self.user, self.team_resource_mapping, - self.auth_mode, + self.access_policy, list(self.resource_images.keys()), ) self.log.debug(f"User '{username}' resolved resources: {available_resources}") diff --git a/runtime/hub/tests/provider_setup_support.py b/runtime/hub/tests/provider_setup_support.py new file mode 100644 index 00000000..9a64c3f8 --- /dev/null +++ b/runtime/hub/tests/provider_setup_support.py @@ -0,0 +1,25 @@ +import types + +GITHUB_SETTINGS = { + "hub.config.GitHubOAuthenticator.app_id": "app-id", + "hub.config.GitHubOAuthenticator.installation_id": "installation-id", + "hub.config.GitHubOAuthenticator.private_key": "private-key", + "hub.config.GitHubOAuthenticator.private_key_file": "private-key-file", + "hub.config.GitHubOAuthenticator.team_sync_ttl_seconds": 123, +} + + +def make_config(auth: object, access_policy: str) -> types.SimpleNamespace: + return types.SimpleNamespace( + auth=auth, + auth_mode=auth.effective_mode, + resources=types.SimpleNamespace(effective_access_policy=access_policy), + accelerators={}, + build_quota_rates=lambda: {}, + quota_enabled=True, + quota=types.SimpleNamespace(minimumToStart=0, defaultQuota=0), + teams=types.SimpleNamespace(mapping={"learners": ["cpu"]}), + github_org_name="example-org", + platform_display_name="AUP Learning Cloud", + cluster_name="", + ) diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py index f5f99eca..76c15a7c 100644 --- a/runtime/hub/tests/test_auth_provider_setup.py +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -7,17 +7,11 @@ import anyio import pytest +from provider_setup_support import GITHUB_SETTINGS, make_config ROOT = Path(__file__).resolve().parents[1] SETUP = ROOT / "core" / "setup.py" CONFIG = ROOT / "core" / "config.py" -GITHUB_SETTINGS = { - "hub.config.GitHubOAuthenticator.app_id": "app-id", - "hub.config.GitHubOAuthenticator.installation_id": "installation-id", - "hub.config.GitHubOAuthenticator.private_key": "private-key", - "hub.config.GitHubOAuthenticator.private_key_file": "private-key-file", - "hub.config.GitHubOAuthenticator.team_sync_ttl_seconds": 123, -} MODULE_NAMES = tuple( ( "bcrypt|core|core.z2jh|core.config|core.authenticators|core.database|core.handlers|core.metrics_updater|" @@ -30,24 +24,13 @@ _module = types.ModuleType -def _config_for(auth: object) -> types.SimpleNamespace: - return types.SimpleNamespace( - auth=auth, - auth_mode=auth.effective_mode, - accelerators={}, - build_quota_rates=lambda: {}, - quota_enabled=True, - quota=types.SimpleNamespace(minimumToStart=0, defaultQuota=0), - teams=types.SimpleNamespace(mapping={"learners": ["cpu"]}), - github_org_name="example-org", - platform_display_name="AUP Learning Cloud", - cluster_name="", - ) - - @contextmanager def _loaded_setup( - monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool], *, fail_setup: bool = False + monkeypatch: pytest.MonkeyPatch, + providers: tuple[bool, bool, bool, bool], + *, + access_policy: str = "group-mapped", + fail_setup: bool = False, ) -> Iterator[types.SimpleNamespace]: with monkeypatch.context() as module_patch: for variable in ("JUPYTERHUB_ADMIN_PASSWORD", "JUPYTERHUB_ADMIN_USERNAME", "JUPYTERHUB_API_TOKEN"): @@ -70,7 +53,7 @@ def _loaded_setup( core.config = config_module config_spec.loader.exec_module(config_module) auth = config_module.AuthCapabilities(*providers) - config = _config_for(auth) + config = make_config(auth, access_policy) config_module.HubConfig._instance, config_module.HubConfig._initialized = config, True settings_reads: list[str] = [] @@ -119,14 +102,19 @@ def create_authenticator(_input: object) -> type | str: database = _module("core.database") database.init_database = database.create_all_tables = lambda *_args: None module_patch.setitem(sys.modules, "core.database", database) + handler_configs: list[dict[str, object]] = [] handlers = _module("core.handlers") - handlers.configure_handlers, handlers.get_handlers = lambda **_kwargs: None, lambda: [] + handlers.configure_handlers = lambda **kwargs: handler_configs.append(kwargs) + handlers.get_handlers = lambda: [] module_patch.setitem(sys.modules, "core.handlers", handlers) metrics = _module("core.metrics_updater") metrics.start_metrics_updater = lambda: None module_patch.setitem(sys.modules, "core.metrics_updater", metrics) + spawner_configs: list[object] = [] spawner = _module("core.spawner") - spawner.RemoteLabKubeSpawner = type("RemoteLabKubeSpawner", (), {"configure_from_config": lambda _config: None}) + spawner.RemoteLabKubeSpawner = type( + "RemoteLabKubeSpawner", (), {"configure_from_config": lambda config: spawner_configs.append(config)} + ) module_patch.setitem(sys.modules, "core.spawner", spawner) group_assignments: list[tuple[str, str]] = [] @@ -178,12 +166,15 @@ async def sync_github_teams_for_user(*args: object, **kwargs: object) -> bool: ) yield types.SimpleNamespace( auth=auth, + config=config, c=c, factory_inputs=factory_inputs, group_assignments=group_assignments, team_syncs=team_syncs, authenticator_types=authenticator_types, settings_reads=settings_reads, + handler_configs=handler_configs, + spawner_configs=spawner_configs, setup=setup_module, ) @@ -225,6 +216,15 @@ def test_native_only_setup_never_reads_github_settings(monkeypatch: pytest.Monke assert not any(key.startswith("hub.config.GitHubOAuthenticator") for key in state.settings_reads) +@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) +def test_setup_propagates_effective_access_policy(monkeypatch: pytest.MonkeyPatch, access_policy: str) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True), access_policy=access_policy) as state: + state.setup.setup_hub(state.c) + + assert state.spawner_configs == [state.config] + assert state.handler_configs[0]["access_policy"] == access_policy + + @pytest.mark.parametrize("providers", ((False, False, False, True), (False, False, True, True))) def test_github_prefixed_users_sync_teams_for_each_github_capability( monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool] diff --git a/runtime/hub/tests/test_resource_access_runtime.py b/runtime/hub/tests/test_resource_access_runtime.py new file mode 100644 index 00000000..6317d61f --- /dev/null +++ b/runtime/hub/tests/test_resource_access_runtime.py @@ -0,0 +1,143 @@ +import asyncio +import importlib.util +import json +import sys +import types +from pathlib import Path + +import pytest +from onboarding_handlers_support import load_handlers + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" +GROUP_TEST = ROOT / "tests" / "test_groups.py" + + +def load_groups_test_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + spec = importlib.util.spec_from_file_location("task7_groups_test", GROUP_TEST) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module + + +def load_spawner(monkeypatch: pytest.MonkeyPatch, groups: types.ModuleType) -> type: + core = types.ModuleType("core") + core.__path__ = [str(CORE)] + metrics = types.ModuleType("core.metrics") + metric = type( + "Metric", + (), + {"labels": lambda self, **_kwargs: self, "inc": lambda self: None, "observe": lambda self, _value: None}, + )() + for name in ( + "pod_failure_total", + "repo_clone_failed_total", + "session_runtime_minutes", + "spawn_duration_seconds", + "spawn_failed_total", + "spawn_gpu_total", + ): + setattr(metrics, name, metric) + jupyterhub = types.ModuleType("jupyterhub") + jupyterhub.__path__ = [] + user = types.ModuleType("jupyterhub.user") + user.User = type("User", (), {}) + kubespawner = types.ModuleType("kubespawner") + kubespawner.KubeSpawner = type("KubeSpawner", (), {}) + tornado = types.ModuleType("tornado") + web = types.ModuleType("tornado.web") + web.HTTPError = RuntimeError + monkeypatch.setitem(sys.modules, "core", core) + monkeypatch.setitem(sys.modules, "core.metrics", metrics) + monkeypatch.setitem(sys.modules, "core.groups", groups) + monkeypatch.setitem(sys.modules, "jupyterhub", jupyterhub) + monkeypatch.setitem(sys.modules, "jupyterhub.user", user) + monkeypatch.setitem(sys.modules, "kubespawner", kubespawner) + monkeypatch.setitem(sys.modules, "tornado", tornado) + monkeypatch.setitem(sys.modules, "tornado.web", web) + spec = importlib.util.spec_from_file_location("core.spawner.kubernetes", CORE / "spawner" / "kubernetes.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module.RemoteLabKubeSpawner + + +@pytest.mark.parametrize( + ("access_policy", "expected_resources"), + (("all", ["cpu", "gpu", "code-cpu"]), ("group-mapped", ["gpu"])), +) +def test_spawner_uses_configured_access_policy_for_exact_resource_list( + monkeypatch: pytest.MonkeyPatch, access_policy: str, expected_resources: list[str] +) -> None: + groups_test = load_groups_test_module(monkeypatch) + spawner_type = load_spawner(monkeypatch, groups_test.groups) + spawner = object.__new__(spawner_type) + spawner.user = groups_test.DummyUser([groups_test.DummyGroup("team-gpu")], name="native-user") + spawner.team_resource_mapping = {"team-gpu": ["gpu"], "native-users": ["cpu"]} + spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image", "code-cpu": "code-image"} + spawner.access_policy = access_policy + spawner.log = types.SimpleNamespace(debug=lambda _message: None) + + assert asyncio.run(spawner.get_user_resources()) == expected_resources + + +@pytest.mark.parametrize( + ("access_policy", "expected_resources"), + (("all", ["code-cpu", "cpu", "gpu"]), ("group-mapped", ["gpu"])), +) +def test_resources_api_uses_configured_access_policy_for_exact_resource_list( + monkeypatch: pytest.MonkeyPatch, access_policy: str, expected_resources: list[str] +) -> None: + groups_test = load_groups_test_module(monkeypatch) + monkeypatch.delitem(sys.modules, "tornado", raising=False) + monkeypatch.delitem(sys.modules, "tornado.web", raising=False) + with load_handlers(monkeypatch) as loaded: + config = types.SimpleNamespace( + resources=types.SimpleNamespace( + images={"cpu": "cpu-image", "gpu": "gpu-image", "code-cpu": "code-image"}, groupOrder=[] + ), + accelerators={}, + git_clone=types.SimpleNamespace( + allowedProviders=[], githubAppName="", allowPersistenceChoice=False, defaultPersistence=True + ), + get_resource_image=lambda key: {"cpu": "cpu-image", "gpu": "gpu-image", "code-cpu": "code-image"}.get(key), + get_resource_requirements=lambda _key: None, + get_resource_metadata=lambda _key: None, + ) + config_module = types.ModuleType("core.config") + config_module.HubConfig = type("HubConfig", (), {"get": staticmethod(lambda: config)}) + monkeypatch.setitem(sys.modules, "core.config", config_module) + monkeypatch.setitem(sys.modules, "core.groups", groups_test.groups) + loaded.handlers.configure_handlers( + team_resource_mapping={"team-gpu": ["gpu"], "native-users": ["cpu"]}, access_policy=access_policy + ) + handler = object.__new__(loaded.handlers.ResourcesAPIHandler) + handler.current_user = groups_test.DummyUser([groups_test.DummyGroup("team-gpu")], name="native-user") + response: dict[str, str] = {} + handler.set_header = lambda _key, _value: None + handler.finish = lambda body: response.setdefault("body", body) + + asyncio.run(handler.get()) + + assert [resource["key"] for resource in json.loads(response["body"])["resources"]] == expected_resources + + +def test_configure_handlers_replaces_optional_policy_and_mapping_state(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(sys.modules, "tornado", raising=False) + monkeypatch.delitem(sys.modules, "tornado.web", raising=False) + with load_handlers(monkeypatch) as loaded: + loaded.handlers.configure_handlers( + accelerator_options={"gpu": {}}, + quota_rates={"gpu": 2}, + team_resource_mapping={"team": ["gpu"]}, + access_policy="all", + ) + loaded.handlers.configure_handlers(access_policy="group-mapped") + + assert loaded.handlers._handler_config["accelerator_options"] == {} + assert loaded.handlers._handler_config["quota_rates"] == {} + assert loaded.handlers._handler_config["team_resource_mapping"] == {} + assert loaded.handlers._handler_config["access_policy"] == "group-mapped" From 0146cc7e21f611a8fc0b36da14084f11497ec60e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:39:11 +0800 Subject: [PATCH 147/180] refactor(chart): validate resource access policy --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 9 ++++ tests/installer/test_access_policy_schema.py | 51 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/installer/test_access_policy_schema.py diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 79bb90be..9b0a5be0 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":["string","null"],"enum":[null,"auto-login","dummy","github","local","multi"]},"auth":{"type":"object","additionalProperties":false,"properties":{"autoLogin":{"type":"boolean"},"dummy":{"type":"boolean"},"native":{"type":"boolean"},"github":{"type":"boolean"}},"oneOf":[{"required":["autoLogin"],"properties":{"autoLogin":{"const":true},"dummy":{"const":false},"native":{"const":false},"github":{"const":false}}},{"required":["dummy"],"properties":{"autoLogin":{"const":false},"dummy":{"const":true},"native":{"const":false},"github":{"const":false}}},{"required":["native"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":false}}},{"required":["github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":false},"github":{"const":true}}},{"required":["native","github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":true}}}]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"not":{"required":["authMode","auth"]}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"oneOf":[{"required":["auth"],"properties":{"auth":{"required":["native"],"properties":{"native":{"const":true}}}}},{"required":["authMode"],"properties":{"authMode":{"enum":["local","multi"]}}}],"properties":{"adminUser":{"required":["username"],"properties":{"username":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":["string","null"],"enum":[null,"auto-login","dummy","github","local","multi"]},"auth":{"type":"object","additionalProperties":false,"properties":{"autoLogin":{"type":"boolean"},"dummy":{"type":"boolean"},"native":{"type":"boolean"},"github":{"type":"boolean"}},"oneOf":[{"required":["autoLogin"],"properties":{"autoLogin":{"const":true},"dummy":{"const":false},"native":{"const":false},"github":{"const":false}}},{"required":["dummy"],"properties":{"autoLogin":{"const":false},"dummy":{"const":true},"native":{"const":false},"github":{"const":false}}},{"required":["native"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":false}}},{"required":["github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":false},"github":{"const":true}}},{"required":["native","github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":true}}}]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"accessPolicy":{"type":"string","enum":["all","group-mapped"]},"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"not":{"required":["authMode","auth"]}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"oneOf":[{"required":["auth"],"properties":{"auth":{"required":["native"],"properties":{"native":{"const":true}}}}},{"required":["authMode"],"properties":{"authMode":{"enum":["local","multi"]}}}],"properties":{"adminUser":{"required":["username"],"properties":{"username":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 04026014..d9b1eb15 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3429,6 +3429,15 @@ properties: Resource images and requirements configuration. Defines available container images and their resource requirements. properties: + accessPolicy: + type: string + enum: + - all + - group-mapped + description: | + Controls resource visibility independently of authentication providers. + `all` exposes every configured resource; `group-mapped` uses the + configured team mapping and native/GitHub fallback behavior. images: type: object additionalProperties: diff --git a/tests/installer/test_access_policy_schema.py b/tests/installer/test_access_policy_schema.py new file mode 100644 index 00000000..4ee4b14a --- /dev/null +++ b/tests/installer/test_access_policy_schema.py @@ -0,0 +1,51 @@ +import json +import subprocess +from pathlib import Path + +import pytest +import yaml + +from scripts.generate_values_schema import remove_descriptions + +ROOT = Path(__file__).resolve().parents[2] +CHART = "runtime/chart" + + +def render(*settings: str, string_settings: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: + command = ["helm", "template", "jupyterhub", CHART] + for setting in settings: + command.extend(("--set", setting)) + for setting in string_settings: + command.extend(("--set-string", setting)) + return subprocess.run(command, cwd=ROOT, check=False, capture_output=True, text=True) + + +@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) +def test_chart_accepts_each_access_policy_literal(access_policy: str) -> None: + result = render(f"custom.resources.accessPolicy={access_policy}") + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("setting", "string_settings"), + [ + ("custom.resources.accessPolicy=unknown", ()), + ("custom.resources.accessPolicy=true", ()), + ("custom.resources.accessPolicy=1", ()), + ("", ("custom.resources.accessPolicy=unknown",)), + ], +) +def test_chart_rejects_invalid_access_policy(setting: str, string_settings: tuple[str, ...]) -> None: + settings = (setting,) if setting else () + result = render(*settings, string_settings=string_settings) + + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr + + +def test_chart_yaml_and_json_schema_remain_exactly_in_sync() -> None: + yaml_schema = yaml.safe_load((ROOT / "runtime/chart/values.schema.yaml").read_text()) + json_schema = json.loads((ROOT / "runtime/chart/values.schema.json").read_text()) + + assert json_schema == remove_descriptions(yaml_schema) From cf30d421c64df4dcd7b206b1c9825c22d6fad6a4 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:03:37 +0800 Subject: [PATCH 148/180] fix(auth): render native login in composed auth --- runtime/hub/core/authenticators/multi.py | 21 ++++--- .../tests/test_multi_authenticator_html.py | 59 +++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 runtime/hub/tests/test_multi_authenticator_html.py diff --git a/runtime/hub/core/authenticators/multi.py b/runtime/hub/core/authenticators/multi.py index 7f509763..81e1caba 100644 --- a/runtime/hub/core/authenticators/multi.py +++ b/runtime/hub/core/authenticators/multi.py @@ -28,7 +28,7 @@ from multiauthenticator import MultiAuthenticator from multiauthenticator.multiauthenticator import PREFIX_SEPARATOR -LOCAL_ACCOUNT_PREFIX = "LocalAccount" +from core.authenticators.firstuse import CustomFirstUseAuthenticator class CustomMultiAuthenticator(MultiAuthenticator): @@ -83,34 +83,37 @@ def get_custom_html(self, base_url): login_service = getattr(authenticator, "login_service", name) url = authenticator.login_url(base_url) - if name == LOCAL_ACCOUNT_PREFIX: - html.append(f""" + match authenticator: + case CustomFirstUseAuthenticator(): + html.append(f""" <div class="login-option mb-6 bg-white rounded-xl shadow-lg p-6"> - <form action="{url}" method="post"> + <form action="{url}{{% if next is defined and next|length %}}?next={{{{ next | urlencode }}}}{{% endif %}}" method="post"> <input type="hidden" name="_xsrf" value="{{{{ xsrf }}}}" /> <div class="mb-4"> <input type="text" name="username" placeholder="Username" + aria-label="Username" class="block w-full px-4 py-2 border rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" required /> </div> <div class="mb-4"> <input type="password" name="password" placeholder="Password" + aria-label="Password" class="block w-full px-4 py-2 border rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" required /> </div> <button type="submit" - class="w-full py-2 px-4 bg-blue-600 hover:bg-blue-700 text-white font-medium rounded-md"> + class="login-submit w-full py-2 px-4 bg-blue-600 hover:bg-blue-700 text-white font-medium rounded-md"> Use LocalAccount Login </button> </form> </div> """) - else: - html.append(f""" + case _: + html.append(f""" <div class="login-option mb-4"> - <a role="button" class="w-full inline-block text-center py-3 px-4 bg-gray-800 text-white + <a role="button" class="login-github-button w-full inline-block text-center py-3 px-4 bg-gray-800 rounded-md hover:bg-gray-900 font-medium" - href="{url}{{% if next is defined and next|length %}}?next={{{{next}}}}{{% endif %}}"> + href="{url}{{% if next is defined and next|length %}}?next={{{{ next }}}}{{% endif %}}"> Use {login_service} Login </a> </div> diff --git a/runtime/hub/tests/test_multi_authenticator_html.py b/runtime/hub/tests/test_multi_authenticator_html.py new file mode 100644 index 00000000..44968764 --- /dev/null +++ b/runtime/hub/tests/test_multi_authenticator_html.py @@ -0,0 +1,59 @@ +import pytest +from auth_template_support import loaded_multi_authenticator, probe_html, render_multi_html + +NEXT_CASES = ( + ( + "/hub/spawn?x=1&y=two words", + "%2Fhub%2Fspawn%3Fx%3D1%26y%3Dtwo+words", + "%252Fhub%252Fspawn%253Fx%253D1%2526y%253Dtwo%2Bwords", + ), + ( + "/路径?值=你好 世界", + "%2F%E8%B7%AF%E5%BE%84%3F%E5%80%BC%3D%E4%BD%A0%E5%A5%BD+%E4%B8%96%E7%95%8C", + "%252F%25E8%25B7%25AF%25E5%25BE%2584%253F%25E5%2580%25BC%253D%25E4%25BD%25A0%25E5%25A5%25BD%2B%25E4%25B8%2596%25E7%2595%258C", + ), + ("", "", ""), +) + + +@pytest.mark.parametrize(("next_value", "escaped_next", "form_next"), NEXT_CASES) +def test_native_child_renders_inline_form_with_encoded_next( + monkeypatch: pytest.MonkeyPatch, + next_value: str, + escaped_next: str, + form_next: str, +) -> None: + with loaded_multi_authenticator(monkeypatch) as state: + state.multi._authenticators = [state.native] + probe = probe_html(render_multi_html(state, next_value)) + + expected_action = "/hub/native/login" + (f"?next={form_next}" if escaped_next else "") + assert [form.get("action") for form in probe.forms] == [expected_action] + fields = {field.get("name"): field for field in probe.inputs} + assert fields["_xsrf"].get("value") == "csrf-token" + assert fields["username"].get("placeholder") == "Username" + assert fields["username"].get("aria-label") == "Username" + assert "required" in fields["username"] + assert fields["password"].get("placeholder") == "Password" + assert fields["password"].get("aria-label") == "Password" + assert "required" in fields["password"] + assert "login-submit" in (probe.buttons[0].get("class") or "").split() + + +@pytest.mark.parametrize(("next_value", "escaped_next", "form_next"), NEXT_CASES) +def test_external_child_renders_encoded_link_even_with_empty_prefix( + monkeypatch: pytest.MonkeyPatch, + next_value: str, + escaped_next: str, + form_next: str, +) -> None: + with loaded_multi_authenticator(monkeypatch) as state: + state.multi._authenticators = [state.external] + probe = probe_html(render_multi_html(state, next_value)) + + expected_href = "/hub/github/oauth_login" + (f"?next={escaped_next}" if form_next else "") + assert probe.hrefs == [expected_href] + assert probe.forms == [] + classes = (probe.anchors[0].get("class") or "").split() + assert "login-github-button" in classes + assert "text-white" not in classes From c38f5d39f1c68b9c2364f4168d0d1004c7a02c73 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:06:18 +0800 Subject: [PATCH 149/180] refactor(ui): render authentication from capabilities --- runtime/hub/core/authenticators/multi.py | 10 +- runtime/hub/core/setup.py | 28 ++- .../templates/admin-reset-password.html | 2 + .../frontend/templates/change-password.html | 2 + runtime/hub/frontend/templates/login.html | 57 +----- runtime/hub/frontend/templates/page.html | 6 +- runtime/hub/tests/auth_template_support.py | 189 +++++++++++++++++ runtime/hub/tests/test_auth_templates.py | 190 ++++++++++++++++++ .../tests/test_multi_authenticator_html.py | 10 +- 9 files changed, 432 insertions(+), 62 deletions(-) create mode 100644 runtime/hub/tests/auth_template_support.py create mode 100644 runtime/hub/tests/test_auth_templates.py diff --git a/runtime/hub/core/authenticators/multi.py b/runtime/hub/core/authenticators/multi.py index 81e1caba..f8013df4 100644 --- a/runtime/hub/core/authenticators/multi.py +++ b/runtime/hub/core/authenticators/multi.py @@ -95,11 +95,15 @@ def get_custom_html(self, base_url): class="block w-full px-4 py-2 border rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" required /> </div> - <div class="mb-4"> + <div class="mb-4 relative"> <input type="password" name="password" placeholder="Password" - aria-label="Password" - class="block w-full px-4 py-2 border rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" + aria-label="Password" autocomplete="current-password" + class="login-input block w-full pl-4 pr-10 py-2 rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" required /> + <button type="button" class="password-toggle absolute inset-y-0 right-0 flex items-center pr-3 text-gray-400 hover:text-gray-600" aria-label="Show password"> + <svg class="eye-open w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z"/></svg> + <svg class="eye-closed w-5 h-5 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13.875 18.825A10.05 10.05 0 0112 19c-4.478 0-8.268-2.943-9.543-7a9.97 9.97 0 011.563-3.029m5.858.908a3 3 0 114.243 4.243M9.878 9.878l4.242 4.242M9.88 9.88l-3.29-3.29m7.532 7.532l3.29 3.29M3 3l3.59 3.59m0 0A9.953 9.953 0 0112 5c4.478 0 8.268 2.943 9.543 7a10.025 10.025 0 01-4.132 5.411m0 0L21 21"/></svg> + </button> </div> <button type="submit" class="login-submit w-full py-2 px-4 bg-blue-600 hover:bg-blue-700 text-white font-medium rounded-md"> diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index d55b5e94..1a2cc164 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -41,12 +41,33 @@ import os from contextlib import suppress -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypedDict import bcrypt if TYPE_CHECKING: - pass + from core.config import AuthCapabilities + + +class AuthTemplateVars(TypedDict): + auth_auto_login: bool + auth_dummy: bool + auth_native: bool + auth_github: bool + password_management_enabled: bool + hide_logout: bool + + +def _build_auth_template_vars(auth: AuthCapabilities) -> AuthTemplateVars: + _ = auth.effective_mode + return { + "auth_auto_login": auth.auto_login, + "auth_dummy": auth.dummy, + "auth_native": auth.native, + "auth_github": auth.github, + "password_management_enabled": auth.native, + "hide_logout": auth.auto_login, + } def _bootstrap_admin_password(admin_username: str, admin_password: str) -> None: @@ -405,8 +426,7 @@ async def delete(self, group_name): if not isinstance(c.JupyterHub.template_vars, dict): c.JupyterHub.template_vars = {} - c.JupyterHub.template_vars["authenticator_mode"] = config.auth_mode # type: ignore[assignment] - c.JupyterHub.template_vars["hide_logout"] = config.auth_mode == "auto-login" # type: ignore[assignment] + c.JupyterHub.template_vars.update(_build_auth_template_vars(auth)) c.JupyterHub.template_vars["cluster_name"] = config.cluster_name # type: ignore[assignment] c.JupyterHub.template_vars["platform_name"] = config.platform_display_name # type: ignore[assignment] diff --git a/runtime/hub/frontend/templates/admin-reset-password.html b/runtime/hub/frontend/templates/admin-reset-password.html index 538df5f6..1d8e1e80 100644 --- a/runtime/hub/frontend/templates/admin-reset-password.html +++ b/runtime/hub/frontend/templates/admin-reset-password.html @@ -22,6 +22,7 @@ {% extends "page.html" %} {% block main %} +{% if password_management_enabled %} <div class="container"> <div class="row"> <div class="col-md-6 col-md-offset-3"> @@ -98,4 +99,5 @@ <h1>Reset Password: {{ target_user }}</h1> </div> </div> </div> +{% endif %} {% endblock %} diff --git a/runtime/hub/frontend/templates/change-password.html b/runtime/hub/frontend/templates/change-password.html index 70ddc5fd..c7ce3d05 100644 --- a/runtime/hub/frontend/templates/change-password.html +++ b/runtime/hub/frontend/templates/change-password.html @@ -22,6 +22,7 @@ {% extends "page.html" %} {% block main %} +{% if password_management_enabled %} <div class="container" style="max-width: 480px; margin-top: 2rem;"> <div class="card bg-body border shadow-sm" style="border-radius: 12px;"> <div class="card-body p-4"> @@ -187,4 +188,5 @@ <h1 class="card-title h4 mb-4 text-body">Change Password</h1> </div> </div> </div> +{% endif %} {% endblock %} diff --git a/runtime/hub/frontend/templates/login.html b/runtime/hub/frontend/templates/login.html index 7a820507..57a9a432 100755 --- a/runtime/hub/frontend/templates/login.html +++ b/runtime/hub/frontend/templates/login.html @@ -42,7 +42,6 @@ {% set announcement = announcement_login %} {% endif %} {% set github_helper_text = login_github_helper_text|default("", true)|trim %} -{% set auth_mode = authenticator_mode|default('multi') %} {% block login_widget %} {% endblock login_widget %} @@ -106,7 +105,7 @@ </svg> <h2 class="text-3xl md:text-4xl font-bold text-white mb-4">{{ platform_name or 'AUP Learning Cloud' }}</h2> <p class="text-blue-100 mb-8">Experience the next generation of AI acceleration with AMD ROCm™.</p> - {% if login_service and auth_mode != 'multi' %} + {% if auth_github and not auth_native %} <a role="button" class='inline-block bg-white hover:bg-gray-100 text-black font-medium py-2 px-4 rounded transition duration-300' href='{{ authenticator_login_url | safe }}'> @@ -145,7 +144,7 @@ <h2 class="text-3xl md:text-4xl font-bold text-white mb-4">{{ platform_name or ' <div class="login-card rounded-xl shadow-lg p-8"> <div class="text-center mb-8"> <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP Learning Cloud' }}</h1> - {% if authenticator_mode == 'dummy' %} + {% if auth_dummy %} <p class="login-dev-mode text-sm mt-2">⚠️ Development Mode - Any username/password accepted</p> {% endif %} </div> @@ -154,7 +153,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L <p class="login-error font-medium mb-4 text-center">{{ login_error }}</p> {% endif %} - {% if authenticator_mode in ['dummy', 'local'] %} + {% if auth_dummy or (auth_native and not auth_github) %} <!-- Dummy Authenticator: Simple login form --> <form action="{{ base_url }}login?next={{ next | urlencode }}" method="post" role="form" class="space-y-6"> <input type="hidden" name="_xsrf" value="{{ xsrf }}" /> @@ -176,7 +175,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L </div> </form> - {% elif authenticator_mode == 'github' %} + {% elif auth_github and not auth_native %} <!-- GitHub App Only --> <!-- NOTE: Do NOT add "| urlencode" to App links! JupyterHub already URL-escapes the "next" variable. App stores next in a cookie as-is, so double-encoding @@ -195,52 +194,8 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L {% endif %} </div> - {% else %} - <!-- Multi Authenticator: GitHub App + Native accounts --> - <!-- GitHub App Button --> - <!-- NOTE: Do NOT add "| urlencode" here - see comment above for explanation --> - <div class="mb-6"> - <a href="{{ base_url }}github/oauth_login?next={{ next }}" - class="login-github-button w-full flex justify-center items-center py-3 px-4 rounded-md shadow-sm text-sm font-medium focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> - <svg class="w-5 h-5 mr-2" fill="currentColor" viewBox="0 0 20 20"> - <path fill-rule="evenodd" d="M10 0C4.477 0 0 4.484 0 10.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0110 4.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.203 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.942.359.31.678.921.678 1.856 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0020 10.017C20 4.484 15.522 0 10 0z" clip-rule="evenodd"/> - </svg> - Sign in with GitHub - </a> - {% if github_helper_text %} - <p class="login-helper text-sm text-center mt-3 mb-0">{{ github_helper_text }}</p> - {% endif %} - </div> - - <div class="login-divider relative mb-6"> - <div class="absolute inset-0 flex items-center"> - <div class="w-full border-t"></div> - </div> - <div class="relative flex justify-center text-sm"> - <span class="px-2">Or use local account</span> - </div> - </div> - - <!-- Local Account Login Form --> - <form action="{{ base_url }}native/login?next={{ next | urlencode }}" method="post" role="form" class="space-y-6"> - <input type="hidden" name="_xsrf" value="{{ xsrf }}" /> - - <div> - <label for="username_input" class="login-field-label block text-sm font-medium mb-1">Username</label> - <input id="username_input" type="text" autocapitalize="off" autocorrect="off" autocomplete="username" - name="username" value="{{ username }}" autofocus="autofocus" - class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> - </div> - - {{ password_field() }} - - <div class="mt-6"> - <button id="login_submit" type="submit" - class="login-submit w-full flex justify-center py-3 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> - Login - </button> - </div> - </form> + {% elif auth_native and auth_github %} + {{ custom_html | safe }} {% endif %} </div> diff --git a/runtime/hub/frontend/templates/page.html b/runtime/hub/frontend/templates/page.html index 6f186605..49f9cbd0 100755 --- a/runtime/hub/frontend/templates/page.html +++ b/runtime/hub/frontend/templates/page.html @@ -371,7 +371,7 @@ <h2 class="modal-title" id="{{ key }}-label">{{ title }}</h2> {% if user %} <span class="me-1">{{ user.name }}</span> {% if not hide_logout %} - {% if not user.name.startswith('github:') %} + {% if password_management_enabled and not user.name.startswith('github:') %} <a id="change-password" role="button" class="btn btn-sm btn-outline-secondary me-1" @@ -384,7 +384,7 @@ <h2 class="modal-title" id="{{ key }}-label">{{ title }}</h2> class="btn btn-sm btn-outline-secondary" href="{{ logout_url }}"> <i aria-hidden="true" class="fa fa-sign-out"></i> Logout</a> {% endif %} - {% else %} + {% elif not hide_logout %} <a id="login" role="button" class="btn btn-sm btn-outline-secondary" @@ -542,7 +542,7 @@ <h2 class="modal-title" id="{{ key }}-label">{{ title }}</h2> }); })(); </script> - {% if user and not user.name.startswith('github:') and not hide_logout %} + {% if user and password_management_enabled and not user.name.startswith('github:') %} <script type="text/javascript"> // Check if user needs to change password (only for native users) (function() { diff --git a/runtime/hub/tests/auth_template_support.py b/runtime/hub/tests/auth_template_support.py new file mode 100644 index 00000000..37debcc6 --- /dev/null +++ b/runtime/hub/tests/auth_template_support.py @@ -0,0 +1,189 @@ +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from html.parser import HTMLParser +from pathlib import Path + +import pytest +from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template +from tornado.escape import url_escape + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATES = ROOT / "frontend" / "templates" +FIRSTUSE = ROOT / "core" / "authenticators" / "firstuse.py" +MULTI = ROOT / "core" / "authenticators" / "multi.py" + + +class HtmlProbe(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.ids: set[str] = set() + self.hrefs: list[str] = [] + self.anchors: list[dict[str, str | None]] = [] + self.forms: list[dict[str, str | None]] = [] + self.inputs: list[dict[str, str | None]] = [] + self.buttons: list[dict[str, str | None]] = [] + self.text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attributes = dict(attrs) + if element_id := attributes.get("id"): + self.ids.add(element_id) + if tag == "a" and (href := attributes.get("href")): + self.hrefs.append(href) + self.anchors.append(attributes) + if tag == "form": + self.forms.append(attributes) + if tag == "input": + self.inputs.append(attributes) + if tag == "button": + self.buttons.append(attributes) + + def handle_data(self, data: str) -> None: + if text := " ".join(data.split()): + self.text.append(text) + + +def template_environment() -> Environment: + environment = Environment( + loader=FileSystemLoader(TEMPLATES), + autoescape=True, + undefined=StrictUndefined, + ) + environment.globals["static_url"] = lambda value, **_kwargs: f"/hub/static/{value}" + return environment + + +def base_context() -> dict[str, object]: + return { + "admin_access": False, + "announcement": "", + "authenticator_login_url": "/hub/oauth_login?next=/hub/home", + "base_url": "/hub/", + "custom_html": "", + "github_helper_text": "", + "login_error": "", + "login_service": "", + "login_url": "/hub/login", + "logo_url": "", + "logout_url": "/hub/logout", + "next": "/hub/home", + "no_spawner_check": True, + "parsed_scopes": [], + "platform_name": "AUP Learning Cloud", + "powered_by": "AUP Learning Cloud", + "prefix": "/hub/", + "services": [], + "user": None, + "username": "", + "version_hash": "", + "xsrf": "csrf-token", + "xsrf_token": "csrf-token", + "auth_auto_login": False, + "auth_dummy": False, + "auth_native": False, + "auth_github": False, + "password_management_enabled": False, + "hide_logout": False, + } + + +def probe_html(html: str) -> HtmlProbe: + probe = HtmlProbe() + probe.feed(html) + return probe + + +@contextmanager +def loaded_multi_authenticator(monkeypatch: pytest.MonkeyPatch) -> Iterator[types.SimpleNamespace]: + with monkeypatch.context() as module_patch: + core = types.ModuleType("core") + core.__path__ = [str(ROOT / "core")] + authenticators = types.ModuleType("core.authenticators") + authenticators.__path__ = [str(ROOT / "core" / "authenticators")] + core.authenticators = authenticators + module_patch.setitem(sys.modules, "core", core) + module_patch.setitem(sys.modules, "core.authenticators", authenticators) + + bcrypt = types.ModuleType("bcrypt") + firstuseauthenticator = types.ModuleType("firstuseauthenticator") + + class FirstUseAuthenticator: + def login_url(self, base_url: str) -> str: + return f"{base_url}native/login" + + firstuseauthenticator.FirstUseAuthenticator = FirstUseAuthenticator + models = types.ModuleType("core.authenticators.models") + models.UserPassword = type("UserPassword", (), {}) + database = types.ModuleType("core.database") + database.get_session = lambda: None + database.session_scope = lambda: None + for module in (bcrypt, firstuseauthenticator, models, database): + module_patch.setitem(sys.modules, module.__name__, module) + + firstuse_spec = importlib.util.spec_from_file_location("core.authenticators.firstuse", FIRSTUSE) + assert firstuse_spec is not None and firstuse_spec.loader is not None + firstuse = importlib.util.module_from_spec(firstuse_spec) + module_patch.setitem(sys.modules, "core.authenticators.firstuse", firstuse) + firstuse_spec.loader.exec_module(firstuse) + + multiauthenticator = types.ModuleType("multiauthenticator") + + class MultiAuthenticator: + def __init__(self) -> None: + self._authenticators = [] + + multiauthenticator.MultiAuthenticator = MultiAuthenticator + multiauthenticator_module = types.ModuleType("multiauthenticator.multiauthenticator") + multiauthenticator_module.PREFIX_SEPARATOR = ":" + module_patch.setitem(sys.modules, "multiauthenticator", multiauthenticator) + module_patch.setitem(sys.modules, "multiauthenticator.multiauthenticator", multiauthenticator_module) + + multi_spec = importlib.util.spec_from_file_location("core.authenticators.multi", MULTI) + assert multi_spec is not None and multi_spec.loader is not None + multi = importlib.util.module_from_spec(multi_spec) + module_patch.setitem(sys.modules, "core.authenticators.multi", multi) + multi_spec.loader.exec_module(multi) + + class ExternalAuthenticator: + service_name = "GitHub" + login_service = "GitHub" + username_prefix = "" + + def login_url(self, base_url: str) -> str: + return f"{base_url}github/oauth_login" + + yield types.SimpleNamespace( + multi=multi.CustomMultiAuthenticator(), + native=firstuse.CustomFirstUseAuthenticator(), + external=ExternalAuthenticator(), + ) + + +def render_multi_html(state: types.SimpleNamespace, next_value: str) -> str: + return Template(state.multi.get_custom_html("/hub/")).render(xsrf="csrf-token", next=url_escape(next_value)) + + +@contextmanager +def loaded_auth_modules(monkeypatch: pytest.MonkeyPatch) -> Iterator[types.SimpleNamespace]: + with monkeypatch.context() as module_patch: + bcrypt = types.ModuleType("bcrypt") + module_patch.setitem(sys.modules, "bcrypt", bcrypt) + + config_name = "task9_auth_config" + config_spec = importlib.util.spec_from_file_location(config_name, ROOT / "core" / "config.py") + assert config_spec is not None and config_spec.loader is not None + config = importlib.util.module_from_spec(config_spec) + module_patch.setitem(sys.modules, config_name, config) + config_spec.loader.exec_module(config) + + setup_name = "task9_auth_setup" + setup_spec = importlib.util.spec_from_file_location(setup_name, ROOT / "core" / "setup.py") + assert setup_spec is not None and setup_spec.loader is not None + setup = importlib.util.module_from_spec(setup_spec) + module_patch.setitem(sys.modules, setup_name, setup) + setup_spec.loader.exec_module(setup) + + yield types.SimpleNamespace(config=config, setup=setup) diff --git a/runtime/hub/tests/test_auth_templates.py b/runtime/hub/tests/test_auth_templates.py new file mode 100644 index 00000000..0cf2bb42 --- /dev/null +++ b/runtime/hub/tests/test_auth_templates.py @@ -0,0 +1,190 @@ +from itertools import product +from types import SimpleNamespace + +import pytest +from auth_template_support import ( + TEMPLATES, + base_context, + loaded_auth_modules, + loaded_multi_authenticator, + probe_html, + render_multi_html, + template_environment, +) + +VALID_VARIANTS = { + "auto-login": (True, False, False, False), + "dummy": (False, True, False, False), + "native": (False, False, True, False), + "github": (False, False, False, True), + "native-github": (False, False, True, True), +} +INVALID_VARIANTS = tuple(values for values in product((False, True), repeat=4) if values not in VALID_VARIANTS.values()) + + +def projected_context(monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool]) -> dict[str, object]: + with loaded_auth_modules(monkeypatch) as modules: + auth = modules.config.AuthCapabilities(*providers) + return dict(modules.setup._build_auth_template_vars(auth)) + + +@pytest.mark.parametrize(("variant", "providers"), VALID_VARIANTS.items()) +def test_setup_projects_explicit_auth_template_capabilities( + monkeypatch: pytest.MonkeyPatch, + variant: str, + providers: tuple[bool, bool, bool, bool], +) -> None: + context = projected_context(monkeypatch, providers) + + assert context == { + "auth_auto_login": variant == "auto-login", + "auth_dummy": variant == "dummy", + "auth_native": variant in {"native", "native-github"}, + "auth_github": variant in {"github", "native-github"}, + "password_management_enabled": variant in {"native", "native-github"}, + "hide_logout": variant == "auto-login", + } + + +@pytest.mark.parametrize("providers", INVALID_VARIANTS) +def test_invalid_auth_capabilities_are_rejected_before_render( + monkeypatch: pytest.MonkeyPatch, + providers: tuple[bool, bool, bool, bool], +) -> None: + with loaded_auth_modules(monkeypatch) as modules: + auth = modules.config.AuthCapabilities(*providers) + rendered = False + + with pytest.raises(modules.config.AuthConfigurationError): + context = modules.setup._build_auth_template_vars(auth) + template_environment().get_template("login.html").render(**base_context(), **context) + rendered = True + + assert rendered is False + + +def test_auth_templates_do_not_branch_on_legacy_mode_names() -> None: + for name in ("login.html", "page.html", "change-password.html", "admin-reset-password.html"): + source = (TEMPLATES / name).read_text(encoding="utf-8") + assert "authenticator_mode" not in source + assert "auth_mode" not in source + + +@pytest.mark.parametrize(("variant", "providers"), VALID_VARIANTS.items()) +def test_login_renders_enabled_authentication_controls( + monkeypatch: pytest.MonkeyPatch, + variant: str, + providers: tuple[bool, bool, bool, bool], +) -> None: + context = base_context() | projected_context(monkeypatch, providers) + if variant == "github": + context |= {"login_service": "GitHub", "github_helper_text": "Use your approved GitHub account."} + if variant == "native-github": + with loaded_multi_authenticator(monkeypatch) as state: + state.multi._authenticators = [state.external, state.native] + context["custom_html"] = render_multi_html(state, str(context["next"])) + + probe = probe_html(template_environment().get_template("login.html").render(**context)) + form_actions = {form.get("action") for form in probe.forms} + input_names = {field.get("name") for field in probe.inputs} + password_toggles = [ + button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split() + ] + visible_text = " ".join(probe.text) + + assert ("username" in input_names and "password" in input_names) is (variant in {"dummy", "native", "native-github"}) + assert len(password_toggles) == (1 if variant in {"dummy", "native", "native-github"} else 0) + assert all(button.get("aria-label") == "Show password" for button in password_toggles) + if variant == "dummy": + assert "Development Mode - Any username/password accepted" in visible_text + else: + assert "Development Mode" not in visible_text + assert ("/hub/login?next=/hub/home" in form_actions) is (variant in {"dummy", "native"}) + assert probe.hrefs.count("/hub/oauth_login?next=/hub/home") == (2 if variant == "github" else 0) + assert ("/hub/github/oauth_login?next=%2Fhub%2Fhome" in probe.hrefs) is (variant == "native-github") + assert ("/hub/native/login?next=%252Fhub%252Fhome" in form_actions) is (variant == "native-github") + assert "auplc-powered-by-footer" in probe.ids + if variant in {"dummy", "native", "native-github"}: + assert any(field.get("name") == "_xsrf" and field.get("value") == "csrf-token" for field in probe.inputs) + + +@pytest.mark.parametrize(("variant", "providers"), VALID_VARIANTS.items()) +def test_page_controls_follow_capabilities( + monkeypatch: pytest.MonkeyPatch, + variant: str, + providers: tuple[bool, bool, bool, bool], +) -> None: + context = base_context() | projected_context(monkeypatch, providers) + context["user"] = SimpleNamespace( + name="learner", + json_escaped_name="learner", + spawner=SimpleNamespace(options_form=False), + ) + + html = template_environment().get_template("page.html").render(**context) + probe = probe_html(html) + + assert ("logout" in probe.ids) is (variant != "auto-login") + assert ("change-password" in probe.ids) is (variant in {"native", "native-github"}) + assert ("auth/check-force-password-change" in html) is (variant in {"native", "native-github"}) + + +@pytest.mark.parametrize(("variant", "providers"), VALID_VARIANTS.items()) +def test_anonymous_login_link_follows_auto_login_capability( + monkeypatch: pytest.MonkeyPatch, + variant: str, + providers: tuple[bool, bool, bool, bool], +) -> None: + context = base_context() | projected_context(monkeypatch, providers) + + probe = probe_html(template_environment().get_template("page.html").render(**context)) + + assert ("login" in probe.ids) is (variant != "auto-login") + + +def test_composed_github_user_has_no_native_password_controls(monkeypatch: pytest.MonkeyPatch) -> None: + context = base_context() | projected_context(monkeypatch, VALID_VARIANTS["native-github"]) + context["user"] = SimpleNamespace( + name="github:octo", + json_escaped_name="github:octo", + spawner=SimpleNamespace(options_form=False), + ) + + html = template_environment().get_template("page.html").render(**context) + probe = probe_html(html) + + assert "logout" in probe.ids + assert "change-password" not in probe.ids + assert "auth/check-force-password-change" not in html + + +@pytest.mark.parametrize("template_name", ("change-password.html", "admin-reset-password.html")) +@pytest.mark.parametrize("variant", tuple(VALID_VARIANTS)) +def test_password_templates_render_controls_only_for_native_capability( + monkeypatch: pytest.MonkeyPatch, + template_name: str, + variant: str, +) -> None: + context = base_context() | projected_context(monkeypatch, VALID_VARIANTS[variant]) + context |= { + "error": "", + "error_message": "", + "forced_change": False, + "password_changed": False, + "success": False, + "target_user": "learner", + } + + probe = probe_html(template_environment().get_template(template_name).render(**context)) + + assert bool(probe.forms) is (variant in {"native", "native-github"}) + + +def test_attribution_footer_is_after_all_template_blocks_and_renders() -> None: + source = (TEMPLATES / "page.html").read_text(encoding="utf-8") + footer_offset = source.index('<footer id="auplc-powered-by-footer">') + + assert footer_offset > source.rfind("{% endblock") + assert "auplc-powered-by-footer" in probe_html( + template_environment().get_template("page.html").render(**base_context()) + ).ids diff --git a/runtime/hub/tests/test_multi_authenticator_html.py b/runtime/hub/tests/test_multi_authenticator_html.py index 44968764..2bb3040b 100644 --- a/runtime/hub/tests/test_multi_authenticator_html.py +++ b/runtime/hub/tests/test_multi_authenticator_html.py @@ -36,8 +36,16 @@ def test_native_child_renders_inline_form_with_encoded_next( assert "required" in fields["username"] assert fields["password"].get("placeholder") == "Password" assert fields["password"].get("aria-label") == "Password" + assert fields["password"].get("autocomplete") == "current-password" assert "required" in fields["password"] - assert "login-submit" in (probe.buttons[0].get("class") or "").split() + button_classes = [(button.get("class") or "").split() for button in probe.buttons] + assert any("login-submit" in classes for classes in button_classes) + password_toggles = [ + button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split() + ] + assert len(password_toggles) == 1 + assert password_toggles[0].get("type") == "button" + assert password_toggles[0].get("aria-label") == "Show password" @pytest.mark.parametrize(("next_value", "escaped_next", "form_next"), NEXT_CASES) From 4ea9c8b1301ef783b861acca6120c752e134f393 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:17:53 +0800 Subject: [PATCH 150/180] refactor(runtime): remove provider-derived resource access --- runtime/hub/core/authenticators/__init__.py | 21 ++- runtime/hub/core/groups.py | 24 +--- runtime/hub/core/handlers.py | 13 +- runtime/hub/core/setup.py | 23 ++-- .../apps/spawn/src/hooks/useResources.ts | 1 - runtime/hub/tests/provider_setup_support.py | 4 +- .../hub/tests/test_access_policy_config.py | 121 ------------------ runtime/hub/tests/test_auth_provider_setup.py | 10 +- runtime/hub/tests/test_auth_templates.py | 11 +- .../hub/tests/test_authenticator_factory.py | 30 +---- runtime/hub/tests/test_groups.py | 12 +- runtime/hub/tests/test_password_handlers.py | 15 +-- .../hub/tests/test_resource_access_runtime.py | 32 ++--- tests/installer/test_access_policy_schema.py | 51 -------- 14 files changed, 63 insertions(+), 305 deletions(-) delete mode 100644 runtime/hub/tests/test_access_policy_config.py delete mode 100644 tests/installer/test_access_policy_schema.py diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index 92799af6..29015db9 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -28,35 +28,30 @@ from core.authenticators.github_app import GITHUB_USERNAME_PREFIX, CustomGitHubOAuthenticator from core.authenticators.jwt import RemoteLabAuthenticator from core.authenticators.multi import CustomMultiAuthenticator -from core.config import AuthCapabilities, AuthConfigurationError, LegacyAuthMode +from core.config import AuthCapabilities, AuthConfigurationError LOCAL_ACCOUNT_PREFIX = "LocalAccount" -def create_authenticator(auth: AuthCapabilities | LegacyAuthMode) -> type | str: +def create_authenticator(auth: AuthCapabilities) -> type | str: """Select the JupyterHub authenticator class for validated capabilities.""" match auth: - case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False) | "auto-login": + case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False): return AutoLoginAuthenticator - case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False) | "dummy": + case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False): return "dummy" - case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False) | "local": + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False): return CustomFirstUseAuthenticator - case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True) | "github": + case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True): return CustomGitHubOAuthenticator - case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True) | "multi": + case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True): return CustomMultiAuthenticator case AuthCapabilities(): raise AuthConfigurationError("auth must enable one exclusive provider or native + github") - # Todo 13: remove the effective-mode compatibility boundary after Todo 6 consumes config.auth. - case str(): - raise ValueError(f"Unknown authentication mode: {auth}") - case bool(): - raise AuthConfigurationError("authentication capabilities cannot be boolean values") case unsupported: raise AuthConfigurationError( - f"authentication capabilities must be AuthCapabilities or a supported effective mode, got {type(unsupported).__name__}" + f"authentication capabilities must be AuthCapabilities, got {type(unsupported).__name__}" ) diff --git a/runtime/hub/core/groups.py b/runtime/hub/core/groups.py index 2c09fdbe..1370acdb 100644 --- a/runtime/hub/core/groups.py +++ b/runtime/hub/core/groups.py @@ -32,7 +32,6 @@ import logging import time from contextlib import suppress -from typing import TYPE_CHECKING, assert_never import aiohttp import jwt @@ -42,9 +41,6 @@ from core.authenticators.github_app import GITHUB_USERNAME_PREFIX -if TYPE_CHECKING: - from core.config import ResourceAccessPolicy - log = logging.getLogger("jupyterhub.groups") GITHUB_TEAM_SOURCE = "github-team" @@ -707,24 +703,16 @@ def get_resources_for_user( def resolve_resources_for_user( user: JupyterHubUser, team_resource_mapping: dict[str, list[str]], - access_policy: ResourceAccessPolicy, - all_resources: list[str], ) -> list[str]: """Resolve the resources visible to a user for UI and spawn flows.""" username = user.name.strip() - match access_policy: - case "all": - return all_resources - case "group-mapped": - available_resources = get_resources_for_user(user, team_resource_mapping) - if available_resources: - return available_resources - if not username.startswith(GITHUB_USERNAME_PREFIX): - return team_resource_mapping.get("native-users", team_resource_mapping.get("official", [])) - return ["none"] - case unreachable: - assert_never(unreachable) + available_resources = get_resources_for_user(user, team_resource_mapping) + if available_resources: + return available_resources + if not username.startswith(GITHUB_USERNAME_PREFIX): + return team_resource_mapping.get("native-users", team_resource_mapping.get("official", [])) + return ["none"] def is_readonly_group(group: ORMGroup) -> bool: diff --git a/runtime/hub/core/handlers.py b/runtime/hub/core/handlers.py index 4d7605db..42358754 100644 --- a/runtime/hub/core/handlers.py +++ b/runtime/hub/core/handlers.py @@ -32,7 +32,7 @@ import asyncio import json from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any +from typing import Any from urllib.parse import urlencode, urlparse, urlunparse from jupyterhub.apihandlers import APIHandler @@ -62,9 +62,6 @@ StatsUserHandler, ) -if TYPE_CHECKING: - from core.config import ResourceAccessPolicy - # ============================================================================= # Module-level configuration (set via configure_handlers) # ============================================================================= @@ -76,8 +73,6 @@ "minimum_quota_to_start": 10, "default_quota": 0, "team_resource_mapping": {}, - "auth_mode": "auto-login", - "access_policy": "group-mapped", "platform_name": "AUP Learning Cloud", } @@ -140,8 +135,6 @@ def configure_handlers( default_quota: int = 0, team_resource_mapping: dict[str, list[str]] | None = None, github_org: str = "", - auth_mode: str = "auto-login", - access_policy: ResourceAccessPolicy = "group-mapped", platform_name: str = "AUP Learning Cloud", ) -> None: """Configure handler module with runtime settings.""" @@ -152,8 +145,6 @@ def configure_handlers( _handler_config["default_quota"] = default_quota _handler_config["team_resource_mapping"] = team_resource_mapping or {} _handler_config["github_org"] = github_org - _handler_config["auth_mode"] = auth_mode - _handler_config["access_policy"] = access_policy _handler_config["platform_name"] = platform_name @@ -1140,8 +1131,6 @@ async def get(self): resolve_resources_for_user( self.current_user, _handler_config.get("team_resource_mapping", {}), - _handler_config["access_policy"], - list(config.resources.images.keys()), ) ) diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index 1a2cc164..fedcc003 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -59,7 +59,7 @@ class AuthTemplateVars(TypedDict): def _build_auth_template_vars(auth: AuthCapabilities) -> AuthTemplateVars: - _ = auth.effective_mode + auth.validate() return { "auth_auto_login": auth.auto_login, "auth_dummy": auth.dummy, @@ -263,8 +263,6 @@ async def auth_state_hook(spawner, auth_state): default_quota=config.quota.defaultQuota, team_resource_mapping=dict(config.teams.mapping), github_org=config.github_org_name, - auth_mode=config.auth_mode, - access_policy=config.resources.effective_access_policy, platform_name=config.platform_display_name, ) @@ -401,13 +399,6 @@ async def delete(self, group_name): if admin_password and not auth.native: raise RuntimeError("Administrator password bootstrap requires native authentication") - # ========================================================================= - # Template Paths - # ========================================================================= - - template_path = os.environ.get("JUPYTERHUB_TEMPLATE_PATH", "/tmp/custom_templates") - c.JupyterHub.template_paths = [template_path] - if admin_password: try: _bootstrap_admin_password(admin_username, admin_password) @@ -420,6 +411,13 @@ async def delete(self, group_name): c.Authenticator.admin_users = {admin_username} print(f"[SETUP] Admin user configured: {admin_username}") + # ========================================================================= + # Template Paths + # ========================================================================= + + template_path = os.environ.get("JUPYTERHUB_TEMPLATE_PATH", "/tmp/custom_templates") + c.JupyterHub.template_paths = [template_path] + # ========================================================================= # Template Vars # ========================================================================= @@ -430,5 +428,8 @@ async def delete(self, group_name): c.JupyterHub.template_vars["cluster_name"] = config.cluster_name # type: ignore[assignment] c.JupyterHub.template_vars["platform_name"] = config.platform_display_name # type: ignore[assignment] - print(f"[SETUP] Hub setup complete: auth_mode={config.auth_mode}") + print( + "[SETUP] Hub setup complete: auth=" + f"auto_login:{auth.auto_login},dummy:{auth.dummy},native:{auth.native},github:{auth.github}" + ) print(f"[SETUP] template_vars: {c.JupyterHub.template_vars}") diff --git a/runtime/hub/frontend/apps/spawn/src/hooks/useResources.ts b/runtime/hub/frontend/apps/spawn/src/hooks/useResources.ts index a3310730..388faa99 100644 --- a/runtime/hub/frontend/apps/spawn/src/hooks/useResources.ts +++ b/runtime/hub/frontend/apps/spawn/src/hooks/useResources.ts @@ -25,7 +25,6 @@ import { getResources } from '@auplc/shared'; declare global { interface Window { AVAILABLE_RESOURCES?: string[]; - SINGLE_NODE_MODE?: boolean; } } diff --git a/runtime/hub/tests/provider_setup_support.py b/runtime/hub/tests/provider_setup_support.py index 9a64c3f8..97e4d90a 100644 --- a/runtime/hub/tests/provider_setup_support.py +++ b/runtime/hub/tests/provider_setup_support.py @@ -9,11 +9,9 @@ } -def make_config(auth: object, access_policy: str) -> types.SimpleNamespace: +def make_config(auth: object) -> types.SimpleNamespace: return types.SimpleNamespace( auth=auth, - auth_mode=auth.effective_mode, - resources=types.SimpleNamespace(effective_access_policy=access_policy), accelerators={}, build_quota_rates=lambda: {}, quota_enabled=True, diff --git a/runtime/hub/tests/test_access_policy_config.py b/runtime/hub/tests/test_access_policy_config.py deleted file mode 100644 index c26fc186..00000000 --- a/runtime/hub/tests/test_access_policy_config.py +++ /dev/null @@ -1,121 +0,0 @@ -import importlib.util -import sys -import types -import warnings -from pathlib import Path - -import pytest -import yaml -from pydantic import ValidationError - -ROOT = Path(__file__).resolve().parents[1] -CONFIG = ROOT / "core" / "config.py" -AUTH_FLAGS = ("autoLogin", "dummy", "native", "github") -CANONICAL_PROVIDERS = ( - (True, False, False, False), - (False, True, False, False), - (False, False, True, False), - (False, False, False, True), - (False, False, True, True), -) -LEGACY_POLICIES = { - "auto-login": "all", - "dummy": "all", - "local": "all", - "github": "group-mapped", - "multi": "group-mapped", -} - - -@pytest.fixture -def config_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: - spec = importlib.util.spec_from_file_location("task7_access_policy_config", CONFIG) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, spec.name, module) - spec.loader.exec_module(module) - return module - - -def write_config(tmp_path: Path, data: dict[str, object]) -> Path: - path = tmp_path / "hub-config.yaml" - path.write_text(yaml.safe_dump(data), encoding="utf-8") - return path - - -def canonical_auth(flags: tuple[bool, bool, bool, bool]) -> dict[str, dict[str, bool]]: - return {"auth": dict(zip(AUTH_FLAGS, flags, strict=True))} - - -@pytest.mark.parametrize("providers", CANONICAL_PROVIDERS) -def test_canonical_providers_default_to_group_mapped_access( - config_module: types.ModuleType, tmp_path: Path, providers: tuple[bool, bool, bool, bool] -) -> None: - hub_config = config_module.HubConfig.init(write_config(tmp_path, canonical_auth(providers))) - - assert hub_config.resources.effective_access_policy == "group-mapped" - - -def test_absent_auth_forms_default_to_group_mapped_access(config_module: types.ModuleType, tmp_path: Path) -> None: - hub_config = config_module.HubConfig.init(write_config(tmp_path, {"resources": {}})) - - assert hub_config.resources.effective_access_policy == "group-mapped" - - -@pytest.mark.parametrize(("auth_mode", "expected_policy"), LEGACY_POLICIES.items()) -def test_legacy_modes_preserve_implicit_resource_access_policy( - config_module: types.ModuleType, tmp_path: Path, auth_mode: str, expected_policy: str -) -> None: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - hub_config = config_module.HubConfig.init(write_config(tmp_path, {"authMode": auth_mode})) - - assert hub_config.resources.effective_access_policy == expected_policy - - -@pytest.mark.parametrize("providers", CANONICAL_PROVIDERS) -@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) -def test_explicit_policy_is_independent_of_canonical_provider_and_runtime_policy( - config_module: types.ModuleType, - tmp_path: Path, - providers: tuple[bool, bool, bool, bool], - access_policy: str, -) -> None: - raw_config: dict[str, object] = canonical_auth(providers) - raw_config.update( - { - "singleNodeMode": True, - "quota": {"enabled": False}, - "resources": {"accessPolicy": access_policy}, - } - ) - - hub_config = config_module.HubConfig.init(write_config(tmp_path, raw_config)) - - assert hub_config.resources.effective_access_policy == access_policy - assert hub_config.single_node_mode is True - assert hub_config.quota_enabled is False - - -@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) -@pytest.mark.parametrize("auth_mode", tuple(LEGACY_POLICIES)) -def test_explicit_policy_overrides_legacy_implicit_policy( - config_module: types.ModuleType, tmp_path: Path, auth_mode: str, access_policy: str -) -> None: - raw_config = {"authMode": auth_mode, "resources": {"accessPolicy": access_policy}} - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - hub_config = config_module.HubConfig.init(write_config(tmp_path, raw_config)) - - assert hub_config.resources.effective_access_policy == access_policy - - -@pytest.mark.parametrize("invalid_policy", ("unknown", None, True, 1, ["all"])) -def test_direct_parser_rejects_invalid_explicit_access_policy( - config_module: types.ModuleType, tmp_path: Path, invalid_policy: object -) -> None: - raw_config = {"resources": {"accessPolicy": invalid_policy}} - - with pytest.raises(ValidationError): - config_module.HubConfig.init(write_config(tmp_path, raw_config)) diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py index 76c15a7c..50ce9b3a 100644 --- a/runtime/hub/tests/test_auth_provider_setup.py +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -29,7 +29,6 @@ def _loaded_setup( monkeypatch: pytest.MonkeyPatch, providers: tuple[bool, bool, bool, bool], *, - access_policy: str = "group-mapped", fail_setup: bool = False, ) -> Iterator[types.SimpleNamespace]: with monkeypatch.context() as module_patch: @@ -53,7 +52,7 @@ def _loaded_setup( core.config = config_module config_spec.loader.exec_module(config_module) auth = config_module.AuthCapabilities(*providers) - config = make_config(auth, access_policy) + config = make_config(auth) config_module.HubConfig._instance, config_module.HubConfig._initialized = config, True settings_reads: list[str] = [] @@ -216,13 +215,12 @@ def test_native_only_setup_never_reads_github_settings(monkeypatch: pytest.Monke assert not any(key.startswith("hub.config.GitHubOAuthenticator") for key in state.settings_reads) -@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) -def test_setup_propagates_effective_access_policy(monkeypatch: pytest.MonkeyPatch, access_policy: str) -> None: - with _loaded_setup(monkeypatch, (False, False, False, True), access_policy=access_policy) as state: +def test_setup_configures_consumers_without_effective_auth_mode(monkeypatch: pytest.MonkeyPatch) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True)) as state: state.setup.setup_hub(state.c) assert state.spawner_configs == [state.config] - assert state.handler_configs[0]["access_policy"] == access_policy + assert "auth_mode" not in state.handler_configs[0] @pytest.mark.parametrize("providers", ((False, False, False, True), (False, False, True, True))) diff --git a/runtime/hub/tests/test_auth_templates.py b/runtime/hub/tests/test_auth_templates.py index 0cf2bb42..74a05dd0 100644 --- a/runtime/hub/tests/test_auth_templates.py +++ b/runtime/hub/tests/test_auth_templates.py @@ -92,7 +92,9 @@ def test_login_renders_enabled_authentication_controls( ] visible_text = " ".join(probe.text) - assert ("username" in input_names and "password" in input_names) is (variant in {"dummy", "native", "native-github"}) + assert ("username" in input_names and "password" in input_names) is ( + variant in {"dummy", "native", "native-github"} + ) assert len(password_toggles) == (1 if variant in {"dummy", "native", "native-github"} else 0) assert all(button.get("aria-label") == "Show password" for button in password_toggles) if variant == "dummy": @@ -185,6 +187,7 @@ def test_attribution_footer_is_after_all_template_blocks_and_renders() -> None: footer_offset = source.index('<footer id="auplc-powered-by-footer">') assert footer_offset > source.rfind("{% endblock") - assert "auplc-powered-by-footer" in probe_html( - template_environment().get_template("page.html").render(**base_context()) - ).ids + assert ( + "auplc-powered-by-footer" + in probe_html(template_environment().get_template("page.html").render(**base_context())).ids + ) diff --git a/runtime/hub/tests/test_authenticator_factory.py b/runtime/hub/tests/test_authenticator_factory.py index 5048b61a..e865376c 100644 --- a/runtime/hub/tests/test_authenticator_factory.py +++ b/runtime/hub/tests/test_authenticator_factory.py @@ -57,23 +57,8 @@ def _loaded_factory(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[types.Mod yield authenticator_factory, config -@pytest.mark.parametrize( - ("mode", "expected_name"), - [ - ("auto-login", "AutoLoginAuthenticator"), - ("dummy", "dummy"), - ("local", "CustomFirstUseAuthenticator"), - ("github", "CustomGitHubOAuthenticator"), - ("multi", "CustomMultiAuthenticator"), - ], -) -def test_factory_preserves_legacy_projection_and_prefix_contract( - monkeypatch: pytest.MonkeyPatch, mode: str, expected_name: str -) -> None: +def test_factory_preserves_identity_prefix_contract(monkeypatch: pytest.MonkeyPatch) -> None: with _loaded_factory(monkeypatch) as (factory, _config): - selected = factory.create_authenticator(mode) - - assert selected == "dummy" if expected_name == "dummy" else selected.__name__ == expected_name assert factory.GITHUB_USERNAME_PREFIX == "github:" assert factory.CustomGitHubOAuthenticator.prefix == "github:" assert factory.CustomFirstUseAuthenticator.prefix == "" @@ -115,7 +100,10 @@ def test_factory_rejects_invalid_capabilities_before_authenticator_construction( factory.create_authenticator(config.AuthCapabilities(*capabilities)) -@pytest.mark.parametrize("malformed_auth", (None, 1, True, (), object())) +@pytest.mark.parametrize( + "malformed_auth", + (None, 1, True, (), object(), "auto-login", "dummy", "local", "github", "multi", "unexpected"), +) def test_factory_rejects_malformed_runtime_inputs(monkeypatch: pytest.MonkeyPatch, malformed_auth) -> None: with _loaded_factory(monkeypatch) as (factory, config), pytest.raises(config.AuthConfigurationError): factory.create_authenticator(malformed_auth) @@ -143,11 +131,3 @@ def test_factory_module_cleanup_survives_a_forced_test_failure(monkeypatch: pyte assert name not in sys.modules else: assert sys.modules[name] is original_module - - -def test_authenticator_factory_rejects_unknown_mode(monkeypatch: pytest.MonkeyPatch) -> None: - with ( - _loaded_factory(monkeypatch) as (factory, _config), - pytest.raises(ValueError, match="Unknown authentication mode"), - ): - factory.create_authenticator("unexpected") diff --git a/runtime/hub/tests/test_groups.py b/runtime/hub/tests/test_groups.py index 3a1d4ac8..26ac599b 100644 --- a/runtime/hub/tests/test_groups.py +++ b/runtime/hub/tests/test_groups.py @@ -242,8 +242,6 @@ def test_resolve_resources_for_user_uses_group_mapping(): resources = resolve_resources_for_user( user, {"team-a": ["cpu", "course-a"], "team-b": ["course-a", "course-b"]}, - "group-mapped", - ["cpu", "gpu", "code-cpu", "course-a", "course-b"], ) assert set(resources) == {"cpu", "course-a", "course-b"} @@ -256,8 +254,6 @@ def test_resolve_resources_for_group_mapped_native_user_uses_native_users_mappin resources = resolve_resources_for_user( user, {"official": ["cpu"], "native-users": ["code-cpu"]}, - "group-mapped", - ["cpu", "gpu", "code-cpu"], ) assert resources == ["code-cpu"] @@ -266,14 +262,14 @@ def test_resolve_resources_for_group_mapped_native_user_uses_native_users_mappin def test_resolve_resources_for_user_denies_unmapped_github_users(): user = DummyUser([]) - resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "group-mapped", ["cpu", "gpu"]) + resources = resolve_resources_for_user(user, {"official": ["cpu"]}) assert resources == ["none"] -def test_resolve_resources_for_user_uses_all_resources_for_all_policy(): +def test_resolve_resources_for_auto_login_user_uses_native_fallback(): user = DummyUser([], name="demo-user") - resources = resolve_resources_for_user(user, {"official": ["cpu"]}, "all", ["cpu", "gpu", "code-cpu"]) + resources = resolve_resources_for_user(user, {"official": ["cpu"]}) - assert resources == ["cpu", "gpu", "code-cpu"] + assert resources == ["cpu"] diff --git a/runtime/hub/tests/test_password_handlers.py b/runtime/hub/tests/test_password_handlers.py index 4e8611bc..273a5951 100644 --- a/runtime/hub/tests/test_password_handlers.py +++ b/runtime/hub/tests/test_password_handlers.py @@ -34,14 +34,13 @@ def batch_set_passwords(self, users, force_change=True): return {"success": len(users), "failed": 0, "results": []} -def configure_local_bootstrap(monkeypatch, handlers) -> None: - monkeypatch.setitem(handlers._handler_config, "auth_mode", "local") +def configure_local_bootstrap(monkeypatch) -> None: monkeypatch.setenv("JUPYTERHUB_ADMIN_USERNAME", "operator") def test_bootstrap_admin_can_change_own_password(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.ChangePasswordHandler) handler.current_user = DummyUser("operator") @@ -65,7 +64,7 @@ def test_bootstrap_admin_can_change_own_password(loaded_handlers, monkeypatch) - def test_admin_can_reset_bootstrap_administrator(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.AdminResetPasswordHandler) handler.current_user = DummyUser("manager", admin=True) @@ -113,7 +112,7 @@ async def render_template(_name, **kwargs): def test_admin_api_can_set_bootstrap_administrator_password(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) handler.current_user = DummyUser("manager", admin=True) @@ -132,7 +131,7 @@ def test_admin_api_can_set_bootstrap_administrator_password(loaded_handlers, mon def test_admin_api_keeps_other_local_users_changeable(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) handler.current_user = DummyUser("manager", admin=True) @@ -150,7 +149,7 @@ def test_admin_api_keeps_other_local_users_changeable(loaded_handlers, monkeypat def test_admin_api_batch_can_set_bootstrap_administrator_password(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.AdminAPIBatchSetPasswordHandler) handler.current_user = DummyUser("manager", admin=True) @@ -169,7 +168,7 @@ def test_admin_api_batch_can_set_bootstrap_administrator_password(loaded_handler def test_github_users_remain_blocked_from_native_password_changes(loaded_handlers, monkeypatch) -> None: authenticator = PasswordAuthenticator() - configure_local_bootstrap(monkeypatch, loaded_handlers.handlers) + configure_local_bootstrap(monkeypatch) monkeypatch.setattr(loaded_handlers.handlers, "_find_firstuse_authenticator", lambda _auth: authenticator) handler = object.__new__(loaded_handlers.handlers.AdminAPISetPasswordHandler) handler.current_user = DummyUser("manager", admin=True) diff --git a/runtime/hub/tests/test_resource_access_runtime.py b/runtime/hub/tests/test_resource_access_runtime.py index 6317d61f..5c1e6844 100644 --- a/runtime/hub/tests/test_resource_access_runtime.py +++ b/runtime/hub/tests/test_resource_access_runtime.py @@ -65,32 +65,19 @@ def load_spawner(monkeypatch: pytest.MonkeyPatch, groups: types.ModuleType) -> t return module.RemoteLabKubeSpawner -@pytest.mark.parametrize( - ("access_policy", "expected_resources"), - (("all", ["cpu", "gpu", "code-cpu"]), ("group-mapped", ["gpu"])), -) -def test_spawner_uses_configured_access_policy_for_exact_resource_list( - monkeypatch: pytest.MonkeyPatch, access_policy: str, expected_resources: list[str] -) -> None: +def test_spawner_uses_shared_group_mapping_without_auth_mode(monkeypatch: pytest.MonkeyPatch) -> None: groups_test = load_groups_test_module(monkeypatch) spawner_type = load_spawner(monkeypatch, groups_test.groups) spawner = object.__new__(spawner_type) spawner.user = groups_test.DummyUser([groups_test.DummyGroup("team-gpu")], name="native-user") spawner.team_resource_mapping = {"team-gpu": ["gpu"], "native-users": ["cpu"]} spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image", "code-cpu": "code-image"} - spawner.access_policy = access_policy spawner.log = types.SimpleNamespace(debug=lambda _message: None) - assert asyncio.run(spawner.get_user_resources()) == expected_resources + assert asyncio.run(spawner.get_user_resources()) == ["gpu"] -@pytest.mark.parametrize( - ("access_policy", "expected_resources"), - (("all", ["code-cpu", "cpu", "gpu"]), ("group-mapped", ["gpu"])), -) -def test_resources_api_uses_configured_access_policy_for_exact_resource_list( - monkeypatch: pytest.MonkeyPatch, access_policy: str, expected_resources: list[str] -) -> None: +def test_resources_api_uses_shared_group_mapping_without_auth_mode(monkeypatch: pytest.MonkeyPatch) -> None: groups_test = load_groups_test_module(monkeypatch) monkeypatch.delitem(sys.modules, "tornado", raising=False) monkeypatch.delitem(sys.modules, "tornado.web", raising=False) @@ -111,9 +98,7 @@ def test_resources_api_uses_configured_access_policy_for_exact_resource_list( config_module.HubConfig = type("HubConfig", (), {"get": staticmethod(lambda: config)}) monkeypatch.setitem(sys.modules, "core.config", config_module) monkeypatch.setitem(sys.modules, "core.groups", groups_test.groups) - loaded.handlers.configure_handlers( - team_resource_mapping={"team-gpu": ["gpu"], "native-users": ["cpu"]}, access_policy=access_policy - ) + loaded.handlers.configure_handlers(team_resource_mapping={"team-gpu": ["gpu"], "native-users": ["cpu"]}) handler = object.__new__(loaded.handlers.ResourcesAPIHandler) handler.current_user = groups_test.DummyUser([groups_test.DummyGroup("team-gpu")], name="native-user") response: dict[str, str] = {} @@ -122,10 +107,10 @@ def test_resources_api_uses_configured_access_policy_for_exact_resource_list( asyncio.run(handler.get()) - assert [resource["key"] for resource in json.loads(response["body"])["resources"]] == expected_resources + assert [resource["key"] for resource in json.loads(response["body"])["resources"]] == ["gpu"] -def test_configure_handlers_replaces_optional_policy_and_mapping_state(monkeypatch: pytest.MonkeyPatch) -> None: +def test_configure_handlers_replaces_mapping_state_without_auth_mode(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delitem(sys.modules, "tornado", raising=False) monkeypatch.delitem(sys.modules, "tornado.web", raising=False) with load_handlers(monkeypatch) as loaded: @@ -133,11 +118,10 @@ def test_configure_handlers_replaces_optional_policy_and_mapping_state(monkeypat accelerator_options={"gpu": {}}, quota_rates={"gpu": 2}, team_resource_mapping={"team": ["gpu"]}, - access_policy="all", ) - loaded.handlers.configure_handlers(access_policy="group-mapped") + loaded.handlers.configure_handlers() assert loaded.handlers._handler_config["accelerator_options"] == {} assert loaded.handlers._handler_config["quota_rates"] == {} assert loaded.handlers._handler_config["team_resource_mapping"] == {} - assert loaded.handlers._handler_config["access_policy"] == "group-mapped" + assert "auth_mode" not in loaded.handlers._handler_config diff --git a/tests/installer/test_access_policy_schema.py b/tests/installer/test_access_policy_schema.py deleted file mode 100644 index 4ee4b14a..00000000 --- a/tests/installer/test_access_policy_schema.py +++ /dev/null @@ -1,51 +0,0 @@ -import json -import subprocess -from pathlib import Path - -import pytest -import yaml - -from scripts.generate_values_schema import remove_descriptions - -ROOT = Path(__file__).resolve().parents[2] -CHART = "runtime/chart" - - -def render(*settings: str, string_settings: tuple[str, ...] = ()) -> subprocess.CompletedProcess[str]: - command = ["helm", "template", "jupyterhub", CHART] - for setting in settings: - command.extend(("--set", setting)) - for setting in string_settings: - command.extend(("--set-string", setting)) - return subprocess.run(command, cwd=ROOT, check=False, capture_output=True, text=True) - - -@pytest.mark.parametrize("access_policy", ("all", "group-mapped")) -def test_chart_accepts_each_access_policy_literal(access_policy: str) -> None: - result = render(f"custom.resources.accessPolicy={access_policy}") - - assert result.returncode == 0, result.stderr - - -@pytest.mark.parametrize( - ("setting", "string_settings"), - [ - ("custom.resources.accessPolicy=unknown", ()), - ("custom.resources.accessPolicy=true", ()), - ("custom.resources.accessPolicy=1", ()), - ("", ("custom.resources.accessPolicy=unknown",)), - ], -) -def test_chart_rejects_invalid_access_policy(setting: str, string_settings: tuple[str, ...]) -> None: - settings = (setting,) if setting else () - result = render(*settings, string_settings=string_settings) - - assert result.returncode != 0 - assert "values don't meet the specifications" in result.stderr - - -def test_chart_yaml_and_json_schema_remain_exactly_in_sync() -> None: - yaml_schema = yaml.safe_load((ROOT / "runtime/chart/values.schema.yaml").read_text()) - json_schema = json.loads((ROOT / "runtime/chart/values.schema.json").read_text()) - - assert json_schema == remove_descriptions(yaml_schema) From 0190f87c511e1ed4f50b91f470c748b616f9560a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:19:11 +0800 Subject: [PATCH 151/180] feat(runtime): configure explicit runtime limits --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 47 +++-- runtime/hub/core/config.py | 114 +++++------- runtime/hub/core/spawner/kubernetes.py | 45 ++--- .../tests/test_config_resource_metadata.py | 104 +++++++---- .../tests/test_spawner_runtime_metadata.py | 70 ++++++- tests/installer/test_chart_local_auth.py | 171 +++++++++++++++++- 7 files changed, 399 insertions(+), 154 deletions(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index 9b0a5be0..7ef8d846 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":["string","null"],"enum":[null,"auto-login","dummy","github","local","multi"]},"auth":{"type":"object","additionalProperties":false,"properties":{"autoLogin":{"type":"boolean"},"dummy":{"type":"boolean"},"native":{"type":"boolean"},"github":{"type":"boolean"}},"oneOf":[{"required":["autoLogin"],"properties":{"autoLogin":{"const":true},"dummy":{"const":false},"native":{"const":false},"github":{"const":false}}},{"required":["dummy"],"properties":{"autoLogin":{"const":false},"dummy":{"const":true},"native":{"const":false},"github":{"const":false}}},{"required":["native"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":false}}},{"required":["github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":false},"github":{"const":true}}},{"required":["native","github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":true}}}]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"accessPolicy":{"type":"string","enum":["all","group-mapped"]},"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"not":{"required":["authMode","auth"]}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"oneOf":[{"required":["auth"],"properties":{"auth":{"required":["native"],"properties":{"native":{"const":true}}}}},{"required":["authMode"],"properties":{"authMode":{"enum":["local","multi"]}}}],"properties":{"adminUser":{"required":["username"],"properties":{"username":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":["string","null"],"enum":[null,"auto-login","dummy","github","local","multi"]},"auth":{"type":"object","additionalProperties":false,"properties":{"autoLogin":{"type":"boolean"},"dummy":{"type":"boolean"},"native":{"type":"boolean"},"github":{"type":"boolean"}},"oneOf":[{"required":["autoLogin"],"properties":{"autoLogin":{"const":true},"dummy":{"const":false},"native":{"const":false},"github":{"const":false}}},{"required":["dummy"],"properties":{"autoLogin":{"const":false},"dummy":{"const":true},"native":{"const":false},"github":{"const":false}}},{"required":["native"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":false}}},{"required":["github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":false},"github":{"const":true}}},{"required":["native","github"],"properties":{"autoLogin":{"const":false},"dummy":{"const":false},"native":{"const":true},"github":{"const":true}}}]},"runtimeLimitEnabled":{"type":"boolean"},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"username":{"type":"string","pattern":"^[a-z0-9][a-z0-9._-]{0,63}$"},"existingSecret":{"type":"string"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}},"allOf":[{"not":{"required":["authMode","auth"]}},{"if":{"required":["runtimeLimitEnabled"],"properties":{"runtimeLimitEnabled":{"const":false}}},"then":{"required":["quota"],"properties":{"quota":{"required":["enabled"],"properties":{"enabled":{"const":false}}}}}},{"if":{"required":["authMode","quota"],"properties":{"authMode":{"enum":["auto-login","local"]},"quota":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"required":["runtimeLimitEnabled"],"properties":{"runtimeLimitEnabled":{"const":true}}}},{"if":{"required":["adminUser"],"properties":{"adminUser":{"required":["enabled"],"properties":{"enabled":{"const":true}}}}},"then":{"oneOf":[{"required":["auth"],"properties":{"auth":{"required":["native"],"properties":{"native":{"const":true}}}}},{"required":["authMode"],"properties":{"authMode":{"enum":["local","multi"]}}}],"properties":{"adminUser":{"required":["username"],"properties":{"username":{"minLength":1}}}}}}]},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index d9b1eb15..e1ab3462 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3252,6 +3252,12 @@ properties: github: const: true + runtimeLimitEnabled: + type: boolean + description: | + Controls enforcement of the selected session runtime and automatic Pod + shutdown. Set to false only when quota is also disabled. + adminUser: type: object additionalProperties: false @@ -3429,15 +3435,6 @@ properties: Resource images and requirements configuration. Defines available container images and their resource requirements. properties: - accessPolicy: - type: string - enum: - - all - - group-mapped - description: | - Controls resource visibility independently of authentication providers. - `all` exposes every configured resource; `group-mapped` uses the - configured team mapping and native/GitHub fallback behavior. images: type: object additionalProperties: @@ -3651,8 +3648,8 @@ properties: type: [boolean, "null"] description: | Enable/disable quota system. - Set to `null` for auto-detection based on authMode - (disabled for auto-login/dummy, enabled otherwise). + Canonical configurations set this explicitly. A null or omitted value + retains historical defaults only for one-release `authMode` migration. cpuRate: type: integer minimum: 1 @@ -3870,6 +3867,34 @@ properties: allOf: - not: required: [authMode, auth] + - if: + required: [runtimeLimitEnabled] + properties: + runtimeLimitEnabled: + const: false + then: + required: [quota] + properties: + quota: + required: [enabled] + properties: + enabled: + const: false + - if: + required: [authMode, quota] + properties: + authMode: + enum: [auto-login, local] + quota: + required: [enabled] + properties: + enabled: + const: true + then: + required: [runtimeLimitEnabled] + properties: + runtimeLimitEnabled: + const: true - if: required: [adminUser] properties: diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index e8e234a6..497474af 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -33,7 +33,7 @@ # In business logic: from core.config import HubConfig config = HubConfig.get() - if config.auth_mode == "multi": + if config.auth.github: ... """ @@ -42,10 +42,10 @@ import warnings from dataclasses import dataclass from pathlib import Path -from typing import Any, Final, Literal, assert_never +from typing import Any, Literal import yaml -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, StrictBool, ValidationError, field_validator # ============================================================================= # YAML Configuration Models @@ -79,7 +79,7 @@ class AcceleratorConfig(BaseModel): class QuotaSettings(BaseModel): """Quota system configuration.""" - enabled: bool | None = None # None = auto-detect based on auth_mode + enabled: StrictBool | None = None cpuRate: int = 1 minimumToStart: int = 10 defaultQuota: int = 0 @@ -149,10 +149,6 @@ def validate_default_path(cls, value: str | None) -> str | None: model_config = {"extra": "allow"} -ResourceAccessPolicy = Literal["all", "group-mapped"] -DEFAULT_RESOURCE_ACCESS_POLICY: Final[ResourceAccessPolicy] = "group-mapped" - - class ResourcesConfig(BaseModel): """Resources configuration (images, requirements, and metadata).""" @@ -160,19 +156,6 @@ class ResourcesConfig(BaseModel): requirements: dict[str, ResourceRequirements] = Field(default_factory=dict) metadata: dict[str, ResourceMetadata] = Field(default_factory=dict) groupOrder: list[str] = Field(default_factory=list) - accessPolicy: ResourceAccessPolicy | None = None - - @model_validator(mode="before") - @classmethod - def reject_explicit_null_access_policy(cls, value: Any) -> Any: - if isinstance(value, dict) and "accessPolicy" in value and value["accessPolicy"] is None: - raise ValueError("accessPolicy must be all or group-mapped") - return value - - @property - def effective_access_policy(self) -> ResourceAccessPolicy: - return self.accessPolicy or DEFAULT_RESOURCE_ACCESS_POLICY - model_config = {"extra": "allow"} @@ -287,21 +270,16 @@ class AuthCapabilities: native: bool github: bool - @property - def effective_mode(self) -> LegacyAuthMode: - """Project capabilities onto the temporary legacy mode consumed downstream.""" - + def validate(self) -> AuthCapabilities: match self: - case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False): - return "auto-login" - case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False): - return "dummy" - case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False): - return "local" - case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True): - return "github" - case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True): - return "multi" + case ( + AuthCapabilities(auto_login=True, dummy=False, native=False, github=False) + | AuthCapabilities(auto_login=False, dummy=True, native=False, github=False) + | AuthCapabilities(auto_login=False, dummy=False, native=True, github=False) + | AuthCapabilities(auto_login=False, dummy=False, native=False, github=True) + | AuthCapabilities(auto_login=False, dummy=False, native=True, github=True) + ): + return self case _: raise AuthConfigurationError("auth must enable one exclusive provider or native + github") @@ -355,7 +333,7 @@ def _legacy_auth_capabilities(mode: str) -> AuthCapabilities: raise AuthConfigurationError("authMode must be one of auto-login, dummy, github, local, or multi") -def _parse_auth_capabilities(raw_config: dict[str, Any]) -> tuple[AuthCapabilities, bool]: +def _parse_auth_capabilities(raw_config: dict[str, Any]) -> tuple[AuthCapabilities, LegacyAuthMode | None]: """Parse explicit configuration form presence before defaulted models erase it.""" canonical_present = "auth" in raw_config @@ -367,24 +345,11 @@ def _parse_auth_capabilities(raw_config: dict[str, Any]) -> tuple[AuthCapabiliti capabilities = CanonicalAuthConfig.model_validate(raw_config["auth"]).capabilities() except ValidationError as error: raise AuthConfigurationError(f"auth must be a strict provider mapping: {error}") from error - try: - _ = capabilities.effective_mode - except AuthConfigurationError as error: - raise AuthConfigurationError("auth must enable one exclusive provider or native + github") from error - return capabilities, False + return capabilities.validate(), None if legacy_present and raw_config["authMode"] is not None: - return _legacy_auth_capabilities(raw_config["authMode"]), True - return AuthCapabilities(True, False, False, False), False - - -def _legacy_resource_access_policy(mode: LegacyAuthMode) -> ResourceAccessPolicy: - match mode: - case "auto-login" | "dummy" | "local": - return "all" - case "github" | "multi": - return "group-mapped" - case unreachable: - assert_never(unreachable) + legacy_mode = raw_config["authMode"] + return _legacy_auth_capabilities(legacy_mode), legacy_mode + return AuthCapabilities(True, False, False, False), None # ============================================================================= @@ -408,9 +373,8 @@ class HubConfig: def __init__(self): # Runtime settings - self.auth_mode: str = "auto-login" self._auth: AuthCapabilities = AuthCapabilities(True, False, False, False) - self.single_node_mode: bool = False + self.runtime_limit_enabled: bool = True self.github_org_name: str = "" self.cluster_name: str = "" self.admin_username: str = "admin" @@ -450,10 +414,8 @@ def init(cls, config_path: str | Path) -> HubConfig: print(f"[CONFIG] Loaded configuration from {config_path}") # Extract runtime settings - instance._auth, legacy_auth = _parse_auth_capabilities(raw_config) - effective_mode = instance._auth.effective_mode - instance.auth_mode = effective_mode - if legacy_auth: + instance._auth, legacy_mode = _parse_auth_capabilities(raw_config) + if legacy_mode is not None: warnings.warn( "authMode is deprecated; configure authentication with auth provider flags instead", DeprecationWarning, @@ -465,13 +427,15 @@ def init(cls, config_path: str | Path) -> HubConfig: if isinstance(admin_user, dict): instance.admin_username = admin_user.get("username", "admin") - # Canonical providers use neutral policy defaults; legacy input retains historical defaults. - if "singleNodeMode" in raw_config: - instance.single_node_mode = raw_config["singleNodeMode"] - elif legacy_auth: - instance.single_node_mode = instance.auth_mode in ("auto-login", "local") + if "runtimeLimitEnabled" in raw_config: + runtime_limit_enabled = raw_config["runtimeLimitEnabled"] + if type(runtime_limit_enabled) is not bool: + raise AuthConfigurationError("runtimeLimitEnabled must be a boolean") + instance.runtime_limit_enabled = runtime_limit_enabled + elif legacy_mode is not None: + instance.runtime_limit_enabled = legacy_mode not in ("auto-login", "local") else: - instance.single_node_mode = False + instance.runtime_limit_enabled = True # Parse structured configuration instance._config = ParsedConfig.from_dicts( @@ -486,25 +450,29 @@ def init(cls, config_path: str | Path) -> HubConfig: notifications=raw_config.get("notifications"), ) - if instance._config.resources.accessPolicy is None: - instance._config.resources.accessPolicy = ( - _legacy_resource_access_policy(effective_mode) if legacy_auth else DEFAULT_RESOURCE_ACCESS_POLICY - ) - # Canonical providers use neutral policy defaults; legacy input retains historical defaults. if instance._config.quota.enabled is not None: instance.quota_enabled = instance._config.quota.enabled else: - instance.quota_enabled = instance.auth_mode not in ("auto-login", "dummy", "local") if legacy_auth else True + instance.quota_enabled = ( + legacy_mode not in ("auto-login", "dummy", "local") if legacy_mode is not None else True + ) instance._config.quota.enabled = instance.quota_enabled + if instance.quota_enabled and not instance.runtime_limit_enabled: + raise AuthConfigurationError("quota.enabled requires runtimeLimitEnabled: true") + cls._instance = instance cls._initialized = True # Log configuration print("[CONFIG] HubConfig initialized:") - print(f"[CONFIG] auth_mode={instance.auth_mode}") - print(f"[CONFIG] single_node_mode={instance.single_node_mode}") + print( + "[CONFIG] auth=" + f"auto_login:{instance.auth.auto_login},dummy:{instance.auth.dummy}," + f"native:{instance.auth.native},github:{instance.auth.github}" + ) + print(f"[CONFIG] runtime_limit_enabled={instance.runtime_limit_enabled}") print(f"[CONFIG] quota_enabled={instance.quota_enabled}") print(f"[CONFIG] resources={len(instance._config.resources.images)} images") print(f"[CONFIG] accelerators={list(instance._config.accelerators.keys())}") diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index dba21dae..e8533cfb 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -50,7 +50,7 @@ ) if TYPE_CHECKING: - from core.config import HubConfig, ResourceAccessPolicy + from core.config import HubConfig # NPU Security Config @@ -90,9 +90,7 @@ class RemoteLabKubeSpawner(KubeSpawner): # Runtime settings (set by jupyterhub_config.py) github_org_name: str = "" - auth_mode: str = "auto-login" - access_policy: ResourceAccessPolicy = "group-mapped" - single_node_mode: bool = False + runtime_limit_enabled: bool = True quota_enabled: bool | None = False # Resource configuration (set from config) @@ -132,9 +130,7 @@ def configure_from_config(cls, config: HubConfig) -> None: cls._hub_config = config # Basic spawner settings - cls.auth_mode = config.auth_mode - cls.access_policy = config.resources.effective_access_policy - cls.single_node_mode = config.single_node_mode + cls.runtime_limit_enabled = config.runtime_limit_enabled cls.github_org_name = config.github_org_name # Extract resource images and requirements @@ -174,9 +170,8 @@ def configure_from_config(cls, config: HubConfig) -> None: def _resolve_user_resources(self) -> list[str]: """Resolve available resources for the current user from server-side policy. - For auto-login/dummy modes, returns all configured resources. - For all other users, resolves resources from JupyterHub groups - (which are synced from GitHub teams or assigned to native users + Resolves resources from JupyterHub groups, which are synced from GitHub teams + or assigned to native users via the auth_state_hook). Falls back to legacy pattern matching for native users with no group assignments. @@ -191,8 +186,6 @@ def _resolve_user_resources(self) -> list[str]: available_resources = resolve_resources_for_user( self.user, self.team_resource_mapping, - self.access_policy, - list(self.resource_images.keys()), ) self.log.debug(f"User '{username}' resolved resources: {available_resources}") return available_resources @@ -237,8 +230,8 @@ def _resolve_accelerator_selection(self, resource_type: str, gpu_selection: Any) async def options_form(self, _) -> str: """Generate the HTML form for resource selection. - Returns a <script> tag that injects ``window.AVAILABLE_RESOURCES`` - and ``window.SINGLE_NODE_MODE`` for the React spawn app. The custom + Returns a <script> tag that injects ``window.AVAILABLE_RESOURCES`` for + the React spawn app. The custom ``spawn.html`` template renders this via ``{{ spawner_options_form | safe }}``. """ try: @@ -246,14 +239,7 @@ async def options_form(self, _) -> str: self.log.debug(f"Providing users with following resources: {available_resource_names}") available_resources_js = json.dumps(available_resource_names) - single_node_mode_js = "true" if self.single_node_mode else "false" - - return ( - "<script>" - f"window.AVAILABLE_RESOURCES={available_resources_js};" - f"window.SINGLE_NODE_MODE={single_node_mode_js};" - "</script>" - ) + return f"<script>window.AVAILABLE_RESOURCES={available_resources_js};</script>" except Exception as e: self.log.error(f"Failed to load options form: {e}", exc_info=True) @@ -750,17 +736,17 @@ def _build_runtime_metadata_env( start_time: int, runtime_minutes: int, quota_rate: int, - runtime_unlimited: bool, + runtime_limit_enabled: bool, ) -> dict[str, str]: env = { "JOB_START_TIME": str(start_time), "QUOTA_RATE": str(quota_rate), } - if runtime_unlimited: - env["AUPLC_RUNTIME_UNLIMITED"] = "true" - else: + if runtime_limit_enabled: env["JOB_RUN_TIME"] = str(runtime_minutes) + else: + env["AUPLC_RUNTIME_UNLIMITED"] = "true" return env @@ -1098,7 +1084,7 @@ async def start(self): start_time=start_time, runtime_minutes=runtime_minutes, quota_rate=quota_rate, - runtime_unlimited=self.single_node_mode, + runtime_limit_enabled=self.runtime_limit_enabled, ) ) @@ -1244,11 +1230,10 @@ async def start(self): self.start_time = start_time self._resource_type = resource_type - # In single-node mode, skip auto-shutdown timer - if self.single_node_mode: + if not self.runtime_limit_enabled: self.shutdown_time = None self.check_timer = None - self.log.debug(f"Container for {self.user.name} started (single-node mode, no time limit)") + self.log.debug(f"Container for {self.user.name} started without a runtime limit") else: self.shutdown_time = start_time + (runtime_minutes * 60) loop = asyncio.get_event_loop() diff --git a/runtime/hub/tests/test_config_resource_metadata.py b/runtime/hub/tests/test_config_resource_metadata.py index a5688036..be4763b8 100644 --- a/runtime/hub/tests/test_config_resource_metadata.py +++ b/runtime/hub/tests/test_config_resource_metadata.py @@ -51,11 +51,11 @@ def load_module(name: str, path: Path): ProviderFlags = tuple[bool, bool, bool, bool] AUTH_FLAG_NAMES = ("autoLogin", "dummy", "native", "github") VALID_CANONICAL_AUTH = ( - ((True, False, False, False), "auto-login"), - ((False, True, False, False), "dummy"), - ((False, False, True, False), "local"), - ((False, False, False, True), "github"), - ((False, False, True, True), "multi"), + (True, False, False, False), + (False, True, False, False), + (False, False, True, False), + (False, False, False, True), + (False, False, True, True), ) INVALID_CANONICAL_AUTH = ( (False, False, False, False), @@ -151,26 +151,26 @@ def test_code_server_extra_trusted_domains_parse_from_config(): def test_legacy_github_mode_preserves_existing_runtime_defaults(tmp_path: Path): hub_config = config.HubConfig.init(write_hub_config(tmp_path, "authMode: github\n")) - assert hub_config.auth_mode == "github" - assert hub_config.single_node_mode is False + assert hub_config.auth.github is True + assert not hasattr(hub_config, "auth_mode") + assert hub_config.runtime_limit_enabled is True assert hub_config.quota_enabled is True def test_absent_auth_forms_preserve_existing_auto_login_compatibility(tmp_path: Path): hub_config = config.HubConfig.init(write_hub_config(tmp_path, "resources: {}\n")) - assert hub_config.auth_mode == "auto-login" + assert hub_config.auth.auto_login is True + assert not hasattr(hub_config, "auth_mode") -@pytest.mark.parametrize(("flags", "expected_mode"), VALID_CANONICAL_AUTH) -def test_canonical_auth_flags_normalize_to_capabilities_and_neutral_policy( - tmp_path: Path, flags: ProviderFlags, expected_mode: str -): +@pytest.mark.parametrize("flags", VALID_CANONICAL_AUTH) +def test_canonical_auth_flags_normalize_to_capabilities_and_runtime_limit_default(tmp_path: Path, flags: ProviderFlags): hub_config = config.HubConfig.init(write_hub_config(tmp_path, canonical_auth_yaml(flags))) - assert hub_config.auth_mode == expected_mode assert (hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github) == flags - assert hub_config.single_node_mode is False + assert not hasattr(hub_config, "auth_mode") + assert hub_config.runtime_limit_enabled is True assert hub_config.quota_enabled is True @@ -180,30 +180,30 @@ def test_canonical_auth_rejects_each_invalid_boolean_combination(tmp_path: Path, @pytest.mark.parametrize( - ("legacy_mode", "expected_flags", "expected_single_node", "expected_quota"), + ("legacy_mode", "expected_flags", "expected_runtime_limit", "expected_quota"), [ - ("auto-login", (True, False, False, False), True, False), - ("dummy", (False, True, False, False), False, False), - ("github", (False, False, False, True), False, True), - ("local", (False, False, True, False), True, False), - ("multi", (False, False, True, True), False, True), + ("auto-login", (True, False, False, False), False, False), + ("dummy", (False, True, False, False), True, False), + ("github", (False, False, False, True), True, True), + ("local", (False, False, True, False), False, False), + ("multi", (False, False, True, True), True, True), ], ) def test_explicit_legacy_modes_map_to_capabilities_and_preserve_policy_defaults( - tmp_path: Path, legacy_mode: str, expected_flags: ProviderFlags, expected_single_node: bool, expected_quota: bool + tmp_path: Path, legacy_mode: str, expected_flags: ProviderFlags, expected_runtime_limit: bool, expected_quota: bool ): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) hub_config = config.HubConfig.init(write_hub_config(tmp_path, f"authMode: {legacy_mode}\n")) - assert hub_config.auth_mode == legacy_mode assert ( hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github, ) == expected_flags - assert hub_config.single_node_mode is expected_single_node + assert not hasattr(hub_config, "auth_mode") + assert hub_config.runtime_limit_enabled is expected_runtime_limit assert hub_config.quota_enabled is expected_quota @@ -226,14 +226,14 @@ def test_absent_auth_forms_use_compatibility_auto_login_with_neutral_defaults(tm hub_config = config.HubConfig.init(write_hub_config(tmp_path, "resources: {}\n")) assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] - assert hub_config.auth_mode == "auto-login" assert (hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github) == ( True, False, False, False, ) - assert hub_config.single_node_mode is False + assert not hasattr(hub_config, "auth_mode") + assert hub_config.runtime_limit_enabled is True assert hub_config.quota_enabled is True @@ -255,23 +255,62 @@ def test_malformed_canonical_auth_is_rejected_before_hub_setup(tmp_path: Path, c @pytest.mark.parametrize( - ("contents", "expected_single_node", "expected_quota"), + ("contents", "expected_runtime_limit", "expected_quota"), [ - ("auth:\n native: true\nsingleNodeMode: true\nquota:\n enabled: false\n", True, False), - ("authMode: local\nsingleNodeMode: false\nquota:\n enabled: true\n", False, True), + ("auth:\n native: true\nruntimeLimitEnabled: true\nquota:\n enabled: true\n", True, True), + ("auth:\n native: true\nruntimeLimitEnabled: true\nquota:\n enabled: false\n", True, False), + ("auth:\n native: true\nruntimeLimitEnabled: false\nquota:\n enabled: false\n", False, False), ], ) -def test_explicit_runtime_policy_values_override_auth_compatibility_defaults( - tmp_path: Path, contents: str, expected_single_node: bool, expected_quota: bool +def test_explicit_runtime_limit_and_quota_values_are_independent( + tmp_path: Path, contents: str, expected_runtime_limit: bool, expected_quota: bool ): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) hub_config = config.HubConfig.init(write_hub_config(tmp_path, contents)) - assert hub_config.single_node_mode is expected_single_node + assert hub_config.runtime_limit_enabled is expected_runtime_limit assert hub_config.quota_enabled is expected_quota +def test_hub_rejects_enabled_quota_with_unlimited_runtime_before_setup(tmp_path: Path): + assert_auth_configuration_rejected( + tmp_path, + "auth:\n native: true\nruntimeLimitEnabled: false\nquota:\n enabled: true\n", + "quota.enabled requires runtimeLimitEnabled: true", + ) + + +@pytest.mark.parametrize("quota_enabled", ['"false"', '"yes"', "1", "[]"]) +def test_hub_rejects_malformed_quota_enabled_values(tmp_path: Path, quota_enabled: str): + with pytest.raises(ValidationError): + config.HubConfig.init( + write_hub_config(tmp_path, f"auth:\n native: true\nquota:\n enabled: {quota_enabled}\n") + ) + + +def test_legacy_local_rejects_enabled_quota_when_runtime_limit_is_omitted(tmp_path: Path): + assert_auth_configuration_rejected( + tmp_path, + "authMode: local\nquota:\n enabled: true\n", + "quota.enabled requires runtimeLimitEnabled: true", + ) + + +def test_legacy_local_accepts_enabled_quota_with_explicit_runtime_limit(tmp_path: Path): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config.HubConfig.init( + write_hub_config( + tmp_path, + "authMode: local\nruntimeLimitEnabled: true\nquota:\n enabled: true\n", + ) + ) + + assert hub_config.runtime_limit_enabled is True + assert hub_config.quota_enabled is True + + def test_hub_config_singleton_is_reset_before_each_case(): assert config.HubConfig.is_initialized() is False with pytest.raises(RuntimeError): @@ -288,7 +327,8 @@ def test_null_legacy_mode_is_absent_compatibility_without_warning(tmp_path: Path warnings.simplefilter("always") hub_config = config.HubConfig.init(write_hub_config(tmp_path, "authMode: null\n")) - assert hub_config.auth_mode == "auto-login" + assert hub_config.auth.auto_login is True + assert not hasattr(hub_config, "auth_mode") assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] diff --git a/runtime/hub/tests/test_spawner_runtime_metadata.py b/runtime/hub/tests/test_spawner_runtime_metadata.py index fc82fdad..64a92491 100644 --- a/runtime/hub/tests/test_spawner_runtime_metadata.py +++ b/runtime/hub/tests/test_spawner_runtime_metadata.py @@ -22,6 +22,8 @@ import types from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] CORE = ROOT / "core" @@ -87,17 +89,17 @@ def load_module(name: str, path: Path): RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner -def build_env(runtime_minutes: int, runtime_unlimited: bool, quota_rate: int = 3): +def build_env(runtime_minutes: int, runtime_limit_enabled: bool, quota_rate: int = 3): return RemoteLabKubeSpawner._build_runtime_metadata_env( start_time=1_717_171_717, runtime_minutes=runtime_minutes, quota_rate=quota_rate, - runtime_unlimited=runtime_unlimited, + runtime_limit_enabled=runtime_limit_enabled, ) def test_finite_runtime_metadata_includes_positive_job_run_time(): - env = build_env(runtime_minutes=120, runtime_unlimited=False) + env = build_env(runtime_minutes=120, runtime_limit_enabled=True) assert env == { "JOB_START_TIME": "1717171717", @@ -108,15 +110,15 @@ def test_finite_runtime_metadata_includes_positive_job_run_time(): def test_quota_unlimited_finite_runtime_metadata_stays_finite(): - env = build_env(runtime_minutes=120, runtime_unlimited=False, quota_rate=0) + env = build_env(runtime_minutes=120, runtime_limit_enabled=True, quota_rate=0) assert env["JOB_RUN_TIME"] == "120" assert env["QUOTA_RATE"] == "0" assert "AUPLC_RUNTIME_UNLIMITED" not in env -def test_single_node_no_limit_runtime_metadata_uses_unlimited_flag(): - env = build_env(runtime_minutes=120, runtime_unlimited=True) +def test_runtime_limit_disabled_metadata_uses_unlimited_flag(): + env = build_env(runtime_minutes=120, runtime_limit_enabled=False) assert env == { "JOB_START_TIME": "1717171717", @@ -125,3 +127,59 @@ def test_single_node_no_limit_runtime_metadata_uses_unlimited_flag(): } assert "JOB_RUN_TIME" not in env assert "4320" not in env.values() + + +@pytest.mark.parametrize("runtime_limit_enabled", [True, False]) +def test_start_schedules_shutdown_only_when_runtime_limit_enabled( + monkeypatch: pytest.MonkeyPatch, runtime_limit_enabled: bool +) -> None: + class QuotaManager: + def start_usage_session(self, *_args: str) -> str: + return "usage-session" + + class TimerLoop: + def __init__(self) -> None: + self.calls: list[tuple[int, object]] = [] + + def call_later(self, delay: int, callback: object) -> str: + self.calls.append((delay, callback)) + return "timer" + + quota_module = types.ModuleType("core.quota") + quota_module.get_quota_manager = lambda: QuotaManager() + monkeypatch.setitem(sys.modules, "core.quota", quota_module) + + async def base_start(_spawner: object) -> str: + return "started" + + timer_loop = TimerLoop() + monkeypatch.setattr(kubernetes.KubeSpawner, "start", base_start, raising=False) + monkeypatch.setattr(kubernetes.time, "time", lambda: 1_717_171_717) + monkeypatch.setattr(kubernetes.asyncio, "get_event_loop", lambda: timer_loop) + + spawner = object.__new__(RemoteLabKubeSpawner) + spawner.user = types.SimpleNamespace(name="student") + spawner.user_options = {"runtime_minutes": 120, "resource_type": "cpu"} + spawner.quota_enabled = False + spawner.runtime_limit_enabled = runtime_limit_enabled + spawner.environment = {} + spawner.notebook_allowed_origins = [] + spawner._hub_config = None + spawner.log = types.SimpleNamespace(debug=lambda _message: None) + spawner._launches_code_server = lambda _resource_type: False + + result = kubernetes.asyncio.run(spawner.start()) + + assert result == "started" + if runtime_limit_enabled: + assert spawner.shutdown_time == 1_717_178_917 + assert spawner.check_timer == "timer" + assert timer_loop.calls == [(60, spawner.check_timeout)] + assert spawner.environment["JOB_RUN_TIME"] == "120" + assert "AUPLC_RUNTIME_UNLIMITED" not in spawner.environment + else: + assert spawner.shutdown_time is None + assert spawner.check_timer is None + assert timer_loop.calls == [] + assert spawner.environment["AUPLC_RUNTIME_UNLIMITED"] == "true" + assert "JOB_RUN_TIME" not in spawner.environment diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index 22445c16..1be3ca4a 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -86,6 +86,175 @@ def test_chart_accepts_absent_auth_forms_without_injecting_a_default() -> None: assert "authMode" not in custom +def test_runtime_values_keep_auth_absent_and_resolve_to_compatibility_auto_login( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + result = subprocess.run( + ["helm", "template", "jupyterhub", CHART, "-f", "runtime/values.yaml"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + config_map = document_by_kind(rendered_documents(result.stdout), "ConfigMap") + rendered_config = config_map["data"]["hub-config.yaml"] + custom = yaml.safe_load(rendered_config) + assert "auth" not in custom + assert "authMode" not in custom + assert "runtimeLimitEnabled" not in custom + + config_path = tmp_path / "hub-config.yaml" + config_path.write_text(rendered_config, encoding="utf-8") + spec = importlib.util.spec_from_file_location("runtime_values_config", ROOT / "runtime/hub/core/config.py") + assert spec is not None + assert spec.loader is not None + config_module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, config_module) + spec.loader.exec_module(config_module) + hub_config = config_module.HubConfig.init(config_path) + + assert ( + hub_config.auth.auto_login, + hub_config.auth.dummy, + hub_config.auth.native, + hub_config.auth.github, + ) == (True, False, False, False) + assert not hasattr(hub_config, "auth_mode") + assert hub_config.runtime_limit_enabled is True + assert hub_config.quota_enabled is True + + +def test_multi_node_example_emits_canonical_auth_and_runtime_policy() -> None: + values = yaml.safe_load((ROOT / "runtime/values-multi-nodes.yaml.example").read_text(encoding="utf-8")) + custom = values["custom"] + + assert custom["auth"] == {"native": True, "github": True} + assert "authMode" not in custom + assert custom["runtimeLimitEnabled"] is True + assert custom["quota"]["enabled"] is True + + +@pytest.mark.parametrize( + ("runtime_limit_enabled", "quota_enabled"), + [(True, True), (True, False), (False, False)], +) +def test_chart_accepts_each_valid_quota_runtime_combination(runtime_limit_enabled: bool, quota_enabled: bool) -> None: + result = render( + f"custom.runtimeLimitEnabled={str(runtime_limit_enabled).lower()}", + f"custom.quota.enabled={str(quota_enabled).lower()}", + ) + + assert result.returncode == 0, result.stderr + + +def test_chart_rejects_enabled_quota_with_unlimited_runtime() -> None: + result = render("custom.runtimeLimitEnabled=false", "custom.quota.enabled=true") + + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr + + +@pytest.mark.parametrize("quota_enabled", ("false", "yes")) +def test_chart_rejects_string_quota_enabled_values(quota_enabled: str) -> None: + result = render(string_settings=(f"custom.quota.enabled={quota_enabled}",)) + + assert result.returncode != 0 + assert "got string, want null or boolean" in result.stderr + + +def test_chart_rejects_integer_quota_enabled_value() -> None: + result = render("custom.quota.enabled=1") + + assert result.returncode != 0 + assert "got number, want null or boolean" in result.stderr + + +def test_chart_rejects_array_quota_enabled_value() -> None: + result = subprocess.run( + ["helm", "template", "jupyterhub", CHART, "--set-json", "custom.quota.enabled=[]"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "got array, want null or boolean" in result.stderr + + +def test_chart_rejects_legacy_local_enabled_quota_without_runtime_limit() -> None: + result = render("custom.authMode=local", "custom.quota.enabled=true") + + assert result.returncode != 0 + assert "values don't meet the specifications" in result.stderr + + +def test_chart_accepts_legacy_local_enabled_quota_with_explicit_runtime_limit() -> None: + result = render( + "custom.authMode=local", + "custom.runtimeLimitEnabled=true", + "custom.quota.enabled=true", + ) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "legacy_case", + [ + ("local", False, False), + ("multi", True, True), + ], +) +def test_legacy_auth_overlay_preserves_runtime_defaults_after_shared_values_render( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, legacy_case: tuple[str, bool, bool] +) -> None: + auth_mode, expected_runtime_limit, expected_quota = legacy_case + overlay = tmp_path / "legacy-auth.yaml" + overlay.write_text(f"custom:\n authMode: {auth_mode}\n", encoding="utf-8") + result = subprocess.run( + [ + "helm", + "template", + "jupyterhub", + CHART, + "-f", + "runtime/values.yaml", + "-f", + str(overlay), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + config_map = document_by_kind(rendered_documents(result.stdout), "ConfigMap") + rendered_config = config_map["data"]["hub-config.yaml"] + custom = yaml.safe_load(rendered_config) + assert custom["authMode"] == auth_mode + assert "auth" not in custom + assert "runtimeLimitEnabled" not in custom + + config_path = tmp_path / "hub-config.yaml" + config_path.write_text(rendered_config, encoding="utf-8") + spec = importlib.util.spec_from_file_location("legacy_chart_contract_config", ROOT / "runtime/hub/core/config.py") + assert spec is not None + assert spec.loader is not None + config_module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, config_module) + spec.loader.exec_module(config_module) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + hub_config = config_module.HubConfig.init(config_path) + + assert hub_config.runtime_limit_enabled is expected_runtime_limit + assert hub_config.quota_enabled is expected_quota + + def test_null_legacy_mode_renders_as_compatibility_absent_without_warning( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -117,13 +286,13 @@ def test_null_legacy_mode_renders_as_compatibility_absent_without_warning( warnings.simplefilter("always") hub_config = config_module.HubConfig.init(config_path) - assert hub_config.auth_mode == "auto-login" assert ( hub_config.auth.auto_login, hub_config.auth.dummy, hub_config.auth.native, hub_config.auth.github, ) == (True, False, False, False) + assert not hasattr(hub_config, "auth_mode") assert not [warning for warning in caught if issubclass(warning.category, DeprecationWarning)] From a04b37b87ebd02143af6e430f3fa286489b09649 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:20:23 +0800 Subject: [PATCH 152/180] fix(auth): equalize native authentication timing --- runtime/hub/core/authenticators/firstuse.py | 11 ++++++++--- runtime/hub/tests/test_native_authenticator.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/runtime/hub/core/authenticators/firstuse.py b/runtime/hub/core/authenticators/firstuse.py index 685b82b4..f83b3c14 100644 --- a/runtime/hub/core/authenticators/firstuse.py +++ b/runtime/hub/core/authenticators/firstuse.py @@ -49,6 +49,7 @@ class CustomFirstUseAuthenticator(FirstUseAuthenticator): service_name = "Native" login_service = "Native" create_users = False + DUMMY_PASSWORD_HASH = b"$2b$12$HxnZoJ.V..l/07wvD0EsOOBq14vGDBJ0ls0k8uKDH/PTVjFK.tXVi" def normalize_username(self, username): """Normalize username to lowercase.""" @@ -56,14 +57,14 @@ def normalize_username(self, username): return username return username.lower() - def _user_exists(self, username): + def _user_exists(self, username: str) -> bool | None: """Check if user exists in JupyterHub database.""" db = getattr(self, "db", None) if db is None: db = getattr(getattr(self, "parent", None), "db", None) if db is None: self.log.warning("Native authentication denied because Hub database is unavailable") - return False + return None from jupyterhub.orm import User @@ -255,7 +256,11 @@ async def authenticate(self, _handler, data): return None # Check if user exists in JupyterHub - if not self._user_exists(username): + user_exists = self._user_exists(username) + if user_exists is None: + return None + if not user_exists: + bcrypt.checkpw(password.encode("utf8"), self.DUMMY_PASSWORD_HASH) self.log.warning(f"User {username} not found in JupyterHub database") return None diff --git a/runtime/hub/tests/test_native_authenticator.py b/runtime/hub/tests/test_native_authenticator.py index b9515540..cc4bfc21 100644 --- a/runtime/hub/tests/test_native_authenticator.py +++ b/runtime/hub/tests/test_native_authenticator.py @@ -72,7 +72,13 @@ def _loaded_firstuse_authenticator(monkeypatch: pytest.MonkeyPatch) -> Iterator[ bcrypt = types.ModuleType("bcrypt") bcrypt.gensalt = lambda: b"salt" bcrypt.hashpw = lambda password, _salt: b"hash:" + password - bcrypt.checkpw = lambda password, password_hash: password_hash == b"hash:" + password + bcrypt.checkpw_calls = [] + + def checkpw(password, password_hash): + bcrypt.checkpw_calls.append((password, password_hash)) + return password_hash == b"hash:" + password + + bcrypt.checkpw = checkpw class FakeFirstUseAuthenticator: def __init__(self) -> None: @@ -180,6 +186,7 @@ def test_existing_user_authentication_checks_normalized_username( ("has_password", "learner"), ("check_password", "learner", submitted_password), ] + assert sys.modules["bcrypt"].checkpw_calls == [] def test_missing_child_and_parent_database_rejects_without_password_side_effect( @@ -199,6 +206,7 @@ def test_missing_child_and_parent_database_rejects_without_password_side_effect( assert authenticated is None assert password_changes == [] assert authenticator.log.warnings + assert sys.modules["bcrypt"].checkpw_calls == [] def test_missing_parent_database_rejects_without_password_side_effect(monkeypatch: pytest.MonkeyPatch) -> None: @@ -214,6 +222,7 @@ def test_missing_parent_database_rejects_without_password_side_effect(monkeypatc assert authenticated is None assert password_changes == [] assert authenticator.log.warnings + assert sys.modules["bcrypt"].checkpw_calls == [] @pytest.mark.parametrize("query_result", [None, False], ids=["none", "falsey"]) @@ -230,6 +239,7 @@ def test_unknown_user_query_result_rejects_without_password_side_effect( assert authenticated is None assert password_changes == [] + assert sys.modules["bcrypt"].checkpw_calls == [(b"Password1!", authenticator_type.DUMMY_PASSWORD_HASH)] def test_database_query_error_propagates_without_password_side_effect(monkeypatch: pytest.MonkeyPatch) -> None: @@ -243,6 +253,7 @@ def test_database_query_error_propagates_without_password_side_effect(monkeypatc asyncio.run(authenticator.authenticate(None, {"username": "learner", "password": "Password1!"})) assert password_changes == [] + assert sys.modules["bcrypt"].checkpw_calls == [] def test_parent_database_fallback_supports_multiauth_child(monkeypatch: pytest.MonkeyPatch) -> None: From 2bfe638906448cf00cb1ca4a357338c5a94e7f87 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:21:39 +0800 Subject: [PATCH 153/180] refactor(installer): add canonical access profiles --- auplc_installer/cli.py | 22 +-- auplc_installer/overlay.py | 54 +++--- auplc_installer/profiles.py | 68 ++++++++ tests/installer/test_access_profiles.py | 157 ++++++++++++++++++ tests/installer/test_admin_secret_contract.py | 33 ++++ tests/installer/test_cli_install_options.py | 12 ++ tests/installer/test_local_auth.py | 38 ++--- tests/installer/test_profile_ownership.py | 108 ++++++++++++ 8 files changed, 419 insertions(+), 73 deletions(-) create mode 100644 auplc_installer/profiles.py create mode 100644 tests/installer/test_access_profiles.py create mode 100644 tests/installer/test_admin_secret_contract.py create mode 100644 tests/installer/test_profile_ownership.py diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 9a8b2428..777a3a42 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -11,7 +11,6 @@ import argparse import contextlib -import re import sys import time from collections.abc import Sequence @@ -19,7 +18,6 @@ from typing import NoReturn from auplc_installer import __version__ -from auplc_installer.auth import validate_local_admin_username from auplc_installer.catalog import parse_selection_spec from auplc_installer.gpu import ( detect_and_configure_gpu, @@ -46,6 +44,7 @@ try_load_courses_from_overlay, ) from auplc_installer.pack import pack_bundle +from auplc_installer.profiles import resolve_access_settings from auplc_installer.progress import stage from auplc_installer.rocm import deploy_rocm_gpu_device_plugin from auplc_installer.state import InstallerState @@ -767,16 +766,6 @@ def _preserve_courses_for_upgrade(state: InstallerState, overlay_path: Path) -> def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: Path) -> None: - if overlay_path.is_file(): - text = overlay_path.read_text(encoding="utf-8") - if re.search( - r"^\s*authMode\s*:\s*(?:\"(?:github|multi|dummy)\"|'(?:github|multi|dummy)'|github|multi|dummy)\s*(?:#.*)?$", - text, - re.MULTILINE, - ): - raise InstallerError( - "Existing overlay uses an advanced authMode; use operator-managed Helm values instead of installer upgrade" - ) previous = try_load_access_settings_from_overlay(overlay_path) if state.access_mode: if state.access_mode == "local" and not state.admin_username and previous and previous[0] == "local": @@ -789,13 +778,8 @@ def _preserve_access_settings_for_upgrade(state: InstallerState, overlay_path: P def _resolve_access_settings(state: InstallerState) -> tuple[str, str]: - access_mode = state.access_mode or "personal" - if access_mode not in ("local", "personal"): - raise InstallerError("--access-mode must be local or personal") - admin_username = state.admin_username or "admin" - if access_mode == "local": - admin_username = validate_local_admin_username(admin_username) - return access_mode, admin_username + settings = resolve_access_settings(state.access_mode, state.admin_username) + return settings.access_mode, settings.admin_username def cmd_rt_remove(state: InstallerState) -> None: diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 2abdf9d3..d3b45ad5 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -14,8 +14,8 @@ import re from io import StringIO from pathlib import Path +from typing import assert_never -from auplc_installer.auth import validate_local_admin_username from auplc_installer.catalog import ( BASE_TEAM_MAPPING, NONE_SENTINEL, @@ -23,6 +23,7 @@ parse_selection_spec, ) from auplc_installer.gpu import GpuConfig, is_curated_sku +from auplc_installer.profiles import AccessProfile, detect_installer_profile, resolve_access_settings from auplc_installer.util import InstallerError, log # Resource name → image basename (used by acceleratorOverrides emission @@ -52,6 +53,7 @@ def emit_overlay( offline_mode: bool = False, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" + settings = resolve_access_settings(access_mode, admin_username) buf = StringIO() primary_tag = f"{image_tag}-{cfg.gpu_target}" homogeneous_target = cfg.homogeneous_target @@ -67,20 +69,30 @@ def emit_overlay( targets = " ".join(s.gpu_target for s in cfg.skus) buf.write(f"# Mixed gfx targets: {targets}\n") buf.write(f"# Env selection : {courses.description()}\n") - buf.write(f"# Access mode : {access_mode}\n") - buf.write(f"# Admin username: {admin_username}\n") + buf.write(f"# Access mode : {settings.access_mode}\n") + buf.write(f"# Admin username: {settings.admin_username}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") - auth_mode = "local" if access_mode == "local" else "auto-login" - if access_mode == "local": - admin_username = validate_local_admin_username(admin_username) - buf.write(f" authMode: {auth_mode}\n") - buf.write(" singleNodeMode: true\n") + match settings.profile: + case AccessProfile.PERSONAL: + buf.write(" auth:\n") + buf.write(" autoLogin: true\n") + case AccessProfile.LOCAL: + buf.write(" auth:\n") + buf.write(" native: true\n") + buf.write(" runtimeLimitEnabled: false\n") buf.write(" adminUser:\n") - buf.write(f" enabled: {'true' if access_mode == 'local' else 'false'}\n") - buf.write(f' username: "{admin_username}"\n') - if access_mode == "local": - buf.write(' existingSecret: "jupyterhub-admin-credentials"\n') + match settings.profile: + case AccessProfile.LOCAL: + buf.write(" enabled: true\n") + buf.write(f' username: "{settings.admin_username}"\n') + buf.write(' existingSecret: "jupyterhub-admin-credentials"\n') + case AccessProfile.PERSONAL: + buf.write(" enabled: false\n") + case unreachable: + assert_never(unreachable) + buf.write(" quota:\n") + buf.write(f" enabled: {str(settings.quota_enabled).lower()}\n") # --- accelerators --- any_accel_emitted = False @@ -123,7 +135,6 @@ def emit_overlay( buf.write(" env: {}\n") buf.write(f" quotaRate: {sku.quota_rate}\n") - # --- resources block: GPU course images + metadata --- emit_resources = [r for r in GPU_RESOURCE_KEYS if not filter_courses or courses.is_selected(r)] if emit_resources: buf.write(" resources:\n") @@ -204,8 +215,6 @@ def generate_values_overlay( # ``rt upgrade`` (no ``--courses=`` flag) preserves whatever the user # originally installed with instead of silently expanding to "all". _COURSE_HEADER_RE = re.compile(r"^# (?:Env selection|Course selection)\s*:\s*(.+?)\s*$") -_ACCESS_MODE_HEADER_RE = re.compile(r"^# Access mode\s*:\s*(local|personal)\s*$") -_ADMIN_USERNAME_HEADER_RE = re.compile(r"^# Admin username\s*:\s*(.+?)\s*$") def try_load_courses_from_overlay(overlay_path: Path) -> CourseSelection | None: @@ -252,19 +261,10 @@ def try_load_access_settings_from_overlay(overlay_path: Path) -> tuple[str, str] text = overlay_path.read_text(encoding="utf-8") except OSError: return None - access_mode = "" - admin_username = "" - for line in text.splitlines(): - mode_match = _ACCESS_MODE_HEADER_RE.match(line) - if mode_match: - access_mode = mode_match.group(1) - continue - username_match = _ADMIN_USERNAME_HEADER_RE.match(line) - if username_match: - admin_username = username_match.group(1) - if access_mode == "" or admin_username == "": + settings = detect_installer_profile(text) + if settings is None: return None - return access_mode, admin_username + return settings.access_mode, settings.admin_username # Re-exported so callers can import ``NONE_SENTINEL`` from a single module diff --git a/auplc_installer/profiles.py b/auplc_installer/profiles.py new file mode 100644 index 00000000..16486e01 --- /dev/null +++ b/auplc_installer/profiles.py @@ -0,0 +1,68 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import assert_never + +from auplc_installer.auth import validate_local_admin_username +from auplc_installer.util import InstallerError + + +class AccessProfile(str, Enum): + PERSONAL = "personal" + LOCAL = "local" + + +@dataclass(frozen=True, slots=True) +class AccessSettings: + profile: AccessProfile + admin_username: str + + @property + def access_mode(self) -> str: + return self.profile.value + + @property + def quota_enabled(self) -> bool: + match self.profile: + case AccessProfile.PERSONAL: + return False + case AccessProfile.LOCAL: + return False + case unreachable: + assert_never(unreachable) + + +_ACCESS_HEADER_RE = re.compile(r"^# Access mode\s*:\s*(.+?)\s*$") +_ADMIN_HEADER_RE = re.compile(r"^# Admin username\s*:\s*(.+?)\s*$") + + +def resolve_access_settings(access_mode: str, admin_username: str) -> AccessSettings: + username = validate_local_admin_username(admin_username or "admin") + match access_mode or AccessProfile.PERSONAL.value: + case AccessProfile.PERSONAL.value: + return AccessSettings(AccessProfile.PERSONAL, "admin") + case AccessProfile.LOCAL.value: + return AccessSettings(AccessProfile.LOCAL, username) + case _: + raise InstallerError("--access-mode must be local or personal") + + +def detect_installer_profile(text: str) -> AccessSettings | None: + access_mode = _header_value(text, _ACCESS_HEADER_RE) + admin_username = _header_value(text, _ADMIN_HEADER_RE) + if access_mode is None or admin_username is None: + return None + try: + return resolve_access_settings(access_mode, admin_username) + except InstallerError: + return None + + +def _header_value(text: str, pattern: re.Pattern[str]) -> str | None: + matches = [match.group(1) for line in text.splitlines() if (match := pattern.match(line))] + if len(matches) != 1: + return None + return matches[0] diff --git a/tests/installer/test_access_profiles.py b/tests/installer/test_access_profiles.py new file mode 100644 index 00000000..5117994d --- /dev/null +++ b/tests/installer/test_access_profiles.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import itertools +from pathlib import Path + +import pytest +import yaml + +from auplc_installer.catalog import COURSE_CATALOG, NONE_SENTINEL, CourseSelection +from auplc_installer.cli import _preserve_access_settings_for_upgrade +from auplc_installer.gpu import GpuConfig, append_product +from auplc_installer.overlay import GPU_RESOURCE_KEYS, emit_overlay, try_load_access_settings_from_overlay +from auplc_installer.state import InstallerState + + +def _gpu_config() -> GpuConfig: + config = GpuConfig() + append_product(config, "AMD_Radeon_8060S_Graphics") + return config + + +def _render(*, courses: CourseSelection, access_mode: str = "personal") -> str: + return emit_overlay( + _gpu_config(), + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=courses, + access_mode=access_mode, + admin_username="operator", + ) + + +class _UniqueKeyLoader(yaml.SafeLoader): + pass + + +def _construct_unique_mapping( + loader: _UniqueKeyLoader, + node: yaml.MappingNode, + deep: bool = False, +) -> dict[str, object]: + mapping: dict[str, object] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + assert isinstance(key, str) + assert key not in mapping, f"duplicate YAML key: {key}" + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_UniqueKeyLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping) + + +def _load_unique(text: str) -> dict[str, object]: + rendered = yaml.load(text, Loader=_UniqueKeyLoader) + assert isinstance(rendered, dict) + return rendered + + +def _all_course_selections() -> list[CourseSelection]: + keys = tuple(course.key for course in COURSE_CATALOG) + selections = [CourseSelection.default(), CourseSelection(picks=[NONE_SENTINEL])] + selections.extend( + CourseSelection(picks=list(picks)) + for count in range(1, len(keys) + 1) + for picks in itertools.combinations(keys, count) + ) + return selections + + +def test_personal_profile_emits_minimal_canonical_provider() -> None: + rendered = _load_unique(_render(courses=CourseSelection.default())) + custom = rendered["custom"] + assert isinstance(custom, dict) + + assert custom["auth"] == {"autoLogin": True} + assert "authMode" not in custom + assert custom["runtimeLimitEnabled"] is False + assert custom["quota"] == {"enabled": False} + assert custom["adminUser"] == {"enabled": False} + + +def test_local_profile_emits_minimal_native_provider() -> None: + rendered = _load_unique(_render(courses=CourseSelection.default(), access_mode="local")) + custom = rendered["custom"] + assert isinstance(custom, dict) + + assert custom["auth"] == {"native": True} + assert "authMode" not in custom + assert custom["runtimeLimitEnabled"] is False + assert custom["quota"] == {"enabled": False} + assert custom["adminUser"] == { + "enabled": True, + "username": "operator", + "existingSecret": "jupyterhub-admin-credentials", + } + + +@pytest.mark.parametrize("courses", _all_course_selections()) +@pytest.mark.parametrize("access_mode", ("personal", "local")) +def test_profile_resources_are_emitted_only_for_selected_courses(courses: CourseSelection, access_mode: str) -> None: + text = _render(courses=courses, access_mode=access_mode) + rendered = _load_unique(text) + custom = rendered["custom"] + assert isinstance(custom, dict) + if not any(courses.is_selected(resource) for resource in GPU_RESOURCE_KEYS): + assert "resources" not in custom + else: + resources = custom["resources"] + assert isinstance(resources, dict) + assert text.count("\n resources:\n") == 1 + + +def test_upgrade_preserves_canonical_local_profile_and_admin_username(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text(_render(courses=CourseSelection.default(), access_mode="local"), encoding="utf-8") + state = InstallerState() + + _preserve_access_settings_for_upgrade(state, overlay) + + assert (state.access_mode, state.admin_username) == ("local", "operator") + assert try_load_access_settings_from_overlay(overlay) == ("local", "operator") + + +def test_upgrade_migrates_legacy_personal_profile_with_headers(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text( + "# Access mode : personal\n# Admin username: admin\ncustom:\n authMode: auto-login\n", + encoding="utf-8", + ) + state = InstallerState() + + _preserve_access_settings_for_upgrade(state, overlay) + + assert (state.access_mode, state.admin_username) == ("personal", "admin") + migrated = _load_unique(_render(courses=CourseSelection.default(), access_mode=state.access_mode)) + custom = migrated["custom"] + assert isinstance(custom, dict) + assert custom["auth"] == {"autoLogin": True} + assert "authMode" not in custom + + +def test_upgrade_migrates_legacy_local_profile_with_headers(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text( + '# Access mode : local\n# Admin username: operator\ncustom:\n authMode: "local"\n', + encoding="utf-8", + ) + state = InstallerState() + + _preserve_access_settings_for_upgrade(state, overlay) + + migrated = _load_unique(_render(courses=CourseSelection.default(), access_mode=state.access_mode)) + custom = migrated["custom"] + assert isinstance(custom, dict) + assert custom["auth"] == {"native": True} + assert "authMode" not in custom diff --git a/tests/installer/test_admin_secret_contract.py b/tests/installer/test_admin_secret_contract.py new file mode 100644 index 00000000..c112cecf --- /dev/null +++ b/tests/installer/test_admin_secret_contract.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json +import subprocess + +import pytest + +from auplc_installer.helm import ensure_local_admin_secret +from auplc_installer.util import InstallerError + + +def test_existing_secret_rejects_a_different_administrator_username(monkeypatch) -> None: + def fake_run(command, *, check=True, input_text=None, capture_output=False): + if command[2] == "secret": + return subprocess.CompletedProcess( + command, + 0, + json.dumps( + { + "data": { + "admin-username": "b3RoZXI=", + "admin-password": "cGFzc3dvcmQ=", + "api-token": "dG9rZW4=", + } + } + ), + ) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr("auplc_installer.helm.run", fake_run) + + with pytest.raises(InstallerError, match="different administrator username"): + ensure_local_admin_secret("operator") diff --git a/tests/installer/test_cli_install_options.py b/tests/installer/test_cli_install_options.py index 4d7a0c8c..9c4a39ca 100644 --- a/tests/installer/test_cli_install_options.py +++ b/tests/installer/test_cli_install_options.py @@ -173,6 +173,18 @@ def test_install_dry_run_defaults_to_pull() -> None: assert " Image source : pull" in out +def test_install_dry_run_local_summary_includes_profile_and_admin_username() -> None: + state = InstallerState(access_mode="local", admin_username="operator") + buf = io.StringIO() + + with redirect_stdout(buf): + cmd_install_plan(state, legacy_pull=False) + + out = buf.getvalue() + assert " Access mode : local" in out + assert " Admin username : operator" in out + + @pytest.mark.parametrize("username", ["Admin", "admin:name", 'admin"name']) @patch("auplc_installer.cli._resolve_source_root") @patch("auplc_installer.cli.InstallerState.from_environment") diff --git a/tests/installer/test_local_auth.py b/tests/installer/test_local_auth.py index 05588bda..533a3f37 100644 --- a/tests/installer/test_local_auth.py +++ b/tests/installer/test_local_auth.py @@ -36,7 +36,8 @@ def test_overlay_emits_local_auth_and_round_trips_generated_headers(tmp_path: Pa settings = try_load_access_settings_from_overlay(overlay) rendered = json.loads(json.dumps(__import__("yaml").safe_load(overlay.read_text()))) assert settings == ("local", "operator") - assert rendered["custom"]["authMode"] == "local" + assert rendered["custom"]["auth"] == {"native": True} + assert "authMode" not in rendered["custom"] assert rendered["custom"]["adminUser"] == { "enabled": True, "username": "operator", @@ -109,7 +110,10 @@ def test_local_admin_username_rejects_unsafe_values(username: str) -> None: def test_explicit_local_upgrade_without_username_preserves_previous_username(tmp_path: Path) -> None: overlay = tmp_path / "values.local.yaml" - overlay.write_text("# Access mode : local\n# Admin username: operator\n", encoding="utf-8") + overlay.write_text( + "# Access mode : local\n# Admin username: operator\ncustom:\n authMode: local\n", + encoding="utf-8", + ) state = InstallerState(access_mode="local") _preserve_access_settings_for_upgrade(state, overlay) @@ -117,29 +121,6 @@ def test_explicit_local_upgrade_without_username_preserves_previous_username(tmp assert _resolve_access_settings(state) == ("local", "operator") -def test_upgrade_rejects_unmanaged_advanced_auth_overlay(tmp_path: Path) -> None: - overlay = tmp_path / "values.local.yaml" - overlay.write_text("custom:\n authMode: github\n", encoding="utf-8") - - with pytest.raises(Exception, match="operator-managed Helm values"): - _preserve_access_settings_for_upgrade(InstallerState(), overlay) - - -@pytest.mark.parametrize( - "auth_mode", - ['"github"', "'multi' # operator-managed", '"dummy"'], -) -def test_upgrade_rejects_quoted_or_commented_advanced_auth_overlay(tmp_path: Path, auth_mode: str) -> None: - overlay = tmp_path / "values.local.yaml" - overlay.write_text( - f"# Access mode : local\n# Admin username: operator\ncustom:\n authMode: {auth_mode}\n", - encoding="utf-8", - ) - - with pytest.raises(Exception, match="operator-managed Helm values"): - _preserve_access_settings_for_upgrade(InstallerState(), overlay) - - @pytest.mark.parametrize( ("menu", "action", "command"), [ @@ -178,7 +159,10 @@ def test_reinstall_preserves_local_access_before_removing_release( monkeypatch, tmp_path: Path, reinstall, install: str ) -> None: overlay = tmp_path / "values.local.yaml" - overlay.write_text("# Access mode : local\n# Admin username: operator\n", encoding="utf-8") + overlay.write_text( + "# Access mode : local\n# Admin username: operator\ncustom:\n authMode: local\n", + encoding="utf-8", + ) state = InstallerState() monkeypatch.setattr(state, "runtime_paths", lambda: RuntimePaths(Path("chart"), Path("values"), overlay)) observed = [] @@ -210,4 +194,4 @@ def test_local_overlay_retains_single_node_runtime_behavior(tmp_path: Path) -> N ) rendered = __import__("yaml").safe_load(overlay.read_text()) - assert rendered["custom"]["singleNodeMode"] is True + assert rendered["custom"]["runtimeLimitEnabled"] is False diff --git a/tests/installer/test_profile_ownership.py b/tests/installer/test_profile_ownership.py new file mode 100644 index 00000000..adf7bfc5 --- /dev/null +++ b/tests/installer/test_profile_ownership.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from auplc_installer.catalog import CourseSelection +from auplc_installer.cli import _preserve_access_settings_for_upgrade +from auplc_installer.gpu import GpuConfig, append_product +from auplc_installer.overlay import emit_overlay, generate_values_overlay +from auplc_installer.state import InstallerState +from auplc_installer.util import InstallerError + + +def _overlay(access_mode: str = "local") -> str: + config = GpuConfig() + append_product(config, "AMD_Radeon_8060S_Graphics") + return emit_overlay( + config, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=CourseSelection.default(), + access_mode=access_mode, + admin_username="operator", + ) + + +def test_upgrade_may_overwrite_user_modified_body_when_headers_are_recoverable(tmp_path: Path) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text( + "# Access mode : local\n# Admin username: operator\ncustom:\n auth:\n github: true\n", + encoding="utf-8", + ) + state = InstallerState() + + _preserve_access_settings_for_upgrade(state, overlay) + config = GpuConfig() + append_product(config, "AMD_Radeon_8060S_Graphics") + generate_values_overlay( + config, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=CourseSelection.default(), + access_mode=state.access_mode, + admin_username=state.admin_username, + overlay_path=overlay, + ) + + assert (state.access_mode, state.admin_username) == ("local", "operator") + assert " native: true\n" in overlay.read_text(encoding="utf-8") + assert " github: true\n" not in overlay.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "text", + ( + "custom:\n auth:\n github: true\n", + "# Access mode : local\ncustom:\n auth:\n native: true\n", + "# Access mode : local\n# Access mode : personal\n# Admin username: operator\ncustom: {}\n", + "# Access mode : github\n# Admin username: operator\ncustom: {}\n", + "# Access mode : local\n# Admin username: Admin\ncustom: {}\n", + ), +) +def test_upgrade_skips_profile_recovery_when_headers_are_unusable(tmp_path: Path, text: str) -> None: + overlay = tmp_path / "values.local.yaml" + overlay.write_text(text, encoding="utf-8") + state = InstallerState() + + _preserve_access_settings_for_upgrade(state, overlay) + + assert (state.access_mode, state.admin_username) == ("", "") + + +def test_upgrade_ignores_auth_like_values_under_hub(tmp_path: Path) -> None: + state = InstallerState() + overlay = tmp_path / "values.local.yaml" + overlay.write_text(_overlay("personal") + "hub:\n authMode: ignored\n", encoding="utf-8") + + _preserve_access_settings_for_upgrade(state, overlay) + + assert (state.access_mode, state.admin_username) == ("personal", "admin") + + +@pytest.mark.parametrize("username", ("Admin", "admin:name", 'admin"name', "admin\nname", "admin name", "a" * 65)) +def test_personal_profile_rejects_unsafe_admin_username(username: str) -> None: + with pytest.raises(InstallerError, match="lowercase ASCII"): + _overlay("personal") if username == "operator" else emit_overlay( + GpuConfig(), + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=CourseSelection.default(), + admin_username=username, + ) + + +def test_personal_profile_canonicalizes_safe_admin_username_to_admin() -> None: + config = GpuConfig() + append_product(config, "AMD_Radeon_8060S_Graphics") + + rendered = emit_overlay( + config, + image_registry="ghcr.io/amdresearch", + image_tag="latest", + courses=CourseSelection.default(), + admin_username="operator", + ) + + assert "# Admin username: admin\n" in rendered From 060695c6a2c0c53c5aaaf78a1c10c5b1be75d5e0 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:22:43 +0800 Subject: [PATCH 154/180] refactor(deploy): emit canonical runtime values --- runtime/values-multi-nodes.yaml.example | 17 +++---- runtime/values.yaml | 12 +---- .../scripts/README.md | 4 ++ .../scripts/config_generation.py | 25 ++++++++-- .../scripts/config_rendering.py | 11 +++-- tests/skills/test_deploy_scripts.py | 46 +++++++++++++++++++ 6 files changed, 91 insertions(+), 24 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 4af6d259..e292578e 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -51,12 +51,13 @@ # ============================================================================ custom: - # Authentication mode - # - auto-login: No credentials required, auto-login as 'student' (for single-node dev) - # - dummy: Accept any username/password (for testing) - # - github: GitHub App authentication - # - multi: GitHub App + native first-use accounts - authMode: "multi" + # Authentication providers: native accounts and GitHub App sign-in. + auth: + native: true + github: true + + # Enforce selected session runtimes and automatically shut down expired Pods. + runtimeLimitEnabled: true # Cluster display name (optional). Appended to "AUP Learning Cloud" in the UI. # Example: "City/University" → "AUP Learning Cloud City/University" @@ -464,7 +465,7 @@ custom: - Course-LLM - Course-PhySim - # Native users in multi auth mode are managed by the built-in admin UI or the + # Native users are managed by the built-in admin UI or the # batch scripts under scripts/. New native users are automatically assigned to # the native-users group, which controls their default resource access above. @@ -472,7 +473,7 @@ custom: # Quota Management # -------------------------------------------------------------------------- quota: - enabled: null # auto-disabled for auto-login/dummy modes + enabled: true cpuRate: 1 minimumToStart: 10 defaultQuota: 0 diff --git a/runtime/values.yaml b/runtime/values.yaml index 4ab78f9a..58ad6ef6 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -42,16 +42,8 @@ # ============================================================================ custom: - # Authentication mode - # - auto-login: No credentials required, auto-login as 'student' (for single-node dev) - # - dummy: Accept any username/password (for testing) - # - github: GitHub App authentication - # - local: Closed local accounts managed by an administrator - # - multi: GitHub App + Local accounts - authMode: "auto-login" - # GitHub organization name for team-based resource access - # Required for github and multi auth modes. + # Required when custom.auth.github is enabled. # githubOrgName: "<YOUR-ORG-NAME>" # Cluster display name (optional). Appended to "AUP Learning Cloud" in the UI. @@ -563,7 +555,7 @@ custom: # Quota Management # ============================================================================ quota: - # Enable quota system (auto-disabled for auto-login/dummy modes if not set) + # Enable quota enforcement independently from the session runtime limit. enabled: null # CPU-only quota consumption rate cpuRate: 1 diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 83946594..16651f76 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -131,6 +131,10 @@ consistency validation, where inventory and resolution values must be strict booleans. The generator-first skill workflow supplies both and never generates `auto`. +The spec's historical `auth_mode` field is a one-release generator compatibility +input. It emits only canonical `custom.auth` provider flags; see the deploy +skill reference migration table before creating or updating a spec. + ## Conventions - Detection data goes to stdout as JSON. Diagnostics go to stderr. diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py index c0e881ff..97bc8cac 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -8,12 +8,14 @@ import re from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote -from config_rendering import render_inventory, render_pxe_vars, render_values +from config_rendering import render_inventory, render_pxe_vars, render_values as _render_values __all__ = [ "DEFAULT_ACCEL_LABELS", "HEADER_HASH", + "AUTH_MODE_PROVIDERS", "SCHEMA", + "auth_providers", "die", "render_inventory", "render_pxe_vars", @@ -55,6 +57,24 @@ ) K3S_VERSION_PATTERN = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+\Z") IMAGE_KEY_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_-]*\Z") +AUTH_MODE_PROVIDERS = { + "auto-login": ("autoLogin",), + "dummy": ("dummy",), + "github": ("github",), + "local": ("native",), + "multi": ("native", "github"), +} + + +def auth_providers(spec: dict) -> tuple[str, ...]: + auth_mode = spec.get("auth_mode", "auto-login") + if not isinstance(auth_mode, str) or auth_mode not in AUTH_MODE_PROVIDERS: + die("spec.auth_mode must be one of: auto-login, dummy, github, local, multi") + return AUTH_MODE_PROVIDERS[auth_mode] + + +def render_values(spec: dict) -> str: + return _render_values(spec, auth_providers(spec)) def validate_accelerators(spec: dict) -> None: @@ -142,8 +162,7 @@ def _validate_agents(spec: dict, server_name: str) -> None: def _validate_rendered_options(spec: dict) -> None: - if "auth_mode" in spec: - _safe_text(spec["auth_mode"], "spec.auth_mode") + auth_providers(spec) if "storage" in spec and "class" in spec["storage"]: _safe_text(spec["storage"]["class"], "spec.storage.class") if "proxy" in spec and "node_port" in spec["proxy"]: diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py index b66a0064..707642c4 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -89,11 +89,10 @@ def render_pxe_vars(spec: dict, pxe_gpu_access_enabled: bool) -> str: return "\n".join(lines) + "\n" -def render_values(spec: dict) -> str: +def render_values(spec: dict, auth_providers: tuple[str, ...]) -> str: accel = spec.get("accelerators") or {} storage_class = (spec.get("storage") or {}).get("class", "nfs-client") node_port = (spec.get("proxy") or {}).get("node_port", 30890) - auth_mode = spec.get("auth_mode", "auto-login") images = spec.get("images") or {} lines = [ "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", @@ -102,7 +101,13 @@ def render_values(spec: dict) -> str: "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", "# --create-namespace -f runtime/values.yaml -f <this file>", "custom:", - f" authMode: {yaml_quote(auth_mode)}", + " auth:", + ] + lines.extend(f" {provider}: true" for provider in auth_providers) + lines += [ + " runtimeLimitEnabled: true", + " quota:", + " enabled: true", ] if accel: lines.append(" accelerators:") diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index c4901737..45c41dd4 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -14,6 +14,7 @@ from pathlib import Path import pytest +import yaml ROOT = Path(__file__).resolve().parents[2] DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" @@ -958,6 +959,51 @@ def test_generator_retains_known_accelerator_product_name_overrides(tmp_path: Pa assert 'amd.com/gpu.product-name: "AMD_Custom_8060S"' in values +@pytest.mark.parametrize( + ("auth_mode", "expected_auth"), + [ + ("auto-login", {"autoLogin": True}), + ("dummy", {"dummy": True}), + ("github", {"github": True}), + ("local", {"native": True}), + ("multi", {"native": True, "github": True}), + ], +) +def test_generator_emits_canonical_auth_and_runtime_policy( + tmp_path: Path, auth_mode: str, expected_auth: dict[str, bool] +) -> None: + spec = generator_spec() + spec["auth_mode"] = auth_mode + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stdout + result.stderr + values = yaml.safe_load((out_dir / "values-basic-example.yaml").read_text(encoding="utf-8")) + custom = values["custom"] + assert custom["auth"] == expected_auth + assert "authMode" not in custom + assert custom["runtimeLimitEnabled"] is True + assert custom["quota"]["enabled"] is True + + +@pytest.mark.parametrize("auth_mode", [None, 42, "unsupported"]) +def test_generator_rejects_invalid_auth_mode_before_discovery( + tmp_path: Path, auth_mode: str | int | None +) -> None: + spec = generator_spec() + spec["auth_mode"] = auth_mode + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "spec.auth_mode must be one of: auto-login, dummy, github, local, multi" in result.stderr + assert not out_dir.exists() + + def test_generator_rejects_a_non_mapping_accelerators_field_before_writing_artifacts(tmp_path: Path) -> None: spec = write_file( tmp_path / "spec.json", From c1c3d5df76135eb1247ec6df2dec398cb3c8c6ff Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:24:03 +0800 Subject: [PATCH 155/180] docs(auth): document canonical provider configuration --- README-SKILL.md | 2 +- README.md | 39 +++- runtime/chart/templates/NOTES.txt | 30 ++- .../SKILL.md | 105 ++++++----- .../reference.md | 147 ++++++++++++--- .../skill-card.md | 2 +- skills/deploy-aup-learning-cloud/SKILL.md | 13 ++ skills/deploy-aup-learning-cloud/reference.md | 33 ++++ .../SKILL.md | 31 +++- .../reference.md | 27 ++- .../manage-aup-learning-cloud-users/SKILL.md | 13 +- .../reference.md | 6 + .../troubleshoot-aup-learning-cloud/SKILL.md | 8 +- .../reference.md | 4 +- tests/skills/test_auth_docs.py | 173 ++++++++++++++++++ 15 files changed, 501 insertions(+), 132 deletions(-) create mode 100644 tests/skills/test_auth_docs.py diff --git a/README-SKILL.md b/README-SKILL.md index a854d736..5af95d42 100644 --- a/README-SKILL.md +++ b/README-SKILL.md @@ -43,7 +43,7 @@ logins, manage users, and control network/storage exposure. | [`upgrade-aup-learning-cloud`](skills/upgrade-aup-learning-cloud/SKILL.md) | Upgrade the JupyterHub chart/values/images and the k3s cluster on a running deployment, in a safe order with rollback. | in-repo | | [`troubleshoot-aup-learning-cloud`](skills/troubleshoot-aup-learning-cloud/SKILL.md) | Diagnose netboot, node-join, GPU scheduling, storage, and auth failures from runtime evidence, then hand off the fix. | in-repo | | [`monitor-aup-learning-cloud`](skills/monitor-aup-learning-cloud/SKILL.md) | Wire the Hub into Prometheus + Grafana: ServiceMonitor, authenticated metrics, dashboards, alert rules, and the metrics NetworkPolicy. | in-repo | -| [`configure-aup-learning-cloud-auth`](skills/configure-aup-learning-cloud-auth/SKILL.md) | Configure the auth mode (auto-login/dummy/github/multi), the GitHub App / OAuth + team sync, native accounts, and admin bootstrap. | in-repo | +| [`configure-aup-learning-cloud-auth`](skills/configure-aup-learning-cloud-auth/SKILL.md) | Configure auto-login, dummy, native, GitHub, or native plus GitHub providers, along with GitHub team sync and first-run admin bootstrap. | in-repo | | [`manage-aup-learning-cloud-users`](skills/manage-aup-learning-cloud-users/SKILL.md) | Day-2 user/group/quota operations via the admin console and `manage_users.py`: bulk onboarding, passwords, admins, and quota grants/refresh. | in-repo | | [`expose-aup-learning-cloud`](skills/expose-aup-learning-cloud/SKILL.md) | Take a deployment past the local defaults: NodePort/LoadBalancer/ingress + TLS, CORS origins, externally-terminated TLS, and shared NFS storage. | in-repo | diff --git a/README.md b/README.md index 69681fb8..ad860016 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,10 @@ cd aup-learning-cloud ### Single-Node Access -The interactive installer defaults to `personal` access, preserving the shared student session used by earlier single-node installs. Choose `local` when individual managed accounts are required. +The installer offers two UX profiles. `personal` keeps the shared student +session used by earlier single-node installs. `local` selects native accounts +and first-run administrator bootstrap. These names are installer choices, not +Helm authentication values. Both interactive and scripted installs keep `personal` as the compatibility default. Select local access explicitly when credentials are required: @@ -94,9 +97,36 @@ Both interactive and scripted installs keep `personal` as the compatibility defa ./auplc-installer install --access-mode=local --admin-username=admin ``` -The installer generates the administrator password and API token only when it creates `jupyterhub-admin-credentials`. It displays the password once after a successful interactive deployment. Re-running against an existing Secret reuses the credentials without rotating them; recover credentials with `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo`. The configured bootstrap administrator remains Secret-managed and cannot change or reset its password through the UI. Other local users are created and assigned passwords through the Admin UI. +The installer creates `jupyterhub-admin-credentials` for the `local` profile. +Its `admin-password` is first-run input only: the Hub uses it only when the +administrator has no password row. After that, the database hash is +authoritative. Changing the Secret doesn't rotate or reconcile the existing +database password. The separate `api-token` key supplies an API token for +scripts and isn't part of password bootstrap. Other native users are created +and assigned passwords through the Admin UI. + +Installer-generated `values.local.yaml` is operational output. Manual edits to +that file aren't preserved and may be silently overwritten by a later upgrade +or reinstall. + +For direct Helm configuration, select exactly one of these provider +combinations with `custom.auth`: auto-login, dummy, native, GitHub, or native +plus GitHub. Runtime limits and quota are separate settings. A multi-node native plus +GitHub overlay looks like this: + +<!-- auplc-deployment-example: canonical --> +```yaml +custom: + auth: + native: true + github: true + runtimeLimitEnabled: true + quota: + enabled: true +``` -Local mode is an installer MVP for `http://localhost:30890` on a trusted single-node host. It does not configure TLS or restrict the K3s NodePort from LAN reachability; do not treat local credentials as a network exposure control. +Every provider combination uses the existing `custom.teams.mapping` resolver +and its existing fallback groups to determine resource visibility. A successful install looks like this: @@ -180,7 +210,8 @@ Kubernetes provides a robust infrastructure for deploying and managing JupyterHu ### Authentication Seamless integration with GitHub Single Sign-On (SSO) and Native Authenticator for secure and efficient user authentication. -- **Auto-admin on install**: Initial admin created automatically with random password +- **Composable providers**: choose auto-login, dummy, native, GitHub, or native plus GitHub with `custom.auth` +- **Optional admin bootstrap**: native authentication can seed a missing administrator password row from a generated or external Secret - **Dual login**: GitHub App + Native accounts on single login page - **Batch user management**: CSV/Excel-based bulk operations via scripts diff --git a/runtime/chart/templates/NOTES.txt b/runtime/chart/templates/NOTES.txt index e36739f8..e5bef334 100644 --- a/runtime/chart/templates/NOTES.txt +++ b/runtime/chart/templates/NOTES.txt @@ -42,7 +42,7 @@ SOFTWARE. {{- if and .Values.custom .Values.custom.adminUser .Values.custom.adminUser.enabled }} {{- $admin_secret := .Values.custom.adminUser.existingSecret | default "jupyterhub-admin-credentials" }} {{- if .Values.custom.adminUser.existingSecret }} -### Admin Credentials (local mode external Secret) +### Admin Credentials (external Secret) Administrator username: {{ .Values.custom.adminUser.username }} Credential Secret: {{ $admin_secret }} @@ -50,11 +50,14 @@ SOFTWARE. Get admin password: kubectl -n {{ .Release.Namespace }} get secret {{ $admin_secret }} -o go-template='{{"{{index .data \"admin-password\" | base64decode}}"}}' - Local mode requires `custom.adminUser.existingSecret`; this chart does not - create or rotate that Secret. If it includes an `api-token` key, retrieve it - from {{ $admin_secret }} for scripts. + The admin-password key is first-run bootstrap input. It seeds a password only + when the administrator has no password row. The database hash is authoritative + afterward, so changing this Secret does not rotate or reconcile that password. + + The separate api-token key, when present, supplies an API token for scripts. + It is not part of password bootstrap. {{- else }} -### Admin Credentials (chart-created Secret for non-local auth) +### Admin Credentials (chart-created Secret) Admin username: {{ .Values.custom.adminUser.username }} @@ -64,6 +67,11 @@ SOFTWARE. Get API token (for scripts): export JUPYTERHUB_TOKEN=$(kubectl -n {{ .Release.Namespace }} get secret jupyterhub-admin-credentials -o go-template='{{"{{index .data \"api-token\" | base64decode}}"}}') + The admin-password key is first-run bootstrap input. It seeds a password only + when the administrator has no password row. The database hash is authoritative + afterward, so changing this Secret does not rotate or reconcile that password. + The api-token key is separate delivery for scripts, not password bootstrap. + {{- end }} {{- end }} ### Followup links @@ -119,20 +127,8 @@ SOFTWARE. The k8s Service {{ $proxy_service }} is exposed via NodePorts. That means that all the k8s cluster's nodes are exposing the k8s Service via those ports. - {{- if and .Values.custom .Values.custom.authMode (or (eq .Values.custom.authMode "auto-login") (eq .Values.custom.authMode "local")) }} - - Single-node mode detected. To get your node's IP address, run: - - NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') - echo "Access JupyterHub at: http://$NODE_IP:{{ .Values.proxy.service.nodePorts.http | default "no-http-nodeport-set"}}" - - Quick access: - http://$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):{{ .Values.proxy.service.nodePorts.http | default "no-http-nodeport-set"}} - {{- else }} - Try insecure HTTP access: http://<any k8s nodes ip>:{{ .Values.proxy.service.nodePorts.http | default "no-http-nodeport-set"}} Try secure HTTPS access: https://<any k8s nodes address>:{{ .Values.proxy.service.nodePorts.https | default "no-https-nodeport-set" }} - {{- end }} {{- else }} If your computer is outside the k8s cluster, you can port-forward traffic to diff --git a/skills/configure-aup-learning-cloud-auth/SKILL.md b/skills/configure-aup-learning-cloud-auth/SKILL.md index b7b22025..8b498a41 100644 --- a/skills/configure-aup-learning-cloud-auth/SKILL.md +++ b/skills/configure-aup-learning-cloud-auth/SKILL.md @@ -2,29 +2,29 @@ name: configure-aup-learning-cloud-auth description: >- Group: Maintain AUP Learning Cloud. Configures authentication for AUP Learning - Cloud: auth modes (auto-login/dummy/github/local/multi), GitHub App / OAuth, GitHub - team-to-group sync, native local accounts, password policy and forced - first-login change, and admin bootstrap. Use when the user wants to set or - switch custom.authMode, enable GitHub login, create or migrate a GitHub - App, set oauth_callback_url / client_id / client_secret / app_id / - private_key_file, sync GitHub teams into JupyterHub groups, enable native - accounts, bootstrap the initial admin (custom.adminUser), or debug "Resource - not accessible by integration", a login 404, or OAuth callback errors. - Triggers include custom.authMode, GitHubOAuthenticator, custom.githubOrgName, - allowed_organizations, jupyterhub-admin-credentials. Do not use to map which - resources a group sees (configure-aup-learning-cloud-courses), to - bulk-manage users (manage-aup-learning-cloud-users), or to configure - private-repo cloning (configure-aup-learning-cloud-repos). + Cloud with custom.auth provider flags for auto-login, dummy, native, GitHub, + or native plus GitHub. Covers GitHub App OAuth and team sync, native accounts, + password policy, forced first-login change, and custom.adminUser bootstrap. + Use for custom.auth, GitHubOAuthenticator, custom.githubOrgName, + oauth_callback_url, allowed_organizations, jupyterhub-admin-credentials, + login 404s, OAuth callback errors, or "Resource not accessible by + integration". Do not use for resource-to-group mapping + (configure-aup-learning-cloud-courses), bulk users + (manage-aup-learning-cloud-users), or private-repo cloning + (configure-aup-learning-cloud-repos). --- # Configure AUP Learning Cloud authentication -Choose and wire the Hub's login path: pick the `custom.authMode`, set up the -GitHub App (OAuth + server-to-server team sync) and/or native local accounts, -and bootstrap the initial admin — then re-apply with the installer or Helm. +Choose and wire the Hub's providers with `custom.auth`, set up the GitHub App +and/or native accounts, and optionally bootstrap the first administrator. Then +re-apply with the installer or Helm. -Edit a **values overlay** (`runtime/values.yaml`, `values-multi-nodes.yaml`, or -`values.local.yaml`), never hardcode secrets into tracked files. The full +Edit a supported, manually managed values overlay and never hardcode secrets +into tracked files. Don't manually edit installer-generated +`runtime/values.local.yaml`: it is operational output, receives no preservation +guarantee, and may be silently overwritten by upgrade or reinstall. Use +installer flags for that profile or maintain a separate Helm overlay. The full GitHub App walkthrough, value blocks, and troubleshooting are in **[reference.md](reference.md)**. @@ -33,67 +33,71 @@ GitHub App walkthrough, value blocks, and troubleshooting are in - A checkout of `aup-learning-cloud`; a running (or about-to-deploy) Hub. - `helm` + `kubectl` against the cluster, or `./auplc-installer` on a single-node box. -- For `github` / `multi`: a GitHub **organization** you own (the App is created +- For GitHub or native plus GitHub: a GitHub **organization** you own (the App is created under the org, not a personal account) and admin access to its settings. -## Pick the auth mode +## Pick the provider combination -| Mode | When to use | Notes | +| `custom.auth` flags | When to use | Notes | | --- | --- | --- | -| `auto-login` | Local demo / single dev box | No credentials; quota auto-disabled unless forced. The checked-in default. | -| `dummy` | Throwaway testing only | Accepts any user/password; not for real use; its login can 404 in normal setups. | -| `github` | Org-backed SSO | GitHub App only; team membership syncs into Hub groups. | -| `local` | Closed single-node or standalone local auth | Administrator-managed local accounts only; no GitHub setup. | -| `multi` | GitHub + local accounts | Combined login page; native accounts for users without GitHub. | +| `autoLogin: true` | Shared demo session | No credentials. | +| `dummy: true` | Throwaway testing only | Accepts any user/password; not for real use. | +| `native: true` | Managed native accounts | No GitHub setup required. | +| `github: true` | Org-backed SSO | GitHub App and team sync. | +| `native: true`, `github: true` | Both login methods | Combined login page. | -`custom.authMode` is the single switch. Confirm the target mode with the user -before changing a live Hub (a `helm upgrade` restarts the Hub pod, a brief -login blip). +Exactly one row is valid. Confirm the provider combination before changing a +live Hub. Set `custom.runtimeLimitEnabled` and `custom.quota.enabled` explicitly; +neither is inferred from the providers. ## Workflow -1. **Read current state.** Check `custom.authMode`, `custom.adminUser.enabled`, +1. **Read current state.** Check `custom.auth`, `custom.adminUser.enabled`, `custom.githubOrgName`, and `hub.config.GitHubOAuthenticator` in the active overlay. -2. **Set the mode** in the overlay. For `auto-login`/`dummy` you are done with +2. **Set the providers** in the overlay. For auto-login or dummy you are done with credentials; skip to step 6. -3. **GitHub App (github/multi).** Create the App under the org with the exact - callback URL for the mode and `Members: Read-only` + `Contents: Read-only` +3. **GitHub App.** Create the App under the org with the callback URL required + by the selected provider combination and `Members: Read-only` + `Contents: Read-only` permissions, then fill `hub.config.GitHubOAuthenticator` (`app_id`, `client_id`, `client_secret`, `private_key_file`, `allowed_organizations`, `scope: []`) and `custom.githubOrgName`. Step-by-step in [reference.md](reference.md). - - **Callback URL must match the mode exactly:** `multi` uses - `…/hub/github/oauth_callback`; single `github` uses `…/hub/oauth_callback`. + - **Callback URL must match the providers:** native plus GitHub uses + `…/hub/github/oauth_callback`; GitHub-only uses `…/hub/oauth_callback`. 4. **Team sync.** Team-to-group sync uses the App installation token; the org teams are intersected with `custom.teams.mapping`. Mapping *which resource* a group sees stays in the configure-courses skill — this skill only makes the - groups exist. -5. **Native accounts (local/multi).** The first-use authenticator has + groups exist. All provider combinations use this mapping and the existing + fallback groups for resource visibility. +5. **Native accounts.** Native and native plus GitHub use the same first-use + authenticator. It has `create_users = False`, so accounts must be created by an admin before login (see manage-users skill). Password policy: ≥8 chars with upper, lower, digit, and special; users can be forced to change on first login. -6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` with a - canonical `custom.adminUser.username`. Direct Helm local mode requires a - nonempty `custom.adminUser.existingSecret`; create that Secret before Helm - runs. The single-node installer creates and validates its lifecycle Secret - before Helm runs. Chart-managed credentials remain available for non-local - authentication modes. +6. **Admin bootstrap (native providers only).** Set + `custom.adminUser.enabled: true` with a canonical `custom.adminUser.username`. + Leave `existingSecret` empty to have the chart generate + `jupyterhub-admin-credentials`, or name an external Secret with + `admin-password` and optional `api-token` keys. The `admin-password` seeds + only a missing password row. An existing database hash is authoritative, so + changing the Secret doesn't rotate or reconcile the password. The separate + `api-token` key supplies API access for scripts and isn't password bootstrap. 7. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay>` must succeed. 8. **Apply.** Single-node: `./auplc-installer rt upgrade`. For a direct Helm - local deployment, first create the configured existing Secret, then run + deployment with an external Secret, create the configured Secret, then run `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f - runtime/values.yaml -f <overlay>`. Multi/manual non-local deployments can - use chart-managed credentials when no existing Secret is configured. + runtime/values.yaml -f <overlay>`. Native-enabled deployments can use + chart-generated credentials when no existing Secret is configured. 9. **Verify.** Load the Hub: the expected login page appears, a GitHub user lands in the right groups, and (if bootstrapped) the admin can log in. Read the secret with the commands in [reference.md](reference.md). If a Helm install/upgrade fails, inspect `helm status jupyterhub -n jupyterhub` before retrying. On a single-node install, `./auplc-installer rt upgrade` or -`./auplc-installer rt reinstall` preserves `jupyterhub-admin-credentials`; do -not delete that Secret unless intentionally resetting credentials. +`./auplc-installer rt reinstall` reuses `jupyterhub-admin-credentials`. This +doesn't change an existing database password. ## Safety @@ -101,7 +105,8 @@ not delete that Secret unless intentionally resetting credentials. and `jupyterhub-admin-credentials` must come from a mounted K8s secret or an untracked overlay. Never commit them. - **Avoid `dummy` outside isolated testing** — it accepts any credentials. -- **Switching modes is disruptive.** `auto-login` → `github`/`multi` forces +- **Switching providers is disruptive.** Moving from auto-login to GitHub or + native plus GitHub forces every user through login and changes who can spawn; confirm timing for a live class. - A `helm upgrade` / `rt upgrade` restarts the Hub pod (brief auth blip). @@ -111,5 +116,5 @@ not delete that Secret unless intentionally resetting credentials. ## Reference GitHub App creation walkthrough, every `GitHubOAuthenticator` field, the -OAuth-App→GitHub-App migration, native-account/password details, admin secret +OAuth-App→GitHub-App migration, native-account/password details, admin Secret retrieval, and the troubleshooting table: [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md index 46549820..b07cb522 100644 --- a/skills/configure-aup-learning-cloud-auth/reference.md +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -13,24 +13,85 @@ Workflow and gates are in [SKILL.md](SKILL.md). The live `runtime/values.yaml` and `runtime/chart/values.schema.yaml` are the source of truth; verify keys against them. -## 1. Auth modes (`custom.authMode`) +## 1. Provider combinations (`custom.auth`) +Choose exactly one document below. Omitted provider keys are false. + +<!-- auplc-auth-examples: canonical --> ```yaml custom: - authMode: "auto-login" # auto-login | dummy | github | local | multi + auth: + autoLogin: true +--- +custom: + auth: + dummy: true +--- +custom: + auth: + native: true +--- +custom: + auth: + github: true +--- +custom: + auth: + native: true + github: true ``` -- `auto-login` — shared, no credentials. Quota auto-disables unless explicitly - enabled. Checked-in single-node default. -- `dummy` — accepts any username/password. Testing only. -- `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. -- `local` — closed, administrator-managed local accounts. It requires - `custom.adminUser.enabled: true`, a canonical username, and a nonempty - `custom.adminUser.existingSecret` containing `admin-password`. The username - always comes from `custom.adminUser.username`; `api-token` remains optional - for direct Helm startup. -- `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends - in `/hub/github/oauth_callback`. +- Auto-login provides a shared session with no credentials. +- Dummy accepts any username/password and is for testing only. +- Native provides administrator-managed accounts. +- GitHub uses the GitHub App. Its `oauth_callback_url` ends in + `/hub/oauth_callback`. +- Native plus GitHub puts both methods on one page. Its GitHub callback ends in + `/hub/github/oauth_callback`. + +All five combinations use `custom.teams.mapping` and the existing fallback +groups for resource visibility. Provider selection doesn't change that policy. + +## Runtime timer and credit enforcement + +`custom.runtimeLimitEnabled: true` enforces each selected session duration and +automatically shuts down the session when its timer expires. `false` disables +automatic runtime shutdown. `custom.quota.enabled` controls credit enforcement +only: `true` enforces credit balances and `false` disables credit enforcement. +It never enables or disables the session timer. + +The pair order below is always runtime limit first, quota second. The installer +`personal` and `local` profiles use `false/false`. Online deployment examples +use `true/true`. `true/false` keeps the timer without credit enforcement. +`false/true` is rejected by both the chart schema and Hub parser. + +<!-- auplc-runtime-quota-matrix: canonical --> +```yaml +controls: + runtimeLimitEnabled: + true: enforce-session-timer + false: disable-session-timer + quota.enabled: + true: enforce-credits + false: disable-credit-enforcement +runtimeQuotaPairs: + - runtimeLimitEnabled: true + quotaEnabled: true + valid: true + examples: [online] + - runtimeLimitEnabled: true + quotaEnabled: false + valid: true + examples: [] + - runtimeLimitEnabled: false + quotaEnabled: false + valid: true + examples: [installer-personal, installer-local] + - runtimeLimitEnabled: false + quotaEnabled: true + valid: false + examples: [] +``` ## 2. Admin bootstrap (`custom.adminUser`) @@ -40,12 +101,11 @@ custom: enabled: true ``` -Direct Helm local mode requires `existingSecret`; create the external Secret -before Helm runs. The installer creates and validates -`jupyterhub-admin-credentials` for its local lifecycle. With `existingSecret`, -the external Secret is never created or rotated by the chart and must contain -`admin-password`; an `api-token` is optional for direct Helm startup. -Chart-managed credentials remain available for non-local authentication modes. +Native and native plus GitHub accept the same contract. Leave `existingSecret` empty for +the chart-created `jupyterhub-admin-credentials`, or create the named external +Secret before Helm runs. An external Secret must contain `admin-password`; an +`api-token` is optional for direct Helm startup. The installer creates and +retains its external Secret for explicit local installs. Retrieve chart-created credentials: ```bash @@ -55,14 +115,20 @@ kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ -o jsonpath='{.data.api-token}' | base64 -d && echo ``` -## 3. GitHub App — create it (github/multi) +The `admin-password` is first-run bootstrap input. It seeds a password only +when the administrator has no password row. Once that row exists, its database +hash is authoritative. Changing the Secret doesn't rotate, overwrite, or +reconcile the existing password. The separate `api-token` key delivers an API +token for scripts and isn't used by password bootstrap. + +## 3. GitHub App setup 1. **Create the App under the organization** (not a personal account): `https://github.com/organizations/<ORG>/settings/apps/new`. 2. **Basic info:** name (e.g. `auplc-hub`), Homepage = Hub URL, **Callback URL** - matching the mode: - - `multi`: `https://<domain>/hub/github/oauth_callback` - - single `github`: `https://<domain>/hub/oauth_callback` + matching the provider combination: + - native plus GitHub: `https://<domain>/hub/github/oauth_callback` + - GitHub only: `https://<domain>/hub/oauth_callback` 3. Check **Expire user authorization tokens** and **Request user authorization (OAuth) during installation**. Uncheck **Webhook → Active**. 4. **Permissions:** @@ -118,7 +184,10 @@ groups is the configure-courses skill. GitHub users without a matched team fall into a `github-users` fallback group; native users can be assigned `native-users`. -## 6. Native accounts (multi) +The same mapping and fallback resolver applies to auto-login, dummy, native, +GitHub, and native plus GitHub. + +## 6. Native accounts - The first-use authenticator sets `create_users = False` — accounts must exist before login (create them via the manage-users skill or `/hub/admin`). @@ -157,20 +226,38 @@ If Helm reports a failed release, inspect it before retrying: helm status jupyterhub -n jupyterhub ``` -On a single-node host, `rt upgrade` and `rt reinstall` retain the installer -Secret. Do not delete it unless intentionally resetting the local administrator -credentials. +On a single-node host, `rt upgrade` and `rt reinstall` reuse the installer +Secret. Reusing or changing it doesn't replace an existing database password. + +## One-release `authMode` migration + +`custom.authMode` is accepted for one release as migration input. Don't combine +it with `custom.auth`. Translate legacy values as follows, then remove the +legacy field from the overlay: + +| Legacy value | Canonical `custom.auth` | +| --- | --- | +| `auto-login` | `autoLogin: true` | +| `dummy` | `dummy: true` | +| `github` | `github: true` | +| `local` | `native: true` | +| `multi` | `native: true`, `github: true` | + +```yaml +custom: + authMode: multi +``` ## Troubleshooting | Symptom | Likely cause | First checks | | --- | --- | --- | -| Login 404 / no login page | `authMode: dummy`, or wrong mode for the deploy | Set `github`/`multi`/`auto-login`; re-apply | -| OAuth callback error | `oauth_callback_url` mismatch (mode or http/https) | Match the App's Callback URL exactly to the mode | +| Login 404 / no login page | Dummy selected or providers don't match the deployment | Set the intended `custom.auth` flags; re-apply | +| OAuth callback error | `oauth_callback_url` mismatch | Match the App's Callback URL to GitHub-only or native plus GitHub | | `Resource not accessible by integration` | App missing `Members: Read-only` | Add the org permission; an org owner must approve the updated install | | GitHub users see no/wrong resources | `githubOrgName`, `allowed_organizations`, `teams.mapping`, or team membership | Verify all four; confirm the user's GitHub teams | | Configured team skipped in sync | Team doesn't exist on GitHub | The Hub only syncs teams that exist; create it or fix the key | | Installation token unavailable | `app_id`/`private_key_file` wrong or App not installed on org | Verify both and the org installation | | No admin user created | `custom.adminUser.enabled` not true | Set it, re-apply, `kubectl logs … | grep -i admin` | -| Native user can't log in | Not `multi`, user not pre-created, or no local password | Confirm mode + that an admin created the account | +| Native user can't log in | Native isn't enabled, user not pre-created, or no password | Confirm `custom.auth.native: true` and that an admin created the account | | Password change keeps failing | New password fails the strength policy | Re-check length + upper/lower/digit/special | diff --git a/skills/configure-aup-learning-cloud-auth/skill-card.md b/skills/configure-aup-learning-cloud-auth/skill-card.md index 5dde230e..7560832b 100644 --- a/skills/configure-aup-learning-cloud-auth/skill-card.md +++ b/skills/configure-aup-learning-cloud-auth/skill-card.md @@ -2,7 +2,7 @@ ## Description -Configure AUP Learning Cloud authentication — auth mode, GitHub App / OAuth, team-to-group sync, native accounts, and admin bootstrap — for operators standing up or securing a Hub. +Configure AUP Learning Cloud authentication providers, GitHub App and team sync, native accounts, and first-run admin bootstrap for operators standing up or securing a Hub. ## Owner diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index cc73af5f..7737f57c 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -49,6 +49,8 @@ Then collect and confirm: diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU policy input because agent hardware can't be inferred from the controller. 5. Shared storage location and the Hub access method. +6. Authentication providers: auto-login, dummy, native, GitHub, or native plus + GitHub. The canonical multi-node example uses native plus GitHub. Confirm detected GPU product labels before mapping them to accelerator keys in the runtime values. @@ -58,6 +60,11 @@ the runtime values. Create a fresh schema and fill only its current fields. Run the generator rather than writing inventory or GPU policy by hand. +The schema temporarily accepts `auth_mode` as a one-release generator +compatibility input. It isn't Helm configuration. The generator always writes +the selected providers as canonical `custom.auth` flags; see the migration +table in [reference.md](reference.md). + For SSH, generation performs read-only discovery on every managed host and publishes canonical artifacts after GPU evidence is consistent. @@ -67,6 +74,12 @@ does not prove rootfs provisioning succeeded. Review, install, and validate thos files, then run the controller playbook with the canonical inventory and PXE vars; the playbook must complete successfully before proceeding. +The generated runtime overlay includes canonical `custom.auth`, +`custom.runtimeLimitEnabled: true`, and `custom.quota.enabled: true`. It also +maps detected GPU labels to accelerator selectors, defines notebook images and +shared storage, and keeps resource visibility tied to `custom.teams.mapping` +and its fallback groups regardless of the selected authentication providers. + Follow the complete topology command sequence in the [skill scripts guide](scripts/README.md). Don't substitute the human direct-edit SSH workflow from `deploy/README.md`; the skill's SSH path remains diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index b1df45e6..952cea5e 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -18,6 +18,39 @@ GPU policy, including for SSH. Create deployment specs from the current `--print-schema` output. Generation resolves hosts to strict `true` or `false` values and never writes `auto`. +## One-release generator auth migration + +`gen_configs.py` temporarily accepts `auth_mode` as a one-release compatibility +input for generator specs. It isn't a Helm value. Generated overlays always use +canonical `custom.auth` flags. + +| Temporary `auth_mode` input | Generated `custom.auth` flags | +| --- | --- | +| `auto-login` | `autoLogin: true` | +| `dummy` | `dummy: true` | +| `github` | `github: true` | +| `local` | `native: true` | +| `multi` | `native: true`, `github: true` | + +## Values field guide + +| Field | Purpose | +| --- | --- | +| `custom.auth` | Select exactly one supported combination: auto-login, dummy, native, GitHub, or native plus GitHub. | +| `custom.runtimeLimitEnabled` | Enforce the selected session timer. Generated multi-node overlays set this to `true`. | +| `custom.quota.enabled` | Enforce credit balances. Generated multi-node overlays set this to `true`. | +| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | Configure GitHub OAuth when GitHub is selected. | +| `custom.adminUser` | Name the Hub administrator. | +| `custom.accelerators.*.nodeSelector` | Match the AMD GPU labels found through discovery and confirmed by the user. | +| `custom.resources.images` | Define CPU, GPU, and course notebook images. | +| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Define per-team resources and quotas. | +| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | Select shared storage, normally `nfs-client` for multi-node deployments. | +| `proxy.service`, `ingress` | Expose the Hub through a NodePort or ingress. | + +Authentication doesn't select runtime limits, quota, or resource visibility. +Every provider combination uses `custom.teams.mapping` and its existing +fallback groups to resolve visible resources. + ## Canonical validation inputs Use the topology's validator command from the diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md index 1b224ee2..64aad916 100644 --- a/skills/install-aup-learning-cloud-single-node/SKILL.md +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -51,8 +51,15 @@ table, offline flow, and troubleshooting are in **[reference.md](reference.md)** 4. **Online or offline**: a normal machine with internet, or an air-gapped one that needs a `pack` bundle (see reference). 5. **Access mode**: interactive and scripted installs default to the `personal` - shared student session. Select local managed accounts explicitly with - `--access-mode=local --admin-username=<name>`. + shared student session. Select the `local` installer profile for managed + accounts with `--access-mode=local --admin-username=<name>`. + +`personal` and `local` are installer UX profiles, not values for the runtime +authentication configuration. The generated overlay emits canonical +`custom.auth` provider flags plus explicit `custom.runtimeLimitEnabled: false` +and `custom.quota.enabled: false` settings. In runtime/quota order, this +`false/false` pair disables both automatic session shutdown and credit +enforcement. ## Phase 2 — Verify the environment @@ -87,7 +94,7 @@ kubectl get nodes # the node is Ready kubectl get pods -n jupyterhub # hub + proxy Running, no CrashLoop/ImagePull ``` -Open `http://localhost:30890` — local interactive installs display a login form; +Open `http://localhost:30890`. Installs using the `local` profile display a login form; sign in with the configured administrator credentials. Scripted `personal` installs retain the compatibility shared student session. The NodePort is 30890, storage is `local-path`, and ingress is disabled. Spawn a CPU notebook, then a @@ -95,8 +102,11 @@ GPU notebook, and confirm the GPU pod schedules. If Helm fails, inspect `helm status jupyterhub -n jupyterhub` before retrying. For a Hub-only retry, use `./auplc-installer rt upgrade` or -`./auplc-installer rt reinstall`; both retain `jupyterhub-admin-credentials`. -Do not delete the Secret unless intentionally resetting local credentials. +`./auplc-installer rt reinstall`; both reuse `jupyterhub-admin-credentials`. +The `admin-password` seeds only a missing administrator password row. An +existing database hash is authoritative, so changing the Secret doesn't rotate +or reconcile that password. Its separate `api-token` key supplies API access +for scripts. ## Safety @@ -107,12 +117,13 @@ Stop and get explicit confirmation before: - Switching `--runtime` (docker ↔ containerd) on an existing install. - Any `--image-source=build` run on a slow/low-disk box (large local builds). -Never commit changes to the checkout. The installer writes a local values -overlay (e.g. `values.local.yaml`); do not commit it. +Never commit changes to the checkout. The installer writes +`values.local.yaml` as generated operational output. User edits are unsupported +across upgrade or reinstall and may be silently overwritten. -Local mode remains localhost-oriented MVP guidance only. It does not configure -TLS or restrict NodePort LAN reachability, so credentials are not a network -authorization boundary. +The `local` installer profile remains localhost-oriented MVP guidance only. It +does not configure TLS or restrict NodePort LAN reachability, so credentials +are not a network authorization boundary. ## Reference diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index a6e03079..84d3a00b 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -53,6 +53,8 @@ sudo apt install python3-questionary python3-prompt-toolkit | `--image-registry=PREFIX` | default `ghcr.io/amdresearch` | Env `IMAGE_REGISTRY`. | | `--image-tag=TAG` | default `latest` | GPU suffix appended automatically. Env `IMAGE_TAG`. Use `develop` for the preview UI. | | `--runtime=MODE` | `docker` (default) or `containerd` | `docker` makes images visible to k3s immediately; `containerd` exports for offline. | +| `--access-mode=PROFILE` | `personal` (default) or `local` | Installer UX profile. `personal` emits auto-login; `local` emits native authentication and admin bootstrap. | +| `--admin-username=NAME` | `admin` | Administrator name for the `local` installer profile. | | `--courses`, `--mirror=`, `--mirror-pip=`, `--mirror-npm=` | — | Registry / PyPI / npm mirrors for restricted networks. | | `-y`, `--yes` | — | Assume yes (scripted/CI). Env `AUPLC_YES=1`. | | `--dry-run` (`--try-run`) | — | Preview only. | @@ -86,16 +88,23 @@ sudo apt install python3-questionary python3-prompt-toolkit ## Default deployment facts -The checked-in chart and interactive installer default to `personal` access via -`custom.authMode: auto-login`. Selecting `--access-mode=local` creates +`personal` and `local` are installer UX profiles only. The generated overlay +uses canonical `custom.auth` flags and always writes explicit +`custom.runtimeLimitEnabled: false` and `custom.quota.enabled: false`. `personal` +selects auto-login. `local` selects native authentication and creates `jupyterhub-admin-credentials` with `admin-username`, `admin-password`, and -`api-token`. -The single-node NodePort is not a TLS or LAN exposure boundary; use local mode only -on a trusted host/network. To change generated installer values, run -`./auplc-installer rt upgrade`; it preserves and validates an existing local Secret. +`api-token`. In runtime/quota order, `false/false` means neither automatic +session shutdown nor credit enforcement is active. +The single-node NodePort is not a TLS or LAN exposure boundary; use the `local` +profile only on a trusted host/network. `values.local.yaml` is installer-generated output. +Manual edits aren't preserved and may be silently overwritten by upgrade or +reinstall. If a release fails, run `helm status jupyterhub -n jupyterhub` before retrying. -`rt upgrade` and `rt reinstall` retain `jupyterhub-admin-credentials`; do not -delete it unless intentionally resetting local credentials. +`rt upgrade` and `rt reinstall` reuse `jupyterhub-admin-credentials`. The +`admin-password` seeds only a missing administrator password row. Once that row +exists, the database hash is authoritative. Changing the Secret doesn't rotate +or reconcile the password. The `api-token` key is separate delivery for API +scripts and isn't password bootstrap. ## Offline / air-gapped (pack) @@ -129,7 +138,7 @@ installation verifies and installs it from the bundle. | `localhost:30890` refused | Proxy not up or NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | | `docker` permission denied | User not in docker group | re-run `usermod -aG docker $USER` then re-login / `newgrp docker` | | Need to re-apply values only | Changed the overlay, not images | `./auplc-installer rt upgrade` (don't reinstall k3s) | -| Local administrator password unavailable | Existing Secret is intentionally preserved | `kubectl -n jupyterhub get secret jupyterhub-admin-credentials -o jsonpath='{.data.admin-password}' | base64 -d && echo` | +| Bootstrap password doesn't match after first login | A database password row already exists | Use supported native password management; changing the Secret won't rotate the database password | ## Out of scope diff --git a/skills/manage-aup-learning-cloud-users/SKILL.md b/skills/manage-aup-learning-cloud-users/SKILL.md index 0ca2cd0c..4c072e78 100644 --- a/skills/manage-aup-learning-cloud-users/SKILL.md +++ b/skills/manage-aup-learning-cloud-users/SKILL.md @@ -12,7 +12,7 @@ description: >- manage_users.py, generate_users_template.py, users.csv, passwords_output.csv, /hub/admin, jupyterhub-admin-credentials, JUPYTERHUB_URL, JUPYTERHUB_TOKEN, set-admin, set-passwords, set-quota, add-quota, list-quota, refreshRules, - "onboard a class", and "bulk users". Do not use to choose auth mode, configure + "onboard a class", and "bulk users". Do not use to choose auth providers, configure course visibility/quota rates, or install/deploy a cluster. --- @@ -47,8 +47,11 @@ in **[reference.md](reference.md)**. - Quota subcommands use the Hub admin API. `kubectl` is only needed to bootstrap an API token from `jupyterhub-admin-credentials` or inspect scheduled quota refresh CronJobs. -- Native-user creation/password reset requires `authMode: multi` (or another - mode with native accounts). Password actions never apply to GitHub identities. +- Native-user creation and password reset require `custom.auth.native: true`. + Password actions never apply to GitHub identities. The admin Secret's + `admin-password` seeds only a missing administrator password row. The database + hash is authoritative afterward, and changing the Secret doesn't rotate or + reconcile it. The separate `api-token` key supplies CLI API access. ## Two surfaces @@ -141,8 +144,8 @@ refresh). Quota **rates and enable/disable knobs** (`custom.quota.*`, - **Quota refresh rules apply broadly.** A global Refresh Quota or a broad `refreshRules` filter touches many users; confirm before applying. - CLI quota commands call the Hub admin API; they need a valid API token and a - reachable Hub, not `kubectl` access. Use `kubectl` only for the secret - bootstrap or scheduled-refresh CronJob inspection described above. + reachable Hub, not the administrator password. `kubectl` reads the separately + delivered `api-token` from the Secret or inspects scheduled-refresh CronJobs. ## Reference diff --git a/skills/manage-aup-learning-cloud-users/reference.md b/skills/manage-aup-learning-cloud-users/reference.md index b43afe7c..1e729a7c 100644 --- a/skills/manage-aup-learning-cloud-users/reference.md +++ b/skills/manage-aup-learning-cloud-users/reference.md @@ -48,6 +48,12 @@ CLI **quota** commands call the Hub admin API, so they need a valid API token and a reachable Hub. `kubectl` is only needed to bootstrap the token from the secret above or inspect scheduled quota refresh CronJobs. +The Secret's `admin-password` is first-run input used only when the +administrator has no password row. An existing database hash is authoritative; +changing the Secret doesn't rotate or reconcile it. The separate `api-token` +key delivers the token used for `JUPYTERHUB_TOKEN` and isn't part of password +bootstrap. + ## Python dependencies ```bash diff --git a/skills/troubleshoot-aup-learning-cloud/SKILL.md b/skills/troubleshoot-aup-learning-cloud/SKILL.md index 8bc51964..cb68def5 100644 --- a/skills/troubleshoot-aup-learning-cloud/SKILL.md +++ b/skills/troubleshoot-aup-learning-cloud/SKILL.md @@ -61,7 +61,8 @@ the loaded deploy skill's `SKILL.md`, then set | --- | --- | | PXE rootfs vars / rebuild, agent netboot, NFS rootfs, k3s token publish | deploy-aup-learning-cloud | | Single-node install / GPU detect / `localhost:30890` | install-aup-learning-cloud-single-node | -| `nodeSelector` ↔ GPU label, course/team/quota, auth mode | configure-aup-learning-cloud-courses | +| `nodeSelector` ↔ GPU label, course/team/quota | configure-aup-learning-cloud-courses | +| Authentication providers, GitHub callback, native login | configure-aup-learning-cloud-auth | | Image tag / `ImagePullBackOff` from a missing build | build-aup-learning-cloud-images | | Version mismatch after a bump, chart rollback | upgrade-aup-learning-cloud | @@ -77,8 +78,9 @@ the loaded deploy skill's `SKILL.md`, then set `custom.accelerators.*.nodeSelector`. - **Storage:** `kubectl get pvc -A`, provisioner logs, `showmount -e <NFS>`, `/etc/exports`. -- **Auth:** Hub logs (`kubectl logs -n jupyterhub deploy/hub`), `custom.authMode` - (avoid `dummy`, whose login 404s), GitHub OAuth callback URL. +- **Auth:** Hub logs (`kubectl logs -n jupyterhub deploy/hub`), `custom.auth` + provider flags, and the GitHub OAuth callback URL. Check resource visibility + separately through `custom.teams.mapping` and the user's fallback group. ## Safety diff --git a/skills/troubleshoot-aup-learning-cloud/reference.md b/skills/troubleshoot-aup-learning-cloud/reference.md index cbdee482..74f93244 100644 --- a/skills/troubleshoot-aup-learning-cloud/reference.md +++ b/skills/troubleshoot-aup-learning-cloud/reference.md @@ -49,9 +49,9 @@ Method and safety gates are in [SKILL.md](SKILL.md). | Symptom | Likely cause | First checks | | --- | --- | --- | -| Login page 404s | `custom.authMode: dummy` | Use `auto-login` (single machine) or a real OAuth mode | +| Login page 404s | Dummy provider selected or invalid provider combination | Check `custom.auth`; use exactly auto-login, dummy, native, GitHub, or native plus GitHub | | GitHub login loops/fails | OAuth callback URL or org/team config | `hub.config.GitHubOAuthenticator`, `custom.githubOrgName`, callback URL matches host | -| User sees no courses | Team mapping empty for their group | `custom.teams.mapping`, group membership in Admin console | +| User sees no courses | Team mapping empty for their group | `custom.teams.mapping`, group membership and existing fallback group in Admin console; providers don't bypass mapping | | Can't reach admin console | Wrong admin user | `custom.adminUser`, `/hub/admin` | ## kubeconfig / access diff --git a/tests/skills/test_auth_docs.py b/tests/skills/test_auth_docs.py new file mode 100644 index 00000000..34b8536b --- /dev/null +++ b/tests/skills/test_auth_docs.py @@ -0,0 +1,173 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Structural checks for public authentication documentation.""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +PUBLIC_AUTH_DOCS = ( + ROOT / "README.md", + ROOT / "README-SKILL.md", + ROOT / "runtime/chart/templates/NOTES.txt", + ROOT / "skills/configure-aup-learning-cloud-auth/SKILL.md", + ROOT / "skills/configure-aup-learning-cloud-auth/reference.md", + ROOT / "skills/install-aup-learning-cloud-single-node/SKILL.md", + ROOT / "skills/install-aup-learning-cloud-single-node/reference.md", + ROOT / "skills/manage-aup-learning-cloud-users/SKILL.md", + ROOT / "skills/manage-aup-learning-cloud-users/reference.md", + ROOT / "skills/deploy-aup-learning-cloud/SKILL.md", + ROOT / "skills/deploy-aup-learning-cloud/reference.md", + ROOT / "skills/troubleshoot-aup-learning-cloud/SKILL.md", + ROOT / "skills/troubleshoot-aup-learning-cloud/reference.md", +) +AUTH_EXAMPLE_MARKER = "auplc-auth-examples: canonical" +DEPLOYMENT_EXAMPLE_MARKER = "auplc-deployment-example: canonical" +RUNTIME_QUOTA_MARKER = "auplc-runtime-quota-matrix: canonical" +VALID_PROVIDER_SETS = { + frozenset({"autoLogin"}), + frozenset({"dummy"}), + frozenset({"native"}), + frozenset({"github"}), + frozenset({"native", "github"}), +} + + +def marked_yaml_documents(text: str, marker: str) -> list[dict]: + pattern = re.compile( + rf"<!--\s*{re.escape(marker)}\s*-->\s*```yaml\s*\n(.*?)```", + re.DOTALL, + ) + return [document for block in pattern.findall(text) for document in yaml.safe_load_all(block)] + + +def legacy_auth_mode_outside_migration(text: str) -> list[int]: + heading = "" + invalid_lines: list[int] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + if match := re.match(r"^#{1,6}\s+(.+)$", line): + heading = match.group(1).casefold() + if "authMode" in line and not ({"migration", "deprecation"} & set(heading.split())): + invalid_lines.append(line_number) + return invalid_lines + + +def render_example(tmp_path: Path, index: int, document: dict) -> subprocess.CompletedProcess[str]: + overlay = tmp_path / f"auth-doc-example-{index}.yaml" + overlay.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") + return subprocess.run( + [ + "helm", + "template", + "jupyterhub", + "runtime/chart", + "-f", + "runtime/values.yaml", + "-f", + str(overlay), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def test_canonical_provider_examples_match_truth_table_and_chart_schema(tmp_path: Path) -> None: + reference = (ROOT / "skills/configure-aup-learning-cloud-auth/reference.md").read_text(encoding="utf-8") + examples = marked_yaml_documents(reference, AUTH_EXAMPLE_MARKER) + + provider_sets = { + frozenset(key for key, enabled in example["custom"]["auth"].items() if enabled) for example in examples + } + assert provider_sets == VALID_PROVIDER_SETS + + for index, example in enumerate(examples): + result = render_example(tmp_path, index, example) + assert result.returncode == 0, result.stderr + + +def test_canonical_deployment_examples_set_provider_topology_and_quota() -> None: + examples = [ + example + for path in PUBLIC_AUTH_DOCS + for example in marked_yaml_documents(path.read_text(encoding="utf-8"), DEPLOYMENT_EXAMPLE_MARKER) + ] + + assert examples + for example in examples: + custom = example["custom"] + assert custom["auth"] == {"native": True, "github": True} + assert custom["runtimeLimitEnabled"] is True + assert custom["quota"]["enabled"] is True + assert "authMode" not in custom + + +def test_runtime_quota_matrix_defines_controls_and_runtime_first_pairs() -> None: + reference = (ROOT / "skills/configure-aup-learning-cloud-auth/reference.md").read_text(encoding="utf-8") + matrices = marked_yaml_documents(reference, RUNTIME_QUOTA_MARKER) + + assert len(matrices) == 1 + matrix = matrices[0] + assert matrix["controls"] == { + "runtimeLimitEnabled": { + True: "enforce-session-timer", + False: "disable-session-timer", + }, + "quota.enabled": { + True: "enforce-credits", + False: "disable-credit-enforcement", + }, + } + assert [ + (entry["runtimeLimitEnabled"], entry["quotaEnabled"], entry["valid"]) for entry in matrix["runtimeQuotaPairs"] + ] == [ + (True, True, True), + (True, False, True), + (False, False, True), + (False, True, False), + ] + assert matrix["runtimeQuotaPairs"][0]["examples"] == ["online"] + assert matrix["runtimeQuotaPairs"][2]["examples"] == ["installer-personal", "installer-local"] + + +def test_legacy_auth_mode_is_confined_to_migration_sections() -> None: + invalid_occurrences = { + str(path.relative_to(ROOT)): legacy_auth_mode_outside_migration(path.read_text(encoding="utf-8")) + for path in PUBLIC_AUTH_DOCS + if legacy_auth_mode_outside_migration(path.read_text(encoding="utf-8")) + } + + assert invalid_occurrences == {} + + +def test_legacy_auth_mode_classifier_rejects_canonical_and_accepts_migration() -> None: + canonical = """## Canonical configuration +```yaml +custom: + authMode: multi +``` +""" + migration = """## One-release migration +```yaml +custom: + authMode: multi +``` +""" + + assert legacy_auth_mode_outside_migration(canonical) == [4] + assert legacy_auth_mode_outside_migration(migration) == [] + + +def test_removed_resource_visibility_configuration_is_absent_from_public_auth_docs() -> None: + removed_field = "access" + "Policy" + occurrences = [ + str(path.relative_to(ROOT)) for path in PUBLIC_AUTH_DOCS if removed_field in path.read_text(encoding="utf-8") + ] + + assert occurrences == [] From e2a4efa2ed86bbafc6544d6543ca7a354bd7608c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:49:15 +0800 Subject: [PATCH 156/180] refactor(auth): centralize authenticator configuration --- runtime/hub/core/authenticators/__init__.py | 28 ++++++++++---- runtime/hub/core/setup.py | 21 +--------- runtime/hub/tests/test_auth_provider_setup.py | 30 ++++++++++----- .../hub/tests/test_authenticator_factory.py | 38 +++++++++++++------ 4 files changed, 69 insertions(+), 48 deletions(-) diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index 29015db9..56150569 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -23,6 +23,8 @@ Provides various authentication methods for JupyterHub. """ +from typing import Any + from core.authenticators.auto_login import AutoLoginAuthenticator from core.authenticators.firstuse import CustomFirstUseAuthenticator from core.authenticators.github_app import GITHUB_USERNAME_PREFIX, CustomGitHubOAuthenticator @@ -33,20 +35,30 @@ LOCAL_ACCOUNT_PREFIX = "LocalAccount" -def create_authenticator(auth: AuthCapabilities) -> type | str: - """Select the JupyterHub authenticator class for validated capabilities.""" +def configure_authenticator(c: Any, auth: AuthCapabilities) -> None: + """Configure the JupyterHub authenticator for validated capabilities.""" match auth: case AuthCapabilities(auto_login=True, dummy=False, native=False, github=False): - return AutoLoginAuthenticator + c.JupyterHub.authenticator_class = AutoLoginAuthenticator + c.Authenticator.allow_all = True case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False): - return "dummy" + c.JupyterHub.authenticator_class = "dummy" case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False): - return CustomFirstUseAuthenticator + c.JupyterHub.authenticator_class = CustomFirstUseAuthenticator + c.Authenticator.allow_all = True case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True): - return CustomGitHubOAuthenticator + c.JupyterHub.authenticator_class = CustomGitHubOAuthenticator case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True): - return CustomMultiAuthenticator + c.JupyterHub.authenticator_class = CustomMultiAuthenticator + c.MultiAuthenticator.authenticators = [ + {"authenticator_class": CustomGitHubOAuthenticator, "url_prefix": "/github"}, + { + "authenticator_class": CustomFirstUseAuthenticator, + "url_prefix": "/native", + "config": {"prefix": "", "allow_all": True}, + }, + ] case AuthCapabilities(): raise AuthConfigurationError("auth must enable one exclusive provider or native + github") case unsupported: @@ -61,7 +73,7 @@ def create_authenticator(auth: AuthCapabilities) -> type | str: "CustomGitHubOAuthenticator", "CustomFirstUseAuthenticator", "CustomMultiAuthenticator", - "create_authenticator", + "configure_authenticator", "LOCAL_ACCOUNT_PREFIX", "GITHUB_USERNAME_PREFIX", ] diff --git a/runtime/hub/core/setup.py b/runtime/hub/core/setup.py index fedcc003..8b01221f 100644 --- a/runtime/hub/core/setup.py +++ b/runtime/hub/core/setup.py @@ -117,9 +117,7 @@ def setup_hub(c: Any) -> None: from core import z2jh from core.authenticators import ( GITHUB_USERNAME_PREFIX, - CustomFirstUseAuthenticator, - CustomGitHubOAuthenticator, - create_authenticator, + configure_authenticator, ) from core.config import HubConfig from core.database import create_all_tables, init_database @@ -234,22 +232,7 @@ async def auth_state_hook(spawner, auth_state): c.Spawner.auth_state_hook = auth_state_hook - c.JupyterHub.authenticator_class = create_authenticator(auth) - - if auth.auto_login or (auth.native and not auth.github): - c.Authenticator.allow_all = True - if auth.native and auth.github: - c.MultiAuthenticator.authenticators = [ - { - "authenticator_class": CustomGitHubOAuthenticator, - "url_prefix": "/github", - }, - { - "authenticator_class": CustomFirstUseAuthenticator, - "url_prefix": "/native", - "config": {"prefix": "", "allow_all": True}, - }, - ] + configure_authenticator(c, auth) # ========================================================================= # Configure Handlers diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py index 50ce9b3a..88fd94c4 100644 --- a/runtime/hub/tests/test_auth_provider_setup.py +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -79,22 +79,34 @@ def get_config(key: str, default: object = None) -> object: factory_inputs: list[object] = [] authenticators = _module("core.authenticators") authenticators.GITHUB_USERNAME_PREFIX = "github:" - authenticators.CustomGitHubOAuthenticator = authenticator_types["github"] - authenticators.CustomFirstUseAuthenticator = authenticator_types["native"] - def create_authenticator(_input: object) -> type | str: + def configure_authenticator(c: object, _input: object) -> None: factory_inputs.append(_input) if auth.auto_login: - return authenticator_types["auto"] + c.JupyterHub.authenticator_class = authenticator_types["auto"] + c.Authenticator.allow_all = True + return if auth.dummy: - return "dummy" + c.JupyterHub.authenticator_class = "dummy" + return if auth.native and auth.github: - return authenticator_types["multi"] + c.JupyterHub.authenticator_class = authenticator_types["multi"] + c.MultiAuthenticator.authenticators = [ + {"authenticator_class": authenticator_types["github"], "url_prefix": "/github"}, + { + "authenticator_class": authenticator_types["native"], + "url_prefix": "/native", + "config": {"prefix": "", "allow_all": True}, + }, + ] + return if auth.github: - return authenticator_types["github"] - return authenticator_types["native"] + c.JupyterHub.authenticator_class = authenticator_types["github"] + return + c.JupyterHub.authenticator_class = authenticator_types["native"] + c.Authenticator.allow_all = True - authenticators.create_authenticator = create_authenticator + authenticators.configure_authenticator = configure_authenticator core.authenticators = authenticators module_patch.setitem(sys.modules, "core.authenticators", authenticators) diff --git a/runtime/hub/tests/test_authenticator_factory.py b/runtime/hub/tests/test_authenticator_factory.py index e865376c..938f7495 100644 --- a/runtime/hub/tests/test_authenticator_factory.py +++ b/runtime/hub/tests/test_authenticator_factory.py @@ -66,22 +66,36 @@ def test_factory_preserves_identity_prefix_contract(monkeypatch: pytest.MonkeyPa @pytest.mark.parametrize( - ("capabilities", "expected_name"), + ("capabilities", "expected_name", "allow_all"), [ - ((True, False, False, False), "AutoLoginAuthenticator"), - ((False, True, False, False), "dummy"), - ((False, False, True, False), "CustomFirstUseAuthenticator"), - ((False, False, False, True), "CustomGitHubOAuthenticator"), - ((False, False, True, True), "CustomMultiAuthenticator"), + ((True, False, False, False), "AutoLoginAuthenticator", True), + ((False, True, False, False), "dummy", None), + ((False, False, True, False), "CustomFirstUseAuthenticator", True), + ((False, False, False, True), "CustomGitHubOAuthenticator", None), + ((False, False, True, True), "CustomMultiAuthenticator", None), ], ) -def test_factory_selects_authenticator_for_canonical_capabilities( - monkeypatch: pytest.MonkeyPatch, capabilities: tuple[bool, bool, bool, bool], expected_name: str +def test_factory_configures_authenticator_for_canonical_capabilities( + monkeypatch: pytest.MonkeyPatch, + capabilities: tuple[bool, bool, bool, bool], + expected_name: str, + allow_all: bool | None, ) -> None: with _loaded_factory(monkeypatch) as (factory, config): - selected = factory.create_authenticator(config.AuthCapabilities(*capabilities)) - + c = types.SimpleNamespace( + JupyterHub=types.SimpleNamespace(), + Authenticator=types.SimpleNamespace(), + MultiAuthenticator=types.SimpleNamespace(), + ) + factory.configure_authenticator(c, config.AuthCapabilities(*capabilities)) + + selected = c.JupyterHub.authenticator_class assert selected == "dummy" if expected_name == "dummy" else selected.__name__ == expected_name + if allow_all is not None: + assert c.Authenticator.allow_all is allow_all + if capabilities == (False, False, True, True): + assert c.MultiAuthenticator.authenticators[0]["url_prefix"] == "/github" + assert c.MultiAuthenticator.authenticators[1]["url_prefix"] == "/native" @pytest.mark.parametrize( @@ -97,7 +111,7 @@ def test_factory_rejects_invalid_capabilities_before_authenticator_construction( monkeypatch: pytest.MonkeyPatch, capabilities: tuple[bool, bool, bool, bool] ) -> None: with _loaded_factory(monkeypatch) as (factory, config), pytest.raises(config.AuthConfigurationError): - factory.create_authenticator(config.AuthCapabilities(*capabilities)) + factory.configure_authenticator(types.SimpleNamespace(), config.AuthCapabilities(*capabilities)) @pytest.mark.parametrize( @@ -106,7 +120,7 @@ def test_factory_rejects_invalid_capabilities_before_authenticator_construction( ) def test_factory_rejects_malformed_runtime_inputs(monkeypatch: pytest.MonkeyPatch, malformed_auth) -> None: with _loaded_factory(monkeypatch) as (factory, config), pytest.raises(config.AuthConfigurationError): - factory.create_authenticator(malformed_auth) + factory.configure_authenticator(types.SimpleNamespace(), malformed_auth) def test_factory_module_cleanup_survives_a_forced_test_failure(monkeypatch: pytest.MonkeyPatch) -> None: From c3cfb106b7d6e5d62f2a6e520642e96cafeff7ee Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:12:28 +0800 Subject: [PATCH 157/180] fix(auth): namespace GitHub identities --- runtime/hub/core/authenticators/github_app.py | 41 +++- runtime/hub/core/authenticators/multi.py | 16 ++ .../hub/tests/github_authenticator_support.py | 178 ++++++++++++++++++ runtime/hub/tests/test_auth_provider_setup.py | 20 ++ .../hub/tests/test_github_authenticator.py | 175 +++++++++++++++++ .../reference.md | 24 ++- 6 files changed, 446 insertions(+), 8 deletions(-) create mode 100644 runtime/hub/tests/github_authenticator_support.py create mode 100644 runtime/hub/tests/test_github_authenticator.py diff --git a/runtime/hub/core/authenticators/github_app.py b/runtime/hub/core/authenticators/github_app.py index 6e04b2c7..3b86a4d5 100644 --- a/runtime/hub/core/authenticators/github_app.py +++ b/runtime/hub/core/authenticators/github_app.py @@ -28,6 +28,7 @@ import logging import time +from types import SimpleNamespace from oauthenticator.github import GitHubOAuthenticator from oauthenticator.oauth2 import OAuthCallbackHandler @@ -60,6 +61,7 @@ class CustomGitHubOAuthenticator(GitHubOAuthenticator): name = "github" prefix = GITHUB_USERNAME_PREFIX + url_scope = "/github" callback_handler = _GitHubAppInstallCallbackHandler app_id = Unicode( @@ -92,6 +94,43 @@ class CustomGitHubOAuthenticator(GitHubOAuthenticator): help="TTL in seconds for GitHub team membership sync caches.", ) + def _with_github_username_prefix(self, auth_model): + auth_model = auth_model.copy() + if not auth_model["name"].startswith(self.prefix): + auth_model["name"] = f"{self.prefix}{auth_model['name']}" + return auth_model + + async def run_post_auth_hook(self, handler, auth_model): + auth_model = await super().run_post_auth_hook(handler, auth_model) + return self._with_github_username_prefix(auth_model) + + def add_user(self, user): + return super().add_user(SimpleNamespace(name=user.name.removeprefix(self.prefix))) + + def delete_user(self, user): + return super().delete_user(SimpleNamespace(name=user.name.removeprefix(self.prefix))) + + def login_url(self, base_url): + if type(self) is CustomGitHubOAuthenticator: + base_url = f"{base_url.rstrip('/')}{self.url_scope}" + return super().login_url(base_url) + + def get_handlers(self, app): + handlers = super().get_handlers(app) + if type(self) is CustomGitHubOAuthenticator: + return [(f"{self.url_scope}{path}", handler) for path, handler in handlers] + return handlers + + def get_callback_url(self, handler=None): + if self.oauth_callback_url: + if not self.oauth_callback_url.endswith(f"{self.url_scope}/oauth_callback"): + raise ValueError("GitHub oauth_callback_url must end in /hub/github/oauth_callback") + return self.oauth_callback_url + callback_url = super().get_callback_url(handler) + if callback_url.endswith(f"{self.url_scope}/oauth_callback"): + return callback_url + return f"{callback_url.removesuffix('/oauth_callback')}{self.url_scope}/oauth_callback" + async def authenticate(self, handler, data=None): result = await super().authenticate(handler, data) if not result: @@ -174,7 +213,7 @@ async def refresh_user(self, user, handler=None, **kwargs): if expires_in is not None: auth_model["auth_state"]["expires_at"] = time.time() + int(expires_in) - return auth_model + return self._with_github_username_prefix(auth_model) # Not close to expiry. Avoid the parent refresh path here because it # may make external GitHub validation calls for every auth_refresh_age diff --git a/runtime/hub/core/authenticators/multi.py b/runtime/hub/core/authenticators/multi.py index f8013df4..44456a20 100644 --- a/runtime/hub/core/authenticators/multi.py +++ b/runtime/hub/core/authenticators/multi.py @@ -75,6 +75,22 @@ async def refresh_user(self, user, handler=None): return True return await authenticator.refresh_user(user, handler) + def add_user(self, user): + from core.authenticators.github_app import GITHUB_USERNAME_PREFIX + + authenticator = self._find_authenticator_for_user(user) + if user.name.startswith(GITHUB_USERNAME_PREFIX) and authenticator is not None: + authenticator.add_user(user) + return super().add_user(user) + + def delete_user(self, user): + from core.authenticators.github_app import GITHUB_USERNAME_PREFIX + + authenticator = self._find_authenticator_for_user(user) + if user.name.startswith(GITHUB_USERNAME_PREFIX) and authenticator is not None: + authenticator.delete_user(user) + return super().delete_user(user) + def get_custom_html(self, base_url): html = [] diff --git a/runtime/hub/tests/github_authenticator_support.py b/runtime/hub/tests/github_authenticator_support.py new file mode 100644 index 00000000..451793ed --- /dev/null +++ b/runtime/hub/tests/github_authenticator_support.py @@ -0,0 +1,178 @@ +import importlib.util +import sys +import types +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +GITHUB_APP = ROOT / "core" / "authenticators" / "github_app.py" +MULTI = ROOT / "core" / "authenticators" / "multi.py" + + +class _Logger: + def warning(self, *_args, **_kwargs) -> None: + pass + + def info(self, *_args, **_kwargs) -> None: + pass + + def error(self, *_args, **_kwargs) -> None: + pass + + +@contextmanager +def loaded_authenticators(monkeypatch: pytest.MonkeyPatch) -> Iterator[types.SimpleNamespace]: + with monkeypatch.context() as module_patch: + core = types.ModuleType("core") + core.__path__ = [str(ROOT / "core")] + authenticators = types.ModuleType("core.authenticators") + authenticators.__path__ = [str(ROOT / "core" / "authenticators")] + core.authenticators = authenticators + firstuse = types.ModuleType("core.authenticators.firstuse") + firstuse.CustomFirstUseAuthenticator = type("CustomFirstUseAuthenticator", (), {}) + oauthenticator = types.ModuleType("oauthenticator") + github = types.ModuleType("oauthenticator.github") + oauth2 = types.ModuleType("oauthenticator.oauth2") + + class GitHubOAuthenticator: + def __init__(self) -> None: + self.enable_auth_state = True + self.allow_all = False + self.allow_existing_users = True + self.allowed_users: set[str] = set() + self.admin_users: set[str] = set() + self.blocked_users: set[str] = set() + self.allowed_organizations: set[str] = set() + self.organization_members: dict[str, set[str]] = {} + self.policy_names: list[str] = [] + self.post_auth_models: list[dict] = [] + self.child_add_names: list[str] = [] + self.child_delete_names: list[str] = [] + self.refresh_token_response: dict = {} + self.refreshed_auth_model: dict = {} + self.oauth_callback_url = "" + self.log = _Logger() + + def login_url(self, base_url: str) -> str: + return f"{base_url.rstrip('/')}/oauth_login" + + def get_handlers(self, _app) -> list[tuple[str, type]]: + return [ + ("/oauth_login", type("LoginHandler", (), {})), + ("/oauth_callback", type("CallbackHandler", (), {})), + ("/logout", type("LogoutHandler", (), {})), + ] + + def get_callback_url(self, handler=None) -> str: + if self.oauth_callback_url: + return self.oauth_callback_url + if handler is not None: + return f"{handler.request.protocol}://{handler.request.host}{handler.hub.server.base_url}oauth_callback" + return "https://hub.example/hub/oauth_callback" + + async def authenticate(self, _handler, data=None): + data = data or {} + username = data["login"].lower() + self.policy_names.append(username) + if username in self.blocked_users: + return None + organization_allowed = any( + username in self.organization_members.get(organization, set()) + for organization in self.allowed_organizations + ) + if ( + not self.allow_all + and (self.allowed_users or self.allowed_organizations) + and username not in self.allowed_users + and not organization_allowed + ): + return None + return { + "name": username, + "admin": username in self.admin_users, + "auth_state": {"token_response": data.get("token_response", {})}, + } + + async def run_post_auth_hook(self, _handler, auth_model): + self.post_auth_models.append(auth_model) + return auth_model + + def add_user(self, user) -> None: + self.child_add_names.append(user.name) + if self.allow_existing_users and not self.allow_all: + self.allowed_users.add(user.name) + + def delete_user(self, user) -> None: + self.child_delete_names.append(user.name) + self.allowed_users.discard(user.name) + + def build_refresh_token_request_params(self, refresh_token: str) -> dict[str, str]: + return {"refresh_token": refresh_token} + + async def get_token_info(self, _handler, _params) -> dict: + return self.refresh_token_response.copy() + + async def _token_to_auth_model(self, _token_info) -> dict: + return self.refreshed_auth_model.copy() + + class OAuthCallbackHandler: + def get_argument(self, name: str, default: str = "") -> str: + return self.arguments.get(name, default) + + def redirect(self, url: str) -> None: + self.redirected_to = url + + async def get(self) -> None: + self.parent_get_called = True + + class MultiAuthenticator: + def __init__(self) -> None: + self._authenticators = [] + self.outer_add_names: list[str] = [] + self.outer_delete_names: list[str] = [] + + def validate_username(self, _username: str) -> bool: + return True + + def add_user(self, user) -> None: + self.outer_add_names.append(user.name) + + def delete_user(self, user) -> None: + self.outer_delete_names.append(user.name) + + github.GitHubOAuthenticator = GitHubOAuthenticator + oauth2.OAuthCallbackHandler = OAuthCallbackHandler + oauthenticator.github, oauthenticator.oauth2 = github, oauth2 + multiauthenticator = types.ModuleType("multiauthenticator") + multiauthenticator.MultiAuthenticator = MultiAuthenticator + multiauthenticator_module = types.ModuleType("multiauthenticator.multiauthenticator") + multiauthenticator_module.PREFIX_SEPARATOR = ":" + modules = { + "core": core, + "core.authenticators": authenticators, + "core.authenticators.firstuse": firstuse, + "oauthenticator": oauthenticator, + "oauthenticator.github": github, + "oauthenticator.oauth2": oauth2, + "multiauthenticator": multiauthenticator, + "multiauthenticator.multiauthenticator": multiauthenticator_module, + } + for name, module in modules.items(): + module_patch.setitem(sys.modules, name, module) + + github_spec = importlib.util.spec_from_file_location("core.authenticators.github_app", GITHUB_APP) + assert github_spec is not None and github_spec.loader is not None + github_module = importlib.util.module_from_spec(github_spec) + module_patch.setitem(sys.modules, "core.authenticators.github_app", github_module) + github_spec.loader.exec_module(github_module) + + multi_spec = importlib.util.spec_from_file_location("core.authenticators.multi", MULTI) + assert multi_spec is not None and multi_spec.loader is not None + multi_module = importlib.util.module_from_spec(multi_spec) + module_patch.setitem(sys.modules, "core.authenticators.multi", multi_module) + multi_spec.loader.exec_module(multi_module) + + yield types.SimpleNamespace(github=github_module, multi=multi_module) diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py index 88fd94c4..40340e24 100644 --- a/runtime/hub/tests/test_auth_provider_setup.py +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -7,6 +7,7 @@ import anyio import pytest +from github_authenticator_support import loaded_authenticators from provider_setup_support import GITHUB_SETTINGS, make_config ROOT = Path(__file__).resolve().parents[1] @@ -251,6 +252,25 @@ def test_github_prefixed_users_sync_teams_for_each_github_capability( assert state.group_assignments == [("github:octo", "github-users")] +def test_github_only_auth_result_syncs_teams_with_the_prefixed_local_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with _loaded_setup(monkeypatch, (False, False, False, True)) as state: + state.setup.setup_hub(state.c) + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + authenticator.allow_all = True + raw_model = anyio.run(authenticator.authenticate, None, {"login": "Octo"}) + auth_model = anyio.run(authenticator.run_post_auth_hook, None, raw_model) + spawner = types.SimpleNamespace(user=types.SimpleNamespace(name=auth_model["name"], db=object())) + + anyio.run(state.c.Spawner.auth_state_hook, spawner, {"access_token": "token"}) + + assert spawner.user.name == "github:octo" + assert len(state.team_syncs) == 1 + assert state.group_assignments == [("github:octo", "github-users")] + + def test_native_user_retains_native_group_without_github_sync(monkeypatch: pytest.MonkeyPatch) -> None: with _loaded_setup(monkeypatch, (False, False, True, True)) as state: state.setup.setup_hub(state.c) diff --git a/runtime/hub/tests/test_github_authenticator.py b/runtime/hub/tests/test_github_authenticator.py new file mode 100644 index 00000000..8195bb35 --- /dev/null +++ b/runtime/hub/tests/test_github_authenticator.py @@ -0,0 +1,175 @@ +from types import SimpleNamespace + +import anyio +import pytest +from github_authenticator_support import loaded_authenticators + + +def test_direct_github_auth_authorizes_raw_login_then_prefixes_accepted_model(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + authenticator.admin_users = {"octo"} + authenticator.allowed_organizations = {"auplc"} + authenticator.organization_members = {"auplc": {"octo"}} + raw_model = anyio.run(authenticator.authenticate, None, {"login": "Octo"}) + prefixed_model = anyio.run(authenticator.run_post_auth_hook, None, raw_model) + + assert authenticator.policy_names == ["octo"] + assert authenticator.post_auth_models == [raw_model] + assert raw_model["name"] == "octo" + assert prefixed_model["name"] == "github:octo" + assert prefixed_model["admin"] is True + + +def test_github_post_auth_prefixing_copies_only_top_level_model_and_is_idempotent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + auth_state = {"github_user": {"login": "octo"}} + raw_model = {"name": "octo", "auth_state": auth_state, "admin": None} + once = anyio.run(authenticator.run_post_auth_hook, None, raw_model) + twice = anyio.run(authenticator.run_post_auth_hook, None, once) + + assert raw_model["name"] == "octo" + assert once is not raw_model + assert once["auth_state"] is auth_state + assert once["name"] == "github:octo" + assert twice["name"] == "github:octo" + + +def test_github_raw_policy_checks_reject_blocked_and_unallowed_logins(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + authenticator.allowed_users = {"octo"} + authenticator.blocked_users = {"blocked"} + + allowed = anyio.run(authenticator.authenticate, None, {"login": "octo"}) + blocked = anyio.run(authenticator.authenticate, None, {"login": "blocked"}) + unallowed = anyio.run(authenticator.authenticate, None, {"login": "other"}) + + assert allowed["name"] == "octo" + assert blocked is None + assert unallowed is None + assert authenticator.policy_names == ["octo", "blocked", "other"] + + +def test_github_refresh_returns_the_same_prefixed_identity(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + modules.github.time.time = lambda: 1_000 + authenticator.refresh_token_response = {"access_token": "fresh", "expires_in": 3_600} + authenticator.refreshed_auth_model = { + "name": "octo", + "auth_state": {"token_response": {"access_token": "fresh"}}, + } + + async def get_auth_state() -> dict[str, int | str]: + return {"refresh_token": "refresh", "expires_at": 1_001} + + user = SimpleNamespace( + name="github:octo", + get_auth_state=get_auth_state, + ) + + result = anyio.run(authenticator.refresh_user, user) + + assert result["name"] == "github:octo" + assert result["auth_state"]["expires_at"] == 4_600 + assert result["auth_state"]["token_response"] == {"access_token": "fresh"} + + +def test_github_allow_existing_users_uses_raw_logins_for_add_and_delete(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + user = SimpleNamespace(name="github:octo") + + authenticator.add_user(user) + authenticator.delete_user(user) + + assert authenticator.child_add_names == ["octo"] + assert authenticator.child_delete_names == ["octo"] + assert authenticator.allowed_users == set() + + +def test_multi_delegates_prefixed_github_lifecycle_to_the_raw_login_child(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + github = modules.github.CustomGitHubOAuthenticator() + github.username_prefix = "github:" + native = SimpleNamespace(username_prefix="") + authenticator = modules.multi.CustomMultiAuthenticator() + authenticator._authenticators = [github, native] + github_user = SimpleNamespace(name="github:octo") + native_user = SimpleNamespace(name="learner") + + authenticator.add_user(github_user) + authenticator.add_user(native_user) + authenticator.delete_user(github_user) + + assert github.child_add_names == ["octo"] + assert github.child_delete_names == ["octo"] + assert authenticator.outer_add_names == ["github:octo", "learner"] + assert authenticator.outer_delete_names == ["github:octo"] + + +def test_github_callback_keeps_normal_oauth_flow_and_handles_app_setup_redirect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with loaded_authenticators(monkeypatch) as modules: + handler = modules.github._GitHubAppInstallCallbackHandler() + handler.hub = SimpleNamespace(base_url="/hub/") + handler.arguments = {"setup_action": "install"} + anyio.run(handler.get) + + normal_handler = modules.github._GitHubAppInstallCallbackHandler() + normal_handler.hub = SimpleNamespace(base_url="/hub/") + normal_handler.arguments = {"state": "oauth-state"} + anyio.run(normal_handler.get) + + assert handler.redirected_to == "/hub/spawn" + assert not hasattr(handler, "parent_get_called") + assert normal_handler.parent_get_called is True + + +def test_github_routes_are_scoped_once_directly_and_when_multi_wrapped(monkeypatch: pytest.MonkeyPatch) -> None: + with loaded_authenticators(monkeypatch) as modules: + direct = modules.github.CustomGitHubOAuthenticator() + + class URLScopeMixin: + url_scope = "/github" + + def login_url(self, base_url: str) -> str: + return super().login_url(f"{base_url.rstrip('/')}{self.url_scope}") + + def get_handlers(self, app): + return [(f"{self.url_scope}{path}", handler) for path, handler in super().get_handlers(app)] + + class WrappedGitHub(URLScopeMixin, modules.github.CustomGitHubOAuthenticator): + pass + + wrapped = WrappedGitHub() + + assert direct.login_url("/hub/") == "/hub/github/oauth_login" + assert [path for path, _handler in direct.get_handlers(None)] == [ + "/github/oauth_login", + "/github/oauth_callback", + "/github/logout", + ] + assert direct.get_callback_url() == "https://hub.example/hub/github/oauth_callback" + handler = SimpleNamespace( + request=SimpleNamespace(protocol="https", host="hub.example"), + hub=SimpleNamespace(server=SimpleNamespace(base_url="/hub/")), + ) + assert direct.get_callback_url(handler) == "https://hub.example/hub/github/oauth_callback" + assert wrapped.login_url("/hub/") == "/hub/github/oauth_login" + assert [path for path, _handler in wrapped.get_handlers(None)] == [ + "/github/oauth_login", + "/github/oauth_callback", + "/github/logout", + ] + assert wrapped.get_callback_url() == "https://hub.example/hub/github/oauth_callback" + direct.oauth_callback_url = "https://configured.example/hub/github/oauth_callback" + assert direct.get_callback_url() == "https://configured.example/hub/github/oauth_callback" + direct.oauth_callback_url = "https://configured.example/hub/oauth_callback" + with pytest.raises(ValueError, match="must end in /hub/github/oauth_callback"): + direct.get_callback_url() diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md index b07cb522..6a544a30 100644 --- a/skills/configure-aup-learning-cloud-auth/reference.md +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -44,10 +44,14 @@ custom: - Auto-login provides a shared session with no credentials. - Dummy accepts any username/password and is for testing only. - Native provides administrator-managed accounts. -- GitHub uses the GitHub App. Its `oauth_callback_url` ends in - `/hub/oauth_callback`. -- Native plus GitHub puts both methods on one page. Its GitHub callback ends in - `/hub/github/oauth_callback`. +- GitHub uses the GitHub App at `/hub/github/oauth_callback` in both GitHub-only + and native-plus-GitHub modes. + +GitHub users always have the local AUP Learning Cloud username +`github:<normalized-login>` in both GitHub-only and native-plus-GitHub modes. +Native users remain unprefixed. Configure GitHub `allowed_users`, `admin_users`, +`blocked_users`, and `allowed_organizations` with raw GitHub logins and +organizations, not the local `github:` username. All five combinations use `custom.teams.mapping` and the existing fallback groups for resource visibility. Provider selection doesn't change that policy. @@ -126,9 +130,7 @@ token for scripts and isn't used by password bootstrap. 1. **Create the App under the organization** (not a personal account): `https://github.com/organizations/<ORG>/settings/apps/new`. 2. **Basic info:** name (e.g. `auplc-hub`), Homepage = Hub URL, **Callback URL** - matching the provider combination: - - native plus GitHub: `https://<domain>/hub/github/oauth_callback` - - GitHub only: `https://<domain>/hub/oauth_callback` + = `https://<domain>/hub/github/oauth_callback`. 3. Check **Expire user authorization tokens** and **Request user authorization (OAuth) during installation**. Uncheck **Webhook → Active**. 4. **Permissions:** @@ -146,8 +148,14 @@ token for scripts and isn't used by password bootstrap. ## 4. GitHub App — configure the Hub +Set `oauth_callback_url` to `https://<domain>/hub/github/oauth_callback` for +both GitHub-only and native-plus-GitHub deployments. + ```yaml custom: + auth: + native: true + github: true githubOrgName: "<YOUR-ORG-NAME>" gitClone: @@ -171,6 +179,8 @@ hub: `scope: []` is correct for a GitHub App. `installation_id` can stay blank when the App is installed on the org (auto-discovered via `GET /orgs/{org}/installation`). +For GitHub-only, set `custom.auth.github: true` without `native`; keep the same +`https://<domain>/hub/github/oauth_callback` callback URL. ## 5. Team-to-group sync From 3519284fd12effa632917b2455a6e5881c90085a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:47:45 +0800 Subject: [PATCH 158/180] test(code): add launcher lifecycle harness --- dockerfiles/Code/tests/fixtures/code-server | 37 +++++ dockerfiles/Code/tests/fixtures/nginx | 36 +++++ dockerfiles/Code/tests/fixtures/npm | 6 + dockerfiles/Code/tests/harness.sh | 156 ++++++++++++++++++++ 4 files changed, 235 insertions(+) create mode 100755 dockerfiles/Code/tests/fixtures/code-server create mode 100755 dockerfiles/Code/tests/fixtures/nginx create mode 100755 dockerfiles/Code/tests/fixtures/npm create mode 100755 dockerfiles/Code/tests/harness.sh diff --git a/dockerfiles/Code/tests/fixtures/code-server b/dockerfiles/Code/tests/fixtures/code-server new file mode 100755 index 00000000..d10700c5 --- /dev/null +++ b/dockerfiles/Code/tests/fixtures/code-server @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +set -euo pipefail +FAKE_CODE_EXIT_STATUS=${FAKE_CODE_EXIT_STATUS-} +FAKE_COMMAND_LOG=${FAKE_COMMAND_LOG:?} +FAKE_EVENT_DIR=${FAKE_EVENT_DIR:?} +FAKE_INSTANCE=${FAKE_INSTANCE:?} +printf -v command_record ' %q' "$@" +printf 'code-server%s\n' "$command_record" >>"$FAKE_COMMAND_LOG" +for argument in "$@"; do + if [ "$argument" = --install-extension ]; then + : >"$FAKE_EVENT_DIR/installer-invoked" + exit 0 + fi +done +if [ -v PORT ]; then + printf '%s\n' "$PORT" >"$FAKE_EVENT_DIR/$FAKE_INSTANCE.code-port-env" +else + printf '<unset>\n' >"$FAKE_EVENT_DIR/$FAKE_INSTANCE.code-port-env" +fi +on_term() { + event="$FAKE_EVENT_DIR/$FAKE_INSTANCE.code-term" + : >"$event.tmp.$$" + mv "$event.tmp.$$" "$event" + exit 0 +} +trap on_term TERM INT +ready="$FAKE_EVENT_DIR/$FAKE_INSTANCE.code-ready" +printf '%s\n' "$$" >"$ready.tmp.$$" +mv "$ready.tmp.$$" "$ready" +if [ -n "$FAKE_CODE_EXIT_STATUS" ]; then + exec 8<>"$FAKE_EVENT_DIR/$FAKE_INSTANCE.code-exit.fifo" + IFS= read -r _ <&8 + exit "$FAKE_CODE_EXIT_STATUS" +fi +exec 9<>"$FAKE_EVENT_DIR/$FAKE_INSTANCE.code.fifo" +IFS= read -r _ <&9 diff --git a/dockerfiles/Code/tests/fixtures/nginx b/dockerfiles/Code/tests/fixtures/nginx new file mode 100755 index 00000000..abf40743 --- /dev/null +++ b/dockerfiles/Code/tests/fixtures/nginx @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +set -euo pipefail +FAKE_COMMAND_LOG=${FAKE_COMMAND_LOG:?} +FAKE_EVENT_DIR=${FAKE_EVENT_DIR:?} +FAKE_INSTANCE=${FAKE_INSTANCE:?} +FAKE_NGINX_EXIT_STATUS=${FAKE_NGINX_EXIT_STATUS-} +printf -v command_record ' %q' "$@" +printf 'nginx%s\n' "$command_record" >>"$FAKE_COMMAND_LOG" +config= +while [ "$#" -gt 0 ]; do + if [ "$1" = -c ]; then + config=$2 + shift 2 + else + shift + fi +done +cp -- "$config" "$FAKE_EVENT_DIR/$FAKE_INSTANCE.nginx.conf" +on_term() { + event="$FAKE_EVENT_DIR/$FAKE_INSTANCE.nginx-term" + : >"$event.tmp.$$" + mv "$event.tmp.$$" "$event" + exit 0 +} +trap on_term TERM INT +ready="$FAKE_EVENT_DIR/$FAKE_INSTANCE.nginx-ready" +printf '%s\n' "$$" >"$ready.tmp.$$" +mv "$ready.tmp.$$" "$ready" +if [ -n "$FAKE_NGINX_EXIT_STATUS" ]; then + exec 8<>"$FAKE_EVENT_DIR/$FAKE_INSTANCE.nginx-exit.fifo" + IFS= read -r _ <&8 + exit "$FAKE_NGINX_EXIT_STATUS" +fi +exec 9<>"$FAKE_EVENT_DIR/$FAKE_INSTANCE.nginx.fifo" +IFS= read -r _ <&9 diff --git a/dockerfiles/Code/tests/fixtures/npm b/dockerfiles/Code/tests/fixtures/npm new file mode 100755 index 00000000..97bdd030 --- /dev/null +++ b/dockerfiles/Code/tests/fixtures/npm @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +set -euo pipefail +FAKE_COMMAND_LOG=${FAKE_COMMAND_LOG:?} +printf -v command_record ' %q' "$@" +printf 'npm%s\n' "$command_record" >>"$FAKE_COMMAND_LOG" diff --git a/dockerfiles/Code/tests/harness.sh b/dockerfiles/Code/tests/harness.sh new file mode 100755 index 00000000..7296507c --- /dev/null +++ b/dockerfiles/Code/tests/harness.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +LAUNCHER=$(cd "$SCRIPT_DIR/.." && pwd)/start-code-server.sh +SYSTEM_PATH=$PATH +NGINX_CONF=/tmp/auplc-code-server-nginx.conf +tmp_root=$(mktemp -d) +fake_bin="$tmp_root/fake-bin" +current_case= +launcher_pid= +last_launcher_status= + +cleanup() { + trap - EXIT INT TERM + if [ -n "$launcher_pid" ]; then + kill -TERM "$launcher_pid" 2>/dev/null || true + wait "$launcher_pid" 2>/dev/null || true + fi + rm -f "$NGINX_CONF" + rm -rf "$tmp_root" +} +trap cleanup EXIT INT TERM + +fail() { + if [ -n "$current_case" ]; then + for file in "$current_case"/*.out "$current_case"/commands.log; do + [ -f "$file" ] || continue + printf '%s\n' "--- $file ---" >&2 + sed -n '1,200p' "$file" >&2 + done + fi + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +assert_eq() { + [ "$1" = "$2" ] || fail "$3: expected '$1', got '$2'" +} + +assert_file_contains() { + grep -Fq -- "$2" "$1" || fail "expected $1 to contain: $2" +} + +assert_file_not_contains() { + if grep -Fq -- "$2" "$1"; then + fail "did not expect $1 to contain: $2" + fi +} + +assert_process_gone() { + if kill -0 "$1" 2>/dev/null; then + fail "$2 process $1 is still running" + fi +} + +wait_for_file() { + local file=$1 + local owner_pid=$2 + local description=$3 + local attempt + for ((attempt = 0; attempt < 500; attempt++)); do + [ -e "$file" ] && return 0 + kill -0 "$owner_pid" 2>/dev/null || fail "$description did not occur before launcher exited" + sleep 0.01 + done + fail "timed out waiting for $description" +} + +new_case() { + current_case="$tmp_root/$1" + mkdir -p "$current_case"/{baked,destination,events,home,npm,pixi,workspace} + : >"$current_case/commands.log" + case_public_port=18888 + case_code_server_port=18889 + case_service_prefix=/user/test/ + case_trusted_domains= + case_code_exit_status= + case_nginx_exit_status= +} + +start_launcher() { + local instance=$1 + mkfifo \ + "$current_case/events/$instance.code.fifo" \ + "$current_case/events/$instance.code-exit.fifo" \ + "$current_case/events/$instance.nginx.fifo" \ + "$current_case/events/$instance.nginx-exit.fifo" + + PATH="$fake_bin:$SYSTEM_PATH" \ + HOME="$current_case/home" \ + NPM_CONFIG_PREFIX="$current_case/npm" \ + PIXI_HOME="$current_case/pixi" \ + PORT="$case_public_port" \ + AUPLC_CODE_SERVER_PORT="$case_code_server_port" \ + JUPYTERHUB_SERVICE_PREFIX="$case_service_prefix" \ + AUPLC_CODE_WORKDIR="$current_case/workspace" \ + AUPLC_CODE_TRUSTED_DOMAINS="$case_trusted_domains" \ + AUPLC_CODE_BAKED_EXTENSIONS_DIR="$current_case/baked" \ + AUPLC_CODE_EXTENSIONS_DIR="$current_case/destination" \ + FAKE_COMMAND_LOG="$current_case/commands.log" \ + FAKE_EVENT_DIR="$current_case/events" \ + FAKE_INSTANCE="$instance" \ + FAKE_CODE_EXIT_STATUS="$case_code_exit_status" \ + FAKE_NGINX_EXIT_STATUS="$case_nginx_exit_status" \ + bash "$LAUNCHER" >"$current_case/$instance.out" 2>&1 & + launcher_pid=$! +} + +wait_launcher() { + local pid=$1 + set +e + wait "$pid" + last_launcher_status=$? + set -e + launcher_pid= +} + +stop_launcher() { + local instance=$1 + local pid=$2 + local code_pid + local nginx_pid + code_pid=$(<"$current_case/events/$instance.code-ready") + nginx_pid=$(<"$current_case/events/$instance.nginx-ready") + kill -TERM "$pid" + wait_for_file "$current_case/events/$instance.code-term" "$pid" "code-server TERM handling" + wait_for_file "$current_case/events/$instance.nginx-term" "$pid" "nginx TERM handling" + wait_launcher "$pid" + assert_eq 143 "$last_launcher_status" "TERM exit status" + assert_process_gone "$code_pid" code-server + assert_process_gone "$nginx_pid" nginx +} + +mkdir -p "$fake_bin" +cp "$SCRIPT_DIR/fixtures/code-server" "$SCRIPT_DIR/fixtures/nginx" "$SCRIPT_DIR/fixtures/npm" "$fake_bin/" +chmod +x "$fake_bin"/* From 6023284e8096f37178cee7b73e317748945493b1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:48:05 +0800 Subject: [PATCH 159/180] fix(code): install default extensions in the system root --- dockerfiles/Code/Dockerfile | 52 ++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/dockerfiles/Code/Dockerfile b/dockerfiles/Code/Dockerfile index 59ade5f8..4c9f1423 100644 --- a/dockerfiles/Code/Dockerfile +++ b/dockerfiles/Code/Dockerfile @@ -82,7 +82,7 @@ COPY dockerfiles/Code/start-code-server.sh /usr/local/bin/start-code-server.sh RUN chmod +x /usr/local/bin/start-code-server.sh && \ mkdir -p \ /opt/auplc/extensions/local \ - /opt/auplc/code-server/extensions \ + /opt/auplc/extensions/staging \ /home/jovyan/.cache \ /home/jovyan/.config \ /home/jovyan/.local/bin \ @@ -96,10 +96,56 @@ USER jovyan RUN set -eu; \ while IFS= read -r extension_id || [ -n "${extension_id}" ]; do \ case "${extension_id}" in ''|'#'*) continue ;; esac; \ - code-server --extensions-dir /opt/auplc/code-server/extensions --install-extension "${extension_id}"; \ + code-server --extensions-dir /opt/auplc/extensions/staging --install-extension "${extension_id}"; \ done < /opt/auplc/extensions/extensions.txt; \ find /opt/auplc/extensions/local -type f -name '*.vsix' -print0 | \ - xargs -0 -r -n 1 code-server --extensions-dir /opt/auplc/code-server/extensions --install-extension + xargs -0 -r -n 1 code-server --extensions-dir /opt/auplc/extensions/staging --install-extension + +USER root + +RUN set -eu; \ + system_root=/usr/lib/code-server/lib/vscode/extensions; \ + staging_root=/opt/auplc/extensions/staging; \ + test -d "${system_root}"; \ + test -d "${staging_root}"; \ + seen_ids=/tmp/auplc-extension-ids; \ + seen_basenames=/tmp/auplc-extension-basenames; \ + : > "${seen_ids}"; \ + : > "${seen_basenames}"; \ + trap 'rm -f "${seen_ids}" "${seen_basenames}"' EXIT; \ + while IFS= read -r -d '' staging_entry; do \ + entry_name="${staging_entry##*/}"; \ + case "${entry_name}" in \ + extensions.json) test -f "${staging_entry}" && [ ! -L "${staging_entry}" ]; continue ;; \ + esac; \ + test -d "${staging_entry}" && [ ! -L "${staging_entry}" ]; \ + test -f "${staging_entry}/package.json" && [ ! -L "${staging_entry}/package.json" ]; \ + extension_id="$(python3 -c 'import json, sys; package = json.load(open(sys.argv[1], encoding="utf-8")); publisher = package.get("publisher"); name = package.get("name"); (isinstance(publisher, str) and publisher and isinstance(name, str) and name) or sys.exit("package.json must define non-empty publisher and name"); print(f"{publisher}.{name}")' "${staging_entry}/package.json")"; \ + if grep -Fxq "${extension_id}" "${seen_ids}"; then \ + printf 'duplicate extension ID in staging: %s\\n' "${extension_id}" >&2; \ + exit 1; \ + fi; \ + if grep -Fxq "${entry_name}" "${seen_basenames}" || [ -e "${system_root}/${entry_name}" ] || [ -L "${system_root}/${entry_name}" ]; then \ + printf 'extension directory basename collision: %s\\n' "${entry_name}" >&2; \ + exit 1; \ + fi; \ + printf '%s\\n' "${extension_id}" >> "${seen_ids}"; \ + printf '%s\\n' "${entry_name}" >> "${seen_basenames}"; \ + cp -a -- "${staging_entry}" "${system_root}/${entry_name}"; \ + chown -R root:root "${system_root}/${entry_name}"; \ + chmod -R u=rwX,go=rX "${system_root}/${entry_name}"; \ + done < <(LC_ALL=C find -P "${staging_root}" -mindepth 1 -maxdepth 1 -print0 | LC_ALL=C sort -z); \ + rm -rf -- "${staging_root}" + +RUN --network=none set -eu; \ + system_root=/usr/lib/code-server/lib/vscode/extensions; \ + staging_root=/opt/auplc/extensions/staging; \ + test -d "${system_root}"; \ + test ! -e "${system_root}/extensions.json"; \ + test ! -e "${staging_root}" && test ! -L "${staging_root}"; \ + python3 -c 'import json, pathlib, sys; system_root = pathlib.Path(sys.argv[1]); extension_list = pathlib.Path(sys.argv[2]); expected = {line.strip() for line in extension_list.read_text(encoding="utf-8").splitlines() if line.strip() and not line.lstrip().startswith("#")} | {"amdresearch.auplc-hub-link"}; manifests = [json.loads((extension_dir / "package.json").read_text(encoding="utf-8")) for extension_dir in system_root.iterdir() if extension_dir.is_dir() and not extension_dir.is_symlink() and (extension_dir / "package.json").is_file()]; ids = [str(manifest.get("publisher")) + "." + str(manifest.get("name")) for manifest in manifests]; invalid = {extension_id: ids.count(extension_id) for extension_id in expected if ids.count(extension_id) != 1}; not invalid or sys.exit(f"system extension manifest count mismatch: {invalid}")' "${system_root}" /opt/auplc/extensions/extensions.txt + +USER jovyan EXPOSE 8888 WORKDIR /home/jovyan From 37ec00a7c19fc58d587d0bc44cfcbc84d7ea2cbb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:48:33 +0800 Subject: [PATCH 160/180] fix(code): remove runtime extension provisioning --- dockerfiles/Code/start-code-server.sh | 69 +++++++------- .../tests/test_runtime_extension_model.sh | 48 ++++++++++ .../Code/tests/test_service_lifecycle.sh | 95 +++++++++++++++++++ 3 files changed, 177 insertions(+), 35 deletions(-) create mode 100755 dockerfiles/Code/tests/test_runtime_extension_model.sh create mode 100755 dockerfiles/Code/tests/test_service_lifecycle.sh diff --git a/dockerfiles/Code/start-code-server.sh b/dockerfiles/Code/start-code-server.sh index 4d188b69..7939ac9d 100755 --- a/dockerfiles/Code/start-code-server.sh +++ b/dockerfiles/Code/start-code-server.sh @@ -12,10 +12,30 @@ code_server_port="${AUPLC_CODE_SERVER_PORT:-8889}" service_prefix="${JUPYTERHUB_SERVICE_PREFIX:-/}" # Without a Hub-provided launch override, open code-server in the image WORKDIR. workdir="${AUPLC_CODE_WORKDIR:-$(pwd)}" -extensions_list="${AUPLC_CODE_EXTENSIONS_LIST:-/opt/auplc/extensions/extensions.txt}" -local_extensions_dir="${AUPLC_CODE_LOCAL_EXTENSIONS_DIR:-/opt/auplc/extensions/local}" extensions_dir="${AUPLC_CODE_EXTENSIONS_DIR:-/home/jovyan/.local/share/code-server/extensions}" trusted_domains="${AUPLC_CODE_TRUSTED_DOMAINS:-}" +code_server_pid= +nginx_pid= +cleanup_status= +pid= + +trap ' + cleanup_status=$? + trap - EXIT INT TERM + for pid in "${code_server_pid}" "${nginx_pid}"; do + if [ -n "${pid}" ]; then + kill -TERM "${pid}" 2>/dev/null || true + fi + done + for pid in "${code_server_pid}" "${nginx_pid}"; do + if [ -n "${pid}" ]; then + wait "${pid}" 2>/dev/null || true + fi + done + exit "${cleanup_status}" +' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM mkdir -p "${NPM_CONFIG_PREFIX}/bin" mkdir -p "${PIXI_HOME}/bin" @@ -26,31 +46,6 @@ url_decode() { printf '%b' "${value//%/\\x}" } -seed_builtin_extensions() { - mkdir -p "${extensions_dir}" - - if [ -f "${extensions_list}" ]; then - while IFS= read -r extension_id || [ -n "${extension_id}" ]; do - case "${extension_id}" in - ''|'#'*) continue ;; - *) ;; - esac - - if ! code-server --extensions-dir "${extensions_dir}" --install-extension "${extension_id}" --force; then - printf 'Warning: failed to install code-server extension %s\n' "${extension_id}" >&2 - fi - done <"${extensions_list}" - fi - - if [ -d "${local_extensions_dir}" ]; then - while IFS= read -r -d '' vsix_path; do - if ! code-server --extensions-dir "${extensions_dir}" --install-extension "${vsix_path}"; then - printf 'Warning: failed to install code-server extension package %s\n' "${vsix_path}" >&2 - fi - done < <(find "${local_extensions_dir}" -type f -name '*.vsix' -print0) - fi -} - trim() { local value="$1" value="${value#"${value%%[![:space:]]*}"}" @@ -88,7 +83,6 @@ regex_prefix="$(printf '%s' "${nginx_prefix}" | sed "s/[.[\\*^\$()+?{}|]/\\\\&/g nginx_conf="/tmp/auplc-code-server-nginx.conf" redirect_block="" -seed_builtin_extensions trusted_domain_args=() build_trusted_domain_args "${trusted_domains}" trusted_domain_args @@ -142,7 +136,7 @@ ${redirect_block} } EOF -code-server \ +env -u PORT code-server \ --auth none \ --bind-addr "127.0.0.1:${code_server_port}" \ --extensions-dir "${extensions_dir}" \ @@ -154,9 +148,14 @@ code_server_pid="$!" nginx -c "${nginx_conf}" -g 'daemon off;' & nginx_pid="$!" -cleanup() { - kill "${nginx_pid}" "${code_server_pid}" 2>/dev/null || true -} -trap cleanup EXIT INT TERM - -wait -n "${nginx_pid}" "${code_server_pid}" +exited_pid= +set +e +wait -n -p exited_pid "${nginx_pid}" "${code_server_pid}" +child_status=$? +set -e +if [ "${exited_pid}" = "${nginx_pid}" ]; then + nginx_pid= +elif [ "${exited_pid}" = "${code_server_pid}" ]; then + code_server_pid= +fi +exit "${child_status}" diff --git a/dockerfiles/Code/tests/test_runtime_extension_model.sh b/dockerfiles/Code/tests/test_runtime_extension_model.sh new file mode 100755 index 00000000..ffb53f4f --- /dev/null +++ b/dockerfiles/Code/tests/test_runtime_extension_model.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR +# shellcheck source=harness.sh +source "$SCRIPT_DIR/harness.sh" + +new_case runtime-extension-model +mkdir -p "$current_case/baked/image.extension-1.0.0" +printf 'image extension\n' >"$current_case/baked/image.extension-1.0.0/package.json" +printf '[{"identifier":{"id":"image.extension"},"version":"1.0.0","location":{"scheme":"file","path":"%s"}}]\n' \ + "$current_case/baked/image.extension-1.0.0" >"$current_case/baked/extensions.json" +printf 'persistent sentinel\n' >"$current_case/destination/sentinel" +cp -a "$current_case/destination" "$current_case/expected-destination" + +start_launcher one +pid=$launcher_pid +wait_for_file "$current_case/events/one.code-ready" "$pid" "code-server startup" +wait_for_file "$current_case/events/one.nginx-ready" "$pid" "nginx startup" +diff -r "$current_case/expected-destination" "$current_case/destination" >/dev/null || \ + fail "launcher mutated the persistent extension tree" +[ ! -e "$current_case/events/installer-invoked" ] || fail "launcher invoked a runtime extension installer" +assert_file_contains "$current_case/commands.log" "--extensions-dir $current_case/destination" +assert_eq 2 "$(grep -c 'extensions_dir' "$LAUNCHER")" "launcher extension-directory references" +for forbidden in --install-extension extensions.json flock seed merge; do + assert_file_not_contains "$LAUNCHER" "$forbidden" +done +stop_launcher one "$pid" +printf 'runtime_extension_model=ok\n' diff --git a/dockerfiles/Code/tests/test_service_lifecycle.sh b/dockerfiles/Code/tests/test_service_lifecycle.sh new file mode 100755 index 00000000..ea163ea6 --- /dev/null +++ b/dockerfiles/Code/tests/test_service_lifecycle.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR +# shellcheck source=harness.sh +source "$SCRIPT_DIR/harness.sh" + +test_launch_contract_and_port_isolation() { + new_case launch-contract + case_public_port=19080 + case_code_server_port=19081 + case_service_prefix=/user/alice%40example/ + case_trusted_domains='hub.example, docs.example' + start_launcher one + local pid=$launcher_pid + wait_for_file "$current_case/events/one.code-ready" "$pid" "code-server startup" + wait_for_file "$current_case/events/one.nginx-ready" "$pid" "nginx startup" + assert_file_contains "$current_case/commands.log" "code-server --auth none" + assert_file_contains "$current_case/commands.log" "--bind-addr 127.0.0.1:19081" + assert_file_contains "$current_case/commands.log" "--extensions-dir $current_case/destination" + assert_file_contains "$current_case/commands.log" "--link-protection-trusted-domains hub.example" + assert_file_contains "$current_case/commands.log" "--link-protection-trusted-domains docs.example" + assert_file_contains "$current_case/commands.log" "--ignore-last-opened $current_case/workspace" + assert_eq '<unset>' "$(<"$current_case/events/one.code-port-env")" "code-server PORT environment" + assert_file_contains "$current_case/events/one.nginx.conf" "listen 0.0.0.0:19080;" + assert_file_contains "$current_case/events/one.nginx.conf" "location /user/alice@example/" + assert_file_contains "$current_case/events/one.nginx.conf" "proxy_pass http://127.0.0.1:19081;" + stop_launcher one "$pid" +} + +test_term_returns_143_and_reaps_services() { + new_case term-cleanup + start_launcher one + local pid=$launcher_pid + wait_for_file "$current_case/events/one.code-ready" "$pid" "code-server startup" + wait_for_file "$current_case/events/one.nginx-ready" "$pid" "nginx startup" + stop_launcher one "$pid" +} + +test_code_server_exit_cleans_nginx() { + new_case code-first-exit + case_code_exit_status=37 + start_launcher one + local pid=$launcher_pid + wait_for_file "$current_case/events/one.code-ready" "$pid" "code-server startup" + wait_for_file "$current_case/events/one.nginx-ready" "$pid" "nginx startup" + local nginx_pid + nginx_pid=$(<"$current_case/events/one.nginx-ready") + printf 'exit\n' >"$current_case/events/one.code-exit.fifo" + wait_for_file "$current_case/events/one.nginx-term" "$pid" "nginx sibling cleanup" + wait_launcher "$pid" + assert_eq 37 "$last_launcher_status" "code-server first exit status" + assert_process_gone "$nginx_pid" nginx +} + +test_nginx_exit_cleans_code_server() { + new_case nginx-first-exit + case_nginx_exit_status=38 + start_launcher one + local pid=$launcher_pid + wait_for_file "$current_case/events/one.code-ready" "$pid" "code-server startup" + wait_for_file "$current_case/events/one.nginx-ready" "$pid" "nginx startup" + local code_pid + code_pid=$(<"$current_case/events/one.code-ready") + printf 'exit\n' >"$current_case/events/one.nginx-exit.fifo" + wait_for_file "$current_case/events/one.code-term" "$pid" "code-server sibling cleanup" + wait_launcher "$pid" + assert_eq 38 "$last_launcher_status" "nginx first exit status" + assert_process_gone "$code_pid" code-server +} + +test_launch_contract_and_port_isolation +test_term_returns_143_and_reaps_services +test_code_server_exit_cleans_nginx +test_nginx_exit_cleans_code_server +printf 'service_lifecycle_tests=ok\n' From ba77d9f71b362809e3b02b168cd972bf847d32b3 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:49:01 +0800 Subject: [PATCH 161/180] fix(chart): preserve bounded spawn failure handling --- runtime/chart/values.yaml | 4 +- runtime/hub/tests/test_spawn_defaults.py | 65 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 runtime/hub/tests/test_spawn_defaults.py diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index a548691f..f83dea19 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -119,6 +119,8 @@ hub: JupyterHub: admin_access: true authenticator_class: dummy + Spawner: + http_timeout: 60 service: type: ClusterIP annotations: {} @@ -133,7 +135,7 @@ hub: nodeSelector: {} tolerations: [] concurrentSpawnLimit: 64 - consecutiveFailureLimit: 5 + consecutiveFailureLimit: 0 activeServerLimit: deploymentStrategy: ## type: Recreate diff --git a/runtime/hub/tests/test_spawn_defaults.py b/runtime/hub/tests/test_spawn_defaults.py new file mode 100644 index 00000000..17fe2eaf --- /dev/null +++ b/runtime/hub/tests/test_spawn_defaults.py @@ -0,0 +1,65 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from pathlib import Path +from typing import Protocol, TypedDict + +import yaml + + +class SpawnerValues(TypedDict): + http_timeout: int + + +class HubConfigValues(TypedDict): + Spawner: SpawnerValues + + +class HubValues(TypedDict): + config: HubConfigValues + consecutiveFailureLimit: int + + +class SingleuserValues(TypedDict): + startTimeout: int + + +class SpawnDefaults(TypedDict): + hub: HubValues + singleuser: SingleuserValues + + +class YamlLoader(Protocol): + def safe_load(self, stream: str, /) -> SpawnDefaults: ... + + +def load_yaml(loader: YamlLoader, stream: str) -> SpawnDefaults: + return loader.safe_load(stream) + + +def test_spawn_defaults() -> None: + values_path = Path(__file__).resolve().parents[3] / "runtime" / "chart" / "values.yaml" + values = load_yaml( + yaml, + values_path.read_text(encoding="utf-8"), + ) + + assert values["hub"]["config"]["Spawner"]["http_timeout"] == 60 + assert values["hub"]["consecutiveFailureLimit"] == 0 + assert values["singleuser"]["startTimeout"] == 300 From 0c2cd1e29a2c71917371ee4779b40c395ecadc10 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:49:19 +0800 Subject: [PATCH 162/180] ci: run focused code-server spawn tests --- .github/workflows/lint.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4e5faac4..8c3b6728 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,8 +22,8 @@ jobs: - name: Run Ruff formatter check run: ruff format --check . - installer-tests: - name: Installer Unit Tests + python-tests: + name: Python Unit Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -36,8 +36,10 @@ jobs: - name: Install test deps run: pip install pytest pyyaml - - name: Run installer unit tests - run: python -m pytest tests/installer -v + - name: Run Python unit tests + run: >- + python -m pytest tests/installer -v + runtime/hub/tests/test_spawn_defaults.py frontend-lint: name: Frontend (ESLint + TypeScript) @@ -84,6 +86,11 @@ jobs: grep -v .git | \ xargs -r shellcheck + - name: Run shell tests + run: | + dockerfiles/Code/tests/test_runtime_extension_model.sh + dockerfiles/Code/tests/test_service_lifecycle.sh + yaml-lint: name: YAML (yamllint) runs-on: ubuntu-latest From 861cf9c27eb681324ae07d2f876d89f11315e060 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:49:42 +0800 Subject: [PATCH 163/180] docs(code): explain system and user extension roots --- dockerfiles/Code/README.md | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/dockerfiles/Code/README.md b/dockerfiles/Code/README.md index 79bb033f..c2be7397 100644 --- a/dockerfiles/Code/README.md +++ b/dockerfiles/Code/README.md @@ -141,15 +141,22 @@ cluster administration for kernel modules, GPU/NPU drivers, device plugins, udev rules, system services, or packages that must write to root-owned system directories. -Extensions are installed into `/opt/auplc/code-server/extensions` during image -build. At runtime, code-server uses the persistent user extension directory -`/home/jovyan/.local/share/code-server/extensions` by default. Before -code-server starts, the launcher seeds the default extension IDs from -`/opt/auplc/extensions/extensions.txt` into that persistent directory by calling -`code-server --install-extension`. Marketplace extensions are installed with -`--force` so code-server handles upgrades instead of the launcher comparing -versions itself; local `.vsix` packages are installed without `--force` to avoid -downgrading a user-installed newer copy. +Extensions resolved from the Marketplace and local `.vsix` packages during the +image build are installed as root-owned system extensions under +`/usr/lib/code-server/lib/vscode/extensions`. User-installed extensions remain +in the persistent directory +`/home/jovyan/.local/share/code-server/extensions`. + +Runtime startup does not install, copy, merge, stage, or lock extension data, +and it does not access an extension marketplace. Existing user data is left +untouched, including extension copies installed or seeded by earlier images. +When the system and user directories contain the same extension ID, native VS +Code extension precedence determines which copy is active. + +`PORT` remains the nginx public-listen input, but the launcher removes it from +the code-server child environment before passing the explicit loopback +`--bind-addr`. This prevents code-server's environment precedence from binding +the nginx-facing public port. `--auth none` is acceptable only because JupyterHub and the JupyterHub proxy remain the authentication boundary. The user pod's port `8888` must stay private to the Hub/proxy path and must not be exposed directly through an unauthenticated service, ingress, or port-forward shared with untrusted users. @@ -160,8 +167,6 @@ the proxied code-server root route. Hub spawn completion, however, redirects the browser to the server base URL, and code-server doesn't consume `JUPYTERHUB_DEFAULT_URL` by itself. AUPLC therefore keeps `AUPLC_CODE_WORKDIR` as the reliable adapter between Hub resource selection and the code-server process. -The local proof is recorded in -`.sisyphus/evidence/task-1-codeserver-default-url-proof.md`. Official Code images are checked by the resource contract verifier: @@ -191,13 +196,15 @@ charliermarsh.ruff This baseline keeps Python and Jupyter support for course work, Debugpy for Python debugging, and Ruff for Python linting and formatting. YAML is retained so users can read and edit course, Kubernetes, and other configuration files without adding their own support first. GitLens is retained on purpose so researchers can learn Git history, blame, and commit discipline inside the same workspace they use for code. -Extension versions are not pinned in this iteration. code-server resolves the current compatible extension releases during each image build, while only the code-server package itself is pinned. +Extension versions are not pinned in this iteration. During each image build, +code-server resolves the current compatible Marketplace releases and installs +them with local `.vsix` packages into the root-owned system extension directory. +Only the code-server package itself is pinned. -User-installed extensions are kept under the user's persistent home volume. When -a new image adds a default extension, existing users receive it on their next -code-server start. Existing marketplace extensions from `extensions.txt` are -updated by code-server's own installer. The launcher does not parse extension -directories or compare semantic versions itself. +User-installed extensions are kept under the user's persistent home volume. +Image startup does not alter this directory. Copies installed by users or seeded +by earlier images remain in place, and native VS Code extension precedence +applies when a user copy and a system copy share an extension ID. Default editor settings are also not baked into the image in this iteration. User workspaces and profiles should keep control over editor preferences. From 95db88fe0c97283caef6c6c80dcef8e7f6463bec Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:58:25 +0800 Subject: [PATCH 164/180] test(installer): align GPU access fixture --- tests/installer/test_cli_gpu_access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index 51854223..1d3967f1 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -39,7 +39,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: monkeypatch.setattr(cli, "deploy_rocm_gpu_device_plugin", lambda **kwargs: events.append("device-plugin")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("runtime")) - monkeypatch.setattr(cli, "_print_success_banner", lambda: events.append("success")) + monkeypatch.setattr(cli, "_print_success_banner", lambda **_kwargs: events.append("success")) cli._cmd_install_inner(state, pull=True) From 467f227e07ba3cc188aaaeab6fc8785cfc8900d8 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:59:14 +0800 Subject: [PATCH 165/180] test(hub): authorize runtime timer fixture --- runtime/hub/tests/test_spawner_runtime_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/hub/tests/test_spawner_runtime_metadata.py b/runtime/hub/tests/test_spawner_runtime_metadata.py index 64a92491..4e6c2441 100644 --- a/runtime/hub/tests/test_spawner_runtime_metadata.py +++ b/runtime/hub/tests/test_spawner_runtime_metadata.py @@ -160,12 +160,17 @@ async def base_start(_spawner: object) -> str: spawner = object.__new__(RemoteLabKubeSpawner) spawner.user = types.SimpleNamespace(name="student") spawner.user_options = {"runtime_minutes": 120, "resource_type": "cpu"} + spawner.resource_images = {"cpu": "cpu-image"} spawner.quota_enabled = False spawner.runtime_limit_enabled = runtime_limit_enabled spawner.environment = {} + spawner.extra_pod_config = {} spawner.notebook_allowed_origins = [] spawner._hub_config = None spawner.log = types.SimpleNamespace(debug=lambda _message: None) + spawner._resolve_user_resources = lambda: ["cpu"] + spawner._resolve_accelerator_selection = lambda _resource_type, _selection: None + spawner._configure_spawner = lambda _resource_type, _selection: None spawner._launches_code_server = lambda _resource_type: False result = kubernetes.asyncio.run(spawner.start()) From 114f356efe0c4f265828a366db76707a2f31c7f2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:15:41 +0800 Subject: [PATCH 166/180] fix(installer): support Python 3.10 exhaustiveness --- auplc_installer/overlay.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index d3b45ad5..c3fe7016 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -14,7 +14,7 @@ import re from io import StringIO from pathlib import Path -from typing import assert_never +from typing import NoReturn from auplc_installer.catalog import ( BASE_TEAM_MAPPING, @@ -42,6 +42,10 @@ GPU_RESOURCE_KEYS: tuple[str, ...] = tuple(_RESOURCE_IMAGE_BASE.keys()) +def _assert_never(value: NoReturn) -> NoReturn: + raise AssertionError(f"Expected unreachable value: {value!r}") + + def emit_overlay( cfg: GpuConfig, *, @@ -80,6 +84,8 @@ def emit_overlay( case AccessProfile.LOCAL: buf.write(" auth:\n") buf.write(" native: true\n") + case unreachable: + _assert_never(unreachable) buf.write(" runtimeLimitEnabled: false\n") buf.write(" adminUser:\n") match settings.profile: @@ -90,7 +96,7 @@ def emit_overlay( case AccessProfile.PERSONAL: buf.write(" enabled: false\n") case unreachable: - assert_never(unreachable) + _assert_never(unreachable) buf.write(" quota:\n") buf.write(f" enabled: {str(settings.quota_enabled).lower()}\n") From eb9ae570da8a61a8bcd125b31e5a3e257dccf822 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:16:07 +0800 Subject: [PATCH 167/180] style(deploy): format modular imports --- skills/deploy-aup-learning-cloud/scripts/config_generation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py index 97bc8cac..4a5e54ae 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -8,7 +8,8 @@ import re from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote -from config_rendering import render_inventory, render_pxe_vars, render_values as _render_values +from config_rendering import render_inventory, render_pxe_vars +from config_rendering import render_values as _render_values __all__ = [ "DEFAULT_ACCEL_LABELS", From 4628f56ca9fbf018cab6884322bff9530dea6c08 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:27:13 +0800 Subject: [PATCH 168/180] fix(installer): centralize Python 3.10 exhaustiveness --- auplc_installer/overlay.py | 10 +++------- auplc_installer/profiles.py | 2 +- auplc_installer/typing_compat.py | 7 +++++++ 3 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 auplc_installer/typing_compat.py diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index c3fe7016..e86c0b88 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -14,7 +14,6 @@ import re from io import StringIO from pathlib import Path -from typing import NoReturn from auplc_installer.catalog import ( BASE_TEAM_MAPPING, @@ -24,6 +23,7 @@ ) from auplc_installer.gpu import GpuConfig, is_curated_sku from auplc_installer.profiles import AccessProfile, detect_installer_profile, resolve_access_settings +from auplc_installer.typing_compat import assert_never from auplc_installer.util import InstallerError, log # Resource name → image basename (used by acceleratorOverrides emission @@ -42,10 +42,6 @@ GPU_RESOURCE_KEYS: tuple[str, ...] = tuple(_RESOURCE_IMAGE_BASE.keys()) -def _assert_never(value: NoReturn) -> NoReturn: - raise AssertionError(f"Expected unreachable value: {value!r}") - - def emit_overlay( cfg: GpuConfig, *, @@ -85,7 +81,7 @@ def emit_overlay( buf.write(" auth:\n") buf.write(" native: true\n") case unreachable: - _assert_never(unreachable) + assert_never(unreachable) buf.write(" runtimeLimitEnabled: false\n") buf.write(" adminUser:\n") match settings.profile: @@ -96,7 +92,7 @@ def emit_overlay( case AccessProfile.PERSONAL: buf.write(" enabled: false\n") case unreachable: - _assert_never(unreachable) + assert_never(unreachable) buf.write(" quota:\n") buf.write(f" enabled: {str(settings.quota_enabled).lower()}\n") diff --git a/auplc_installer/profiles.py b/auplc_installer/profiles.py index 16486e01..13eddb55 100644 --- a/auplc_installer/profiles.py +++ b/auplc_installer/profiles.py @@ -4,9 +4,9 @@ import re from dataclasses import dataclass from enum import Enum -from typing import assert_never from auplc_installer.auth import validate_local_admin_username +from auplc_installer.typing_compat import assert_never from auplc_installer.util import InstallerError diff --git a/auplc_installer/typing_compat.py b/auplc_installer/typing_compat.py new file mode 100644 index 00000000..414f8121 --- /dev/null +++ b/auplc_installer/typing_compat.py @@ -0,0 +1,7 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +from typing import NoReturn + + +def assert_never(value: NoReturn) -> NoReturn: + raise AssertionError(f"Expected unreachable value: {value!r}") From c36f88ccddb08b09bce911f8ffbd97f359e09184 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:27:53 +0800 Subject: [PATCH 169/180] style(test): format rebase integration coverage --- runtime/hub/tests/test_auth_templates.py | 4 +--- runtime/hub/tests/test_multi_authenticator_html.py | 4 +--- tests/skills/test_deploy_scripts.py | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/runtime/hub/tests/test_auth_templates.py b/runtime/hub/tests/test_auth_templates.py index 74a05dd0..9104cbe1 100644 --- a/runtime/hub/tests/test_auth_templates.py +++ b/runtime/hub/tests/test_auth_templates.py @@ -87,9 +87,7 @@ def test_login_renders_enabled_authentication_controls( probe = probe_html(template_environment().get_template("login.html").render(**context)) form_actions = {form.get("action") for form in probe.forms} input_names = {field.get("name") for field in probe.inputs} - password_toggles = [ - button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split() - ] + password_toggles = [button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split()] visible_text = " ".join(probe.text) assert ("username" in input_names and "password" in input_names) is ( diff --git a/runtime/hub/tests/test_multi_authenticator_html.py b/runtime/hub/tests/test_multi_authenticator_html.py index 2bb3040b..54620ef1 100644 --- a/runtime/hub/tests/test_multi_authenticator_html.py +++ b/runtime/hub/tests/test_multi_authenticator_html.py @@ -40,9 +40,7 @@ def test_native_child_renders_inline_form_with_encoded_next( assert "required" in fields["password"] button_classes = [(button.get("class") or "").split() for button in probe.buttons] assert any("login-submit" in classes for classes in button_classes) - password_toggles = [ - button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split() - ] + password_toggles = [button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split()] assert len(password_toggles) == 1 assert password_toggles[0].get("type") == "button" assert password_toggles[0].get("aria-label") == "Show password" diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 45c41dd4..0d1811b5 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -989,9 +989,7 @@ def test_generator_emits_canonical_auth_and_runtime_policy( @pytest.mark.parametrize("auth_mode", [None, 42, "unsupported"]) -def test_generator_rejects_invalid_auth_mode_before_discovery( - tmp_path: Path, auth_mode: str | int | None -) -> None: +def test_generator_rejects_invalid_auth_mode_before_discovery(tmp_path: Path, auth_mode: str | int | None) -> None: spec = generator_spec() spec["auth_mode"] = auth_mode spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) From 78fd941e7b5b772c83c1acf85f383eac88454604 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:37:13 +0800 Subject: [PATCH 170/180] ci(test): install Hub config test dependency --- .github/workflows/lint.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4e5faac4..9ba844cc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,7 +34,7 @@ jobs: python-version: '3.10' - name: Install test deps - run: pip install pytest pyyaml + run: pip install pytest pyyaml "pydantic>=2.0" - name: Run installer unit tests run: python -m pytest tests/installer -v diff --git a/pyproject.toml b/pyproject.toml index fe43c64e..f6d28f46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ installer = [ "prompt_toolkit>=3.0.43", ] test = [ + "pydantic>=2.0", "pytest>=8.0", "pyyaml>=6.0", ] From bbec7240ad6421d6af11b11190da3e632482a7f8 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:32:17 +0800 Subject: [PATCH 171/180] fix(hub): preserve auth template variables at startup --- runtime/hub/core/jupyterhub_config.py | 2 +- .../tests/test_jupyterhub_config_startup.py | 172 ++++++++++++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 runtime/hub/tests/test_jupyterhub_config_startup.py diff --git a/runtime/hub/core/jupyterhub_config.py b/runtime/hub/core/jupyterhub_config.py index 0a2bd1f9..ce3abe3a 100644 --- a/runtime/hub/core/jupyterhub_config.py +++ b/runtime/hub/core/jupyterhub_config.py @@ -151,7 +151,7 @@ def _camel_case(s: str) -> str: # Inject platform identity into every Jinja template context so that # {{ powered_by }} is available in all Hub-rendered pages. -c.JupyterHub.template_vars = {"powered_by": "AUP Learning Cloud"} +c.JupyterHub.template_vars["powered_by"] = "AUP Learning Cloud" # Database configuration db_type = z2jh.get_config("hub.db.type") diff --git a/runtime/hub/tests/test_jupyterhub_config_startup.py b/runtime/hub/tests/test_jupyterhub_config_startup.py new file mode 100644 index 00000000..d9a5bbad --- /dev/null +++ b/runtime/hub/tests/test_jupyterhub_config_startup.py @@ -0,0 +1,172 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import importlib.util +import sys +import types +from pathlib import Path +from typing import final + +import pytest + +CONFIG_PATH = Path(__file__).resolve().parents[1] / "core" / "jupyterhub_config.py" +TemplateValue = bool | str +ConfigValue = bool | int | str | None + + +@final +class ConfigSection: + def __init__(self) -> None: + self.template_vars: dict[str, TemplateValue] = {} + self.tornado_settings: dict[str, int | dict[str, bool | str]] = {} + self.volumes: list[dict[str, ConfigValue]] = [] + self.volume_mounts: list[dict[str, ConfigValue]] = [] + + def get(self, _key: str, default: str) -> str: + return default + + def update(self, _values: dict[str, ConfigValue]) -> None: + return None + + +@final +class StubConfig: + def __init__(self) -> None: + self.JupyterHub = ConfigSection() + self.ConfigurableHTTPProxy = ConfigSection() + self.KubeSpawner = ConfigSection() + self.Spawner = ConfigSection() + self.CryptKeeper = ConfigSection() + + def __getitem__(self, _key: str) -> ConfigSection: + return ConfigSection() + + +def test_startup_preserves_setup_and_deployment_template_vars(monkeypatch: pytest.MonkeyPatch) -> None: + config = StubConfig() + setup_template_vars = { + "auth_auto_login": False, + "auth_dummy": False, + "auth_native": True, + "auth_github": True, + "password_management_enabled": True, + "hide_logout": False, + "cluster_name": "test-cluster", + "platform_name": "Test Platform", + } + deployment_template_vars = {"deployment_marker": "kept"} + + core = types.ModuleType("core") + z2jh = types.ModuleType("core.z2jh") + + def get_config(key: str, default: ConfigValue | dict[str, ConfigValue] = None): + if key == "hub.templateVars": + return deployment_template_vars + if key == "hub.db.type": + return "sqlite-memory" + return default + + def get_config_dict(_key: str) -> dict[str, ConfigValue]: + return {} + + def get_config_list(_key: str) -> list[ConfigValue]: + return [] + + def get_name(name: str) -> str: + return name + + def get_name_env(_name: str, _suffix: str) -> str: + return "8081" + + def get_secret_value(_key: str, default: ConfigValue = None) -> ConfigValue: + return default + + def set_config_if_not_none(_section: ConfigSection, _trait: str, _key: str) -> None: + return None + + z2jh.__dict__.update( + get_config=get_config, + get_config_dict=get_config_dict, + get_config_list=get_config_list, + get_name=get_name, + get_name_env=get_name_env, + get_secret_value=get_secret_value, + set_config_if_not_none=set_config_if_not_none, + ) + core.__dict__["z2jh"] = z2jh + + config_module = types.ModuleType("core.config") + + class StubHubConfig: + @staticmethod + def init(config_path: str) -> None: + assert config_path.endswith("hub-config.yaml") + + @staticmethod + def get(): + return types.SimpleNamespace(hub_network=types.SimpleNamespace(allowedOrigins=[])) + + config_module.__dict__["HubConfig"] = StubHubConfig + setup_module = types.ModuleType("core.setup") + + def setup_hub(hub_config: StubConfig) -> None: + hub_config.JupyterHub.template_vars = dict(setup_template_vars) + + setup_module.__dict__["setup_hub"] = setup_hub + + kubernetes_asyncio = types.ModuleType("kubernetes_asyncio") + kubernetes_client = types.ModuleType("kubernetes_asyncio.client") + kubernetes_asyncio.__dict__["client"] = kubernetes_client + tornado = types.ModuleType("tornado") + tornado_httpclient = types.ModuleType("tornado.httpclient") + + class StubAsyncHTTPClient: + @staticmethod + def configure(_backend: str) -> None: + return None + + tornado_httpclient.__dict__["AsyncHTTPClient"] = StubAsyncHTTPClient + tornado.__dict__["httpclient"] = tornado_httpclient + + for module in ( + core, + z2jh, + config_module, + setup_module, + kubernetes_asyncio, + kubernetes_client, + tornado, + tornado_httpclient, + ): + monkeypatch.setitem(sys.modules, module.__name__, module) + + spec = importlib.util.spec_from_file_location("startup_order_jupyterhub_config", CONFIG_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + module.__dict__["get_config"] = lambda: config + spec.loader.exec_module(module) + + assert dict(config.JupyterHub.template_vars) == { + **setup_template_vars, + "powered_by": "AUP Learning Cloud", + **deployment_template_vars, + } + headers = config.JupyterHub.tornado_settings["headers"] + assert isinstance(headers, dict) + assert headers["X-Powered-By"] == "AUP Learning Cloud" From f1b845ed243b3d50d10dd397af2b01fce05836a4 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:33:02 +0800 Subject: [PATCH 172/180] fix(hub): resolve automatic accelerators before configuration --- runtime/hub/core/spawner/kubernetes.py | 9 +- runtime/hub/tests/test_spawner_gpu_access.py | 57 +++++++++++ .../tests/test_spawner_runtime_metadata.py | 94 +++++++++++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index e8533cfb..32caf88f 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -214,6 +214,11 @@ def _resolve_accelerator_selection(self, resource_type: str, gpu_selection: Any) if not allowed_accelerators: raise RuntimeError(f"GPU resource '{resource_type}' has no authorized accelerators configured") + if selected_accelerator == "auto": + if len(allowed_accelerators) > 1: + return selected_accelerator + raise RuntimeError(f"GPU resource '{resource_type}' requires selecting an accelerator") + if not selected_accelerator: if len(allowed_accelerators) == 1: selected_accelerator = allowed_accelerators[0] @@ -1001,7 +1006,6 @@ async def start(self): self.user_options.get("gpu_selection"), ) self.user_options["gpu_selection"] = gpu_selection - self._configure_spawner(resource_type, gpu_selection) # Ensure pod fails immediately (not retried) when an init container fails. # JupyterHub manages pod lifecycle; Kubernetes should not silently restart pods. @@ -1016,6 +1020,9 @@ async def start(self): metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None eligible = list(metadata.acceleratorKeys) if metadata and metadata.acceleratorKeys else [] gpu_selection = await self._resolve_auto_accelerator(resource_type, eligible) + if not isinstance(gpu_selection, str) or not gpu_selection.strip() or gpu_selection.strip() == "auto": + raise RuntimeError("Auto-selection must return a concrete accelerator") + gpu_selection = self._resolve_accelerator_selection(resource_type, gpu_selection) self.user_options["gpu_selection"] = gpu_selection self.log.info(f"Auto-selected accelerator '{gpu_selection}' for resource '{resource_type}'") diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py index 4ea6f476..58cb2c7b 100644 --- a/runtime/hub/tests/test_spawner_gpu_access.py +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -171,3 +171,60 @@ def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): with pytest.raises(RuntimeError, match="not authorized"): spawner.options_from_form({"runtime": ["20"], "resource_type": ["gpu"], "gpu_selection_gpu": ["gpu-a"]}) + + +def test_auto_accelerator_is_a_gpu_sentinel_only_for_multiple_authorized_keys(): + spawner = make_spawner() + spawner._hub_config = types.SimpleNamespace( + get_resource_metadata=lambda _resource_type: types.SimpleNamespace(acceleratorKeys=["gpu-a", "gpu-b"]) + ) + spawner.accelerator_options = {"gpu-a": {}, "gpu-b": {}} + + assert spawner._resolve_accelerator_selection("gpu", "auto") == "auto" + + +@pytest.mark.parametrize("selection", [None, "", " "]) +def test_single_authorized_accelerator_defaults_blank_selection(selection: str | None): + spawner = make_spawner() + + assert spawner._resolve_accelerator_selection("gpu", selection) == "gpu-a" + + +@pytest.mark.parametrize( + ("resource_type", "accelerator_keys", "selection", "error"), + [ + ("cpu", ["gpu-a", "gpu-b"], "auto", "does not allow GPU selection"), + ("gpu", ["gpu-a"], "auto", "requires selecting an accelerator"), + ], +) +def test_auto_accelerator_is_rejected_outside_multiple_authorized_gpu_keys( + resource_type: str, accelerator_keys: list[str], selection: str, error: str +): + spawner = make_spawner() + spawner._hub_config = types.SimpleNamespace( + get_resource_metadata=lambda _resource_type: types.SimpleNamespace(acceleratorKeys=accelerator_keys) + ) + + with pytest.raises(RuntimeError, match=error): + spawner._resolve_accelerator_selection(resource_type, selection) + + +@pytest.mark.parametrize( + ("accelerator_keys", "accelerator_options", "selection", "error"), + [ + (["gpu-a"], {"gpu-a": {}}, "gpu-x", "not authorized"), + (["gpu-a"], {"gpu-a": {}, "gpu-b": {}}, "gpu-b", "not authorized"), + (["gpu-a", "gpu-b"], {"gpu-a": {}}, "gpu-b", "not configured"), + ], +) +def test_concrete_accelerator_requires_resource_authorization_and_global_configuration( + accelerator_keys: list[str], accelerator_options: dict[str, dict[str, str]], selection: str, error: str +): + spawner = make_spawner() + spawner._hub_config = types.SimpleNamespace( + get_resource_metadata=lambda _resource_type: types.SimpleNamespace(acceleratorKeys=accelerator_keys) + ) + spawner.accelerator_options = accelerator_options + + with pytest.raises(RuntimeError, match=error): + spawner._resolve_accelerator_selection("gpu", selection) diff --git a/runtime/hub/tests/test_spawner_runtime_metadata.py b/runtime/hub/tests/test_spawner_runtime_metadata.py index 4e6c2441..7184bad8 100644 --- a/runtime/hub/tests/test_spawner_runtime_metadata.py +++ b/runtime/hub/tests/test_spawner_runtime_metadata.py @@ -188,3 +188,97 @@ async def base_start(_spawner: object) -> str: assert timer_loop.calls == [] assert spawner.environment["AUPLC_RUNTIME_UNLIMITED"] == "true" assert "JOB_RUN_TIME" not in spawner.environment + + +def test_start_resolves_auto_with_authorized_keys_and_configures_once(monkeypatch: pytest.MonkeyPatch) -> None: + class QuotaManager: + def start_usage_session(self, *_args: str) -> str: + return "usage-session" + + metadata = types.SimpleNamespace(acceleratorKeys=["gpu-a", "gpu-b"], allowGitClone=False) + quota_module = types.SimpleNamespace(get_quota_manager=lambda: QuotaManager()) + monkeypatch.setitem(sys.modules, "core.quota", quota_module) + + async def base_start(_spawner: object) -> str: + return "started" + + monkeypatch.setattr(kubernetes.KubeSpawner, "start", base_start, raising=False) + + spawner = object.__new__(RemoteLabKubeSpawner) + spawner.user = types.SimpleNamespace(name="student") + spawner.user_options = {"runtime_minutes": 20, "resource_type": "gpu", "gpu_selection": "auto"} + spawner.resource_images = {"gpu": "gpu-image"} + spawner.resource_requirements = {"gpu": {"cpu": "1", "memory": "1Gi", "amd.com/gpu": "1"}} + spawner.accelerator_options = {"gpu-a": {}, "gpu-b": {}} + spawner.quota_enabled = False + spawner.runtime_limit_enabled = False + spawner.environment = {} + spawner.extra_pod_config = {} + spawner.notebook_allowed_origins = [] + spawner._hub_config = types.SimpleNamespace(get_resource_metadata=lambda _resource_type: metadata) + spawner.log = types.SimpleNamespace(debug=lambda _message: None, info=lambda _message: None) + spawner._resolve_user_resources = lambda: ["gpu"] + spawner._launches_code_server = lambda _resource_type: False + spawner._resolve_target_path = lambda _resource_type, _custom_repo_path: None + spawner._apply_target_path_mapping = lambda _resource_type, _target_path: None + + auto_calls: list[list[str]] = [] + configure_calls: list[tuple[str, str | None]] = [] + + async def resolve_auto(resource_type: str, eligible_keys: list[str]) -> str: + assert resource_type == "gpu" + auto_calls.append(eligible_keys) + return "gpu-b" + + def configure(resource_type: str, selection: str | None) -> None: + configure_calls.append((resource_type, selection)) + + spawner._resolve_auto_accelerator = resolve_auto + spawner._configure_spawner = configure + + result = kubernetes.asyncio.run(spawner.start()) + + assert result == "started" + assert auto_calls == [["gpu-a", "gpu-b"]] + assert spawner.user_options["gpu_selection"] == "gpu-b" + assert configure_calls == [("gpu", "gpu-b")] + + +@pytest.mark.parametrize( + ("auto_result", "error"), + [ + ("gpu-x", "not authorized"), + ("gpu-unconfigured", "not configured"), + (None, "must return a concrete accelerator"), + ("", "must return a concrete accelerator"), + (" ", "must return a concrete accelerator"), + ("auto", "must return a concrete accelerator"), + ], +) +def test_start_rejects_auto_result_that_is_not_an_authorized_concrete_accelerator( + auto_result: str | None, error: str +) -> None: + metadata = types.SimpleNamespace(acceleratorKeys=["gpu-a", "gpu-unconfigured"], allowGitClone=False) + + spawner = object.__new__(RemoteLabKubeSpawner) + spawner.user = types.SimpleNamespace(name="student") + spawner.user_options = {"runtime_minutes": 20, "resource_type": "gpu", "gpu_selection": "auto"} + spawner.resource_images = {"gpu": "gpu-image"} + spawner.resource_requirements = {"gpu": {"cpu": "1", "memory": "1Gi", "amd.com/gpu": "1"}} + spawner.accelerator_options = {"gpu-a": {}} + spawner.extra_pod_config = {} + spawner._hub_config = types.SimpleNamespace(get_resource_metadata=lambda _resource_type: metadata) + spawner._resolve_user_resources = lambda: ["gpu"] + auto_calls: list[list[str]] = [] + + async def resolve_auto(_resource_type: str, eligible_keys: list[str]) -> str | None: + auto_calls.append(eligible_keys) + return auto_result + + spawner._resolve_auto_accelerator = resolve_auto + spawner._configure_spawner = lambda *_args: pytest.fail("invalid auto result configured the spawner") + + with pytest.raises(RuntimeError, match=error): + kubernetes.asyncio.run(spawner.start()) + + assert auto_calls == [["gpu-a", "gpu-unconfigured"]] From 2e9308efb8c3bbdac2679385c452a3eef523e4e6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:33:40 +0800 Subject: [PATCH 173/180] ci(test): run focused Hub regression tests --- .github/workflows/lint.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9ba844cc..e2536142 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -39,6 +39,23 @@ jobs: - name: Run installer unit tests run: python -m pytest tests/installer -v + hub-regression-tests: + name: Hub Regression Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install test deps + run: pip install pytest + + - name: Run Hub regression tests + run: python -m pytest runtime/hub/tests/test_jupyterhub_config_startup.py runtime/hub/tests/test_spawner_gpu_access.py + frontend-lint: name: Frontend (ESLint + TypeScript) runs-on: ubuntu-latest From e5eb71b7e44e5f85039f6ed61ce2fcb7acb3f760 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:19:35 +0800 Subject: [PATCH 174/180] fix(auth): enforce provider-specific allow policies --- runtime/hub/core/authenticators/__init__.py | 4 + runtime/hub/tests/test_auth_provider_setup.py | 8 ++ .../hub/tests/test_authenticator_factory.py | 86 ++++++++++++++++--- .../hub/tests/test_github_authenticator.py | 15 ++++ 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/runtime/hub/core/authenticators/__init__.py b/runtime/hub/core/authenticators/__init__.py index 56150569..7d552db4 100644 --- a/runtime/hub/core/authenticators/__init__.py +++ b/runtime/hub/core/authenticators/__init__.py @@ -44,13 +44,17 @@ def configure_authenticator(c: Any, auth: AuthCapabilities) -> None: c.Authenticator.allow_all = True case AuthCapabilities(auto_login=False, dummy=True, native=False, github=False): c.JupyterHub.authenticator_class = "dummy" + c.Authenticator.allow_all = True case AuthCapabilities(auto_login=False, dummy=False, native=True, github=False): c.JupyterHub.authenticator_class = CustomFirstUseAuthenticator c.Authenticator.allow_all = True case AuthCapabilities(auto_login=False, dummy=False, native=False, github=True): c.JupyterHub.authenticator_class = CustomGitHubOAuthenticator + c.GitHubOAuthenticator.allow_all = False case AuthCapabilities(auto_login=False, dummy=False, native=True, github=True): c.JupyterHub.authenticator_class = CustomMultiAuthenticator + c.GitHubOAuthenticator.allow_all = False + c.MultiAuthenticator.allow_all = True c.MultiAuthenticator.authenticators = [ {"authenticator_class": CustomGitHubOAuthenticator, "url_prefix": "/github"}, { diff --git a/runtime/hub/tests/test_auth_provider_setup.py b/runtime/hub/tests/test_auth_provider_setup.py index 40340e24..2997be39 100644 --- a/runtime/hub/tests/test_auth_provider_setup.py +++ b/runtime/hub/tests/test_auth_provider_setup.py @@ -89,9 +89,12 @@ def configure_authenticator(c: object, _input: object) -> None: return if auth.dummy: c.JupyterHub.authenticator_class = "dummy" + c.Authenticator.allow_all = True return if auth.native and auth.github: c.JupyterHub.authenticator_class = authenticator_types["multi"] + c.GitHubOAuthenticator.allow_all = False + c.MultiAuthenticator.allow_all = True c.MultiAuthenticator.authenticators = [ {"authenticator_class": authenticator_types["github"], "url_prefix": "/github"}, { @@ -103,6 +106,7 @@ def configure_authenticator(c: object, _input: object) -> None: return if auth.github: c.JupyterHub.authenticator_class = authenticator_types["github"] + c.GitHubOAuthenticator.allow_all = False return c.JupyterHub.authenticator_class = authenticator_types["native"] c.Authenticator.allow_all = True @@ -173,6 +177,7 @@ async def sync_github_teams_for_user(*args: object, **kwargs: object) -> bool: c = types.SimpleNamespace( JupyterHub=hub, Authenticator=types.SimpleNamespace(), + GitHubOAuthenticator=types.SimpleNamespace(), Spawner=types.SimpleNamespace(), MultiAuthenticator=types.SimpleNamespace(), ) @@ -289,6 +294,7 @@ def test_github_only_preserves_direct_callback_path(monkeypatch: pytest.MonkeyPa state.setup.setup_hub(state.c) assert state.c.JupyterHub.authenticator_class is state.authenticator_types["github"] + assert state.c.GitHubOAuthenticator.allow_all is False assert not hasattr(state.c.MultiAuthenticator, "authenticators") @@ -299,6 +305,8 @@ def test_composed_auth_preserves_prefixed_github_and_unprefixed_native_callbacks state.setup.setup_hub(state.c) assert state.c.JupyterHub.authenticator_class is state.authenticator_types["multi"] + assert state.c.GitHubOAuthenticator.allow_all is False + assert state.c.MultiAuthenticator.allow_all is True assert state.c.MultiAuthenticator.authenticators == [ {"authenticator_class": state.authenticator_types["github"], "url_prefix": "/github"}, { diff --git a/runtime/hub/tests/test_authenticator_factory.py b/runtime/hub/tests/test_authenticator_factory.py index 938f7495..7c61d09b 100644 --- a/runtime/hub/tests/test_authenticator_factory.py +++ b/runtime/hub/tests/test_authenticator_factory.py @@ -1,3 +1,4 @@ +import importlib import importlib.util import sys import types @@ -5,6 +6,7 @@ from contextlib import contextmanager from pathlib import Path +import anyio import pytest ROOT = Path(__file__).resolve().parents[1] @@ -66,36 +68,98 @@ def test_factory_preserves_identity_prefix_contract(monkeypatch: pytest.MonkeyPa @pytest.mark.parametrize( - ("capabilities", "expected_name", "allow_all"), + ("capabilities", "expected_name", "expected_allow_all"), [ - ((True, False, False, False), "AutoLoginAuthenticator", True), - ((False, True, False, False), "dummy", None), - ((False, False, True, False), "CustomFirstUseAuthenticator", True), - ((False, False, False, True), "CustomGitHubOAuthenticator", None), - ((False, False, True, True), "CustomMultiAuthenticator", None), + ((True, False, False, False), "AutoLoginAuthenticator", (("Authenticator", True),)), + ((False, True, False, False), "dummy", (("Authenticator", True),)), + ((False, False, True, False), "CustomFirstUseAuthenticator", (("Authenticator", True),)), + ((False, False, False, True), "CustomGitHubOAuthenticator", (("GitHubOAuthenticator", False),)), + ( + (False, False, True, True), + "CustomMultiAuthenticator", + (("GitHubOAuthenticator", False), ("MultiAuthenticator", True)), + ), ], ) def test_factory_configures_authenticator_for_canonical_capabilities( monkeypatch: pytest.MonkeyPatch, capabilities: tuple[bool, bool, bool, bool], expected_name: str, - allow_all: bool | None, + expected_allow_all: tuple[tuple[str, bool], ...], ) -> None: with _loaded_factory(monkeypatch) as (factory, config): c = types.SimpleNamespace( JupyterHub=types.SimpleNamespace(), Authenticator=types.SimpleNamespace(), + GitHubOAuthenticator=types.SimpleNamespace(), MultiAuthenticator=types.SimpleNamespace(), ) factory.configure_authenticator(c, config.AuthCapabilities(*capabilities)) selected = c.JupyterHub.authenticator_class assert selected == "dummy" if expected_name == "dummy" else selected.__name__ == expected_name - if allow_all is not None: - assert c.Authenticator.allow_all is allow_all + for authenticator_name, allow_all in expected_allow_all: + assert getattr(c, authenticator_name).allow_all is allow_all if capabilities == (False, False, True, True): - assert c.MultiAuthenticator.authenticators[0]["url_prefix"] == "/github" - assert c.MultiAuthenticator.authenticators[1]["url_prefix"] == "/native" + assert c.MultiAuthenticator.authenticators == [ + {"authenticator_class": factory.CustomGitHubOAuthenticator, "url_prefix": "/github"}, + { + "authenticator_class": factory.CustomFirstUseAuthenticator, + "url_prefix": "/native", + "config": {"prefix": "", "allow_all": True}, + }, + ] + + +def test_factory_keeps_multi_github_allow_all_available_for_later_operator_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with _loaded_factory(monkeypatch) as (factory, config): + c = types.SimpleNamespace( + JupyterHub=types.SimpleNamespace(), + Authenticator=types.SimpleNamespace(), + GitHubOAuthenticator=types.SimpleNamespace(), + MultiAuthenticator=types.SimpleNamespace(), + ) + factory.configure_authenticator(c, config.AuthCapabilities(False, False, True, True)) + + c.GitHubOAuthenticator.allow_all = True + + assert c.GitHubOAuthenticator.allow_all is True + assert "config" not in c.MultiAuthenticator.authenticators[0] + + +def test_factory_multi_github_child_enforces_org_policy_until_class_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.syspath_prepend(str(ROOT / "tests")) + support_module = importlib.import_module("github_authenticator_support") + loaded_authenticators = support_module.loaded_authenticators + with _loaded_factory(monkeypatch) as (factory, config), loaded_authenticators(monkeypatch) as modules: + c = types.SimpleNamespace( + JupyterHub=types.SimpleNamespace(), + Authenticator=types.SimpleNamespace(), + GitHubOAuthenticator=types.SimpleNamespace(), + MultiAuthenticator=types.SimpleNamespace(), + ) + factory.configure_authenticator(c, config.AuthCapabilities(False, False, True, True)) + github_child = c.MultiAuthenticator.authenticators[0] + authenticator = modules.github.CustomGitHubOAuthenticator() + authenticator.allow_all = c.GitHubOAuthenticator.allow_all + authenticator.allowed_organizations = {"auplc"} + authenticator.organization_members = {"auplc": {"octo"}} + + member = anyio.run(authenticator.authenticate, None, {"login": "octo"}) + outsider = anyio.run(authenticator.authenticate, None, {"login": "outside"}) + + c.GitHubOAuthenticator.allow_all = True + authenticator.allow_all = c.GitHubOAuthenticator.allow_all + overridden_outsider = anyio.run(authenticator.authenticate, None, {"login": "outside"}) + + assert github_child == {"authenticator_class": factory.CustomGitHubOAuthenticator, "url_prefix": "/github"} + assert member["name"] == "octo" + assert outsider is None + assert overridden_outsider["name"] == "outside" @pytest.mark.parametrize( diff --git a/runtime/hub/tests/test_github_authenticator.py b/runtime/hub/tests/test_github_authenticator.py index 8195bb35..15c73ec5 100644 --- a/runtime/hub/tests/test_github_authenticator.py +++ b/runtime/hub/tests/test_github_authenticator.py @@ -21,6 +21,21 @@ def test_direct_github_auth_authorizes_raw_login_then_prefixes_accepted_model(mo assert prefixed_model["admin"] is True +def test_github_organization_policy_rejects_nonmember_when_allow_all_is_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with loaded_authenticators(monkeypatch) as modules: + authenticator = modules.github.CustomGitHubOAuthenticator() + authenticator.allowed_organizations = {"auplc"} + authenticator.organization_members = {"auplc": {"octo"}} + + auth_model = anyio.run(authenticator.authenticate, None, {"login": "outside"}) + + assert authenticator.allow_all is False + assert auth_model is None + assert authenticator.policy_names == ["outside"] + + def test_github_post_auth_prefixing_copies_only_top_level_model_and_is_idempotent( monkeypatch: pytest.MonkeyPatch, ) -> None: From 078266082a2530e27514fb2859e5d23bc48c692f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:20:43 +0800 Subject: [PATCH 175/180] fix(config): remove global authenticator bypass --- runtime/values-multi-nodes.yaml.example | 1 - runtime/values.yaml | 3 --- tests/installer/test_chart_local_auth.py | 15 +++++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index e292578e..9b0bce7e 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -516,7 +516,6 @@ hub: redirect_to_server: false Authenticator: - allow_all: true admin_users: - your-github-username diff --git a/runtime/values.yaml b/runtime/values.yaml index 58ad6ef6..45d51845 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -602,9 +602,6 @@ hub: # Users can access their running server via the "My Server" button on Home. redirect_to_server: false - Authenticator: - allow_all: true - # ---- GitHub App ---- GitHubOAuthenticator: oauth_callback_url: "https://<Your.domain>/hub/github/oauth_callback" diff --git a/tests/installer/test_chart_local_auth.py b/tests/installer/test_chart_local_auth.py index 1be3ca4a..6d783e78 100644 --- a/tests/installer/test_chart_local_auth.py +++ b/tests/installer/test_chart_local_auth.py @@ -136,6 +136,21 @@ def test_multi_node_example_emits_canonical_auth_and_runtime_policy() -> None: assert custom["quota"]["enabled"] is True +@pytest.mark.parametrize("values_file", ["runtime/values.yaml", "runtime/values-multi-nodes.yaml.example"]) +def test_maintained_values_omit_global_authenticator_bypass(values_file: str) -> None: + values = yaml.safe_load((ROOT / values_file).read_text(encoding="utf-8")) + config = values["hub"]["config"] + + assert "allow_all" not in config.get("Authenticator", {}) + assert config["GitHubOAuthenticator"]["allowed_organizations"] == ["<YOUR-ORG-NAME>"] + + +def test_multi_node_example_preserves_admin_users() -> None: + values = yaml.safe_load((ROOT / "runtime/values-multi-nodes.yaml.example").read_text(encoding="utf-8")) + + assert values["hub"]["config"]["Authenticator"]["admin_users"] == ["your-github-username"] + + @pytest.mark.parametrize( ("runtime_limit_enabled", "quota_enabled"), [(True, True), (True, False), (False, False)], From 65ff77a2c8a193e12082bd721efe37db8ca09840 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:22:16 +0800 Subject: [PATCH 176/180] ci(test): cover Hub auth and runtime regressions --- .github/workflows/lint.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e2536142..600c5202 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,10 +51,18 @@ jobs: python-version: '3.10' - name: Install test deps - run: pip install pytest + run: pip install pytest anyio pyyaml traitlets "pydantic>=2.0" - name: Run Hub regression tests - run: python -m pytest runtime/hub/tests/test_jupyterhub_config_startup.py runtime/hub/tests/test_spawner_gpu_access.py + run: | + python -m pytest -v \ + runtime/hub/tests/test_jupyterhub_config_startup.py \ + runtime/hub/tests/test_spawner_gpu_access.py \ + runtime/hub/tests/test_spawner_runtime_metadata.py \ + runtime/hub/tests/test_authenticator_factory.py \ + runtime/hub/tests/test_auth_provider_setup.py \ + runtime/hub/tests/test_github_authenticator.py \ + runtime/hub/tests/test_native_authenticator.py frontend-lint: name: Frontend (ESLint + TypeScript) From 842cea5c9b112bc6baf01920f21d675aaeeb6206 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:36:55 +0800 Subject: [PATCH 177/180] refactor(auth): stop generating login markup --- runtime/hub/core/authenticators/multi.py | 56 +--------------- .../tests/test_multi_authenticator_html.py | 64 ++----------------- 2 files changed, 8 insertions(+), 112 deletions(-) diff --git a/runtime/hub/core/authenticators/multi.py b/runtime/hub/core/authenticators/multi.py index 44456a20..b44c76d7 100644 --- a/runtime/hub/core/authenticators/multi.py +++ b/runtime/hub/core/authenticators/multi.py @@ -28,14 +28,11 @@ from multiauthenticator import MultiAuthenticator from multiauthenticator.multiauthenticator import PREFIX_SEPARATOR -from core.authenticators.firstuse import CustomFirstUseAuthenticator - class CustomMultiAuthenticator(MultiAuthenticator): """ - MultiAuthenticator with custom login page HTML and refresh_user support. + MultiAuthenticator with refresh_user support. - Provides a unified login page supporting multiple authentication methods. Delegates ``refresh_user`` to the sub-authenticator that owns the user. """ @@ -91,52 +88,5 @@ def delete_user(self, user): authenticator.delete_user(user) return super().delete_user(user) - def get_custom_html(self, base_url): - html = [] - - for authenticator in self._authenticators: - name = getattr(authenticator, "service_name", "authenticator") - login_service = getattr(authenticator, "login_service", name) - url = authenticator.login_url(base_url) - - match authenticator: - case CustomFirstUseAuthenticator(): - html.append(f""" - <div class="login-option mb-6 bg-white rounded-xl shadow-lg p-6"> - <form action="{url}{{% if next is defined and next|length %}}?next={{{{ next | urlencode }}}}{{% endif %}}" method="post"> - <input type="hidden" name="_xsrf" value="{{{{ xsrf }}}}" /> - <div class="mb-4"> - <input type="text" name="username" placeholder="Username" - aria-label="Username" - class="block w-full px-4 py-2 border rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" - required /> - </div> - <div class="mb-4 relative"> - <input type="password" name="password" placeholder="Password" - aria-label="Password" autocomplete="current-password" - class="login-input block w-full pl-4 pr-10 py-2 rounded-md shadow-sm focus:ring-2 focus:ring-blue-500" - required /> - <button type="button" class="password-toggle absolute inset-y-0 right-0 flex items-center pr-3 text-gray-400 hover:text-gray-600" aria-label="Show password"> - <svg class="eye-open w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z"/></svg> - <svg class="eye-closed w-5 h-5 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13.875 18.825A10.05 10.05 0 0112 19c-4.478 0-8.268-2.943-9.543-7a9.97 9.97 0 011.563-3.029m5.858.908a3 3 0 114.243 4.243M9.878 9.878l4.242 4.242M9.88 9.88l-3.29-3.29m7.532 7.532l3.29 3.29M3 3l3.59 3.59m0 0A9.953 9.953 0 0112 5c4.478 0 8.268 2.943 9.543 7a10.025 10.025 0 01-4.132 5.411m0 0L21 21"/></svg> - </button> - </div> - <button type="submit" - class="login-submit w-full py-2 px-4 bg-blue-600 hover:bg-blue-700 text-white font-medium rounded-md"> - Use LocalAccount Login - </button> - </form> - </div> - """) - case _: - html.append(f""" - <div class="login-option mb-4"> - <a role="button" class="login-github-button w-full inline-block text-center py-3 px-4 bg-gray-800 - rounded-md hover:bg-gray-900 font-medium" - href="{url}{{% if next is defined and next|length %}}?next={{{{ next }}}}{{% endif %}}"> - Use {login_service} Login - </a> - </div> - """) - - return "\n".join(html) + def get_custom_html(self, base_url: str) -> str: + return "" diff --git a/runtime/hub/tests/test_multi_authenticator_html.py b/runtime/hub/tests/test_multi_authenticator_html.py index 54620ef1..cf3f7eab 100644 --- a/runtime/hub/tests/test_multi_authenticator_html.py +++ b/runtime/hub/tests/test_multi_authenticator_html.py @@ -1,65 +1,11 @@ import pytest -from auth_template_support import loaded_multi_authenticator, probe_html, render_multi_html +from auth_template_support import loaded_multi_authenticator -NEXT_CASES = ( - ( - "/hub/spawn?x=1&y=two words", - "%2Fhub%2Fspawn%3Fx%3D1%26y%3Dtwo+words", - "%252Fhub%252Fspawn%253Fx%253D1%2526y%253Dtwo%2Bwords", - ), - ( - "/路径?值=你好 世界", - "%2F%E8%B7%AF%E5%BE%84%3F%E5%80%BC%3D%E4%BD%A0%E5%A5%BD+%E4%B8%96%E7%95%8C", - "%252F%25E8%25B7%25AF%25E5%25BE%2584%253F%25E5%2580%25BC%253D%25E4%25BD%25A0%25E5%25A5%25BD%2B%25E4%25B8%2596%25E7%2595%258C", - ), - ("", "", ""), -) - -@pytest.mark.parametrize(("next_value", "escaped_next", "form_next"), NEXT_CASES) -def test_native_child_renders_inline_form_with_encoded_next( - monkeypatch: pytest.MonkeyPatch, - next_value: str, - escaped_next: str, - form_next: str, -) -> None: +def test_multi_authenticator_custom_html_is_intentionally_empty(monkeypatch: pytest.MonkeyPatch) -> None: with loaded_multi_authenticator(monkeypatch) as state: - state.multi._authenticators = [state.native] - probe = probe_html(render_multi_html(state, next_value)) - - expected_action = "/hub/native/login" + (f"?next={form_next}" if escaped_next else "") - assert [form.get("action") for form in probe.forms] == [expected_action] - fields = {field.get("name"): field for field in probe.inputs} - assert fields["_xsrf"].get("value") == "csrf-token" - assert fields["username"].get("placeholder") == "Username" - assert fields["username"].get("aria-label") == "Username" - assert "required" in fields["username"] - assert fields["password"].get("placeholder") == "Password" - assert fields["password"].get("aria-label") == "Password" - assert fields["password"].get("autocomplete") == "current-password" - assert "required" in fields["password"] - button_classes = [(button.get("class") or "").split() for button in probe.buttons] - assert any("login-submit" in classes for classes in button_classes) - password_toggles = [button for button in probe.buttons if "password-toggle" in (button.get("class") or "").split()] - assert len(password_toggles) == 1 - assert password_toggles[0].get("type") == "button" - assert password_toggles[0].get("aria-label") == "Show password" + state.multi._authenticators = [state.external, state.native] + custom_html = state.multi.get_custom_html("/hub/") -@pytest.mark.parametrize(("next_value", "escaped_next", "form_next"), NEXT_CASES) -def test_external_child_renders_encoded_link_even_with_empty_prefix( - monkeypatch: pytest.MonkeyPatch, - next_value: str, - escaped_next: str, - form_next: str, -) -> None: - with loaded_multi_authenticator(monkeypatch) as state: - state.multi._authenticators = [state.external] - probe = probe_html(render_multi_html(state, next_value)) - - expected_href = "/hub/github/oauth_login" + (f"?next={escaped_next}" if form_next else "") - assert probe.hrefs == [expected_href] - assert probe.forms == [] - classes = (probe.anchors[0].get("class") or "").split() - assert "login-github-button" in classes - assert "text-white" not in classes + assert custom_html == "" From 187ecb089f56f4c8f4567a52e2b1520fefe122fd Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:37:28 +0800 Subject: [PATCH 178/180] fix(ui): restore composed login experience --- .../hub/frontend/templates/_login_macros.html | 67 ++++++++ runtime/hub/frontend/templates/login.html | 94 ++++++------ runtime/hub/frontend/templates/page.html | 16 +- runtime/hub/tests/auth_template_support.py | 44 +++++- runtime/hub/tests/test_auth_templates.py | 144 ++++++++++++++++-- 5 files changed, 293 insertions(+), 72 deletions(-) create mode 100644 runtime/hub/frontend/templates/_login_macros.html diff --git a/runtime/hub/frontend/templates/_login_macros.html b/runtime/hub/frontend/templates/_login_macros.html new file mode 100644 index 00000000..69e9971e --- /dev/null +++ b/runtime/hub/frontend/templates/_login_macros.html @@ -0,0 +1,67 @@ +{# +Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +#} + +{% macro native_login_form(action, xsrf, username="", autofocus=false) %} +<form action="{{ action }}" method="post" role="form" class="space-y-6"> + <input type="hidden" name="_xsrf" value="{{ xsrf }}" /> + + <div> + <label for="username_input" class="login-field-label block text-sm font-medium mb-1">Username</label> + <input id="username_input" type="text" autocapitalize="off" autocorrect="off" autocomplete="username" + name="username" value="{{ username }}" required{% if autofocus %} autofocus="autofocus"{% endif %} + class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> + </div> + + <div> + <label for="password_input" class="login-field-label block text-sm font-medium mb-1">Password</label> + <div class="relative"> + <input id="password_input" type="password" autocomplete="current-password" name="password" required + class="login-input block w-full pl-3 pr-10 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> + <button type="button" class="password-toggle absolute inset-y-0 right-0 flex items-center pr-3 text-gray-400 hover:text-gray-600" aria-label="Show password"> + <svg class="eye-open w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z"/></svg> + <svg class="eye-closed w-5 h-5 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13.875 18.825A10.05 10.05 0 0112 19c-4.478 0-8.268-2.943-9.543-7a9.97 9.97 0 011.563-3.029m5.858.908a3 3 0 114.243 4.243M9.878 9.878l4.242 4.242M9.88 9.88l-3.29-3.29m7.532 7.532l3.29 3.29M3 3l3.59 3.59m0 0A9.953 9.953 0 0112 5c4.478 0 8.268 2.943 9.543 7a10.025 10.025 0 01-4.132 5.411m0 0L21 21"/></svg> + </button> + </div> + </div> + + <div class="mt-6"> + <button id="login_submit" type="submit" + class="login-submit w-full flex justify-center py-3 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> + Login + </button> + </div> +</form> +{% endmacro %} + +{% macro github_login_button(href, helper_text="") %} +<div class="mb-6"> + <a href="{{ href }}" + class="login-github-button w-full flex justify-center items-center py-3 px-4 rounded-md shadow-sm text-sm font-medium focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> + <svg class="w-5 h-5 mr-2" fill="currentColor" viewBox="0 0 20 20"> + <path fill-rule="evenodd" d="M10 0C4.477 0 0 4.484 0 10.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0110 4.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.203 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.942.359.31.678.921.678 1.856 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0020 10.017C20 4.484 15.522 0 10 0z" clip-rule="evenodd"/> + </svg> + Sign in with GitHub + </a> + {% if helper_text %} + <p class="login-helper text-sm text-center mt-3 mb-0">{{ helper_text }}</p> + {% endif %} +</div> +{% endmacro %} diff --git a/runtime/hub/frontend/templates/login.html b/runtime/hub/frontend/templates/login.html index 57a9a432..bb712164 100755 --- a/runtime/hub/frontend/templates/login.html +++ b/runtime/hub/frontend/templates/login.html @@ -23,20 +23,7 @@ {% extends "page.html" %} - -{% macro password_field(field_id="password_input") %} -<div> - <label for="{{ field_id }}" class="login-field-label block text-sm font-medium mb-1">Password</label> - <div class="relative"> - <input id="{{ field_id }}" type="password" autocomplete="current-password" name="password" - class="login-input block w-full pl-3 pr-10 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> - <button type="button" class="password-toggle absolute inset-y-0 right-0 flex items-center pr-3 text-gray-400 hover:text-gray-600" aria-label="Show password"> - <svg class="eye-open w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z"/></svg> - <svg class="eye-closed w-5 h-5 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13.875 18.825A10.05 10.05 0 0112 19c-4.478 0-8.268-2.943-9.543-7a9.97 9.97 0 011.563-3.029m5.858.908a3 3 0 114.243 4.243M9.878 9.878l4.242 4.242M9.88 9.88l-3.29-3.29m7.532 7.532l3.29 3.29M3 3l3.59 3.59m0 0A9.953 9.953 0 0112 5c4.478 0 8.268 2.943 9.543 7a10.025 10.025 0 01-4.132 5.411m0 0L21 21"/></svg> - </button> - </div> -</div> -{% endmacro %} +{% from "_login_macros.html" import github_login_button, native_login_form %} {% if announcement_login is string %} {% set announcement = announcement_login %} @@ -58,7 +45,7 @@ {% endblock stylesheet %} {% block scripts %} -{# Inherit same-origin scripts (jQuery, Bootstrap bundle, darkmode.js) +{# Inherit same-origin scripts (jQuery and Bootstrap bundle) from page.html. Previously this block replaced the parent entirely and pulled Tailwind + jQuery from public CDNs, which introduced a third-party supply-chain risk without SRI and relied on Tailwind's @@ -67,6 +54,33 @@ {{ super() }} {% endblock scripts %} +{% block darkmode_script %} +<script id="login-theme-init"> + (function() { + var primaryTheme = localStorage.getItem('auplc-theme'); + var legacyTheme = localStorage.getItem('jupyterhub-bs-theme'); + var storedTheme = primaryTheme || legacyTheme; + var systemTheme = window.matchMedia('(prefers-color-scheme: dark)'); + + function applyTheme(theme) { + document.documentElement.setAttribute('data-bs-theme', theme); + } + + if (storedTheme) { + applyTheme(storedTheme); + localStorage.setItem('auplc-theme', storedTheme); + localStorage.setItem('jupyterhub-bs-theme', storedTheme); + return; + } + + applyTheme(systemTheme.matches ? 'dark' : 'light'); + systemTheme.addEventListener('change', function(event) { + applyTheme(event.matches ? 'dark' : 'light'); + }); + })(); +</script> +{% endblock darkmode_script %} + {% block require_config %} <!-- Login page doesn't need RequireJS config --> {% endblock require_config %} @@ -76,8 +90,8 @@ {% block main %} {% block login %} -<div class="min-h-screen flex flex-col md:flex-row"> - <div class="w-full md:w-2/5 bg-black flex flex-col justify-center items-center p-10 md:p-16"> +<div class="min-h-screen flex flex-col lg:flex-row"> + <div class="w-full lg:w-2/5 bg-black flex flex-col justify-center items-center p-10 lg:p-16"> <div class="text-center"> <svg class="w-24 mb-6 mx-auto" id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 139.72 33.32"> @@ -117,7 +131,7 @@ <h2 class="text-3xl md:text-4xl font-bold text-white mb-4">{{ platform_name or ' {% endif %} </div> </div> - <div class="login-main-panel w-full md:w-3/5 flex items-center justify-center p-6 md:p-16"> + <div class="login-main-panel w-full lg:w-3/5 flex items-center justify-center p-6 lg:p-16"> <div class="w-full max-w-md"> {% block login_container %} <div id="announcement-box" class="login-announcement p-4 mb-6 rounded-lg hidden"></div> @@ -155,25 +169,7 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L {% if auth_dummy or (auth_native and not auth_github) %} <!-- Dummy Authenticator: Simple login form --> - <form action="{{ base_url }}login?next={{ next | urlencode }}" method="post" role="form" class="space-y-6"> - <input type="hidden" name="_xsrf" value="{{ xsrf }}" /> - - <div> - <label for="username_input" class="login-field-label block text-sm font-medium mb-1">Username</label> - <input id="username_input" type="text" autocapitalize="off" autocorrect="off" autocomplete="username" - name="username" value="{{ username }}" autofocus="autofocus" - class="login-input block w-full pl-3 pr-3 py-2 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" /> - </div> - - {{ password_field() }} - - <div class="mt-6"> - <button id="login_submit" type="submit" - class="login-submit w-full flex justify-center py-3 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> - Login - </button> - </div> - </form> + {{ native_login_form(base_url ~ "login?next=" ~ (next | urlencode), xsrf, username, true) }} {% elif auth_github and not auth_native %} <!-- GitHub App Only --> @@ -181,21 +177,19 @@ <h1 class="login-heading text-2xl font-bold">Login to {{ platform_name or 'AUP L the "next" variable. App stores next in a cookie as-is, so double-encoding causes redirects to fail (e.g., /hub/%2Fhub%2F instead of /hub/). Form POST actions DO need urlencode due to different browser/server handling. --> - <div class="mb-6"> - <a href="{{ base_url }}oauth_login?next={{ next }}" - class="login-github-button w-full flex justify-center items-center py-3 px-4 rounded-md shadow-sm text-sm font-medium focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 transition duration-300"> - <svg class="w-5 h-5 mr-2" fill="currentColor" viewBox="0 0 20 20"> - <path fill-rule="evenodd" d="M10 0C4.477 0 0 4.484 0 10.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0110 4.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.203 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.942.359.31.678.921.678 1.856 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0020 10.017C20 4.484 15.522 0 10 0z" clip-rule="evenodd"/> - </svg> - Sign in with GitHub - </a> - {% if github_helper_text %} - <p class="login-helper text-sm text-center mt-3 mb-0">{{ github_helper_text }}</p> - {% endif %} - </div> + {{ github_login_button(base_url ~ "oauth_login?next=" ~ next, github_helper_text) }} {% elif auth_native and auth_github %} - {{ custom_html | safe }} + {{ github_login_button((base_url ~ "github/oauth_login?next=" ~ next) if next else (base_url ~ "github/oauth_login"), github_helper_text) }} + <div class="login-divider relative my-6"> + <div class="absolute inset-0 flex items-center" aria-hidden="true"> + <div class="w-full border-t"></div> + </div> + <div class="relative flex justify-center text-sm"> + <span class="px-2">Or use local account</span> + </div> + </div> + {{ native_login_form((base_url ~ "native/login?next=" ~ (next | urlencode)) if next else (base_url ~ "native/login"), xsrf, username, true) }} {% endif %} </div> diff --git a/runtime/hub/frontend/templates/page.html b/runtime/hub/frontend/templates/page.html index 49f9cbd0..da94d99c 100755 --- a/runtime/hub/frontend/templates/page.html +++ b/runtime/hub/frontend/templates/page.html @@ -87,11 +87,15 @@ <h2 class="modal-title" id="{{ key }}-label">{{ title }}</h2> #auplc-powered-by-footer { margin-top: auto; text-align: center; - padding: 6px 0; + padding: 6px var(--bs-gutter-x, 0.75rem); font-size: 0.72rem; - opacity: 0.55; + color: var(--bs-secondary-color); + overflow-wrap: anywhere; border-top: 1px solid rgba(128, 128, 128, 0.15); } + #auplc-powered-by-footer a { + color: var(--bs-link-color); + } #notification-banner-mount { width: 100%; margin: 0 0 1rem; @@ -211,9 +215,11 @@ <h2 class="modal-title" id="{{ key }}-label">{{ title }}</h2> <script src="{{static_url("components/jquery/dist/jquery.min.js") }}" type="text/javascript" charset="utf-8"></script> - <script src="{{static_url("js/darkmode.js") }}" - type="text/javascript" - charset="utf-8"></script> + {% block darkmode_script %} + <script src="{{static_url("js/darkmode.js") }}" + type="text/javascript" + charset="utf-8"></script> + {% endblock darkmode_script %} <script type="text/javascript"> // Keep auplc-theme (React apps) and jupyterhub-bs-theme (darkmode.js) // in sync by observing data-bs-theme attribute changes on <html>. diff --git a/runtime/hub/tests/auth_template_support.py b/runtime/hub/tests/auth_template_support.py index 37debcc6..01831c8c 100644 --- a/runtime/hub/tests/auth_template_support.py +++ b/runtime/hub/tests/auth_template_support.py @@ -7,13 +7,25 @@ from pathlib import Path import pytest -from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template -from tornado.escape import url_escape +from jinja2 import Environment, FileSystemLoader, StrictUndefined ROOT = Path(__file__).resolve().parents[1] TEMPLATES = ROOT / "frontend" / "templates" FIRSTUSE = ROOT / "core" / "authenticators" / "firstuse.py" MULTI = ROOT / "core" / "authenticators" / "multi.py" +LOGIN_NEXT_CASES = ( + ( + "/hub/spawn?x=1&y=two words", + "%2Fhub%2Fspawn%3Fx%3D1%26y%3Dtwo+words", + "%252Fhub%252Fspawn%253Fx%253D1%2526y%253Dtwo%2Bwords", + ), + ( + "/路径?值=你好 世界", + "%2F%E8%B7%AF%E5%BE%84%3F%E5%80%BC%3D%E4%BD%A0%E5%A5%BD+%E4%B8%96%E7%95%8C", + "%252F%25E8%25B7%25AF%25E5%25BE%2584%253F%25E5%2580%25BC%253D%25E4%25BD%25A0%25E5%25A5%25BD%2B%25E4%25B8%2596%25E7%2595%258C", + ), + ("", "", ""), +) class HtmlProbe(HTMLParser): @@ -22,28 +34,50 @@ def __init__(self) -> None: self.ids: set[str] = set() self.hrefs: list[str] = [] self.anchors: list[dict[str, str | None]] = [] + self.divs: list[dict[str, str | None]] = [] self.forms: list[dict[str, str | None]] = [] self.inputs: list[dict[str, str | None]] = [] + self.labels: list[dict[str, str | None]] = [] self.buttons: list[dict[str, str | None]] = [] + self.scripts: list[dict[str, str | None]] = [] + self.events: list[tuple[str, str, dict[str, str | None] | None]] = [] + self.github_button_icon_count = 0 + self._inside_github_button = False self.text: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attributes = dict(attrs) + self.events.append(("start", tag, attributes)) if element_id := attributes.get("id"): self.ids.add(element_id) if tag == "a" and (href := attributes.get("href")): self.hrefs.append(href) self.anchors.append(attributes) + self._inside_github_button = "login-github-button" in (attributes.get("class") or "").split() + if tag == "div": + self.divs.append(attributes) if tag == "form": self.forms.append(attributes) if tag == "input": self.inputs.append(attributes) + if tag == "label": + self.labels.append(attributes) if tag == "button": self.buttons.append(attributes) + if tag == "script": + self.scripts.append(attributes) + if tag == "svg" and self._inside_github_button: + self.github_button_icon_count += 1 + + def handle_endtag(self, tag: str) -> None: + self.events.append(("end", tag, None)) + if tag == "a": + self._inside_github_button = False def handle_data(self, data: str) -> None: if text := " ".join(data.split()): self.text.append(text) + self.events.append(("text", text, None)) def template_environment() -> Environment: @@ -63,7 +97,7 @@ def base_context() -> dict[str, object]: "authenticator_login_url": "/hub/oauth_login?next=/hub/home", "base_url": "/hub/", "custom_html": "", - "github_helper_text": "", + "login_github_helper_text": "", "login_error": "", "login_service": "", "login_url": "/hub/login", @@ -162,10 +196,6 @@ def login_url(self, base_url: str) -> str: ) -def render_multi_html(state: types.SimpleNamespace, next_value: str) -> str: - return Template(state.multi.get_custom_html("/hub/")).render(xsrf="csrf-token", next=url_escape(next_value)) - - @contextmanager def loaded_auth_modules(monkeypatch: pytest.MonkeyPatch) -> Iterator[types.SimpleNamespace]: with monkeypatch.context() as module_patch: diff --git a/runtime/hub/tests/test_auth_templates.py b/runtime/hub/tests/test_auth_templates.py index 9104cbe1..f77ce19c 100644 --- a/runtime/hub/tests/test_auth_templates.py +++ b/runtime/hub/tests/test_auth_templates.py @@ -3,14 +3,15 @@ import pytest from auth_template_support import ( + LOGIN_NEXT_CASES, TEMPLATES, + HtmlProbe, base_context, loaded_auth_modules, - loaded_multi_authenticator, probe_html, - render_multi_html, template_environment, ) +from tornado.escape import url_escape VALID_VARIANTS = { "auto-login": (True, False, False, False), @@ -78,12 +79,7 @@ def test_login_renders_enabled_authentication_controls( ) -> None: context = base_context() | projected_context(monkeypatch, providers) if variant == "github": - context |= {"login_service": "GitHub", "github_helper_text": "Use your approved GitHub account."} - if variant == "native-github": - with loaded_multi_authenticator(monkeypatch) as state: - state.multi._authenticators = [state.external, state.native] - context["custom_html"] = render_multi_html(state, str(context["next"])) - + context |= {"login_service": "GitHub", "login_github_helper_text": "Use your approved GitHub account."} probe = probe_html(template_environment().get_template("login.html").render(**context)) form_actions = {form.get("action") for form in probe.forms} input_names = {field.get("name") for field in probe.inputs} @@ -101,13 +97,134 @@ def test_login_renders_enabled_authentication_controls( assert "Development Mode" not in visible_text assert ("/hub/login?next=/hub/home" in form_actions) is (variant in {"dummy", "native"}) assert probe.hrefs.count("/hub/oauth_login?next=/hub/home") == (2 if variant == "github" else 0) - assert ("/hub/github/oauth_login?next=%2Fhub%2Fhome" in probe.hrefs) is (variant == "native-github") - assert ("/hub/native/login?next=%252Fhub%252Fhome" in form_actions) is (variant == "native-github") + assert ("/hub/github/oauth_login?next=/hub/home" in probe.hrefs) is (variant == "native-github") + assert ("/hub/native/login?next=/hub/home" in form_actions) is (variant == "native-github") assert "auplc-powered-by-footer" in probe.ids if variant in {"dummy", "native", "native-github"}: assert any(field.get("name") == "_xsrf" and field.get("value") == "csrf-token" for field in probe.inputs) +def _field_by_name(probe: HtmlProbe, name: str) -> dict[str, str | None]: + return next(field for field in probe.inputs if field.get("name") == name) + + +def _classes(attributes: dict[str, str | None]) -> set[str]: + return set((attributes.get("class") or "").split()) + + +def test_native_login_controls_share_the_rendered_dom_contract(monkeypatch: pytest.MonkeyPatch) -> None: + native_context = base_context() | projected_context(monkeypatch, VALID_VARIANTS["native"]) + composed_context = base_context() | projected_context(monkeypatch, VALID_VARIANTS["native-github"]) + + native = probe_html(template_environment().get_template("login.html").render(**native_context)) + composed = probe_html(template_environment().get_template("login.html").render(**composed_context)) + + for field_name in ("username", "password"): + native_field = _field_by_name(native, field_name) + composed_field = _field_by_name(composed, field_name) + assert _classes(native_field) == _classes(composed_field) + assert "login-input" in _classes(native_field) + assert "required" in native_field + assert "required" in composed_field + assert native_field.get("autocomplete") == composed_field.get("autocomplete") + assert {label.get("for") for label in native.labels} == {"username_input", "password_input"} + assert {label.get("for") for label in composed.labels} == {"username_input", "password_input"} + assert _field_by_name(native, "username").get("value") == _field_by_name(composed, "username").get("value") + assert "autofocus" in _field_by_name(native, "username") + assert "autofocus" in _field_by_name(composed, "username") + + +def test_composed_login_renders_one_ordered_card_without_nested_options(monkeypatch: pytest.MonkeyPatch) -> None: + context = base_context() | projected_context(monkeypatch, VALID_VARIANTS["native-github"]) + context["login_github_helper_text"] = "Use your approved GitHub account." + + probe = probe_html(template_environment().get_template("login.html").render(**context)) + + assert sum("login-card" in _classes(div) for div in probe.divs) == 1 + assert all("login-option" not in _classes(div) for div in probe.divs) + assert sum("login-divider" in _classes(div) for div in probe.divs) == 1 + assert probe.github_button_icon_count == 1 + visible_text = " ".join(probe.text) + assert "Or use local account" in visible_text + assert "Use your approved GitHub account." in visible_text + assert "Username" in visible_text + assert "Password" in visible_text + assert "Login" in visible_text + + github_offset = next( + index + for index, (event, tag, attributes) in enumerate(probe.events) + if event == "start" and tag == "a" and attributes is not None and "login-github-button" in _classes(attributes) + ) + divider_offset = next( + index + for index, (event, tag, attributes) in enumerate(probe.events) + if event == "start" and tag == "div" and attributes is not None and "login-divider" in _classes(attributes) + ) + form_offset = next( + index for index, (event, tag, _attributes) in enumerate(probe.events) if event == "start" and tag == "form" + ) + assert github_offset < divider_offset < form_offset + + +@pytest.mark.parametrize( + ("raw_next", "template_next", "form_next"), + LOGIN_NEXT_CASES, +) +def test_direct_login_routes_preserve_their_existing_template_behavior( + raw_next: str, template_next: str, form_next: str +) -> None: + environment = template_environment() + assert url_escape(raw_next) == template_next + + native = probe_html( + environment.get_template("login.html").render(**(base_context() | {"auth_native": True, "next": template_next})) + ) + github = probe_html( + environment.get_template("login.html").render(**(base_context() | {"auth_github": True, "next": template_next})) + ) + + github_button = next(anchor for anchor in github.anchors if "login-github-button" in _classes(anchor)) + assert [form.get("action") for form in native.forms] == [f"/hub/login?next={form_next}"] + assert github_button.get("href") == f"/hub/oauth_login?next={template_next}" + + +@pytest.mark.parametrize( + ("template_next", "form_next"), + [(template_next, form_next) for _raw_next, template_next, form_next in LOGIN_NEXT_CASES], +) +def test_composed_login_routes_preserve_multi_authenticator_next_behavior(template_next: str, form_next: str) -> None: + composed = probe_html( + template_environment() + .get_template("login.html") + .render(**(base_context() | {"auth_native": True, "auth_github": True, "next": template_next})) + ) + + github_button = next(anchor for anchor in composed.anchors if "login-github-button" in _classes(anchor)) + expected_suffix = f"?next={template_next}" if template_next else "" + expected_form_suffix = f"?next={form_next}" if template_next else "" + assert github_button.get("href") == f"/hub/github/oauth_login{expected_suffix}" + assert [form.get("action") for form in composed.forms] == [f"/hub/native/login{expected_form_suffix}"] + + +def test_login_omits_darkmode_script_while_normal_pages_keep_it() -> None: + environment = template_environment() + + login = probe_html(environment.get_template("login.html").render(**base_context())) + page = probe_html(environment.get_template("page.html").render(**base_context())) + + assert "/hub/static/js/darkmode.js" not in {script.get("src") for script in login.scripts} + assert "/hub/static/js/darkmode.js" in {script.get("src") for script in page.scripts} + + +def test_login_uses_theme_initializer_without_a_toggle_dependency() -> None: + html = template_environment().get_template("login.html").render(**base_context()) + probe = probe_html(html) + + assert any(script.get("id") == "login-theme-init" for script in probe.scripts) + assert "dark-theme-toggle" not in html + + @pytest.mark.parametrize(("variant", "providers"), VALID_VARIANTS.items()) def test_page_controls_follow_capabilities( monkeypatch: pytest.MonkeyPatch, @@ -189,3 +306,10 @@ def test_attribution_footer_is_after_all_template_blocks_and_renders() -> None: "auplc-powered-by-footer" in probe_html(template_environment().get_template("page.html").render(**base_context())).ids ) + + +def test_composed_login_template_does_not_delegate_markup_to_authenticator_python() -> None: + source = (TEMPLATES / "login.html").read_text(encoding="utf-8") + + assert "custom_html" not in source + assert "_authenticators" not in source From dc2233467f246d7afcf8b2ae19f26f1fbf9e87d6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:38:09 +0800 Subject: [PATCH 179/180] test(ui): lock login visual contracts --- .../hub/tests/test_login_visual_contract.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 runtime/hub/tests/test_login_visual_contract.py diff --git a/runtime/hub/tests/test_login_visual_contract.py b/runtime/hub/tests/test_login_visual_contract.py new file mode 100644 index 00000000..782507f6 --- /dev/null +++ b/runtime/hub/tests/test_login_visual_contract.py @@ -0,0 +1,27 @@ +from pathlib import Path +from typing import Final + +TEMPLATES: Final = Path(__file__).resolve().parents[1] / "frontend" / "templates" + + +def test_login_split_layout_starts_at_large_breakpoint() -> None: + source = (TEMPLATES / "login.html").read_text(encoding="utf-8") + + assert 'class="min-h-screen flex flex-col lg:flex-row"' in source + assert 'class="w-full lg:w-2/5 bg-black flex flex-col justify-center items-center p-10 lg:p-16"' in source + assert 'class="login-main-panel w-full lg:w-3/5 flex items-center justify-center p-6 lg:p-16"' in source + assert all(token not in source for token in ("md:flex-row", "md:w-2/5", "md:w-3/5", "md:p-16")) + + +def test_attribution_footer_uses_accessible_padded_wrapping_styles() -> None: + source = (TEMPLATES / "page.html").read_text(encoding="utf-8") + footer_rule = source.split("#auplc-powered-by-footer {", maxsplit=1)[1].split("}", maxsplit=1)[0] + link_selector = "#auplc-powered-by-footer a {" + + assert "opacity:" not in footer_rule + assert "padding: 6px var(--bs-gutter-x, 0.75rem);" in footer_rule + assert "color: var(--bs-secondary-color);" in footer_rule + assert "overflow-wrap: anywhere;" in footer_rule + assert link_selector in source + link_rule = source.split(link_selector, maxsplit=1)[1].split("}", maxsplit=1)[0] + assert "color: var(--bs-link-color);" in link_rule From 0d38a2fc5fcfd4993f423429644d29c90d5266be Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:52:29 +0800 Subject: [PATCH 180/180] fix(code): update code-server for trusted domains --- dockerfiles/Code/Dockerfile | 5 +++-- dockerfiles/Code/README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dockerfiles/Code/Dockerfile b/dockerfiles/Code/Dockerfile index 4c9f1423..e914f465 100644 --- a/dockerfiles/Code/Dockerfile +++ b/dockerfiles/Code/Dockerfile @@ -20,7 +20,7 @@ ARG NODE_IMAGE=docker.io/library/node:22-bookworm-slim ARG BASE_IMAGE=ghcr.io/amdresearch/auplc-default:latest ARG IMAGE_FLAVOR=cpu -ARG CODE_SERVER_VERSION=4.96.4 +ARG CODE_SERVER_VERSION=4.131.0 FROM ${NODE_IMAGE} AS node-runtime @@ -43,7 +43,7 @@ RUN pnpm --filter @auplc/runtime-status run build && \ FROM ${BASE_IMAGE} -ARG CODE_SERVER_VERSION=4.96.4 +ARG CODE_SERVER_VERSION=4.131.0 ARG IMAGE_FLAVOR ARG NPM_REGISTRY= ARG PNPM_VERSION=10.27.0 @@ -80,6 +80,7 @@ COPY --from=hub-link-builder /build/runtime/code-server/extensions/auplc-hub-lin COPY dockerfiles/Code/start-code-server.sh /usr/local/bin/start-code-server.sh RUN chmod +x /usr/local/bin/start-code-server.sh && \ + code-server --help | grep -F -- '--link-protection-trusted-domains' && \ mkdir -p \ /opt/auplc/extensions/local \ /opt/auplc/extensions/staging \ diff --git a/dockerfiles/Code/README.md b/dockerfiles/Code/README.md index c2be7397..398d6d89 100644 --- a/dockerfiles/Code/README.md +++ b/dockerfiles/Code/README.md @@ -52,7 +52,7 @@ make -C dockerfiles code `code-cpu` builds `ghcr.io/amdresearch/auplc-code-cpu:latest`. `code-gpu` builds `ghcr.io/amdresearch/auplc-code-gpu:latest` and tags the selected GPU target, for example `ghcr.io/amdresearch/auplc-code-gpu:latest-gfx1151`. The aggregate `code` target builds both. -The Dockerfile pins code-server to version `4.96.4` so builds use a known editor runtime instead of silently changing when a new upstream release appears. +The Dockerfile pins code-server to version `4.131.0` so builds use a known editor runtime instead of silently changing when a new upstream release appears. The image build also verifies that the binary supports the trusted-domain CLI option required by the launcher. Additional build arguments customize the shared development toolchain:
0} + checked={allCurrentPageSelected} onChange={toggleSelectAll} + title="Select users on this page" /> handleSort('name')}> @@ -946,8 +1058,8 @@ export function UserList() { onQuotaInputChange={handleQuotaInputChange} onQuotaSave={handleQuotaSave} onQuotaCancel={handleQuotaCancel} - onStartServer={handleStartServerCallback} - onStopServer={handleStopServerCallback} + onStartServer={handleStartServer} + onStopServer={handleStopServer} onEditUser={openEditModal} onPasswordReset={openPasswordModal} onDeleteUser={openDeleteModal} @@ -1099,6 +1211,88 @@ export function UserList() { onHide={() => setShowBatchPasswordModal(false)} /> + {/* Batch Group Membership Modal */} + { + if (actionLoading !== 'batch-group') setShowBatchGroupModal(false); + }} + > + + + {batchGroupMode === 'add' ? 'Add Users to Group' : 'Remove Users from Group'} + + + + + This will {batchGroupMode} {selectedUsernames.length} selected user(s){' '} + {batchGroupMode === 'add' ? 'to' : 'from'} the selected group. + {batchGroupMode === 'remove' && ' Users who are not members are skipped.'} + + + {batchGroupMode === 'remove' && selectedBatchGroup?.source === 'github-team' && ( + + + This is a GitHub-synced group. Members synced from GitHub may be added back after login or group synchronization. + Use this mainly to remove manually added members. + + )} + +
+ Users:{' '} + {selectedUsernames.slice(0, 10).map(name => ( + {name} + ))} + {selectedUsernames.length > 10 && ( + +{selectedUsernames.length - 10} more + )} +
+ + + Group + setBatchGroupName(e.target.value)} + disabled={actionLoading === 'batch-group'} + > + + {mutableGroups.map(group => ( + + ))} + + + System-managed groups are read-only and are not listed here. + + + + + + + + + {/* Batch Delete Confirmation Modal */} Date: Thu, 16 Jul 2026 12:37:38 +0800 Subject: [PATCH 020/180] feat(admin): add group detail management page --- runtime/hub/frontend/apps/admin/src/App.tsx | 2 + .../apps/admin/src/pages/GroupDetail.tsx | 492 ++++++++++++++++++ .../apps/admin/src/pages/GroupList.tsx | 250 ++------- 3 files changed, 541 insertions(+), 203 deletions(-) create mode 100644 runtime/hub/frontend/apps/admin/src/pages/GroupDetail.tsx diff --git a/runtime/hub/frontend/apps/admin/src/App.tsx b/runtime/hub/frontend/apps/admin/src/App.tsx index 0ea4215c..3a092828 100644 --- a/runtime/hub/frontend/apps/admin/src/App.tsx +++ b/runtime/hub/frontend/apps/admin/src/App.tsx @@ -20,6 +20,7 @@ import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom'; import { UserList } from './pages/UserList'; import { GroupList } from './pages/GroupList'; +import { GroupDetail } from './pages/GroupDetail'; import { Dashboard } from './pages/Dashboard'; import { NavBar } from './components/NavBar'; import { useState, useEffect } from 'react'; @@ -43,6 +44,7 @@ function App() { } /> } /> + } /> } /> } /> diff --git a/runtime/hub/frontend/apps/admin/src/pages/GroupDetail.tsx b/runtime/hub/frontend/apps/admin/src/pages/GroupDetail.tsx new file mode 100644 index 00000000..160e8b4e --- /dev/null +++ b/runtime/hub/frontend/apps/admin/src/pages/GroupDetail.tsx @@ -0,0 +1,492 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useNavigate, useParams } from 'react-router-dom'; +import { Alert, Badge, Button, ButtonGroup, Form, InputGroup, Spinner, Table } from 'react-bootstrap'; +import AsyncSelect from 'react-select/async'; +import type { MultiValue, StylesConfig } from 'react-select'; +import type { Group } from '@auplc/shared'; +import * as api from '@auplc/shared'; +import { EditGroupModal } from '../components/EditGroupModal'; + +interface UserOption { + value: string; + label: string; +} + +const getSelectStyles = (isDark: boolean): StylesConfig => ({ + menuPortal: (base) => ({ ...base, zIndex: 9999 }), + control: (base, state) => ({ + ...base, + minHeight: '38px', + backgroundColor: isDark ? '#212529' : base.backgroundColor, + borderColor: isDark ? '#495057' : base.borderColor, + '&:hover': { + borderColor: isDark ? '#6c757d' : base.borderColor, + }, + ...(state.isFocused && { + borderColor: isDark ? '#0d6efd' : '#86b7fe', + boxShadow: '0 0 0 0.25rem rgba(13, 110, 253, 0.25)', + }), + }), + menu: (base) => ({ + ...base, + backgroundColor: isDark ? '#212529' : base.backgroundColor, + border: isDark ? '1px solid #495057' : base.border, + }), + option: (base, state) => ({ + ...base, + backgroundColor: state.isFocused + ? (isDark ? '#495057' : '#deebff') + : (isDark ? '#212529' : base.backgroundColor), + color: isDark ? '#fff' : base.color, + '&:active': { + backgroundColor: isDark ? '#6c757d' : '#b2d4ff', + }, + }), + input: (base) => ({ + ...base, + color: isDark ? '#fff' : base.color, + }), + placeholder: (base) => ({ + ...base, + color: isDark ? '#adb5bd' : base.color, + }), + multiValue: (base) => ({ + ...base, + backgroundColor: '#6c757d', + }), + multiValueLabel: (base) => ({ + ...base, + color: 'white', + }), + multiValueRemove: (base) => ({ + ...base, + color: 'white', + ':hover': { + backgroundColor: '#5a6268', + color: 'white', + }, + }), + noOptionsMessage: (base) => ({ + ...base, + color: isDark ? '#adb5bd' : base.color, + }), + loadingMessage: (base) => ({ + ...base, + color: isDark ? '#adb5bd' : base.color, + }), +}); + +function safeDecode(value: string): string { + try { + return decodeURIComponent(value); + } catch { + return value; + } +} + +function SourceBadge({ group }: { group: Group }) { + if (group.source === 'github-team') { + return GitHub; + } + if (group.source === 'system') { + return System; + } + return Manual; +} + +function ResourceBadges({ resources }: { resources: string[] }) { + if (resources.length === 0) return No mapped resources; + return ( +
+ {resources.map(resource => {resource})} +
+ ); +} + +export function GroupDetail() { + const params = useParams(); + const navigate = useNavigate(); + const groupName = safeDecode(params.groupName ?? ''); + + const [group, setGroup] = useState(null); + const [loading, setLoading] = useState(true); + const [actionLoading, setActionLoading] = useState(null); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [memberSearch, setMemberSearch] = useState(''); + const [selectedMembers, setSelectedMembers] = useState>(new Set()); + const [usersToAdd, setUsersToAdd] = useState([]); + const [showEditModal, setShowEditModal] = useState(false); + const [isDark, setIsDark] = useState(() => + document.documentElement.getAttribute('data-bs-theme') === 'dark' + ); + + const isReadOnly = group?.source === 'system'; + const isGitHubTeam = group?.source === 'github-team'; + + useEffect(() => { + const observer = new MutationObserver(() => { + setIsDark(document.documentElement.getAttribute('data-bs-theme') === 'dark'); + }); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ['data-bs-theme'], + }); + return () => observer.disconnect(); + }, []); + + const loadGroup = useCallback(async (silent = false) => { + try { + if (!silent) setLoading(true); + setError(null); + const response = await api.getGroups(); + const nextGroup = response.groups.find(candidate => candidate.name === groupName) ?? null; + setGroup(nextGroup); + setSelectedMembers(prev => { + if (!nextGroup) return new Set(); + const currentMembers = new Set(nextGroup.users); + return new Set(Array.from(prev).filter(member => currentMembers.has(member))); + }); + if (!nextGroup) { + setError(`Group "${groupName}" was not found.`); + } + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load group'); + } finally { + if (!silent) setLoading(false); + } + }, [groupName]); + + useEffect(() => { + loadGroup(); + }, [loadGroup]); + + useEffect(() => { + setMemberSearch(''); + setSelectedMembers(new Set()); + setUsersToAdd([]); + setNotice(null); + setError(null); + }, [groupName]); + + const filteredMembers = useMemo(() => { + if (!group) return []; + const searchLower = memberSearch.trim().toLowerCase(); + if (!searchLower) return [...group.users].sort(); + return group.users + .filter(member => member.toLowerCase().includes(searchLower)) + .sort(); + }, [group, memberSearch]); + + const selectedVisibleCount = useMemo( + () => filteredMembers.filter(member => selectedMembers.has(member)).length, + [filteredMembers, selectedMembers] + ); + + const allVisibleSelected = filteredMembers.length > 0 && selectedVisibleCount === filteredMembers.length; + + const loadUserOptions = useCallback(async (inputValue: string): Promise => { + if (!inputValue || inputValue.length < 1 || !group) return []; + + try { + const response = await api.getUsers({ offset: 0, limit: 20, nameFilter: inputValue }); + const existingMembers = new Set(group.users); + const pendingAdds = new Set(usersToAdd.map(user => user.value)); + return (response.items || []) + .filter(user => !existingMembers.has(user.name) && !pendingAdds.has(user.name)) + .map(user => ({ + value: user.name, + label: user.admin ? `${user.name} (Admin)` : user.name, + })); + } catch (err) { + console.error('Failed to load users:', err); + return []; + } + }, [group, usersToAdd]); + + const toggleMember = (member: string) => { + setSelectedMembers(prev => { + const next = new Set(prev); + if (next.has(member)) { + next.delete(member); + } else { + next.add(member); + } + return next; + }); + }; + + const toggleVisibleMembers = () => { + setSelectedMembers(prev => { + const next = new Set(prev); + if (allVisibleSelected) { + filteredMembers.forEach(member => next.delete(member)); + } else { + filteredMembers.forEach(member => next.add(member)); + } + return next; + }); + }; + + const handleAddMembers = async () => { + if (!group || usersToAdd.length === 0 || isReadOnly) return; + + try { + setActionLoading('add-members'); + setError(null); + setNotice(null); + const usernames = usersToAdd.map(user => user.value); + const updatedGroup = await api.addUsersToGroup(group.name, usernames); + setGroup(updatedGroup); + setUsersToAdd([]); + setNotice(`Added ${usernames.length} user(s) to ${group.name}.`); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to add members'); + } finally { + setActionLoading(null); + } + }; + + const handleRemoveSelected = async () => { + if (!group || selectedMembers.size === 0 || isReadOnly) return; + + const usernames = Array.from(selectedMembers); + if (!window.confirm(`Remove ${usernames.length} member(s) from "${group.name}"?`)) { + return; + } + + try { + setActionLoading('remove-members'); + setError(null); + setNotice(null); + const updatedGroup = await api.removeUsersFromGroup(group.name, usernames); + setGroup(updatedGroup); + setSelectedMembers(new Set()); + setNotice(`Removed ${usernames.length} member(s) from ${group.name}.`); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to remove members'); + } finally { + setActionLoading(null); + } + }; + + const handleEditUpdate = async () => { + await loadGroup(true); + }; + + const handleDelete = () => { + navigate('/groups'); + }; + + if (loading) { + return ( +
+ + Loading... + +
+ ); + } + + if (!group) { + return ( +
+ + {error && {error}} +
+ ); + } + + return ( +
+
+
+ +
+

{group.name}

+ +
+
+ {group.users.length} {group.users.length === 1 ? 'member' : 'members'} + {(group.resources?.length ?? 0) > 0 && ` · ${group.resources!.length} resources`} +
+
+ + + + +
+ + {error && setError(null)}>{error}} + {notice && setNotice(null)}>{notice}} + + {isReadOnly && ( + + System-managed group membership is read-only. You can view members and edit group properties, but cannot add or remove members. + + )} + + {isGitHubTeam && ( + + + This group is synced from GitHub Teams. Manual additions are allowed, but GitHub-synced members may be added back after login or synchronization. + + )} + +
+
Mapped Resources
+ +
+ + {!isReadOnly && ( +
+
Add Members
+
+
+ + isMulti + cacheOptions + defaultOptions={false} + value={usersToAdd} + loadOptions={loadUserOptions} + onChange={(newValue: MultiValue) => setUsersToAdd([...newValue])} + isDisabled={actionLoading === 'add-members'} + isLoading={actionLoading === 'add-members'} + placeholder="Search users to add..." + noOptionsMessage={({ inputValue }) => inputValue ? 'No users found' : 'Type to search users'} + loadingMessage={() => 'Searching...'} + menuPortalTarget={document.body} + styles={getSelectStyles(isDark)} + /> + + Existing members are hidden from the search results. + +
+ +
+
+ )} + +
+
Members
+
+ {!isReadOnly && ( + + )} + {selectedMembers.size > 0 && ( + + )} +
+
+ + + + setMemberSearch(event.target.value)} + /> + {memberSearch && ( + + )} + + + + + + + + + + + {filteredMembers.map(member => ( + + + + + ))} + +
+ + Username
+ toggleMember(member)} + /> + {member}
+ + {filteredMembers.length === 0 && ( +
+ {memberSearch ? 'No members match your search.' : 'This group has no members.'} +
+ )} + + setShowEditModal(false)} + onUpdate={handleEditUpdate} + onDelete={handleDelete} + /> +
+ ); +} diff --git a/runtime/hub/frontend/apps/admin/src/pages/GroupList.tsx b/runtime/hub/frontend/apps/admin/src/pages/GroupList.tsx index a42e6c6c..476b3c31 100644 --- a/runtime/hub/frontend/apps/admin/src/pages/GroupList.tsx +++ b/runtime/hub/frontend/apps/admin/src/pages/GroupList.tsx @@ -19,89 +19,11 @@ import { useState, useEffect, useCallback, useMemo, memo } from 'react'; import { Table, Button, Form, InputGroup, Alert, Spinner, Modal, Badge } from 'react-bootstrap'; -import AsyncSelect from 'react-select/async'; -import type { MultiValue, ActionMeta, StylesConfig } from 'react-select'; +import { useNavigate } from 'react-router-dom'; import type { Group } from '@auplc/shared'; - -// Dark mode aware styles for react-select -const getSelectStyles = (isDark: boolean): StylesConfig => { - - return { - menuPortal: (base) => ({ ...base, zIndex: 9999 }), - control: (base, state) => ({ - ...base, - minHeight: '38px', - backgroundColor: isDark ? '#212529' : base.backgroundColor, - borderColor: isDark ? '#495057' : base.borderColor, - '&:hover': { - borderColor: isDark ? '#6c757d' : base.borderColor, - }, - ...(state.isFocused && { - borderColor: isDark ? '#0d6efd' : '#86b7fe', - boxShadow: isDark ? '0 0 0 0.25rem rgba(13, 110, 253, 0.25)' : '0 0 0 0.25rem rgba(13, 110, 253, 0.25)', - }), - }), - menu: (base) => ({ - ...base, - backgroundColor: isDark ? '#212529' : base.backgroundColor, - border: isDark ? '1px solid #495057' : base.border, - }), - option: (base, state) => ({ - ...base, - backgroundColor: state.isFocused - ? (isDark ? '#495057' : '#deebff') - : (isDark ? '#212529' : base.backgroundColor), - color: isDark ? '#fff' : base.color, - '&:active': { - backgroundColor: isDark ? '#6c757d' : '#b2d4ff', - }, - }), - input: (base) => ({ - ...base, - color: isDark ? '#fff' : base.color, - }), - placeholder: (base) => ({ - ...base, - color: isDark ? '#adb5bd' : base.color, - }), - singleValue: (base) => ({ - ...base, - color: isDark ? '#fff' : base.color, - }), - multiValue: (base) => ({ - ...base, - backgroundColor: '#6c757d', - }), - multiValueLabel: (base) => ({ - ...base, - color: 'white', - }), - multiValueRemove: (base) => ({ - ...base, - color: 'white', - ':hover': { - backgroundColor: '#5a6268', - color: 'white', - }, - }), - noOptionsMessage: (base) => ({ - ...base, - color: isDark ? '#adb5bd' : base.color, - }), - loadingMessage: (base) => ({ - ...base, - color: isDark ? '#adb5bd' : base.color, - }), - }; -}; import * as api from '@auplc/shared'; import { EditGroupModal } from '../components/EditGroupModal'; -interface UserOption { - value: string; - label: string; -} - const COLLAPSED_LIMIT = 3; function ResourceBadges({ resources }: { resources: string[] }) { @@ -138,73 +60,35 @@ function ResourceBadges({ resources }: { resources: string[] }) { ); } -// Memoized GroupRow component with inline member management +function MemberSummary({ members }: { members: string[] }) { + const preview = members.slice(0, COLLAPSED_LIMIT); + const hidden = members.length - preview.length; + + return ( +
+
+ {members.length} {members.length === 1 ? 'member' : 'members'} +
+ {preview.length > 0 && ( +
+ {preview.map(member => {member})} + {hidden > 0 && +{hidden} more} +
+ )} +
+ ); +} + +// Memoized GroupRow component with compact member summary interface GroupRowProps { group: Group; onEdit: (group: Group) => void; - onMembersChange: (groupName: string, members: string[]) => void; - loadUserOptions: (inputValue: string, excludeUsers: string[]) => Promise; } -const GroupRow = memo(function GroupRow({ group, onEdit, onMembersChange, loadUserOptions }: GroupRowProps) { - const [isUpdating, setIsUpdating] = useState(false); - const [isDark, setIsDark] = useState(() => - document.documentElement.getAttribute('data-bs-theme') === 'dark' - ); - +const GroupRow = memo(function GroupRow({ group, onEdit }: GroupRowProps) { const isGitHubTeam = group.source === 'github-team'; - const isReadOnly = group.source === 'system'; - - // Watch for theme changes - useEffect(() => { - const observer = new MutationObserver(() => { - setIsDark(document.documentElement.getAttribute('data-bs-theme') === 'dark'); - }); - observer.observe(document.documentElement, { - attributes: true, - attributeFilter: ['data-bs-theme'], - }); - return () => observer.disconnect(); - }, []); - - // Convert current members to options - const currentMembers: UserOption[] = group.users.map(name => ({ - value: name, - label: name, - })); - - // Load options excluding current members - const loadOptions = useCallback(async (inputValue: string): Promise => { - return loadUserOptions(inputValue, group.users); - }, [loadUserOptions, group.users]); - - // Handle member changes - const handleChange = useCallback(async ( - _newValue: MultiValue, - actionMeta: ActionMeta - ) => { - if (isUpdating) return; - - setIsUpdating(true); - try { - if (actionMeta.action === 'select-option' && actionMeta.option) { - await api.addUserToGroup(group.name, actionMeta.option.value); - onMembersChange(group.name, [...group.users, actionMeta.option.value]); - } else if (actionMeta.action === 'remove-value' && actionMeta.removedValue) { - await api.removeUserFromGroup(group.name, actionMeta.removedValue.value); - onMembersChange(group.name, group.users.filter(u => u !== actionMeta.removedValue!.value)); - } else if (actionMeta.action === 'clear') { - for (const user of group.users) { - await api.removeUserFromGroup(group.name, user); - } - onMembersChange(group.name, []); - } - } catch (err) { - console.error('Failed to update group members:', err); - } finally { - setIsUpdating(false); - } - }, [group.name, group.users, onMembersChange, isUpdating]); + const navigate = useNavigate(); + const openGroup = () => navigate(`/groups/${encodeURIComponent(group.name)}`); return (
- - isMulti - cacheOptions - defaultOptions={false} - value={currentMembers} - loadOptions={loadOptions} - onChange={handleChange} - isDisabled={isUpdating || isReadOnly} - isClearable={!isReadOnly} - isLoading={isUpdating} - placeholder={isReadOnly ? 'System-managed members' : (isGitHubTeam ? 'Add users (synced members are auto-managed)...' : 'Type to search and add users...')} - noOptionsMessage={({ inputValue }) => - inputValue ? 'No users found' : 'Type to search users' - } - loadingMessage={() => 'Searching...'} - menuPortalTarget={document.body} - styles={getSelectStyles(isDark)} - {...(isReadOnly && { - components: { MultiValueRemove: () => null }, - })} - /> + + + - +
+ + +