From ac70484654570b82e5b60ff7b2f2adc8e9ad16b6 Mon Sep 17 00:00:00 2001 From: vrutz Date: Fri, 3 Jul 2026 16:04:53 +0200 Subject: [PATCH 01/12] Add CI pipeline for running unit tests and linting --- .gitignore | 3 + Jenkinsfile | 84 +++++++++++++++++++ Makefile | 34 +++++++- .../node-pool-request/parameter-set.json | 2 +- python-lib/dku_aws/eksctl_loader.py | 2 +- python-lib/dku_kube/gpu_driver.py | 4 +- python-lib/dku_utils/static_resources.py | 4 +- .../install-alb-controller/runnable.py | 4 +- python-runnables/test-network/runnable.py | 4 +- 9 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 Jenkinsfile diff --git a/.gitignore b/.gitignore index 8c701e1..4b7cbed 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,10 @@ .wlock .ts .git +.venv/ +__pycache__/ *.pyc +dist/ .coverage .htmlcov .DS_Store diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..1a45109 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,84 @@ +pipeline { + options { + disableConcurrentBuilds() + timestamps() + } + + agent { label 'dss-plugin-tests' } + + stages { + stage('Lint') { + steps { + sh 'make lint' + } + } + + stage('Compile Python') { + steps { + sh 'make compile-python' + } + } + + stage('Validate JSON Descriptors') { + steps { + sh 'make validate-json' + } + } + + stage('Check Generated Files') { + steps { + sh 'make check-generated-files' + } + } + + stage('Run Unit Tests') { + steps { + sh 'make tests' + } + } + + stage('Build Plugin') { + steps { + sh 'make plugin' + archiveArtifacts artifacts: 'dist/*.zip', fingerprint: true + } + } + } + + post { + always { + script { + if (fileExists('tests/allure_report')) { + allure([ + includeProperties: false, + jdk: '', + properties: [], + reportBuildPolicy: 'ALWAYS', + results: [[path: 'tests/allure_report']] + ]) + } + + def status = currentBuild.currentResult + def statusFields = [ + env.BUILD_URL, + env.CHANGE_TITLE, + env.CHANGE_AUTHOR, + env.CHANGE_URL, + env.BRANCH_NAME, + status + ].collect { field -> (field ?: '').replaceAll(/[;\r\n]/, ' ') } + + writeFile file: '.build-status-line', text: statusFields.join(';') + ';\n' + + sh ''' + mkdir -p "$HOME/daily-statuses" + file_name=$(echo "$JOB_NAME" | tr '/' '-').status + touch "$HOME/daily-statuses/$file_name" + cat .build-status-line >> "$HOME/daily-statuses/$file_name" + ''' + + cleanWs() + } + } + } +} diff --git a/Makefile b/Makefile index f80dc52..2613255 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ plugin_version=`cat plugin.json | python -c "import sys, json; print(str(json.lo archive_file_name="dss-plugin-${plugin_id}-${plugin_version}.zip" remote_url=`git config --get remote.origin.url` last_commit_id=`git rev-parse HEAD` +PYTHON ?= python3 +VENV ?= .venv +RUFF := $(VENV)/bin/ruff plugin: @echo "[START] Archiving plugin to dist/ folder..." @@ -28,5 +31,34 @@ plugin: dist-clean: rm -rf dist +$(RUFF): + $(PYTHON) -m venv $(VENV) + $(VENV)/bin/python -m pip install --upgrade pip + $(VENV)/bin/python -m pip install ruff + +lint: $(RUFF) + $(RUFF) check . + $(RUFF) format --check . + +compile-python: + $(PYTHON) -m compileall -q python-lib python-runnables python-clusters tests + +validate-json: + @find . -maxdepth 3 -type f -name '*.json' -print0 | xargs -0 -n1 $(PYTHON) -m json.tool >/dev/null + +check-generated-files: + @tracked=$$(git ls-files | grep -E '(__pycache__/|\.pyc$$|^dist/)' || true); \ + if [ -n "$$tracked" ]; then \ + echo "Generated files are tracked and should be removed:"; \ + echo "$$tracked"; \ + exit 1; \ + fi + +validate: compile-python validate-json check-generated-files + test: - python3 -m unittest discover -s tests -v + $(PYTHON) -m unittest discover -s tests -v + +tests: test + +ci: lint compile-python validate-json check-generated-files tests plugin diff --git a/parameter-sets/node-pool-request/parameter-set.json b/parameter-sets/node-pool-request/parameter-set.json index a0459cf..7572232 100644 --- a/parameter-sets/node-pool-request/parameter-set.json +++ b/parameter-sets/node-pool-request/parameter-set.json @@ -87,7 +87,7 @@ "description": "Disk size for the nodes, in GB. Can use 0 for default (200 GB)", "type": "INT", "mandatory" : false, - "defaultValue": 200, /* Repeated in the code in node-pool.py */ + "defaultValue": 200, "minI": 0 }, { diff --git a/python-lib/dku_aws/eksctl_loader.py b/python-lib/dku_aws/eksctl_loader.py index 2159ada..387cc9b 100644 --- a/python-lib/dku_aws/eksctl_loader.py +++ b/python-lib/dku_aws/eksctl_loader.py @@ -22,7 +22,7 @@ def get_eksctl_or_fetch(): r = requests.get( "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_%s_amd64.tar.gz" % arch, stream=True, - headers={"User-Agent": "DSS EKS Plugin"} + headers={"User-Agent": "DSS EKS Plugin"}, ) local_eksctl_archive = os.path.join(local_eksctl_folder, "eksctl.tar.gz") with open(local_eksctl_archive, "wb") as f: diff --git a/python-lib/dku_kube/gpu_driver.py b/python-lib/dku_kube/gpu_driver.py index c4d95b7..a246aa5 100644 --- a/python-lib/dku_kube/gpu_driver.py +++ b/python-lib/dku_kube/gpu_driver.py @@ -66,9 +66,7 @@ def add_gpu_driver_if_needed(cluster_id, kube_config_path, connection_info, tain nvidia_config["spec"]["template"]["spec"]["tolerations"] = Toleration.to_list(tolerations) # Restrict the DaemonSet to nodes that request a GPU, so it doesn't try to run on every CPU node in the cluster - nvidia_config["spec"]["template"]["spec"]["nodeSelector"] = { - GPU_ENABLED_NODEPOOL_LABEL: GPU_ENABLED_NODEPOOL_LABEL_VALUE - } + nvidia_config["spec"]["template"]["spec"]["nodeSelector"] = {GPU_ENABLED_NODEPOOL_LABEL: GPU_ENABLED_NODEPOOL_LABEL_VALUE} # Write the configuration locally local_nvidia_plugin_config = os.path.join(os.environ["DIP_HOME"], "clusters", cluster_id, "nvidia-device-plugin.yml") diff --git a/python-lib/dku_utils/static_resources.py b/python-lib/dku_utils/static_resources.py index dbfd8a6..60eaa54 100644 --- a/python-lib/dku_utils/static_resources.py +++ b/python-lib/dku_utils/static_resources.py @@ -2,6 +2,7 @@ import os import requests + def get_url_or_fallback(url, static_resource_filename): r = requests.get(url, headers={"User-Agent": "DSS EKS Plugin"}) @@ -17,6 +18,7 @@ def get_url_or_fallback(url, static_resource_filename): else: logging.error("No static resource fallback was defined.") + def get_static_resource(static_resource_filename): static_resource_path = os.path.join(os.environ["DKU_CUSTOM_RESOURCE_FOLDER"], static_resource_filename) @@ -26,4 +28,4 @@ def get_static_resource(static_resource_filename): resource_content_raw = f.read() return resource_content_raw else: - logging.warning("Unable to locate the static resource at path: %s" % static_resource_path) \ No newline at end of file + logging.warning("Unable to locate the static resource at path: %s" % static_resource_path) diff --git a/python-runnables/install-alb-controller/runnable.py b/python-runnables/install-alb-controller/runnable.py index ca9e67e..9144e25 100644 --- a/python-runnables/install-alb-controller/runnable.py +++ b/python-runnables/install-alb-controller/runnable.py @@ -154,7 +154,9 @@ def run(self, progress_callback): if command_outputs[-1][1] != 0: return make_html(command_outputs) - alb_controller_url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.4/docs/examples/alb-ingress-controller.yaml" + alb_controller_url = ( + "https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.4/docs/examples/alb-ingress-controller.yaml" + ) service_data = get_url_or_fallback(alb_controller_url, "alb-ingress-controller.yaml") cluster_flag_pattern = "#.*cluster\\-name=.*" cluster_flag_replacement = "- --cluster-name=%s" % cluster_id diff --git a/python-runnables/test-network/runnable.py b/python-runnables/test-network/runnable.py index e370d80..bf08a61 100644 --- a/python-runnables/test-network/runnable.py +++ b/python-runnables/test-network/runnable.py @@ -42,7 +42,7 @@ def run(self, progress_callback): def add_to_result(result, op, cmd, out, err): return ( result - + '
%s
Command
%s
Output
%s
Error
%s
' # noqa E501 + + '
%s
Command
%s
Output
%s
Error
%s
' # noqa E501 % (op, json.dumps(cmd), out, err) ) @@ -84,7 +84,7 @@ def add_to_result(result, op, cmd, out, err): out, err = b.exec_cmd(cmd, timeout=10) result = ( result - + '
Test connection to port
Command
%s
Debug (stderr)
%s
' # noqa E501 + + '
Test connection to port
Command
%s
Debug (stderr)
%s
' # noqa E501 % (json.dumps(cmd), err) ) if "no route to host" in err.lower(): From fcc79ee5a7411b3b046ea04925363ab21e3e42ef Mon Sep 17 00:00:00 2001 From: vrutz Date: Mon, 20 Jul 2026 16:53:20 +0200 Subject: [PATCH 02/12] change test target name --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2613255..60da57b 100644 --- a/Makefile +++ b/Makefile @@ -56,9 +56,9 @@ check-generated-files: validate: compile-python validate-json check-generated-files -test: +unit-tests: $(PYTHON) -m unittest discover -s tests -v -tests: test +tests: unit-tests ci: lint compile-python validate-json check-generated-files tests plugin From 5da690a529d4e0c78cd0b1cb361477ae3e07c00c Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:09:10 +0200 Subject: [PATCH 03/12] Add GitHub Actions workflow for PR checks and remove Jenkinsfile --- .github/workflows/pr-checks.yml | 48 +++++++++++++++++++ Jenkinsfile | 84 --------------------------------- Makefile | 6 +-- 3 files changed, 51 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/pr-checks.yml delete mode 100644 Jenkinsfile diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..5ecc14f --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,48 @@ +name: PR checks + +on: + pull_request: + workflow_dispatch: + +jobs: + checks: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y make wget zip + + - name: Lint + run: make lint + + - name: Compile Python + run: make compile-python + + - name: Validate JSON descriptors + run: make validate-json + + - name: Check generated files + run: make check-generated-files + + - name: Run unit tests + run: make tests + + - name: Build plugin + run: make plugin + + - name: Upload plugin archive + uses: actions/upload-artifact@v4 + with: + name: plugin-archive + path: dist/*.zip + if-no-files-found: error diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 1a45109..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,84 +0,0 @@ -pipeline { - options { - disableConcurrentBuilds() - timestamps() - } - - agent { label 'dss-plugin-tests' } - - stages { - stage('Lint') { - steps { - sh 'make lint' - } - } - - stage('Compile Python') { - steps { - sh 'make compile-python' - } - } - - stage('Validate JSON Descriptors') { - steps { - sh 'make validate-json' - } - } - - stage('Check Generated Files') { - steps { - sh 'make check-generated-files' - } - } - - stage('Run Unit Tests') { - steps { - sh 'make tests' - } - } - - stage('Build Plugin') { - steps { - sh 'make plugin' - archiveArtifacts artifacts: 'dist/*.zip', fingerprint: true - } - } - } - - post { - always { - script { - if (fileExists('tests/allure_report')) { - allure([ - includeProperties: false, - jdk: '', - properties: [], - reportBuildPolicy: 'ALWAYS', - results: [[path: 'tests/allure_report']] - ]) - } - - def status = currentBuild.currentResult - def statusFields = [ - env.BUILD_URL, - env.CHANGE_TITLE, - env.CHANGE_AUTHOR, - env.CHANGE_URL, - env.BRANCH_NAME, - status - ].collect { field -> (field ?: '').replaceAll(/[;\r\n]/, ' ') } - - writeFile file: '.build-status-line', text: statusFields.join(';') + ';\n' - - sh ''' - mkdir -p "$HOME/daily-statuses" - file_name=$(echo "$JOB_NAME" | tr '/' '-').status - touch "$HOME/daily-statuses/$file_name" - cat .build-status-line >> "$HOME/daily-statuses/$file_name" - ''' - - cleanWs() - } - } - } -} diff --git a/Makefile b/Makefile index 60da57b..ae3f10a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -plugin_id=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['id']).replace('/',''))"` -plugin_version=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['version']).replace('/',''))"` +plugin_id=`cat plugin.json | python3 -c "import sys, json; print(str(json.load(sys.stdin)['id']).replace('/',''))"` +plugin_version=`cat plugin.json | python3 -c "import sys, json; print(str(json.load(sys.stdin)['version']).replace('/',''))"` archive_file_name="dss-plugin-${plugin_id}-${plugin_version}.zip" remote_url=`git config --get remote.origin.url` last_commit_id=`git rev-parse HEAD` @@ -9,7 +9,7 @@ RUFF := $(VENV)/bin/ruff plugin: @echo "[START] Archiving plugin to dist/ folder..." - @cat plugin.json | json_pp > /dev/null + @python3 -m json.tool plugin.json > /dev/null @rm -rf dist @mkdir -p resource @wget https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/main/deployments/static/nvidia-device-plugin.yml -P resource From 7f676e6b7b4f5096dd970ff41f5217992446e077 Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:14:16 +0200 Subject: [PATCH 04/12] Make linter non-blocking --- .github/workflows/pr-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5ecc14f..b88090c 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,6 +23,7 @@ jobs: sudo apt-get install -y make wget zip - name: Lint + continue-on-error: true run: make lint - name: Compile Python From 992d3ed37fd09973475ec227fd18376971e333fa Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:22:34 +0200 Subject: [PATCH 05/12] Remove plugin archive upload step from PR checks workflow --- .github/workflows/pr-checks.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index b88090c..6826cdf 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -40,10 +40,3 @@ jobs: - name: Build plugin run: make plugin - - - name: Upload plugin archive - uses: actions/upload-artifact@v4 - with: - name: plugin-archive - path: dist/*.zip - if-no-files-found: error From 72c123d51da607abc961fe891f72e0fb764fe0a3 Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:24:17 +0200 Subject: [PATCH 06/12] Refactor lint step to provide advisory warnings without blocking the workflow --- .github/workflows/pr-checks.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 6826cdf..3638ef2 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,8 +23,11 @@ jobs: sudo apt-get install -y make wget zip - name: Lint - continue-on-error: true - run: make lint + run: | + make lint || { + echo "::warning::Ruff lint/format checks failed. This is advisory for now; see the lint step logs." + exit 0 + } - name: Compile Python run: make compile-python From ded246dfe898fc00527f6c1458cf5ff50976243c Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:26:39 +0200 Subject: [PATCH 07/12] Update lint step to exit with error code for failed checks --- .github/workflows/pr-checks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 3638ef2..83c6f2f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,10 +23,11 @@ jobs: sudo apt-get install -y make wget zip - name: Lint + continue-on-error: true run: | make lint || { echo "::warning::Ruff lint/format checks failed. This is advisory for now; see the lint step logs." - exit 0 + exit 1 } - name: Compile Python From b3589fef428bba429dd9fedf00b83f4df86e7d7f Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:33:24 +0200 Subject: [PATCH 08/12] Refactor lint step to exit with success code and improve formatting in autoscaler.py --- .github/workflows/pr-checks.yml | 2 +- python-lib/dku_kube/autoscaler.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 83c6f2f..fec2ccb 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -27,7 +27,7 @@ jobs: run: | make lint || { echo "::warning::Ruff lint/format checks failed. This is advisory for now; see the lint step logs." - exit 1 + exit 0 } - name: Compile Python diff --git a/python-lib/dku_kube/autoscaler.py b/python-lib/dku_kube/autoscaler.py index f140f42..5298b8b 100644 --- a/python-lib/dku_kube/autoscaler.py +++ b/python-lib/dku_kube/autoscaler.py @@ -332,4 +332,8 @@ def get_autoscaler_config(cluster_id, autoscaler_image_version, autoscaler_regis - name: ssl-certs hostPath: path: "/etc/ssl/certs/ca-bundle.crt" -""" % {"autoscalerimageversion": autoscaler_image_version, "clusterid": cluster_id, "autoscalerregistryurl": autoscaler_registry_url} +""" % { + "autoscalerimageversion": autoscaler_image_version, + "clusterid": cluster_id, + "autoscalerregistryurl": autoscaler_registry_url, + } From 2ee8a73223fc0023fcece95a3eb8cd6454c5a004 Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:35:12 +0200 Subject: [PATCH 09/12] To revert: force fail unit test --- tests/test_autoscaler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_autoscaler.py b/tests/test_autoscaler.py index dc5b6e3..95bca6f 100644 --- a/tests/test_autoscaler.py +++ b/tests/test_autoscaler.py @@ -114,6 +114,10 @@ def test_get_autoscaler_config_uses_selected_registry_and_tag(self): self.assertIn("image: registry.example.com/prefix/autoscaling/cluster-autoscaler:v1.35.0", config) self.assertIn("k8s.io/cluster-autoscaler/cluster-1", config) + def test_failure_on_purpose(self): + with self.assertRaises(ValueError): + autoscaler.select_autoscaler_image("1.33", "registry.k8s.io", "nonexistent-tag") + if __name__ == "__main__": unittest.main() From 638dc825fcaea3759e6141f97b94f3c7804e5f9b Mon Sep 17 00:00:00 2001 From: vrutz Date: Thu, 27 Aug 2026 17:38:46 +0200 Subject: [PATCH 10/12] Revert "To revert: force fail unit test" This reverts commit 2ee8a73223fc0023fcece95a3eb8cd6454c5a004. --- tests/test_autoscaler.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_autoscaler.py b/tests/test_autoscaler.py index 20403ca..7095c0b 100644 --- a/tests/test_autoscaler.py +++ b/tests/test_autoscaler.py @@ -115,10 +115,6 @@ def test_get_autoscaler_config_uses_selected_registry_and_tag(self): self.assertIn("k8s.io/cluster-autoscaler/cluster-1", config) self.assertIn("name: AWS_REGION\n value: eu-west-1", config) - def test_failure_on_purpose(self): - with self.assertRaises(ValueError): - autoscaler.select_autoscaler_image("1.33", "registry.k8s.io", "nonexistent-tag") - if __name__ == "__main__": unittest.main() From 6fdff7d235b689e9f3a7fe97d1720ee024bc5138 Mon Sep 17 00:00:00 2001 From: vrutz Date: Mon, 31 Aug 2026 10:17:18 +0200 Subject: [PATCH 11/12] ruff format --- python-runnables/add-autoscaler/runnable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-runnables/add-autoscaler/runnable.py b/python-runnables/add-autoscaler/runnable.py index 77a9379..723c49b 100644 --- a/python-runnables/add-autoscaler/runnable.py +++ b/python-runnables/add-autoscaler/runnable.py @@ -4,7 +4,6 @@ from dku_utils.config_parser import get_region_fallback_to_metadata - class MyRunnable(Runnable): def __init__(self, project_key, config, plugin_config): self.project_key = project_key From 7f533cb7bf228fb30bf15dad7a72251699246b8d Mon Sep 17 00:00:00 2001 From: vrutz Date: Mon, 31 Aug 2026 10:35:27 +0200 Subject: [PATCH 12/12] make lint errors blocking --- .github/workflows/pr-checks.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index fec2ccb..13e9eed 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,12 +23,7 @@ jobs: sudo apt-get install -y make wget zip - name: Lint - continue-on-error: true - run: | - make lint || { - echo "::warning::Ruff lint/format checks failed. This is advisory for now; see the lint step logs." - exit 0 - } + run: make lint - name: Compile Python run: make compile-python