Add unit tests/linter with GitHub actions to run them on PR

.github/workflows/pr-checks.yml

@@ -0,0 +1,41 @@
+name: PR checks
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y make wget zip
+
+    - name: Lint
+      run: make lint
+
+    - name: Compile Python
+      run: make compile-python
+
+    - name: Validate JSON descriptors
+      run: make validate-json
+
+    - name: Check generated files
+      run: make check-generated-files
+
+    - name: Run unit tests
+      run: make tests
+
+    - name: Build plugin
+      run: make plugin

.gitignore

@@ -3,7 +3,10 @@
 .wlock
 .ts
 .git
+.venv/
+__pycache__/
 *.pyc
+dist/
 .coverage
 .htmlcov
 .DS_Store

Makefile

@@ -1,12 +1,15 @@
-plugin_id=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['id']).replace('/',''))"`
-plugin_version=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['version']).replace('/',''))"`
+plugin_id=`cat plugin.json | python3 -c "import sys, json; print(str(json.load(sys.stdin)['id']).replace('/',''))"`
+plugin_version=`cat plugin.json | python3 -c "import sys, json; print(str(json.load(sys.stdin)['version']).replace('/',''))"`
 archive_file_name="dss-plugin-${plugin_id}-${plugin_version}.zip"
 remote_url=`git config --get remote.origin.url`
 last_commit_id=`git rev-parse HEAD`
+PYTHON ?= python3
+VENV ?= .venv
+RUFF := $(VENV)/bin/ruff
 
 plugin:
 	@echo "[START] Archiving plugin to dist/ folder..."
-	@cat plugin.json | json_pp > /dev/null
+	@python3 -m json.tool plugin.json > /dev/null
 	@rm -rf dist
 	@mkdir -p resource
 	@wget https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/main/deployments/static/nvidia-device-plugin.yml -P resource
@@ -28,5 +31,34 @@ plugin:
 dist-clean:
 	rm -rf dist
 
-test:
-	python3 -m unittest discover -s tests -v
+$(RUFF):
+	$(PYTHON) -m venv $(VENV)
+	$(VENV)/bin/python -m pip install --upgrade pip
+	$(VENV)/bin/python -m pip install ruff
+
+lint: $(RUFF)
+	$(RUFF) check .
+	$(RUFF) format --check .
+
+compile-python:
+	$(PYTHON) -m compileall -q python-lib python-runnables python-clusters tests
+
+validate-json:
+	@find . -maxdepth 3 -type f -name '*.json' -print0 | xargs -0 -n1 $(PYTHON) -m json.tool >/dev/null
+
+check-generated-files:
+	@tracked=$$(git ls-files | grep -E '(__pycache__/|\.pyc$$|^dist/)' || true); \
+	if [ -n "$$tracked" ]; then \
+		echo "Generated files are tracked and should be removed:"; \
+		echo "$$tracked"; \
+		exit 1; \
+	fi
+
+validate: compile-python validate-json check-generated-files
+
+unit-tests:
+	$(PYTHON) -m unittest discover -s tests -v
+
+tests: unit-tests
+
+ci: lint compile-python validate-json check-generated-files tests plugin

parameter-sets/node-pool-request/parameter-set.json

@@ -87,7 +87,7 @@
             "description": "Disk size for the nodes, in GB. Can use 0 for default (200 GB)",
             "type": "INT",
             "mandatory" : false,
-            "defaultValue": 200, /* Repeated in the code in node-pool.py */
+            "defaultValue": 200,
             "minI": 0
         },
         {

python-lib/dku_aws/eksctl_loader.py

@@ -22,7 +22,7 @@ def get_eksctl_or_fetch():
             r = requests.get(
                 "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_%s_amd64.tar.gz" % arch,
                 stream=True,
-                headers={"User-Agent": "DSS EKS Plugin"}
+                headers={"User-Agent": "DSS EKS Plugin"},
             )
             local_eksctl_archive = os.path.join(local_eksctl_folder, "eksctl.tar.gz")
             with open(local_eksctl_archive, "wb") as f:

python-lib/dku_kube/autoscaler.py

@@ -338,4 +338,9 @@ def get_autoscaler_config(cluster_id, autoscaler_image_version, autoscaler_regis
         - name: ssl-certs
           hostPath:
             path: "/etc/ssl/certs/ca-bundle.crt"
-""" % {"autoscalerimageversion": autoscaler_image_version, "clusterid": cluster_id, "autoscalerregistryurl": autoscaler_registry_url, "aws_region": aws_region}
+""" % {
+        "autoscalerimageversion": autoscaler_image_version,
+        "clusterid": cluster_id,
+        "autoscalerregistryurl": autoscaler_registry_url,
+        "aws_region": aws_region,
+    }

python-lib/dku_kube/gpu_driver.py

@@ -66,9 +66,7 @@ def add_gpu_driver_if_needed(cluster_id, kube_config_path, connection_info, tain
     nvidia_config["spec"]["template"]["spec"]["tolerations"] = Toleration.to_list(tolerations)
 
     # Restrict the DaemonSet to nodes that request a GPU, so it doesn't try to run on every CPU node in the cluster
-    nvidia_config["spec"]["template"]["spec"]["nodeSelector"] = {
-        GPU_ENABLED_NODEPOOL_LABEL: GPU_ENABLED_NODEPOOL_LABEL_VALUE
-    }
+    nvidia_config["spec"]["template"]["spec"]["nodeSelector"] = {GPU_ENABLED_NODEPOOL_LABEL: GPU_ENABLED_NODEPOOL_LABEL_VALUE}
 
     # Write the configuration locally
     local_nvidia_plugin_config = os.path.join(os.environ["DIP_HOME"], "clusters", cluster_id, "nvidia-device-plugin.yml")

python-lib/dku_utils/static_resources.py

@@ -2,6 +2,7 @@
 import os
 import requests
 
+
 def get_url_or_fallback(url, static_resource_filename):
     r = requests.get(url, headers={"User-Agent": "DSS EKS Plugin"})
 
@@ -17,6 +18,7 @@ def get_url_or_fallback(url, static_resource_filename):
     else:
         logging.error("No static resource fallback was defined.")
 
+
 def get_static_resource(static_resource_filename):
     static_resource_path = os.path.join(os.environ["DKU_CUSTOM_RESOURCE_FOLDER"], static_resource_filename)
 
@@ -26,4 +28,4 @@ def get_static_resource(static_resource_filename):
             resource_content_raw = f.read()
             return resource_content_raw
     else:
-        logging.warning("Unable to locate the static resource at path: %s" % static_resource_path)
+        logging.warning("Unable to locate the static resource at path: %s" % static_resource_path)

python-runnables/add-autoscaler/runnable.py

@@ -4,7 +4,6 @@
 from dku_utils.config_parser import get_region_fallback_to_metadata
 
 
-
 class MyRunnable(Runnable):
     def __init__(self, project_key, config, plugin_config):
         self.project_key = project_key

python-runnables/install-alb-controller/runnable.py

@@ -154,7 +154,9 @@ def run(self, progress_callback):
         if command_outputs[-1][1] != 0:
             return make_html(command_outputs)
 
-        alb_controller_url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.4/docs/examples/alb-ingress-controller.yaml"
+        alb_controller_url = (
+            "https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.4/docs/examples/alb-ingress-controller.yaml"
+        )
         service_data = get_url_or_fallback(alb_controller_url, "alb-ingress-controller.yaml")
         cluster_flag_pattern = "#.*cluster\\-name=.*"
         cluster_flag_replacement = "- --cluster-name=%s" % cluster_id

python-runnables/test-network/runnable.py

@@ -42,7 +42,7 @@ def run(self, progress_callback):
         def add_to_result(result, op, cmd, out, err):
             return (
                 result
-                + '<h5>%s</h5><div style="margin-left: 20px;"><div>Command</div><pre class="debug">%s</pre><div>Output</div><pre class="debug">%s</pre><div>Error</div><pre class="debug">%s</pre></div>' # noqa E501
+                + '<h5>%s</h5><div style="margin-left: 20px;"><div>Command</div><pre class="debug">%s</pre><div>Output</div><pre class="debug">%s</pre><div>Error</div><pre class="debug">%s</pre></div>'  # noqa E501
                 % (op, json.dumps(cmd), out, err)
             )
 
@@ -84,7 +84,7 @@ def add_to_result(result, op, cmd, out, err):
                 out, err = b.exec_cmd(cmd, timeout=10)
                 result = (
                     result
-                    + '<h5>Test connection to port</h5><div style="margin-left: 20px;"><div>Command</div><pre class="debug">%s</pre><div>Debug (stderr)</div><pre class="debug">%s</pre></div>' # noqa E501
+                    + '<h5>Test connection to port</h5><div style="margin-left: 20px;"><div>Command</div><pre class="debug">%s</pre><div>Debug (stderr)</div><pre class="debug">%s</pre></div>'  # noqa E501
                     % (json.dumps(cmd), err)
                 )
                 if "no route to host" in err.lower():