Improvements

.dockerignore

@@ -0,0 +1,22 @@
+# The NiFi image only copies pyproject.toml and the nifi source tree.
+# Keep the build context small and prevent unrelated repository content,
+# credentials, and local development state from entering image layers.
+**
+!pyproject.toml
+!uv.lock
+!nifi/
+!nifi/**
+
+# Local NiFi runtime state. These files may contain environment-specific
+# endpoints, identifiers, or other data and are not reproducible inputs.
+nifi/conf/flow.json.gz
+nifi/conf/archive/
+
+# Development and operating-system artefacts within the included source tree.
+nifi/**/__pycache__/
+nifi/**/*.py[cod]
+nifi/**/.DS_Store
+nifi/**/.pytest_cache/
+nifi/**/.ruff_cache/
+nifi/**/.mypy_cache/
+nifi/**/*.egg-info/

.github/dependabot.yml

@@ -14,9 +14,15 @@ updates:
     registries:
       - dockerhub
 
+  # Keep the root Python project and lock file up to date.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
 registries:
   dockerhub:
     type: "docker-registry"
     url: "https://index.docker.io"
     username: "${{secrets.DOCKER_HUB_USERNAME}}"
-    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"
+    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"

.github/workflows/dependency_hygiene.yml

@@ -4,11 +4,23 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 6 * * 1" # weekly Monday UTC
+  push:
+    branches: ['**']
+    paths:
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
   pull_request:
     paths:
-      - "docs/requirements.txt"
-      - "nifi/requirements.txt"
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
       - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
 
 permissions:
   contents: read
@@ -23,7 +35,7 @@ jobs:
         uses: actions/cache@v6
         with:
           path: ~/.cache/pip
-          key: pip-${{ runner.os }}-${{ hashFiles('docs/requirements.txt', 'nifi/requirements.txt', 'scripts/maintenance/audit_pip_dependency.sh') }}
+          key: pip-${{ runner.os }}-${{ hashFiles('docs/pyproject.toml', 'docs/uv.lock', 'pyproject.toml', 'uv.lock', 'scripts/maintenance/audit_pip_dependency.sh') }}
           restore-keys: |
             pip-${{ runner.os }}-
 
@@ -32,7 +44,12 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install pip-audit
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version-file: "pyproject.toml"
+
+      - name: Install audit tools
         run: python -m pip install --upgrade pip pip-audit
 
       - name: Ensure script is executable

.github/workflows/doc_build.yml

@@ -24,6 +24,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
+          version-file: "pyproject.toml"
           python-version: "3.12"
           enable-cache: true
           cache-dependency-glob: "docs/uv.lock"
@@ -39,4 +40,4 @@ jobs:
 
       - name: Build docs
         working-directory: docs
-        run: uv run mkdocs build -f ../mkdocs.yml
+        run: uv run mkdocs build --strict -f ../mkdocs.yml

.github/workflows/python_quality.yml

@@ -5,17 +5,15 @@ on:
     branches: ['**']
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
       - ".github/workflows/python_quality.yml"
   pull_request:
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
@@ -39,17 +37,18 @@ jobs:
           python-version: "3.11"
 
       - name: Install uv
-        run: |
-          python -m pip install --upgrade pip uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version-file: "pyproject.toml"
 
-      - name: Install requirements with uv
+      - name: Install locked project dependencies
         run: |
-          uv pip install --system -r nifi/requirements.txt -r nifi/requirements-dev.txt
+          uv sync --locked --dev
 
       - name: Ruff (critical rules)
         run: |
-          python -m ruff check scripts nifi/user_python_extensions --select F,E9
+          uv run --locked ruff check scripts nifi/user_python_extensions --select F,E9
 
       - name: MyPy
         run: |
-          python -m mypy --config-file pyproject.toml nifi/user_python_extensions scripts
+          uv run --locked mypy --config-file pyproject.toml nifi/user_python_extensions scripts

.github/workflows/shell_script_enforce_exec_check.yml

@@ -40,3 +40,39 @@ jobs:
             echo "Fix with: git update-index --chmod=+x <file>"
             exit 1
           fi
+
+  shell-quality:
+    name: ShellCheck and syntax
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Check root-owned shell scripts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          scripts=()
+          while IFS= read -r -d '' file; do
+            scripts+=("$file")
+          done < <(
+            git ls-files -z -- '*.sh' \
+              ':(exclude)services/**' \
+              ':(exclude)deploy/charts/**'
+          )
+
+          if [[ "${#scripts[@]}" -eq 0 ]]; then
+            echo "No root-owned shell scripts found."
+            exit 0
+          fi
+
+          echo "Checking ${#scripts[@]} root-owned shell scripts."
+          docker run --rm \
+            --volume "${GITHUB_WORKSPACE}:/mnt:ro" \
+            --workdir /mnt \
+            koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d \
+            "${scripts[@]}"
+
+          for file in "${scripts[@]}"; do
+            bash -n "$file"
+          done

.github/workflows/smoke_gitea_service.yml

@@ -24,10 +24,11 @@ jobs:
           java-version: '21'
 
       - name: Generate certificates
-        working-directory: security/scripts
         run: |
           set -euo pipefail
-          ./create_root_ca_cert.sh
+          make -C deploy init-security-gitea
+          sudo chown 1000:1000 security/certificates/gitea/gitea.key
+          sudo chmod 600 security/certificates/gitea/gitea.key
 
       - name: Start Gitea service
         run: |

README.md

@@ -16,7 +16,7 @@ All NLP/ML/data services are expected to implement a uniform RESTful API, allowi
 
 ## ⚠️ Important Notice
 
-This project is under active development. New features or services may impact existing deployments. Please review the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) and [documentation](https://cogstack-nifi.readthedocs.io) before upgrading.
+This project is under active development. New features or services may impact existing deployments. Please review the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news/) and [documentation](https://cogstack-nifi.readthedocs.io) before upgrading.
 
 ---
 
@@ -54,6 +54,7 @@ This table describes repository layout. For setup and operations, use the deploy
 git lfs pull
 make -C deploy git-update-submodules
 make -C deploy help
+make -C deploy init-security
 make -C deploy start-data-infra
 ```
 
@@ -79,15 +80,15 @@ make -C deploy stop-data-infra
 
 📖 Official documentation: [cogstack-nifi.readthedocs.io](https://cogstack-nifi.readthedocs.io/en/latest/)
 
-🚀 New to the project? Start with the [deployment guide](https://cogstack-nifi.readthedocs.io/en/latest/deploy/main.html) for example setups and workflows.
+🚀 New to the project? Start with the [deployment guide](https://cogstack-nifi.readthedocs.io/en/latest/deploy/main/) for example setups and workflows.
 
-🐞 For troubleshooting or bug reports, consult the [known issues section](https://cogstack-nifi.readthedocs.io/en/latest/deploy/troubleshooting.html) before opening a ticket.
+🐞 For troubleshooting or bug reports, consult the [known issues section](https://cogstack-nifi.readthedocs.io/en/latest/deploy/troubleshooting/) before opening a ticket.
 
 ---
 
 ## 🛑 Important Updates
 
-Check the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news.html) section regularly for:
+Check the [release notes](https://cogstack-nifi.readthedocs.io/en/latest/news/) section regularly for:
 
 - Major changes to project structure or configuration
 - Security advisories or vulnerabilities affecting deployments

deploy/Makefile

@@ -34,6 +34,11 @@ HELM_GITEA_VALUES_ARG = -f $(HELM_GITEA_VALUES_FILE)
 CNPG_OPERATOR_MINOR ?= 1.28
 CNPG_OPERATOR_VERSION ?= 1.28.1
 CNPG_OPERATOR_MANIFEST ?= https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-$(CNPG_OPERATOR_MINOR)/releases/cnpg-$(CNPG_OPERATOR_VERSION).yaml
+SECURITY_DIR ?= ../security
+SECURITY_SCRIPTS_DIR ?= $(SECURITY_DIR)/scripts
+SECURITY_CERTIFICATES_DIR ?= $(SECURITY_DIR)/certificates
+SECURITY_ENV_DIR ?= $(SECURITY_DIR)/env
+OPENSEARCH_CERT_NAMES ?=
 
 define WITH_ENV
 set -a && source ./export_env_vars.sh;
@@ -95,6 +100,140 @@ load-env: ## Load variables from export_env_vars.sh in a subshell
 show-env: ## Print sorted environment variables after loading export_env_vars.sh
 	${WITH_ENV} >/dev/null 2>&1; printenv | sort
 
+init-security-root-ca: ## Generate the shared root CA only when no CA material exists
+	@set -euo pipefail; \
+	source "$(SECURITY_ENV_DIR)/certificates_general.env"; \
+	root_dir="$(SECURITY_CERTIFICATES_DIR)/root"; \
+	if [[ -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}.key" \
+	   && -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}.pem" \
+	   && -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}.p12" \
+	   && -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}-keystore.jks" \
+	   && -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}-truststore.jks" ]]; then \
+		echo "Root CA certificates already exist; leaving them unchanged."; \
+	elif [[ -d "$$root_dir" && -n "$$(find "$$root_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then \
+		echo "ERROR: $$root_dir contains an incomplete root CA set." >&2; \
+		echo "Preserve and repair or explicitly replace that CA before rerunning this target." >&2; \
+		exit 1; \
+	else \
+		echo "Generating the shared root CA certificates..."; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && ./create_root_ca_cert.sh); \
+	fi
+
+init-security-nifi: init-security-root-ca ## Generate NiFi certificates only when they are missing or incomplete
+	@set -euo pipefail; \
+	nifi_dir="$(SECURITY_CERTIFICATES_DIR)/nifi"; \
+	if [[ -f "$$nifi_dir/nifi.key" \
+	   && -f "$$nifi_dir/nifi.pem" \
+	   && -f "$$nifi_dir/nifi.p12" \
+	   && -f "$$nifi_dir/nifi-keystore.jks" \
+	   && -f "$$nifi_dir/nifi-truststore.jks" ]]; then \
+		echo "NiFi certificates already exist; leaving them unchanged."; \
+	else \
+		echo "Generating NiFi certificates..."; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && ./create_nifi_certs.sh); \
+	fi
+
+init-security-gitea: init-security-root-ca ## Generate a dedicated Gitea leaf certificate when it is missing
+	@set -euo pipefail; \
+	gitea_dir="$(SECURITY_CERTIFICATES_DIR)/gitea"; \
+	if [[ -f "$$gitea_dir/gitea.key" \
+	   && -f "$$gitea_dir/gitea.pem" \
+	   && -f "$$gitea_dir/gitea.crt" ]]; then \
+		echo "Gitea leaf certificate already exists; leaving it unchanged."; \
+	else \
+		echo "Generating Gitea leaf certificate..."; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && bash ./create_gitea_certs.sh); \
+	fi
+
+init-security-opensearch: init-security-root-ca ## Generate OpenSearch node, client, and admin certificates when missing
+	@set -euo pipefail; \
+	source "$(SECURITY_ENV_DIR)/certificates_elasticsearch.env"; \
+	if [[ -n "$(strip $(OPENSEARCH_CERT_NAMES))" ]]; then \
+		read -r -a node_names <<< "$(strip $(OPENSEARCH_CERT_NAMES))"; \
+	else \
+		node_names=("$$ES_INSTANCE_NAME_1" "$$ES_INSTANCE_NAME_2" "$$ES_INSTANCE_NAME_3"); \
+	fi; \
+	opensearch_dir="$(SECURITY_CERTIFICATES_DIR)/elastic/opensearch"; \
+	node_certificates_missing=0; \
+	for node_name in "$${node_names[@]}"; do \
+		node_dir="$$opensearch_dir/elasticsearch/$$node_name"; \
+		for suffix in .crt .key .p12 -keystore.jks -truststore.key; do \
+			if [[ ! -f "$$node_dir/$$node_name$$suffix" ]]; then \
+				node_certificates_missing=1; \
+				break 2; \
+			fi; \
+		done; \
+	done; \
+	if [[ ! -f "$$opensearch_dir/elastic-stack-ca.crt.pem" \
+	   || ! -f "$$opensearch_dir/elastic-stack-ca.p12" ]]; then \
+		node_certificates_missing=1; \
+	fi; \
+	if (( node_certificates_missing )); then \
+		echo "Generating OpenSearch node certificates for: $${node_names[*]}"; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && ./create_opensearch_node_cert.sh "$${node_names[@]}"); \
+	else \
+		echo "OpenSearch node certificates already exist; leaving them unchanged."; \
+	fi; \
+	if [[ -f "$$opensearch_dir/admin.crt" \
+	   && -f "$$opensearch_dir/admin.key.pem" \
+	   && -f "$$opensearch_dir/$${ES_CLIENT_CERT_NAME}.pem" \
+	   && -f "$$opensearch_dir/$${ES_CLIENT_CERT_NAME}.key" ]]; then \
+		echo "OpenSearch client and admin certificates already exist; leaving them unchanged."; \
+	else \
+		echo "Generating OpenSearch client and admin certificates..."; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && ./create_opensearch_client_admin_certs.sh); \
+	fi
+
+init-security-elasticsearch: ## Generate native Elasticsearch certificates when the output directory is absent
+	@set -euo pipefail; \
+	source "$(SECURITY_ENV_DIR)/certificates_elasticsearch.env"; \
+	elasticsearch_dir="$(SECURITY_CERTIFICATES_DIR)/elastic/elasticsearch"; \
+	certificate_set_complete=1; \
+	for required_file in \
+		"elastic-stack-ca.crt.pem" \
+		"elastic-stack-ca.p12" \
+		"elasticsearch/$${ES_INSTANCE_NAME_1}/$${ES_INSTANCE_NAME_1}.crt" \
+		"elasticsearch/$${ES_INSTANCE_NAME_1}/$${ES_INSTANCE_NAME_1}.key" \
+		"elasticsearch/$${ES_INSTANCE_NAME_2}/$${ES_INSTANCE_NAME_2}.crt" \
+		"elasticsearch/$${ES_INSTANCE_NAME_2}/$${ES_INSTANCE_NAME_2}.key" \
+		"elasticsearch/$${ES_INSTANCE_NAME_3}/$${ES_INSTANCE_NAME_3}.crt" \
+		"elasticsearch/$${ES_INSTANCE_NAME_3}/$${ES_INSTANCE_NAME_3}.key"; do \
+		if [[ ! -f "$$elasticsearch_dir/$$required_file" ]]; then \
+			certificate_set_complete=0; \
+			break; \
+		fi; \
+	done; \
+	if (( certificate_set_complete )); then \
+		echo "Native Elasticsearch certificates already exist; leaving them unchanged."; \
+	elif [[ -e "$$elasticsearch_dir" ]]; then \
+		echo "ERROR: $$elasticsearch_dir exists but contains an incomplete certificate set." >&2; \
+		echo "Move or remove that directory after preserving anything needed, then rerun this target." >&2; \
+		exit 1; \
+	else \
+		echo "Generating native Elasticsearch certificates with Docker..."; \
+		(cd "$(SECURITY_SCRIPTS_DIR)" && ./create_es_native_certs.sh); \
+	fi
+
+init-security-search: ## Generate certificates for ELASTICSEARCH_VERSION from elasticsearch.env
+	@set -euo pipefail; \
+	source ./elasticsearch.env; \
+	case "$${ELASTICSEARCH_VERSION}" in \
+		opensearch|elasticsearch) \
+			$(MAKE) --no-print-directory "init-security-$${ELASTICSEARCH_VERSION}" \
+			;; \
+		*) \
+			echo "ERROR: unsupported ELASTICSEARCH_VERSION '$${ELASTICSEARCH_VERSION}'." >&2; \
+			exit 1 \
+			;; \
+	esac
+
+init-security: ## Generate missing NiFi, Gitea, and selected search-backend certificates
+	@$(MAKE) --no-print-directory init-security-nifi
+	@$(MAKE) --no-print-directory init-security-gitea
+	@$(MAKE) --no-print-directory init-security-search
+
+.PHONY: init-security init-security-root-ca init-security-nifi init-security-gitea init-security-opensearch init-security-elasticsearch init-security-search
+
 helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml
 	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG)