[TCL-7759] fix: restrict Slurm health checks to idle nodes

.github/workflows/container-images-1.0.yaml

@@ -91,6 +91,9 @@ jobs:
         with:
           version: v3.16.3
 
+      - name: Run lint
+        run: make golangci-lint
+
       - name: Run tests
         run: make helm-validate test
 

Makefile

@@ -304,6 +304,10 @@ govulncheck: govulncheck-bin ## Run govulncheck
 # https://github.com/golangci/golangci-lint/blob/main/.pre-commit-hooks.yaml
 .PHONY: golangci-lint
 golangci-lint: golangci-lint-bin ## Run golangci-lint.
+	$(GOLANGCI_LINT) run
+
+.PHONY: golangci-lint-fix
+golangci-lint-fix: golangci-lint-bin ## Run golangci-lint and apply fixes.
 	$(GOLANGCI_LINT) run --fix
 
 # https://github.com/golangci/golangci-lint/blob/main/.pre-commit-hooks.yaml

VERSION

@@ -1 +1 @@
-1.0.14
+1.0.15

internal/builder/accounting_app.go

@@ -29,8 +29,8 @@ const (
 	slurmdbdConfFile = "slurmdbd.conf"
 
 	// Volume names for accounting initContainer fix
-	slurmSecretsTmpVolume = "slurm-secrets-tmp"
-	slurmSecretsTmpDir    = "/tmp/slurm-secrets"
+	slurmSecretsTmpVolume = "slurm-secrets-tmp" //nolint:gosec // Kubernetes volume name, not a credential.
+	slurmSecretsTmpDir    = "/tmp/slurm-secrets" //nolint:gosec // Kubernetes mount path, not a credential.
 )
 
 func (b *Builder) BuildAccounting(accounting *slinkyv1beta1.Accounting) (*appsv1.StatefulSet, error) {

internal/builder/controller_config.go

@@ -275,7 +275,7 @@ func buildSlurmConf(
 	conf.AddProperty(config.NewPropertyRaw("### SYSTEM DEFAULTS ###"))
 	conf.AddProperty(config.NewProperty("UnkillableStepTimeout", 600))
 	conf.AddProperty(config.NewProperty("HealthCheckInterval", 60))
-	conf.AddProperty(config.NewProperty("HealthCheckNodeState", "ANY"))
+	conf.AddProperty(config.NewProperty("HealthCheckNodeState", "IDLE"))
 	conf.AddProperty(config.NewProperty("HealthCheckProgram", "/usr/bin/gpu_healthcheck.sh"))
 	conf.AddProperty(config.NewProperty("JobRequeue", 0))
 

internal/builder/controller_config_test.go

@@ -165,14 +165,17 @@ func TestBuilder_BuildControllerConfig(t *testing.T) {
 				for _, directive := range []string{
 					"UnkillableStepTimeout=600",
 					"HealthCheckInterval=60",
-					"HealthCheckNodeState=ANY",
+					"HealthCheckNodeState=IDLE",
 					"HealthCheckProgram=/usr/bin/gpu_healthcheck.sh",
 					"JobRequeue=0",
 				} {
 					if !strings.Contains(slurmConf, directive) {
 						t.Errorf("slurm.conf missing system default %q", directive)
 					}
 				}
+				if strings.Contains(slurmConf, "HealthCheckNodeState=ANY") {
+					t.Errorf("slurm.conf must not run health checks on allocated nodes")
+				}
 				if cgroupConf, ok := got.Data[cgroupConfFile]; ok {
 					if !strings.Contains(cgroupConf, "ConstrainRAMSpace=yes") {
 						t.Errorf("cgroup.conf missing ConstrainRAMSpace=yes")