From 37a0a0a53b1224d827d855d3f0a05c8df45babf5 Mon Sep 17 00:00:00 2001
From: Austin Fuller <afuller@crusoe.ai>
Date: Fri, 12 Jun 2026 16:18:44 -0400
Subject: [PATCH 1/3] fix: avoid socket recreate, reopen socket after EOF

Never rm/recreate an existing FIFO, and reopen after writer EOF; the old
exit-and-recreate cycle raced a respawning slurmctld into a permanently
blocked fifo_open on an unlinked inode.

Signed-off-by: Austin Fuller <afuller@crusoe.ai>
Changelog: Fixed - Fixed cases where slurmctld would get blocked by
 fifo_open on an unlinked inode from logfile sidecar.
---
 internal/builder/scripts/logfile.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/internal/builder/scripts/logfile.sh b/internal/builder/scripts/logfile.sh
index ff6a02f5b..e8f1af735 100755
--- a/internal/builder/scripts/logfile.sh
+++ b/internal/builder/scripts/logfile.sh
@@ -8,10 +8,12 @@ set -eu
 # SOCKET - Named socket to read from
 
 mkdir -v -p "$(dirname "$SOCKET")"
-rm -f "$SOCKET"
-if ! [ -f "$SOCKET" ]; then
+if ! [ -p "$SOCKET" ]; then
+	rm -f "$SOCKET"
 	mkfifo -m 777 "$SOCKET"
 fi
-while IFS="" read data; do
-	echo "$data"
-done <"$SOCKET"
+
+while true; do
+	cat "$SOCKET" || true
+	sleep 1
+done

From 855653757bb89c475ca7289d92fe22684849c146 Mon Sep 17 00:00:00 2001
From: Sarthak Agrawal <sagrawal@together.ai>
Date: Fri, 28 Aug 2026 15:12:53 -0700
Subject: [PATCH 2/3] chore: bump operator image and chart versions

Publish the FIFO fix under new immutable image and chart versions for Helm rollout.
---
 VERSION                        | 2 +-
 helm/slurm-operator/Chart.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/VERSION b/VERSION
index 2ac9634d3..5b09c67c2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.0.13
+1.0.14
diff --git a/helm/slurm-operator/Chart.yaml b/helm/slurm-operator/Chart.yaml
index cce7bd6d3..19b5d1d1b 100644
--- a/helm/slurm-operator/Chart.yaml
+++ b/helm/slurm-operator/Chart.yaml
@@ -6,7 +6,7 @@ name: slurm-operator
 description: Slurm Operator
 type: application
 appVersion: "25.11"
-version: 1.0.1
+version: 1.0.2
 
 annotations:
   license: Apache-2.0

From 9ad0ff46235b497166cec79ad41daaa4e35cfd25 Mon Sep 17 00:00:00 2001
From: Sarthak Agrawal <sagrawal@together.ai>
Date: Wed, 23 Sep 2026 21:28:13 -0700
Subject: [PATCH 3/3] fix(slurm): render login readiness probes

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 hack/test-external-login.sh                      | 3 ++-
 helm/slurm/Chart.yaml                            | 2 +-
 helm/slurm/templates/login/login-deployment.yaml | 4 ++++
 helm/slurm/values.yaml                           | 4 ++++
 4 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/hack/test-external-login.sh b/hack/test-external-login.sh
index 4d5e9cc9f..c986727a0 100644
--- a/hack/test-external-login.sh
+++ b/hack/test-external-login.sh
@@ -12,12 +12,13 @@ rendered="$(
         --set controller.externalConfig.host=controller.example \
         --set controller.externalConfig.port=6817 \
         --set login.enabled=true \
+        --set-json 'login.readinessProbe={"exec":{"command":["/usr/bin/test","-f","/tmp/ready"]},"periodSeconds":5}' \
         --set slurmKeyRef.name=external-auth \
         --set slurmKeyRef.key=custom.key \
         --show-only templates/login/login-deployment.yaml
 )"
 
-for expected in 'key: "custom.key"' 'path: slurm.key'; do
+for expected in 'key: "custom.key"' 'path: slurm.key' 'readinessProbe:' '/tmp/ready'; do
     if [[ "$rendered" != *"$expected"* ]]; then
         printf 'external login render is missing %q\n' "$expected" >&2
         exit 1
diff --git a/helm/slurm/Chart.yaml b/helm/slurm/Chart.yaml
index 9169c4ba0..b7173df7a 100644
--- a/helm/slurm/Chart.yaml
+++ b/helm/slurm/Chart.yaml
@@ -6,7 +6,7 @@ name: slurm
 description: Slurm Cluster
 type: application
 appVersion: "25.11"
-version: 1.0.3
+version: 1.0.4
 
 annotations:
   license: Apache-2.0
diff --git a/helm/slurm/templates/login/login-deployment.yaml b/helm/slurm/templates/login/login-deployment.yaml
index ae8bb13de..15a79fbb9 100644
--- a/helm/slurm/templates/login/login-deployment.yaml
+++ b/helm/slurm/templates/login/login-deployment.yaml
@@ -85,6 +85,10 @@ spec:
         resources:
           {{- toYaml . | nindent 10 }}
         {{- end }}
+        {{- with .Values.login.readinessProbe }}
+        readinessProbe:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         env:
         - name: SLURM_CLUSTER_NAME
           value: {{ include "slurm.fullname" . }}
diff --git a/helm/slurm/values.yaml b/helm/slurm/values.yaml
index 5cd08ebac..3eec56495 100644
--- a/helm/slurm/values.yaml
+++ b/helm/slurm/values.yaml
@@ -742,6 +742,10 @@ login:
     # limits:
     #   cpu: 2
     #   memory: 4Gi
+  #
+  # -- (object)
+  # Configure the login container readiness probe.
+  readinessProbe: {}
 
 #
 # Slurm compute (slurmd) configurations.