From 37a0a0a53b1224d827d855d3f0a05c8df45babf5 Mon Sep 17 00:00:00 2001 From: Austin Fuller Date: Fri, 12 Jun 2026 16:18:44 -0400 Subject: [PATCH 1/2] fix: avoid socket recreate, reopen socket after EOF Never rm/recreate an existing FIFO, and reopen after writer EOF; the old exit-and-recreate cycle raced a respawning slurmctld into a permanently blocked fifo_open on an unlinked inode. Signed-off-by: Austin Fuller Changelog: Fixed - Fixed cases where slurmctld would get blocked by fifo_open on an unlinked inode from logfile sidecar. --- internal/builder/scripts/logfile.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/builder/scripts/logfile.sh b/internal/builder/scripts/logfile.sh index ff6a02f5b..e8f1af735 100755 --- a/internal/builder/scripts/logfile.sh +++ b/internal/builder/scripts/logfile.sh @@ -8,10 +8,12 @@ set -eu # SOCKET - Named socket to read from mkdir -v -p "$(dirname "$SOCKET")" -rm -f "$SOCKET" -if ! [ -f "$SOCKET" ]; then +if ! [ -p "$SOCKET" ]; then + rm -f "$SOCKET" mkfifo -m 777 "$SOCKET" fi -while IFS="" read data; do - echo "$data" -done <"$SOCKET" + +while true; do + cat "$SOCKET" || true + sleep 1 +done From 855653757bb89c475ca7289d92fe22684849c146 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Fri, 28 Aug 2026 15:12:53 -0700 Subject: [PATCH 2/2] chore: bump operator image and chart versions Publish the FIFO fix under new immutable image and chart versions for Helm rollout. --- VERSION | 2 +- helm/slurm-operator/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 2ac9634d3..5b09c67c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.13 +1.0.14 diff --git a/helm/slurm-operator/Chart.yaml b/helm/slurm-operator/Chart.yaml index cce7bd6d3..19b5d1d1b 100644 --- a/helm/slurm-operator/Chart.yaml +++ b/helm/slurm-operator/Chart.yaml @@ -6,7 +6,7 @@ name: slurm-operator description: Slurm Operator type: application appVersion: "25.11" -version: 1.0.1 +version: 1.0.2 annotations: license: Apache-2.0