Hopefully fix the spurious "multinode" test once for all

Framework/script/o2-qc-multinode-test.sh

@@ -4,7 +4,8 @@ set -x
 set -u
 set -m
 # Arguments or expected variables
-# UNIQUE_PORT_1 and UNIQUE_PORT_2 must be set and not occupied by another process
+# UNIQUE_PORT_1 must be set; it is a candidate, and the script draws another
+# pair if it turns out to be occupied. UNIQUE_PORT_2 is derived from it.
 # JSON_DIR must be set and point to the directory containing multinode-test.json.
 
 # this is to make sure that we do not leave child processes behind
@@ -26,23 +27,56 @@ then
   echo "UNIQUE_PORT_1 must be set when calling o2-qc-multinode-test.sh"
   exit 1
 fi
-export UNIQUE_TEST_NAME="multinode-test-${UNIQUE_PORT_1}"
+if [ -z "$JSON_DIR" ]
+then
+  echo "JSON_DIR must be set when calling o2-qc-multinode-test.sh"
+  exit 1
+fi
 
-function check_if_port_in_use() {
-  OS=`uname`
-  if [[ $OS == Linux ]] ; then
-    PORT_PRESENT="$(netstat -tulpn 2>/dev/null | grep LISTEN | grep -w $1)"
-  else #Darwin/BSD
-    PORT_PRESENT="$(netstat -an -ptcp | grep LISTEN | grep -w $1)"
-  fi
-
-  if [[ ! -z "$PORT_PRESENT" ]]; then
-    echo 'Port '$1' is in use, exiting.'
-    echo 'If this port is always used in the build machines, ping the QC developers please'
-    exit 1
-  fi
+# UNIQUE_PORT_1 comes from string(RANDOM) at configure time, so it may be taken
+# by the time the test runs -- and netstat can only look, not claim: binding is
+# the only reliable check. Bind the pair for real, and draw a new even candidate
+# if it is taken. Redraws come from BELOW the ephemeral range (32768+ on Linux,
+# 49152+ on macOS), so the kernel's own outgoing connections cannot land on the
+# port between our probe and FairMQ's bind.
+function pick_port() {
+  python3 - "$1" <<'EOF'
+import random, socket, sys
+
+def bindable(port):
+    held = []
+    try:
+        for p in (port, port + 1):
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.bind(("", p))
+            held.append(s)
+        return True
+    except OSError:
+        return False
+    finally:
+        for s in held:
+            s.close()
+
+for port in [int(sys.argv[1])] + [random.randrange(20000, 30000, 2) for _ in range(50)]:
+    if bindable(port):
+        print(port)
+        sys.exit(0)
+sys.exit(1)
+EOF
 }
 
+configured_port_1=$UNIQUE_PORT_1
+UNIQUE_PORT_1=$(pick_port "$UNIQUE_PORT_1") || { echo "No free port pair found, exiting."; exit 1; }
+if [ "$UNIQUE_PORT_1" != "$configured_port_1" ]; then
+  echo "Port pair ${configured_port_1}/$((configured_port_1+1)) is in use, using ${UNIQUE_PORT_1}/$((UNIQUE_PORT_1+1))."
+fi
+UNIQUE_PORT_2=$((UNIQUE_PORT_1+1))
+export UNIQUE_PORT_1 UNIQUE_PORT_2
+
+# The port also names the QCDB objects (MNLTest$PORT) and the /tmp work area,
+# keeping concurrent runs apart in the database as well as on the socket.
+export UNIQUE_TEST_NAME="multinode-test-${UNIQUE_PORT_1}"
+
 function delete_data() {
   curl -i -L ali-qcdb-test.cern.ch:8083/truncate/qc/TST/MO/MNLTest${UNIQUE_PORT_1}*
   curl -i -L ali-qcdb-test.cern.ch:8083/truncate/qc/TST/MO/MNRTest${UNIQUE_PORT_2}*
@@ -60,14 +94,20 @@ delete_data
 mkdir /tmp/${UNIQUE_TEST_NAME} || { echo "Concurrent usage of the same port ${UNIQUE_PORT_1} detected, exiting"; exit 1; }
 pushd /tmp/${UNIQUE_TEST_NAME}
 
-UNIQUE_PORT_2=$((UNIQUE_PORT_1+1))
-
-check_if_port_in_use $UNIQUE_PORT_1
-check_if_port_in_use $UNIQUE_PORT_2
-if [ -z "$JSON_DIR" ]
-then
-  echo "JSON_DIR must be set when calling o2-qc-multinode-test.sh"
-  exit 1
+# The configure-time port is baked into multinode-test.json, in the bind ports
+# and the task names. If we drew a different one, substitute it in a private
+# copy; the file in JSON_DIR is shared by every run on this machine.
+QC_CONFIG="${JSON_DIR}/multinode-test.json"
+if [ "$UNIQUE_PORT_1" != "$configured_port_1" ]; then
+  QC_CONFIG="/tmp/${UNIQUE_TEST_NAME}/multinode-test.json"
+  python3 - "${JSON_DIR}/multinode-test.json" "$QC_CONFIG" "$configured_port_1" "$UNIQUE_PORT_1" <<'EOF' || { echo "Could not rewrite multinode-test.json."; exit 1; }
+import re, sys
+src, dst, old, new = sys.argv[1:5]
+text = open(src).read()
+for o, n in ((old, new), (str(int(old) + 1), str(int(new) + 1))):
+    text = re.sub(r"(?<!\d)" + o + r"(?!\d)", n, text)
+open(dst, "w").write(text)
+EOF
 fi
 
 # make sure the CCDB is available otherwise we bail (no failure)
@@ -80,8 +120,8 @@ else
 fi
 
 # store data
-timeout -s INT 40s o2-qc --config json://${JSON_DIR}/multinode-test.json -b --remote --run &
-o2-qc-run-producer --producers 2 --message-amount 20  --message-rate 1 -b | timeout -s INT 35s o2-qc --config json://${JSON_DIR}/multinode-test.json -b --local --host localhost --run &
+timeout -s INT 40s o2-qc --config json://${QC_CONFIG} -b --remote --run &
+o2-qc-run-producer --producers 2 --message-amount 20  --message-rate 1 -b | timeout -s INT 35s o2-qc --config json://${QC_CONFIG} -b --local --host localhost --run &
 
 
 # wait until the local QC quits before moving forward.

cmake/GenerateUniquePort.cmake

@@ -12,15 +12,17 @@ include_guard()
 
 
 #
-# o2_generate_unique_port(VAR_NAME) generates a random TCP/UDP port number in
-# the range 30000 - 59999 and puts it under the name specified in the first argument.
+# o2_generate_unique_port(VAR_NAME) generates a random even TCP/UDP port number
+# in the range 20000 - 29998 and puts it under the name specified in the first
+# argument. Below the ephemeral range (32768+ on Linux, 49152+ on macOS), so
+# kernel-assigned source ports cannot collide with it; even, so callers may use
+# port+1 as a companion.
 
 function(o2_generate_unique_port VAR_NAME)
 
-  string(RANDOM LENGTH 1 ALPHABET 345 FIRST_DIGIT)
   string(RANDOM LENGTH 3 ALPHABET 0123456789 OTHER_DIGITS)
   string(RANDOM LENGTH 1 ALPHABET 02468 LAST_DIGIT)
-  string(CONCAT ${VAR_NAME} ${FIRST_DIGIT} ${OTHER_DIGITS} ${LAST_DIGIT})
+  string(CONCAT ${VAR_NAME} 2 ${OTHER_DIGITS} ${LAST_DIGIT})
 
   set(${VAR_NAME} ${${VAR_NAME}} PARENT_SCOPE)