diff --git a/Framework/script/o2-qc-multinode-test.sh b/Framework/script/o2-qc-multinode-test.sh index 340f78aee9..b16eefe882 100755 --- a/Framework/script/o2-qc-multinode-test.sh +++ b/Framework/script/o2-qc-multinode-test.sh @@ -4,7 +4,8 @@ set -x set -u set -m # Arguments or expected variables -# UNIQUE_PORT_1 and UNIQUE_PORT_2 must be set and not occupied by another process +# UNIQUE_PORT_1 must be set; it is a candidate, and the script draws another +# pair if it turns out to be occupied. UNIQUE_PORT_2 is derived from it. # JSON_DIR must be set and point to the directory containing multinode-test.json. # this is to make sure that we do not leave child processes behind @@ -26,23 +27,56 @@ then echo "UNIQUE_PORT_1 must be set when calling o2-qc-multinode-test.sh" exit 1 fi -export UNIQUE_TEST_NAME="multinode-test-${UNIQUE_PORT_1}" +if [ -z "$JSON_DIR" ] +then + echo "JSON_DIR must be set when calling o2-qc-multinode-test.sh" + exit 1 +fi -function check_if_port_in_use() { - OS=`uname` - if [[ $OS == Linux ]] ; then - PORT_PRESENT="$(netstat -tulpn 2>/dev/null | grep LISTEN | grep -w $1)" - else #Darwin/BSD - PORT_PRESENT="$(netstat -an -ptcp | grep LISTEN | grep -w $1)" - fi - - if [[ ! -z "$PORT_PRESENT" ]]; then - echo 'Port '$1' is in use, exiting.' - echo 'If this port is always used in the build machines, ping the QC developers please' - exit 1 - fi +# UNIQUE_PORT_1 comes from string(RANDOM) at configure time, so it may be taken +# by the time the test runs -- and netstat can only look, not claim: binding is +# the only reliable check. Bind the pair for real, and draw a new even candidate +# if it is taken. Redraws come from BELOW the ephemeral range (32768+ on Linux, +# 49152+ on macOS), so the kernel's own outgoing connections cannot land on the +# port between our probe and FairMQ's bind. +function pick_port() { + python3 - "$1" <<'EOF' +import random, socket, sys + +def bindable(port): + held = [] + try: + for p in (port, port + 1): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", p)) + held.append(s) + return True + except OSError: + return False + finally: + for s in held: + s.close() + +for port in [int(sys.argv[1])] + [random.randrange(20000, 30000, 2) for _ in range(50)]: + if bindable(port): + print(port) + sys.exit(0) +sys.exit(1) +EOF } +configured_port_1=$UNIQUE_PORT_1 +UNIQUE_PORT_1=$(pick_port "$UNIQUE_PORT_1") || { echo "No free port pair found, exiting."; exit 1; } +if [ "$UNIQUE_PORT_1" != "$configured_port_1" ]; then + echo "Port pair ${configured_port_1}/$((configured_port_1+1)) is in use, using ${UNIQUE_PORT_1}/$((UNIQUE_PORT_1+1))." +fi +UNIQUE_PORT_2=$((UNIQUE_PORT_1+1)) +export UNIQUE_PORT_1 UNIQUE_PORT_2 + +# The port also names the QCDB objects (MNLTest$PORT) and the /tmp work area, +# keeping concurrent runs apart in the database as well as on the socket. +export UNIQUE_TEST_NAME="multinode-test-${UNIQUE_PORT_1}" + function delete_data() { curl -i -L ali-qcdb-test.cern.ch:8083/truncate/qc/TST/MO/MNLTest${UNIQUE_PORT_1}* curl -i -L ali-qcdb-test.cern.ch:8083/truncate/qc/TST/MO/MNRTest${UNIQUE_PORT_2}* @@ -60,14 +94,20 @@ delete_data mkdir /tmp/${UNIQUE_TEST_NAME} || { echo "Concurrent usage of the same port ${UNIQUE_PORT_1} detected, exiting"; exit 1; } pushd /tmp/${UNIQUE_TEST_NAME} -UNIQUE_PORT_2=$((UNIQUE_PORT_1+1)) - -check_if_port_in_use $UNIQUE_PORT_1 -check_if_port_in_use $UNIQUE_PORT_2 -if [ -z "$JSON_DIR" ] -then - echo "JSON_DIR must be set when calling o2-qc-multinode-test.sh" - exit 1 +# The configure-time port is baked into multinode-test.json, in the bind ports +# and the task names. If we drew a different one, substitute it in a private +# copy; the file in JSON_DIR is shared by every run on this machine. +QC_CONFIG="${JSON_DIR}/multinode-test.json" +if [ "$UNIQUE_PORT_1" != "$configured_port_1" ]; then + QC_CONFIG="/tmp/${UNIQUE_TEST_NAME}/multinode-test.json" + python3 - "${JSON_DIR}/multinode-test.json" "$QC_CONFIG" "$configured_port_1" "$UNIQUE_PORT_1" <<'EOF' || { echo "Could not rewrite multinode-test.json."; exit 1; } +import re, sys +src, dst, old, new = sys.argv[1:5] +text = open(src).read() +for o, n in ((old, new), (str(int(old) + 1), str(int(new) + 1))): + text = re.sub(r"(?