Skip to content

Commit 00ff4d5

Browse files
committed
fix(storage): stabilize benchmark ALTS handshakes and static IRQ affinity
1 parent 98fc304 commit 00ff4d5

1 file changed

Lines changed: 15 additions & 0 deletions

File tree

‎packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ ROUNDS="${ROUNDS:-3}" # Run benchmark 3 rounds by default
1616
BUCKET_TYPE="${BUCKET_TYPE:-rapid}" # "rapid" uses BidiReadObject gRPC DirectPath, "regional" uses REST/gRPC standard
1717
RAPID_BUCKET="${RAPID_BUCKET:-${DEFAULT_RAPID_BUCKET:-gcs-read-bench-zb-us-west4-a}}"
1818
REGIONAL_BUCKET="${REGIONAL_BUCKET:-${DEFAULT_STANDARD_BUCKET:-gcs-read-bench-rb-us-west4}}"
19+
# Limit concurrent ALTS handshakes per worker process to 2 so that 48 worker
20+
# processes never exceed 96 total concurrent streams against the local GCE
21+
# metadata server (169.254.169.254:8080, ~100 stream buffer limit), preventing
22+
# GOAWAY [11] "Buffers full" errors while still handshaking multiple subchannels.
23+
GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES="${GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES:-2}"
1924
if [ -n "${TARGET_BUCKET:-}" ]; then
2025
if [ "${BUCKET_TYPE}" = "regional" ]; then
2126
REGIONAL_BUCKET="${TARGET_BUCKET}"
@@ -39,6 +44,7 @@ echo " Rounds: ${ROUNDS}"
3944
echo " Bucket Type: ${BUCKET_TYPE}"
4045
echo " Rapid Bucket: gs://${RAPID_BUCKET}"
4146
echo " Regional Bucket: gs://${REGIONAL_BUCKET}"
47+
echo " ALTS Handshakes: ${GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES}"
4248
echo " Output JSON Path: ${OUTPUT_JSON_PATH}"
4349
echo " Upload GCS Path: ${UPLOAD_GCS_PREFIX:-None}"
4450
echo "========================================================================"
@@ -51,6 +57,7 @@ export FILE_SIZE_MIB="${FILE_SIZE_MIB}"
5157
export CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB}"
5258
export ROUNDS="${ROUNDS}"
5359
export BUCKET_TYPE="${BUCKET_TYPE}"
60+
export GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES="${GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES}"
5461

5562
# Navigate to package directory
5663
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -78,6 +85,14 @@ if ! python3 -c "import pytest, psutil, yaml, google.cloud.storage" 2>/dev/null;
7885
pip install -e ".[grpc,testing]"
7986
fi
8087

88+
# Stop background irqbalance daemon and perform a single synchronous one-shot
89+
# IRQ balance pass so that NIC IRQ affinities in /proc/irq/*/smp_affinity_list
90+
# are static before _worker_init() pins worker processes to non-IRQ vCPUs.
91+
if command -v irqbalance &>/dev/null; then
92+
sudo systemctl stop irqbalance 2>/dev/null || true
93+
sudo irqbalance --foreground --oneshot -t 1 2>/dev/null || true
94+
fi
95+
8196
echo "--- 2. Executing pytest benchmark suite (${ROUNDS} rounds) ---"
8297
set +e
8398
python3 -m pytest --benchmark-json="${OUTPUT_JSON_PATH}" \

0 commit comments

Comments
 (0)