Skip to content

chore(crc32c): retire direct worker-to-PyPI twine upload in release scripts #18877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ohmayr wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

chore(crc32c): retire direct worker-to-PyPI twine upload in release scripts #18877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions .kokoro/release/crc32c/linux/common.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,3 @@ action {
strip_prefix: "github/google-cloud-python/packages/google-crc32c"
}
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google-cloud-pypi-token-keystore-3"
}
}
}
20 changes: 0 additions & 20 deletions .kokoro/release/crc32c/osx/common.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,3 @@ action {
strip_prefix: "github/google-cloud-python/packages/google-crc32c"
}
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google-cloud-pypi-token-keystore-3"
}
}
}
4 changes: 1 addition & 3 deletions .kokoro/release/crc32c/release-manylinux-wheel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,5 @@ set -eo pipefail

cd github/google-cloud-python/packages/google-crc32c

export PUBLISH_WHEELS="${PUBLISH_WHEELS:-true}"

# Build and publish ManyLinux wheels
# Build ManyLinux wheels
./scripts/manylinux/build.sh
4 changes: 1 addition & 3 deletions .kokoro/release/crc32c/release-osx-wheel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,5 @@ set -eo pipefail
export REPO_ROOT=$(pwd)/github/google-cloud-python/packages/google-crc32c
cd "${REPO_ROOT}"

export PUBLISH_WHEELS="${PUBLISH_WHEELS:-true}"

# Build and publish OSX wheels
# Build OSX wheels
./scripts/osx/build.sh
4 changes: 1 addition & 3 deletions .kokoro/release/crc32c/release-windows-wheel.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@

cd /d %~dp0..\..\..\packages\google-crc32c || goto :error

if "%PUBLISH_WHEELS%"=="" set PUBLISH_WHEELS=true

@echo "Build, Test, and Publish Wheels"
@echo "Build and Test Wheels"
call scripts\windows\build.bat || goto :error

goto :EOF
Expand Down
20 changes: 0 additions & 20 deletions .kokoro/release/crc32c/windows/common.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,3 @@ action {
strip_prefix: "github/google-cloud-python/packages/google-crc32c"
}
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google-cloud-pypi-token-keystore-3"
}
}
}
26 changes: 2 additions & 24 deletions packages/google-crc32c/scripts/manylinux/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,6 @@ MANYLINUX_DIR=$(echo $(cd $(dirname ${0}); pwd))
SCRIPTS_DIR=$(dirname ${MANYLINUX_DIR})
REPO_ROOT=$(dirname ${SCRIPTS_DIR})

if [[ "${PUBLISH_WHEELS}" == "true" ]]; then
KEYSTORE_FILE="${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3"
if [[ ! -s "${KEYSTORE_FILE}" ]]; then
echo "ERROR [pre-flight]: PyPI token missing or empty at '${KEYSTORE_FILE}'." >&2
exit 1
fi
fi

cd $REPO_ROOT
# Add directory as safe to avoid "detected dubious ownership" fatal issue1
git config --global --add safe.directory '*'
Expand All @@ -55,25 +47,11 @@ docker run \
quay.io/pypa/manylinux2014_aarch64 \
/var/code/python-crc32c/scripts/manylinux/build_on_centos.sh

DOCKER_VOLUMES=(--volume "${REPO_ROOT}:/var/code/python-crc32c/")
if [[ -d "/tmpfs" ]]; then
DOCKER_VOLUMES+=(--volume "/tmpfs:/tmpfs")
fi
if [[ -n "${KOKORO_KEYSTORE_DIR}" && -d "${KOKORO_KEYSTORE_DIR}" && "${KOKORO_KEYSTORE_DIR}" != /tmpfs/* ]]; then
DOCKER_VOLUMES+=(--volume "${KOKORO_KEYSTORE_DIR}:${KOKORO_KEYSTORE_DIR}")
fi

docker run \
--rm \
--interactive \
"${DOCKER_VOLUMES[@]}" \
--volume ${REPO_ROOT}:/var/code/python-crc32c/ \
--env REPO_ROOT=/var/code/python-crc32c \
--env BUILD_PYTHON \
--env PUBLISH_WHEELS \
--env KOKORO_KEYSTORE_DIR \
--env KOKORO_GFILE_DIR \
--env KOKORO_BUILD_ID \
--env KOKORO_GITHUB_COMMIT \
--env KOKORO_GITHUB_PULL_REQUEST_NUMBER \
--env BUILD_PYTHON=${BUILD_PYTHON} \
Comment on lines +53 to +55

@gemini-code-assist gemini-code-assist Bot Oct 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The variable ${REPO_ROOT} is not double-quoted, which can lead to word splitting and unexpected behavior if the path contains spaces. Additionally, --env BUILD_PYTHON=${BUILD_PYTHON} can be simplified to --env BUILD_PYTHON to safely pass the environment variable from the host to the container without worrying about quoting or empty values.

Suggested change
--volume ${REPO_ROOT}:/var/code/python-crc32c/ \
--env REPO_ROOT=/var/code/python-crc32c \
--env BUILD_PYTHON \
--env PUBLISH_WHEELS \
--env KOKORO_KEYSTORE_DIR \
--env KOKORO_GFILE_DIR \
--env KOKORO_BUILD_ID \
--env KOKORO_GITHUB_COMMIT \
--env KOKORO_GITHUB_PULL_REQUEST_NUMBER \
--env BUILD_PYTHON=${BUILD_PYTHON} \
--volume "${REPO_ROOT}:/var/code/python-crc32c/" \
--env REPO_ROOT=/var/code/python-crc32c \
--env BUILD_PYTHON \

quay.io/pypa/manylinux2014_x86_64 \
/var/code/python-crc32c/scripts/manylinux/publish_python_wheel.sh
17 changes: 2 additions & 15 deletions packages/google-crc32c/scripts/manylinux/publish_python_wheel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ set -eo pipefail
REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"

# This script runs as a container entrypoint in quay.io/pypa/manylinux2014_x86_64.
# The venv is load-bearing: `releasetool publish-reporter-script` emits bash that
# hardcodes `python3`, which does not exist in this image outside a venv.
# Run twine check inside an isolated Python 3.12 virtual environment.
PYTHON_BIN="/opt/python/cp312-cp312/bin/python"
if [[ ! -x "${PYTHON_BIN}" ]]; then
echo "ERROR: ${PYTHON_BIN} not found; run this inside manylinux2014_x86_64." >&2
Expand Down Expand Up @@ -56,16 +55,4 @@ for VER in ${VERSIONS_TO_CHECK}; do
done

python -m twine check "${REPO_ROOT}/wheels/"*

if [[ "${PUBLISH_WHEELS}" == "true" ]]; then
# Start the releasetool reporter
python -m pip install --upgrade gcp-releasetool
python -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script

# Disable logging
set +x
TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3")
python -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" "${REPO_ROOT}/wheels/"*
else
echo "PUBLISH_WHEELS is not 'true'; skipping PyPI upload after twine check."
fi
echo "Linux wheels are ready for Kokoro / OSS Exit Gate aggregation."
15 changes: 1 addition & 14 deletions packages/google-crc32c/scripts/osx/publish_python_wheel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,4 @@ for VER in $(awk -F': ' '/^versions:/ {print $2}' "${REPO_ROOT}/scripts/python_v
done

"${PYTHON_EXE}" -m twine check "${REPO_ROOT}/wheels/"*

if [[ "${PUBLISH_WHEELS}" == "true" ]]; then
# Start the releasetool reporter
"${PYTHON_EXE}" -m pip install --upgrade gcp-releasetool
"${PYTHON_EXE}" -m releasetool publish-reporter-script > /tmp/publisher-script
source /tmp/publisher-script

# Disable logging
set +x
TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3")
"${PYTHON_EXE}" -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" "${REPO_ROOT}/wheels/"*
else
echo "PUBLISH_WHEELS is not set to 'true'. Skipping releasetool and twine upload (dry-run validation passed)."
fi
echo "macOS wheels are ready for Kokoro / OSS Exit Gate aggregation."
21 changes: 1 addition & 20 deletions packages/google-crc32c/scripts/windows/build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,26 +113,7 @@ py -3 -m pip install --upgrade twine wheel setuptools pkginfo || goto :error
echo "Build the source distribution (sdist) and validate with twine check"
py -3 setup.py sdist || goto :error
py -3 -m twine check dist/* wheels/* || goto :error

if "%PUBLISH_WHEELS%"=="true" (
echo "Start the releasetool reporter"
py -3 -m pip install gcp-releasetool || goto :error
if not exist C:\temp mkdir C:\temp
py -3 -m releasetool publish-reporter-script > C:\temp\publisher-script || goto :error

echo "Disable buffering, so that the logs stream through."
set PYTHONUNBUFFERED=1

echo "## RELEASE WORKFLOW SUCCESSFUL ##"
echo "## Uploading Wheels and sdist ##"

set /p TWINE_PASSWORD=<%KOKORO_KEYSTORE_DIR%/73713_google-cloud-pypi-token-keystore-3
py -3 -m twine upload --skip-existing --username __token__ --password "!TWINE_PASSWORD!" dist/* wheels/* || goto :error
dir wheels
dir dist
) else (
echo "PUBLISH_WHEELS is '%PUBLISH_WHEELS%'. Skipping PyPI upload after sdist and wheel validation."
)
echo "Windows wheels and sdist are ready for Kokoro / OSS Exit Gate aggregation."

goto :EOF

Expand Down
Loading