diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 8b4e05f47d92..9200f5aa5593 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -39,6 +39,7 @@ pipeline { OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3' OUTPUT_DIR_RELATIVE_JDK21_HADOOP3 = 'output-jdk21-hadoop3' + OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica' PROJECT = 'hbase' PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' @@ -137,6 +138,7 @@ pipeline { stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match" stash name: 'jdk21-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/doesn't-match" + stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match" } } stage ('health checks') { @@ -321,7 +323,7 @@ pipeline { else echo "No archiver directory, skipping compressing." fi -''' + ''' sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', transfers: [ @@ -340,7 +342,7 @@ pipeline { else echo "No test_logs.zip, skipping" fi -''' + ''' // Has to be relative to WORKSPACE. archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -434,7 +436,7 @@ pipeline { else echo "No archiver directory, skipping compressing." fi -''' + ''' sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', transfers: [ @@ -453,7 +455,7 @@ pipeline { else echo "No test_logs.zip, skipping" fi -''' + ''' // Has to be relative to WORKSPACE. archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" @@ -582,7 +584,6 @@ pipeline { } } } - stage ('yetus jdk17 hadoop3 checks') { agent { node { @@ -803,6 +804,81 @@ pipeline { } } } + stage ('hbase read-replica feature checks') { + agent { + node { + label 'hbase' + } + } + when { + anyOf { + branch 'master' + branch 'branch-3' + } + } + environment { + BASEDIR = "${env.WORKSPACE}/component" + OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + } + steps { + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + ''' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/hbase_nightly_read_replica_test.sh" ; then + echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + else + echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Nightly_20Test_20Report/]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of + // test output. See HBASE-26339 for more details. + currentBuild.result = 'UNSTABLE' + } + } + } + post { + always { + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true + + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'read-replica-nightly-test-report.html', + reportName : 'Read Replica Nightly Test Report' + ] + + stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true + } + } + } } // parallel } //stage:_health checks } //stages @@ -820,6 +896,7 @@ pipeline { rm -rf ${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK21_HADOOP3} + rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA} ''' unstash 'general-result' unstash 'jdk8-hadoop2-result' @@ -827,6 +904,7 @@ pipeline { unstash 'jdk11-hadoop3-result' unstash 'jdk17-hadoop3-result' unstash 'jdk21-hadoop3-result' + unstash 'read-replica-result' def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile", @@ -834,6 +912,7 @@ pipeline { "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK21_HADOOP3}/commentfile"] + "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"] echo env.BRANCH_NAME echo env.BUILD_URL echo currentBuild.result diff --git a/dev-support/README.md b/dev-support/README.md index 9dc3f3143616..eb6cec921b43 100644 --- a/dev-support/README.md +++ b/dev-support/README.md @@ -60,6 +60,12 @@ configurations in this directory: - `docker/` -- Dockerfile for CI build environment - `hbase_docker/`, `hbase_docker.sh` -- Docker-based local test cluster +- `hbase_nightly_read_replica_test.sh` -- Builds a Docker image and runs two + HBase clusters (primary + read-replica) to execute end-to-end replication + integration tests via pytest. Optional `--keep-containers` and `--keep-image` + flags skip cleanup on exit. +- `read-replica/` -- Python test suite, Docker Compose configuration, and + cluster config used by `hbase_nightly_read_replica_test.sh` - `adhoc_run_tests/` -- Scripts for running test suites outside CI - `integration-test/` -- Integration test support diff --git a/dev-support/hbase_nightly_read_replica_test.sh b/dev-support/hbase_nightly_read_replica_test.sh new file mode 100755 index 000000000000..528f8fa8e514 --- /dev/null +++ b/dev-support/hbase_nightly_read_replica_test.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Run the read-replica Docker integration test suite. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPLICA_DIR="${SCRIPT_DIR}/read-replica" +OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}" +export HBASE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}" + +KEEP_IMAGE=false +KEEP_CONTAINERS=false + +while [[ $# -gt 0 ]]; do + case "$1" in + -i|--keep-image) + KEEP_IMAGE=true + shift + ;; + -c|--keep-containers) + KEEP_CONTAINERS=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [-i|--keep-image] [-c|--keep-containers]" + exit 1 + ;; + esac +done + +echo "Script dir: ${SCRIPT_DIR}" +echo "Replica dir: ${REPLICA_DIR}" +echo "Output dir: ${OUTPUT_DIR}" +echo "HBase root: ${HBASE_ROOT}" + +echo "Changing to replica dir: REPLICA_DIR" +cd "${REPLICA_DIR}" + +echo "Sourcing environment file: $(pwd)/.env" +set -a +source .env +set +a + +echo "HBASE_IMAGE=${HBASE_IMAGE}" +echo "ACTIVE_CLUSTER_CONF_DIR=${ACTIVE_CLUSTER_CONF_DIR}" +echo "REPLICA_CLUSTER_CONF_DIR=${REPLICA_CLUSTER_CONF_DIR}" +echo "DOCKER_COMPOSE_FILE=${DOCKER_COMPOSE_FILE}" +echo "HBASE_DATA_STORE_ROOT=${HBASE_DATA_STORE_ROOT}" +echo "realpath of HBASE_DATA_STORE_ROOT=$(realpath ${HBASE_DATA_STORE_ROOT})" + +echo "Removing HBase log directories from mounted volumes that may exist from a previous test run:" +echo "ACTIVE_CLUSTER_LOGS_DIR=${ACTIVE_CLUSTER_LOGS_DIR}" +echo "REPLICA_CLUSTER_LOGS_DIR=${REPLICA_CLUSTER_LOGS_DIR}" +rm -rf "${ACTIVE_CLUSTER_LOGS_DIR}" "${REPLICA_CLUSTER_LOGS_DIR}" +mkdir -p "${ACTIVE_CLUSTER_LOGS_DIR}" "${REPLICA_CLUSTER_LOGS_DIR}" +chmod 777 "${ACTIVE_CLUSTER_LOGS_DIR}" "${REPLICA_CLUSTER_LOGS_DIR}" + +# Clone HBase source for Docker build context (Docker COPY doesn't follow symlinks) +echo "Cloning HBase source into ${REPLICA_DIR}/hbase for Docker build context..." +rm -rf "${REPLICA_DIR}/hbase" +git clone --local "${HBASE_ROOT}" "${REPLICA_DIR}/hbase" +rm -rf "${REPLICA_DIR}/hbase/.git" + +cleanup() { + local exit_code=$? + if [ ${exit_code} -ne 0 ]; then + echo "=== FAILURE ===" + echo "An error occurred during this stage in the Jenkins run." + fi + if [ "${KEEP_CONTAINERS}" = "false" ]; then + echo "=== Cleanup: Stopping Docker containers ===" + docker compose -f "${DOCKER_COMPOSE_FILE}" down 2>/dev/null || true + else + echo "=== Cleanup: Keeping Docker containers (--keep-containers) ===" + fi + if [ "${KEEP_IMAGE}" = "false" ]; then + echo "=== Cleanup: Removing Docker image: ${HBASE_IMAGE} ===" + docker rmi --force "${HBASE_IMAGE}" 2>/dev/null || true + else + echo "=== Cleanup: Keeping Docker image: ${HBASE_IMAGE} (--keep-image) ===" + fi + echo "=== Cleanup: Deleting cloned HBase directory: ${REPLICA_DIR}/hbase ===" + rm -rf "${REPLICA_DIR}/hbase" + exit "${exit_code}" +} +trap cleanup EXIT + +# Copy latest proto file from source +echo "Copying latest version of ActiveClusterSuffix.proto to $(pwd)/python/proto/" +cp "${HBASE_ROOT}/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto" \ + python/proto/ + +export PYTHONPATH="$(pwd)" +echo "Set PYTHONPATH=${PYTHONPATH}" + +# Create Python environment +echo "Creating Python environment: .venv" +python3 -m venv .venv +source .venv/bin/activate + +# Install Python dependencies +echo "Installing Python libraries" +pip install --upgrade pip +pip install -r requirements.txt + +# Compile protobuf +echo "Compiling Protobuf" +python3 python/proto/proto_compiler.py + +# Build Docker images +echo "Building hbase-docker image" +./build-images.sh + +# Run read-replica integration test suite +echo "Starting read-replica integration test scripts" +echo "Starting read-replica integration test suite via Pytest..." +pytest --html="${OUTPUT_DIR}/read-replica-nightly-test-report.html" \ + --self-contained-html \ + --junitxml="${OUTPUT_DIR}/read-replica-nightly-test-results.xml" \ + python/test/test_read_replica_feature.py + +echo "=== Success: All read-replica integration tests passed. ===" diff --git a/dev-support/read-replica/.env b/dev-support/read-replica/.env new file mode 100644 index 000000000000..d6edb145d58f --- /dev/null +++ b/dev-support/read-replica/.env @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The name of the HBase Docker image +HBASE_IMAGE=${HBASE_IMAGE:-kgeisz/hbase-docker:read-replica-jenkins} +# The name of the HBase docker container +HBASE_CONTAINER_NAME=hbase-docker +# This is the host running the hbase-docker containers. Use localhost if the containers +# are running locally. If they are started by another container, such as a Jenkins +# container in a Docker-out-of-Docker setup, then try setting this to host.docker.internal. +HBASE_HOST=${HBASE_HOST:-localhost} +# The directory within the docker container that contains the config files +HBASE_CONF_DIR=/opt/hbase/conf +# The directory containing the 'data-store/' directory. +# This directory is mounted with the HBase Docker container. +HBASE_DATA_STORE_ROOT=./tmp-read-replica-data +# The port for the active cluster's HBase UI (used for cluster readiness) +ACTIVE_CLUSTER_PORT=16010 +# The port for the replica cluster's HBase UI (used for cluster readiness) +REPLICA_CLUSTER_PORT=26010 +# Local path to the active cluster's HBase config file. +# This file is part of a mounted volume, so modifying it +# locally also modifies it within the container (and vise-versa). +ACTIVE_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/conf +# Local path to docker volume where HBase sends active cluster logs +ACTIVE_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/logs +# Same as above, except for the replica cluster +REPLICA_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/conf +# Local path to docker volume where HBase sends replica cluster logs +REPLICA_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/logs +# The path to the docker-compose file +DOCKER_COMPOSE_FILE=${HBASE_ROOT}/dev-support/read-replica/docker-compose.yml +# The location of the utils directory within the docker container +CONTAINER_UTILS_DIR=/opt/utils +# The log level for the Python scripts +LOG_LEVEL=DEBUG diff --git a/dev-support/read-replica/Dockerfile b/dev-support/read-replica/Dockerfile new file mode 100755 index 000000000000..08999c51dee0 --- /dev/null +++ b/dev-support/read-replica/Dockerfile @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Stage 0: Cache Maven dependencies +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS cache-stage + +# Switch to user root (UID 0) +USER 0 + +# Install necessary packages for building Maven dependencies +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + maven \ + git \ + hostname \ + diffutils \ + && microdnf clean all + +# Copy the entire source code to cache dependencies +COPY ./hbase /opt/hbase-src + +WORKDIR /opt/hbase-src + +# Download and cache all dependencies +RUN mvn clean install -DskipTests -Dskip.license.check=true + +# Switch to default user (UID 1001) at end of stage +USER 1001 + +# Stage 1: Build the HBase source code +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS build-stage + +# Switch to user root (UID 0) +USER 0 + +# Install necessary build packages +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + maven \ + git \ + hostname \ + diffutils \ + && microdnf clean all + +# Copy the cached Maven dependencies +COPY --from=cache-stage /root/.m2 /root/.m2 + +# Copy the HBase source code +COPY ./hbase /opt/hbase-src + +WORKDIR /opt/hbase-src + +# Build HBase source code using cached dependencies and enable parallel build +RUN mvn clean package -DskipTests -Dskip.license.check=true assembly:single -T 1C + +# Switch to default user (UID 1001) at end of stage +USER 1001 + +# Stage 2: Create the final Docker image +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 + +# Switch to user root (UID 0) +USER 0 + +# Set environment variables +ENV HBASE_HOME=/opt/hbase +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk \ + HBASE_USER=hbase \ + HBASE_CONF_DIR=${HBASE_HOME}/conf \ + HBASE_LIB_DIR=${HBASE_HOME}/lib \ + HBASE_LOGS_DIR=${HBASE_HOME}/logs \ + DATA_DIR=/data-store + +# Install necessary runtime packages +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + unzip \ + gzip \ + wget \ + hostname \ + maven \ + git \ + diffutils \ + vim \ + openssh-clients \ + python3 \ + procps \ + && microdnf clean all + +# Copy the built HBase binaries from the build-stage +COPY --from=build-stage /opt/hbase-src/hbase-assembly/target/hbase-*-bin.tar.gz /opt/hbase-dist/ + +# Extract the full HBase binary distribution (not the client-only assembly) +RUN archive="$(find /opt/hbase-dist -maxdepth 1 -type f -name 'hbase-*-bin.tar.gz' ! -name 'hbase-*-client-bin.tar.gz' -print -quit)" \ + && mkdir -p /opt/hbase \ + && tar -xzf "${archive}" -C /opt/hbase --strip-components=1 \ + && rm -rf /opt/hbase-dist + +COPY --from=build-stage /root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/*/hadoop-mapreduce-client-common-*.jar \ + ${HBASE_HOME}/lib/ + +# Apply custom HBase build steps +RUN sed -i "s,^. export JAVA_HOME.*,export JAVA_HOME=$JAVA_HOME," ${HBASE_CONF_DIR}/hbase-env.sh \ + && sed -E -i 's/(.*)hbase\-daemons\.sh(.*zookeeper)/\1hbase-daemon.sh\2/g' ${HBASE_HOME}/bin/start-hbase.sh \ + && printf "JAVA_HOME=%s\nexport JAVA_HOME\nexport PATH=%s/jre/bin:%s\n" "$JAVA_HOME" "$JAVA_HOME" "$PATH" \ + > /etc/profile.d/defaults.sh \ + && ln -sf ${HBASE_HOME}/bin/* /usr/bin + +# Create HBase user and home directory +RUN useradd -u 1000 -m ${HBASE_USER} \ + && mkdir -p /home/${HBASE_USER} \ + && chown -R ${HBASE_USER}:${HBASE_USER} /opt /home/${HBASE_USER} \ + && mkdir "$DATA_DIR" && chown -R ${HBASE_USER}:${HBASE_USER} "$DATA_DIR" + +# Set permissions for jboss home directory +RUN chown -R ${HBASE_USER}:${HBASE_USER} /home/jboss + +# Copy configuration files - Removed in order to mount config files individually # + +# Expose required ports for HBase and related services +EXPOSE 8000 8080 8085 9090 9095 2181 16000 16010 16020 16030 + +# Set up the utils directory as a volume +VOLUME ["/opt/utils"] + +# Switch to the HBase user (UID 1000) +USER 1000 + +# Create necessary directories for HBase to run +RUN mkdir -p "$DATA_DIR"/hbase "$DATA_DIR"/run "$DATA_DIR"/logs && \ + chmod -R 777 "$DATA_DIR" + +# Add the 'ls -l' alias +RUN echo 'alias ll="ls -l"' >> /home/hbase/.bashrc && \ + echo 'alias ll="ls -l"' >> /home/jboss/.bashrc + +# Start HBase and keep it running +ENTRYPOINT ["/bin/bash", "-c", "${HBASE_HOME}/bin/start-hbase.sh && tail -f ${HBASE_LOGS_DIR}/*.log"] diff --git a/dev-support/read-replica/README.md b/dev-support/read-replica/README.md new file mode 100644 index 000000000000..e4b5c3b436a8 --- /dev/null +++ b/dev-support/read-replica/README.md @@ -0,0 +1,284 @@ + + +# Read-Replica Integration Tests + +Integration test framework for HBase's Read-Replica feature, driven by Jenkins and Docker. + +## Why Docker? + +HBase's `MiniHBaseCluster` cannot support multi-cluster Read-Replica testing because the +`META_TABLE_NAME` static variable is shared across JVMs (see HBASE-29691). This framework +sidesteps that limitation by running two fully isolated HBase clusters in Docker containers +that share a filesystem-based `hbase.rootdir`. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ Jenkins Agent │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ hbase-docker │ │ hbase-docker-2 │ │ +│ │ (Active Cluster)│ │ (Replica Cluster)│ │ +│ │ │ │ │ │ +│ │ ZooKeeper │ │ ZooKeeper │ │ +│ │ HMaster │ │ HMaster │ │ +│ │ RegionServer │ │ RegionServer │ │ +│ │ │ │ │ │ +│ │ read-only=false │ │ read-only=true │ │ +│ │ suffix=(empty) │ │ suffix=replica1 │ │ +│ └────────┬─────────┘ └────────┬─────────┘ │ +│ │ │ │ +│ └────────────┬───────────────┘ │ +│ │ │ +│ ┌─────────▼───────────┐ │ +│ │ Shared Data Store │ │ +│ │ (hbase.rootdir) │ │ +│ │ │ │ +│ │ /data-store/hbase │ │ +│ └─────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ Python Test Scripts │ │ +│ │ (communicate via `docker exec` + HBase Shell) │ │ +│ └──────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +Both clusters mount the same `data-store/hbase` directory as their `hbase.rootdir`. The active +cluster writes data and the replica cluster reads it after explicit `refresh_meta` / +`refresh_hfiles` calls. Each cluster has its own ZooKeeper instance and distinct +`hbase.meta.table.suffix` to avoid meta table collisions. + +## Directory Structure + +``` +read-replica/ +├── .env # Environment variables (image name, ports, paths) +├── Dockerfile # Multi-stage build for the hbase-docker image +├── build-images.sh # Builds the Docker image from the local HBase checkout +├── docker-compose.yml # Defines the two container services +├── requirements.txt # Python dependencies +├── cluster1/ # Active cluster configuration +│ └── conf/ +│ ├── hbase-site.xml # hbase.global.readonly.enabled=false +│ ├── log4j2.properties +│ └── zoo.cfg +├── cluster2/ # Replica cluster configuration +│ └── conf/ +│ ├── hbase-site.xml # hbase.global.readonly.enabled=true, suffix=replica1 +│ ├── log4j2.properties +│ └── zoo.cfg +├── python/ +│ ├── proto/ +│ │ ├── ActiveClusterSuffix.proto # Automatically copied to this location by +│ │ │ # hbase_nightly_read_replica_test.sh +│ │ └── proto_compiler.py # Compiles .proto files into python/proto/generated/ +│ ├── scripts/ +│ │ └── verify_hbase_start.py # Standalone startup verification (not part of pytest suite) +│ ├── src/ +│ │ ├── hbase_docker_client.py # Core client — talks to containers via docker exec +│ │ ├── environment_loader.py # Reads env vars with validation +│ │ ├── logger_config.py # Logging setup +│ │ └── utils.py # Shared test utilities +│ └── test/ +│ ├── test_read_replica_feature.py # Pytest entry point (TestReadReplica class) +│ ├── test_dual_active_cluster_startup.py +│ ├── test_create_drop_behavior.py +│ ├── test_put_get_delete_behavior.py +│ ├── test_read_only_flag_flipping.py +│ ├── test_cannot_promote_second_active_cluster.py +│ └── test_bulkloaded_data_and_region_splits.py +└── utils/ + ├── bulkload.sh # In-container script for ImportTsv + completebulkload + └── tsv_generator.py # Generates random TSV data for bulkloading +``` + +## CI: Jenkins Nightly Pipeline + +**Files:** +- `dev-support/Jenkinsfile` — stage definition (`hbase read-replica feature checks`) +- `dev-support/hbase_nightly_read_replica_test.sh` — test driver script + +### When It Runs + +The read-replica stage runs as part of the HBase nightly build on the `master` and `branch-3` +branches. It executes in parallel alongside the other nightly check stages (Yetus, JDK8/11/17). + +### What the Test Driver Does + +`hbase_nightly_read_replica_test.sh` is invoked by the Jenkins stage and performs these steps: + +| # | Step | Description | +|---|------|-------------| +| 1 | Clone HBase source | `git clone --local` into `read-replica/hbase/` for the Docker build context (Docker COPY can't follow symlinks) | +| 2 | Source `.env` and clean old logs | Loads environment variables and removes log directories from prior runs | +| 3 | Register cleanup trap | On exit: runs `docker compose down` (unless `--keep-containers`), removes the Docker image (unless `--keep-image`), and deletes the cloned source | +| 4 | Copy Protobuf | Copies the latest `ActiveClusterSuffix.proto` from the source tree into `python/proto/` | +| 5 | Set up Python environment | Creates a venv, installs dependencies from `requirements.txt` | +| 6 | Compile Protobuf | Runs `python/proto/proto_compiler.py` | +| 7 | Build Docker image | Runs `build-images.sh` (Maven build + Docker multi-stage build) | +| 8 | Run test suite | Runs `pytest` on `python/test/test_read_replica_feature.py`, producing an HTML report and JUnit XML results (see [Test Suite](#test-suite) below) | + +### On Failure + +If any test fails, the build is marked `UNSTABLE` (not `FAILURE`) so that later result +publishing stages are not skipped. HBase logs from both containers are archived as Jenkins +build artifacts for debugging. + +## Test Suite + +Tests live in `python/test/` and are run via pytest through the `TestReadReplica` class in +`test_read_replica_feature.py`. Each test method delegates to a corresponding module's +`run_test()` function. The class is decorated with `@pytest.mark.flaky(reruns=2, reruns_delay=2)`, +so any failing test is automatically retried up to 2 times before being marked as failed. + +| Test | What it verifies | +|------|-----------------| +| `test_dual_active_cluster_startup` | Exactly one of two active clusters fails to start with an error about another active cluster already existing | +| `test_create_drop_behavior` | `create` and `drop` are rejected on the replica; `refresh_meta` propagates DDL changes | +| `test_put_get_delete_behavior` | `put`, `delete`, and `flush` are rejected on the replica; data propagates after `refresh_hfiles` | +| `test_read_only_flag_flipping` | Clusters can swap roles (15 iterations); `active.cluster.suffix.id` protobuf file stays consistent | +| `test_cannot_promote_second_active_cluster` | Disabling read-only on the replica raises `ReadOnlyTransitionException` while an active cluster exists | +| `test_bulkloaded_data_and_region_splits` | `ImportTsv` + `completebulkload` works on active, is rejected on replica; region splits propagate | + +To run a specific test: + +```bash +pytest python/test/test_read_replica_feature.py::TestReadReplica::test_create_drop_behavior +``` + +### Test Results + +Pytest produces two output files in the `output/` directory: + +- **HTML report** (`read-replica-nightly-test-report.html`) — a self-contained HTML report + published via `publishHTML` in the Jenkinsfile, viewable from the Jenkins build page +- **JUnit XML** (`read-replica-nightly-test-results.xml`) — parsed by Jenkins to display + individual test results in the build UI + +## Key Components + +### HBaseDockerClient + +`python/src/hbase_docker_client.py` — The core interface to HBase containers. It: + +- Executes commands inside containers via `docker exec ... bash -c` +- Wraps HBase Shell commands with retry logic and timeout handling +- Provides assertion helpers (`assert_read_only_error_occurs`, `assert_table_row_count`, etc.) +- Manages cluster lifecycle (start/stop containers, wait for readiness) +- Manipulates `hbase-site.xml` to toggle `hbase.global.readonly.enabled` at runtime + +### .env File + +Defines environment variables consumed by Docker Compose, the build script, and Python tests: + +- `HBASE_IMAGE` — Docker image tag +- `HBASE_DATA_STORE_ROOT` — Host path for the shared data store +- `ACTIVE_CLUSTER_PORT` / `REPLICA_CLUSTER_PORT` — Mapped HBase Master UI ports (16010 / 26010) +- `DOCKER_COMPOSE_FILE` — Absolute path to `docker-compose.yml` + +### Protobuf Verification + +The `ActiveClusterSuffix.proto` message defines the format of the `active.cluster.suffix.id` +file written to the shared data store. Tests compile this proto and deserialize the file to +verify that the recorded active cluster matches the expected configuration after role swaps. + +## Running Locally + +The easiest way to run the tests locally is with `hbase_nightly_read_replica_test.sh`, which +handles environment setup, Docker image builds, and pytest execution automatically. This is also +useful for reproducing test failures seen in CI. + +```bash +# From the repo root — run the full suite +dev-support/hbase_nightly_read_replica_test.sh +``` + +The script accepts two flags for local debugging: + +| Flag | Description | +|------|-------------| +| `-i` / `--keep-image` | Preserves the Docker image after tests finish. This lets you spin up fresh containers with `docker compose up` from the `dev-support/read-replica/` directory without rebuilding the image. | +| `-c` / `--keep-containers` | Preserves the running containers after tests finish. This lets you `docker exec` into the containers to inspect state, logs, and HBase Shell. | + +```bash +# Keep the image and containers for debugging +dev-support/hbase_nightly_read_replica_test.sh --keep-image --keep-containers +``` + +### Running Manually + +If you prefer to run the setup steps yourself (e.g. to skip the Docker image rebuild after the +first run): + +```bash +cd dev-support/read-replica + +# Override CI-specific paths +export HBASE_ROOT="$(cd ../.. && pwd)" + +# Load remaining variables +set -a && source .env && set +a + +# Set Python path +export PYTHONPATH="$(pwd)" + +# Install Python dependencies +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +# Compile protobuf +cp "${HBASE_ROOT}/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto" python/proto/ +python3 python/proto/proto_compiler.py + +# Build Docker image (requires Maven + Docker — skip if reusing a previous image) +./build-images.sh + +# Run the full test suite +pytest python/test/test_read_replica_feature.py + +# Or run a single test +pytest python/test/test_read_replica_feature.py::TestReadReplica::test_create_drop_behavior + +# Clean up +docker compose -f docker-compose.yml down +``` + +### Mounted Volumes + +The data-store directory (`tmp-read-replica-data/`) and log directories (`cluster1/logs/`, +`cluster2/logs/`) are mounted into the containers. Between `docker compose down` and +`docker compose up`, consider removing these directories to start with a clean state: + +```bash +rm -rf tmp-read-replica-data cluster1/logs cluster2/logs +``` + +The shell script automatically cleans the log directories on each run, but the data-store +directory persists across runs. + +**Prerequisites:** Docker, Docker Compose, Python 3, Maven, JDK 17. + +## Related + +- **JIRA:** [HBASE-30087](https://issues.apache.org/jira/browse/HBASE-30087) +- **Read-Replica feature PR:** [#8364](https://github.com/apache/hbase/pull/8364) +- **Background (MiniHBaseCluster limitation):** [HBASE-29691](https://issues.apache.org/jira/browse/HBASE-29691) diff --git a/dev-support/read-replica/build-images.sh b/dev-support/read-replica/build-images.sh new file mode 100755 index 000000000000..fc123dff3391 --- /dev/null +++ b/dev-support/read-replica/build-images.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Load environment variables from .env file +set -a +. ./.env +set +a + +# Check if required environment variables are set +if [ -z "$HBASE_IMAGE" ]; then + echo "Error: HBASE_IMAGE is not set in .env file." + exit 1 +fi + +# HBase source directory +HBASE_SOURCE_DIR="./hbase" + +# Check if HBase source directory exists +if [ ! -d "$HBASE_SOURCE_DIR" ]; then + echo "Error: HBase source directory does not exist at $HBASE_SOURCE_DIR." + exit 1 +fi + +# Run Maven clean to remove previous build artifacts +echo "Running 'mvn clean' in $HBASE_SOURCE_DIR..." +cd "$HBASE_SOURCE_DIR" || exit 1 +mvn clean + +if [ $? -ne 0 ]; then + echo "Error: 'mvn clean' failed." + exit 1 +else + echo "'mvn clean' completed successfully." +fi + +# Return to the original directory +cd - || exit 1 + +# Build HBase Docker image +echo "Building HBase Docker image: ${HBASE_IMAGE}" +docker build -t "${HBASE_IMAGE}" ./ + +if [ $? -ne 0 ]; then + echo "Error: Failed to build HBase Docker image." + exit 1 +else + echo "HBase Docker image built successfully." +fi diff --git a/dev-support/read-replica/cluster1/conf/hbase-site.xml b/dev-support/read-replica/cluster1/conf/hbase-site.xml new file mode 100755 index 000000000000..d47936b45aca --- /dev/null +++ b/dev-support/read-replica/cluster1/conf/hbase-site.xml @@ -0,0 +1,73 @@ + + + + + + hbase.zookeeper.quorum + hbase-docker + + + hbase.zookeeper.property.dataDir + /data-store/zk + + + hbase.rootdir + file:///data-store/hbase + + + hbase.wal.dir + file:///data-store/wal + + + hbase.store.file-tracker.impl + FILE + + + + hbase.master.info.bindAddress + hbase-docker + + + hbase.regionserver.info.bindAddress + hbase-docker + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.rest.port + 8000 + + + + hbase.global.readonly.enabled + false + + + hbase.master.hbck.chore.interval + 0 + + diff --git a/dev-support/read-replica/cluster1/conf/log4j2.properties b/dev-support/read-replica/cluster1/conf/log4j2.properties new file mode 100644 index 000000000000..91957fdc8a31 --- /dev/null +++ b/dev-support/read-replica/cluster1/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t%notEmpty{ %X}] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/dev-support/read-replica/cluster1/conf/zoo.cfg b/dev-support/read-replica/cluster1/conf/zoo.cfg new file mode 100755 index 000000000000..8e281c60a6a6 --- /dev/null +++ b/dev-support/read-replica/cluster1/conf/zoo.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +clientPort=2181 +clientPortAddress=hbase-docker +server.1=hbase-docker:2888:3888 diff --git a/dev-support/read-replica/cluster2/conf/hbase-site.xml b/dev-support/read-replica/cluster2/conf/hbase-site.xml new file mode 100755 index 000000000000..efadf178a22c --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/hbase-site.xml @@ -0,0 +1,77 @@ + + + + + + hbase.zookeeper.quorum + hbase-docker-2 + + + hbase.zookeeper.property.dataDir + /data-store/zk + + + hbase.rootdir + file:///data-store/hbase + + + hbase.wal.dir + file:///data-store/wal + + + hbase.store.file-tracker.impl + FILE + + + + hbase.master.info.bindAddress + hbase-docker-2 + + + hbase.regionserver.info.bindAddress + hbase-docker-2 + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.rest.port + 8000 + + + + hbase.meta.table.suffix + replica1 + + + hbase.global.readonly.enabled + true + + + hbase.master.hbck.chore.interval + 0 + + diff --git a/dev-support/read-replica/cluster2/conf/log4j2.properties b/dev-support/read-replica/cluster2/conf/log4j2.properties new file mode 100644 index 000000000000..91957fdc8a31 --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t%notEmpty{ %X}] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/dev-support/read-replica/cluster2/conf/zoo.cfg b/dev-support/read-replica/cluster2/conf/zoo.cfg new file mode 100755 index 000000000000..49243987a4cb --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/zoo.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +clientPort=2181 +clientPortAddress=hbase-docker-2 +server.1=hbase-docker-2:2888:3888 diff --git a/dev-support/read-replica/docker-compose.yml b/dev-support/read-replica/docker-compose.yml new file mode 100644 index 000000000000..2fd3143f8bd5 --- /dev/null +++ b/dev-support/read-replica/docker-compose.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# version: '3.8' + +# Variables are created in .env file +services: + hbase: + image: ${HBASE_IMAGE} + container_name: ${HBASE_CONTAINER_NAME} + hostname: ${HBASE_CONTAINER_NAME} + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - ${ACTIVE_CLUSTER_PORT}:16010 # Master UI + volumes: + - ./utils:${CONTAINER_UTILS_DIR} + - ${HBASE_DATA_STORE_ROOT}/data-store/hbase:/data-store/hbase + - ${ACTIVE_CLUSTER_CONF_DIR}/hbase-site.xml:${HBASE_CONF_DIR}/hbase-site.xml + - ${ACTIVE_CLUSTER_CONF_DIR}/zoo.cfg:${HBASE_CONF_DIR}/zoo.cfg + - ${ACTIVE_CLUSTER_CONF_DIR}/log4j2.properties:${HBASE_CONF_DIR}/log4j2.properties + - ${ACTIVE_CLUSTER_LOGS_DIR}:/opt/hbase/logs + + hbase2: + image: ${HBASE_IMAGE} + container_name: ${HBASE_CONTAINER_NAME}-2 + hostname: ${HBASE_CONTAINER_NAME}-2 + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - ${REPLICA_CLUSTER_PORT}:16010 # Master UI + volumes: + - ./utils:${CONTAINER_UTILS_DIR} + - ${HBASE_DATA_STORE_ROOT}/data-store/hbase:/data-store/hbase + - ${REPLICA_CLUSTER_CONF_DIR}/hbase-site.xml:${HBASE_CONF_DIR}/hbase-site.xml + - ${REPLICA_CLUSTER_CONF_DIR}/zoo.cfg:${HBASE_CONF_DIR}/zoo.cfg + - ${REPLICA_CLUSTER_CONF_DIR}/log4j2.properties:${HBASE_CONF_DIR}/log4j2.properties + - ${REPLICA_CLUSTER_LOGS_DIR}:/opt/hbase/logs diff --git a/dev-support/read-replica/python/__init__.py b/dev-support/read-replica/python/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/proto/proto_compiler.py b/dev-support/read-replica/python/proto/proto_compiler.py new file mode 100644 index 000000000000..b3f9962c8493 --- /dev/null +++ b/dev-support/read-replica/python/proto/proto_compiler.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Compiles all *.proto files in the 'python/proto' directory. The compiled output is sent to 'python/proto/generated'. +This script removes any existing 'generated' directory and creates a new one every time. +""" +import os +import shutil + +from grpc_tools import protoc +from python.src.logger_config import get_logger + +logger = get_logger(__name__) + + +if __name__ == '__main__': + proto_dir = os.path.dirname(__file__) + generated_dir = os.path.join(proto_dir, 'generated') + + if os.path.exists(generated_dir): + shutil.rmtree(generated_dir) + os.mkdir(generated_dir) + + proto_files = [file for file in os.listdir(proto_dir) if file.endswith('.proto')] + for file in proto_files: + logger.info(f"Compiling {file} and sending output to {generated_dir}") + protoc.main(( + '', + f'-I{proto_dir}', + f'--python_out={generated_dir}/.', + f'--pyi_out={generated_dir}/.', + os.path.join(proto_dir, file), + )) diff --git a/dev-support/read-replica/python/scripts/__init__.py b/dev-support/read-replica/python/scripts/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/scripts/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/scripts/verify_hbase_start.py b/dev-support/read-replica/python/scripts/verify_hbase_start.py new file mode 100644 index 000000000000..4b96f0a0a3db --- /dev/null +++ b/dev-support/read-replica/python/scripts/verify_hbase_start.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies the hbase-docker containers have started properly. For each cluster, the script first +curls the HBase UI until it receives a 200 response and then gets the server status to verify +there are no dead clusters +""" +import argparse + +from python.src import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger +from python.src.utils import (load_env_and_set_up_clients, log_script_start, log_script_end, + add_common_skip_container_stop_or_restart_arg) + +logger = get_logger(__name__) + + +def main(): + start_time = log_script_start(__file__, logger) + + parser = argparse.ArgumentParser() + parser = add_common_skip_container_stop_or_restart_arg(parser) + args = parser.parse_args() + + active_cluster, replica_cluster = load_env_and_set_up_clients(cluster1_name="Active Cluster", + cluster2_name="Read-Replica Cluster") + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + if not args.skip_container_start_or_restart: + HBaseDockerClient.start_or_restart_containers(docker_compose_file=docker_compose_file, + data_store_root=f'{data_store_root}') + + HBaseDockerClient.wait_for_clusters_to_start([active_cluster, replica_cluster]) + + log_script_end(__file__, logger, start_time) + + +if __name__ == "__main__": + main() diff --git a/dev-support/read-replica/python/src/__init__.py b/dev-support/read-replica/python/src/__init__.py new file mode 100644 index 000000000000..380ba83067c2 --- /dev/null +++ b/dev-support/read-replica/python/src/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .hbase_docker_client import HBaseDockerClient +from .environment_loader import get_env +from .logger_config import get_logger diff --git a/dev-support/read-replica/python/src/environment_loader.py b/dev-support/read-replica/python/src/environment_loader.py new file mode 100644 index 000000000000..458f917821c1 --- /dev/null +++ b/dev-support/read-replica/python/src/environment_loader.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + + +def get_env(key, default=None): + """Retrieve environment variables, ensuring they are loaded from the GitHub Actions runner.""" + val = os.environ.get(key, default) + if val is None: + raise RuntimeError(f"Error: Environment variable {key} is not set.") + return val diff --git a/dev-support/read-replica/python/src/hbase_docker_client.py b/dev-support/read-replica/python/src/hbase_docker_client.py new file mode 100644 index 000000000000..8ded58080195 --- /dev/null +++ b/dev-support/read-replica/python/src/hbase_docker_client.py @@ -0,0 +1,659 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ast +import logging +import re +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError + +import docker +import requests +import subprocess +import time +import xml.etree.ElementTree as ET + +from .logger_config import get_logger + +logger = get_logger(__name__) + + +class DockerExecCommandError(Exception): + pass + + +class HBaseShellCommandError(DockerExecCommandError): + pass + + +class DockerExecCommandTimeoutError(DockerExecCommandError): + pass + + +class HBaseInitializationError(Exception): + pass + + +class HBaseDockerClient: + def __init__(self, container_name: str, local_conf: str, hbase_ui_port: int = 16010, + cluster_name: str = "HBase Cluster", max_retries: int = 12, sleep_time: int = 5, + hbase_host: str = "localhost") -> None: + self._container_name = container_name + self._local_conf = local_conf + self._hbase_ui_port = hbase_ui_port + self._cluster_name = cluster_name + self._max_retries = max_retries + self._sleep_time = sleep_time + self._hbase_host = hbase_host + self._docker_client = docker.from_env() + + @property + def name(self) -> str: + return self._cluster_name + + def run_docker_exec_command(self, bash_cmd: str, timeout: int | None = None) -> str: + """ + Uses the Docker SDK to exec a Bash command in the object's Docker container. + Equivalent to: docker exec bash -c + """ + cmd = ["bash", "-c", bash_cmd] + cmd_str = f"docker exec {self._container_name} bash -c {bash_cmd}" + logger.debug(f"Running command on {self._cluster_name}: {cmd_str}") + + try: + container = self._docker_client.containers.get(self._container_name) + + if timeout is not None: + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(container.exec_run, cmd, demux=True) + try: + result = future.result(timeout=timeout) + except FuturesTimeoutError: + raise DockerExecCommandTimeoutError( + f"Command timed out after {timeout}s on {self._cluster_name} " + f"({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + ) + finally: + pool.shutdown(wait=False, cancel_futures=True) + else: + result = container.exec_run(cmd, demux=True) + except DockerExecCommandError: + raise + except docker.errors.DockerException as e: + raise DockerExecCommandError( + f"The following command failed on {self._cluster_name} ({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + f"Docker error: {e}\n" + ) + + exit_code, (stdout, stderr) = result + stdout_str = (stdout or b'').decode('utf-8') + if exit_code != 0: + raise DockerExecCommandError( + f"The following command failed on {self._cluster_name} ({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + f"The command's STDERR was:\n{(stderr or b'').decode('utf-8')}\n" + f"The command's STDOUT was:\n{stdout_str}\n" + ) + return stdout_str + + def run_hbase_shell_command(self, hbase_cmd: str, timeout: int | None = None) -> str: + """ + Uses 'docker exec' to run the provided HBase shell command in the object's Docker container. + The command looks like: docker exec bash -c hbase shell -n <<< "" + """ + hbase_shell_cmd = f'''hbase shell -n <<< "{hbase_cmd}"''' + try: + return self.run_docker_exec_command(hbase_shell_cmd, timeout=timeout) + except DockerExecCommandTimeoutError: + # DockerExecCommandTimeoutError is a subclass of DockerExecCommandError, so we need to make sure + # it's specifically caught and re-raised. Otherwise, it's swallowed when catching DockerExecCommandError + raise + except DockerExecCommandError as e: + raise HBaseShellCommandError(e) + + def _get_pid_from_jps(self, process_name: str) -> int | None: + """Runs jps inside the container and returns the PID of the named process, or None.""" + try: + output = self.run_docker_exec_command("jps") + for line in output.strip().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1] == process_name: + return int(parts[0]) + except DockerExecCommandError: + pass + return None + + def wait_for_hbase_ui(self) -> bool: + """Checks for a 200 OK on the HBase Master UI.""" + # Read HBASE_HOST from environment, falling back to 'localhost' for host-native execution + url = f"http://{self._hbase_host}:{self._hbase_ui_port}" + logger.info(f"Waiting for HBase UI: {self._cluster_name} on {url}") + last_exception = None + for attempt in range(1, self._max_retries + 1): + try: + response = requests.get(url, timeout=self._sleep_time) + if response.status_code == 200: + logger.info(f"SUCCESS: {self._cluster_name} UI is up.") + return True + except requests.exceptions.RequestException as e: + last_exception = e + logging.info(f"Waiting {self._sleep_time} seconds before requesting HBase UI again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} UI failed to respond after " + f"{self._max_retries} attempts. " + f"Last raised exception was: {last_exception}" + ) + + def wait_for_master_initialization(self) -> bool: + """Waits for the current HMaster process to log 'Master has completed initialization'.""" + logger.info(f"Waiting for Master initialization: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + pid = self._get_pid_from_jps("HMaster") + if pid is not None: + awk_cmd = ( + f"awk '/env:JVM_PID={pid}/{{seen=1; found=0}} " + f"seen && /Master has completed initialization/{{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-master-*.log" + ) + try: + self.run_docker_exec_command(awk_cmd) + logger.info(f"SUCCESS: {self._cluster_name} Master has completed initialization.") + return True + except DockerExecCommandError: + pass + logging.info(f"Waiting {self._sleep_time} seconds before checking Master initialization again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} Master failed to initialize after " + f"{self._max_retries} attempts." + ) + + def wait_for_region_server_initialization(self) -> bool: + """Waits for the current HRegionServer process to log 'Serving as' message.""" + logger.info(f"Waiting for RegionServer initialization: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + pid = self._get_pid_from_jps("HRegionServer") + if pid is not None: + awk_cmd = ( + f"awk '/env:JVM_PID={pid}/{{seen=1; found=0}} " + f"seen && /Serving as {self._container_name},/{{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-regionserver-*.log" + ) + try: + self.run_docker_exec_command(awk_cmd) + logger.info(f"SUCCESS: {self._cluster_name} RegionServer is serving.") + return True + except DockerExecCommandError: + pass + logging.info(f"Waiting {self._sleep_time} seconds before checking RegionServer initialization again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} RegionServer failed to initialize after " + f"{self._max_retries} attempts." + ) + + def check_server_status(self, desired_status: dict | None = None) -> bool: + """Runs 'status' inside the HBase shell and validates the output.""" + if desired_status is None: + desired_status = {'masters': '1', 'region_servers': '1', 'dead_servers': '0'} + logger.info(f"Validating Cluster Status: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + try: + output = self.get_hbase_status() + + # The cluster's status should have 1 active master, 1 region server, + # and no dead servers + validations = { + "Active Master": f"{desired_status['masters']} active master" in output, + "Region Server": f"{desired_status['region_servers']} servers" in output, + "No Dead Servers": f"{desired_status['dead_servers']} dead" in output + } + + if all(validations.values()): + for check, status in validations.items(): + logger.info(f" [PASS] {check}") + logger.info(f"SUCCESS: {self._cluster_name} is fully operational.") + return True + else: + logger.warning(f"{self._cluster_name} is responding, but not all " + f"components are ready...") + logger.info(f"HBase 'status' command output:\n{output}") + + except HBaseShellCommandError: + pass + + logging.info(f"Waiting {self._sleep_time} seconds before getting status on {self.name} again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} shell check failed after {self._max_retries} attempts." + ) + + def get_hbase_status(self) -> str: + logger.debug(f"Getting status of {self.name}") + return self.run_hbase_shell_command("status") + + def wait_for_cluster_to_start(self) -> None: + """curls the cluster's HBase UI to make sure it is up and then makes sure all desired servers are up""" + self.wait_for_hbase_ui() + self.wait_for_master_initialization() + self.wait_for_region_server_initialization() + self.check_server_status() + + def create_table(self, table_name: str, column_family: str) -> bool: + logger.info(f"Creating table '{table_name}' on {self._cluster_name}") + create_cmd = f"create '{table_name}', '{column_family}'" + output = self.run_hbase_shell_command(create_cmd) + + if f"Created table {table_name}" not in output: + logger.error(f"Could not create table '{table_name}' on {self._cluster_name}") + return False + return True + + def disable_table(self, table_name: str) -> None: + logger.debug(f"Disabling table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"disable '{table_name}'") + + def drop_table(self, table_name: str) -> None: + logger.info(f"Dropping table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"drop '{table_name}'") + + def list_tables(self) -> list: + """Gets the list of HBase tables and returns it as a Python list""" + logger.debug(f"Getting the list of tables in HBase on {self.name}") + pattern = r'\[(.*?)\]' + output = self.run_hbase_shell_command("list") + output = output.replace('\n', ' ') + match = re.search(pattern, output) + return ast.literal_eval(match.group(0)) + + def list_regions(self, table_name: str) -> str: + """Gets list of regions and their info for the provided table""" + logger.info(f"Getting list of regions for table '{table_name}'") + return self.run_hbase_shell_command(f"list_regions '{table_name}'") + + def put(self, table_name: str, row: str, column: str, data: str, spec_map: str | None = None) -> None: + """ + Performs an HBase put command. + :param table_name: the table we are inserting data into + :param row: row of the table we are inserting data into + :param column: column of the table we are inserting data into + :param data: the actual data we are inserting (as a string) + :param spec_map: additional attributes input as a string + (e.g. "{ATTRIBUTES=>{'my-key'=>'my-value'}}") + """ + logger.info(f"Adding data to table '{table_name}' on {self.name}") + put_cmd = f"put '{table_name}', '{row}', '{column}', '{data}'" + if spec_map: + put_cmd += f", {spec_map}" + self.run_hbase_shell_command(put_cmd) + + def get(self, table_name: str, row: str, column: str | None = None, spec_map: str | None = None) -> str: + logger.info(f"Getting data from table '{table_name}' on {self.name}") + get_cmd = f"get '{table_name}', '{row}'" + if column: + get_cmd += f", '{column}'" + if spec_map: + get_cmd += f", {spec_map}" + output = self.run_hbase_shell_command(get_cmd) + logger.debug(f"Got data:\n{output}") + return output + + def delete(self, table_name: str, row: str, column: str, timestamp: int | None = None, + spec_map: str | None = None) -> None: + logger.info(f"Deleting data from table '{table_name}' on {self.name}") + delete_cmd = f"delete '{table_name}', '{row}', '{column}'" + if timestamp: + delete_cmd += f", {timestamp}" + if spec_map: + delete_cmd += f", {spec_map}" + self.run_hbase_shell_command(delete_cmd) + + def scan(self, table_name: str, spec_map: str | None = None) -> str: + log_msg = f"Scanning table '{table_name}' on {self.name}" + scan_cmd = f"scan '{table_name}'" + if spec_map: + scan_cmd += f", {spec_map}" + log_msg += f" with spec_map {spec_map}" + logging.info(log_msg) + return self.run_hbase_shell_command(scan_cmd) + + def count(self, table_name: str, spec: str | None = None) -> str: + logger.info(f"Counting rows for table '{table_name}' on {self.name}") + count_cmd = f"count '{table_name}'" + if spec: + count_cmd += f"{spec}" + return self.run_hbase_shell_command(count_cmd) + + def flush(self, table_name: str, timeout: int | None = None) -> None: + logger.debug(f"Flushing table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"flush '{table_name}'", timeout=timeout) + + def split(self, thing_to_split: str, split_key: str | None = None) -> None: + log_msg = f"Splitting '{thing_to_split}'" + split_cmd = f"split '{thing_to_split}'" + + if split_key: + log_msg += f" on key '{split_key}'" + split_cmd += f", '{split_key}'" + + log_msg += f" on {self.name}" + + logger.info(log_msg) + self.run_hbase_shell_command(split_cmd) + + def flush_and_split(self, thing_to_split: str, split_key: str | None = None) -> None: + """ + Flushes the table and triggers an asynchronous region split. Split the entire table or pass a region to split an + individual region. With the second parameter, you can specify an explicit split key for the region. + + thing_to_split - TABLENAME, REGIONNAME, or ENCODED_REGIONNAME + split_key - where to have the region split + """ + self.flush(thing_to_split) + self.split(thing_to_split, split_key) + + def major_compact(self, table_or_region: str, column_family: str | None = None, mob: str | None = None) -> None: + log_msg = f"Running major_compact on '{table_or_region}'" + command = f"major_compact '{table_or_region}'" + + if column_family: + log_msg += f" for column family '{column_family}'" + command += f", '{column_family}'" + + if mob and column_family: + log_msg += " with MOB" + command += f", 'MOB'" + elif mob and not column_family: + log_msg += " with MOB" + command += ", nil, 'MOB'" + + self.run_hbase_shell_command(command) + + def major_compact_and_wait(self, table_or_region: str, column_family: str | None = None, mob: str | None = None, + timeout: int = 30, sleep_time: int = 1) -> bool: + """Triggers major compaction on a table and blocks until it completes.""" + logger.info(f"Triggering major compaction on '{table_or_region}' on {self.name}...") + self.major_compact(table_or_region, column_family, mob) + + start_time = time.time() + while time.time() - start_time < timeout: + output = self.run_hbase_shell_command(f"compaction_state '{table_or_region}'") + + # When all regions finish compacting, compaction_state returns NONE + if "NONE" in output: + logger.info(f"SUCCESS: Major compaction completed for '{table_or_region}'.") + return True + + logger.debug(f"Compaction still in progress for '{table_or_region}'... waiting {sleep_time}s") + time.sleep(sleep_time) + + raise RuntimeError( + f"TIMEOUT: Major compaction on table '{table_or_region}' failed to complete within {timeout} seconds." + ) + + def catalogjanitor_run(self) -> None: + """Forces the CatalogJanitor to immediately clean up split parent regions in hbase:meta.""" + logger.info(f"Running catalogjanitor_run on {self.name}") + self.run_hbase_shell_command("catalogjanitor_run") + + def refresh_meta(self) -> None: + logger.info(f"Refreshing meta on {self.name}") + self.run_hbase_shell_command("refresh_meta") + + def refresh_hfiles(self) -> None: + logger.info(f"Refreshing HFiles on {self.name}") + self.run_hbase_shell_command("refresh_hfiles") + + def refresh_meta_and_hfiles(self) -> None: + """Consecutively runs refresh_meta and refresh_hfiles in the HBase shell""" + self.refresh_meta() + self.refresh_hfiles() + + def enable_read_only_mode(self, run_update_all_config: bool = True) -> None: + """ + Sets hbase.global.readonly.enabled to 'true' in the local hbase-site.xml file and runs update_all_config + to dynamically update the configuration. This method assumes the hbase-site.xml file is a mounted volume + in the docker-compose file, which allows the config file within the docker container to be updated as well. + """ + self._set_read_only_mode(new_read_only_flag=True, run_update_all_config=run_update_all_config) + + def disable_read_only_mode(self, run_update_all_config: bool = True) -> None: + """ + Sets hbase.global.readonly.enabled to 'false' in the local hbase-site.xml file and runs update_all_config + to dynamically update the configuration. This method assumes the hbase-site.xml file is a mounted volume + in the docker-compose file, which allows the config file within the docker container to be updated as well. + """ + self._set_read_only_mode(new_read_only_flag=False, run_update_all_config=run_update_all_config) + + def _set_read_only_mode(self, new_read_only_flag: bool, run_update_all_config: bool = True) -> None: + action = "Enabling" if new_read_only_flag else "Disabling" + conjunction_adverb = "and then" if run_update_all_config else "but not" + logger.info(f"{action} read-only mode in conf for {self.name} " + f"{conjunction_adverb} running update_all_config after") + + new_read_only_flag = str(new_read_only_flag).lower() + self.set_hbase_conf_property_value('hbase.global.readonly.enabled', new_read_only_flag) + actual = self.get_hbase_conf_property_value('hbase.global.readonly.enabled') + assert actual == new_read_only_flag, ( + f"Expected hbase.global.readonly.enabled={new_read_only_flag} on {self.name}, but got '{actual}'" + ) + if run_update_all_config: + self.update_all_config() + + def update_all_config(self) -> None: + logger.debug(f"Running update_all_config on {self.name} to dynamically update the configuration") + self.run_hbase_shell_command("update_all_config") + + def get_hbase_conf_property_value(self, conf_prop: str) -> str | None: + tree = ET.parse(self._local_conf) + root = tree.getroot() + for prop in root.findall('property'): + name_elem = prop.find('name') + if name_elem is not None and name_elem.text == conf_prop: + return prop.find('value').text + + def set_hbase_conf_property_value(self, conf_prop: str, value: str) -> None: + """Sets hbase.global.readonly.enabled to a new value in a local hbase-site.xml file""" + tree = ET.parse(self._local_conf) + root = tree.getroot() + for prop in root.findall('property'): + name_elem = prop.find('name') + if name_elem is not None and name_elem.text == conf_prop: + value_elem = prop.find('value') + if value_elem is not None: + value_elem.text = str(value) + break + tree.write(self._local_conf, encoding='utf-8', xml_declaration=True) + # The conf file is a Docker volume - wait for the updated version to sync + time.sleep(1) + + def assert_read_only_error_occurs(self, cmd_type: str, table_name: str, column: str, + row: str | None = None, data: str | None = None) -> None: + """ + Runs a command on read-only cluster and expects an error to occur as a result. + """ + logger.info(f"Verifying we cannot perform a '{cmd_type}' on {self.name} " + f"since it is in read-only mode") + try: + # This should throw an exception + match cmd_type.lower(): + case 'create': + self.create_table(table_name, column) + case 'drop': + self.drop_table(table_name) + case 'put': + self.put(table_name, row, column, data) + case 'delete': + self.delete(table_name, row, column) + case _: + raise RuntimeError(f"Unexpected command type: {cmd_type}") + + # If we get here, then the command succeeded on the read-replica cluster, which should + # not have happened. + raise RuntimeError(f"Expected {cmd_type} attempt on {self.name} " + f"to result in an error") + except HBaseShellCommandError as e: + # Verify the command we ran on the read-replica cluster produced the expected exception + expected_error = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error in str(e), (f"Expected exception to contain the following: " + f"{expected_error}\n" + f"The actual exception was:\n{e}") + logger.info(f"{cmd_type.capitalize()} attempt on {self.name} failed as expected") + + def assert_table_does_not_exist(self, table_name: str) -> None: + logger.info(f"Verifying '{table_name}' is not in the list of tables on {self.name}") + assert table_name not in self.list_tables(), \ + f"Expected table '{table_name}' to not exist on {self.name}" + + def assert_table_exists(self, table_name: str) -> None: + logger.info(f"Verifying '{table_name}' is in the list of tables on {self.name}") + assert table_name in self.list_tables(), \ + f"Expected table '{table_name}' to exist on {self.name}" + + def assert_table_row_count(self, table_name: str, expected_row_count: int) -> None: + logger.info(f"Verifying table '{table_name}' on {self.name} has {expected_row_count} row(s)") + output = self.count(table_name) + match = re.search(r'^(\d+) row\(s\)$', output, re.MULTILINE) + actual_row_count = int(match.group(1)) if match else None + assert actual_row_count == expected_row_count, \ + (f"Expected table '{table_name}' on {self.name} to have {expected_row_count} row(s). " + f"Instead got {actual_row_count}") + + def assert_get_output(self, table: str, row: str, cf: str, expected_data: str) -> str: + output = self.get(table, row, cf) + assert f"value={expected_data}" in output, \ + f"Expected get command to retrieve a row with value={expected_data}. Output instead was:\n{output}" + return output + + def assert_region_count_for_table(self, table_name: str, expected_region_count: int) -> None: + logger.info(f"Verifying table '{table_name}' has {expected_region_count} region(s)") + output = self.list_regions(table_name) + match = re.search(r'^ (\d+) rows$', output, re.MULTILINE) + actual_region_count = int(match.group(1)) if match else None + assert actual_region_count == expected_region_count, \ + (f"Expected table '{table_name}' on {self.name} to have {expected_region_count} region(s). " + f"Instead got {actual_region_count}") + + @staticmethod + def __run_subprocess_command(command: list | str, error_msg: str, + shell: bool = False) -> subprocess.CompletedProcess: + if shell: + cmd_msg = command + else: + cmd_msg = f"{' '.join(command)}" + logger.info(f"Running: {cmd_msg}") + result = subprocess.run(command, capture_output=True, text=True, shell=shell) + if result.returncode != 0: + raise RuntimeError( + f"Command failed: {cmd_msg}\n" + f"{error_msg} (exit {result.returncode}):\n" + f"STDOUT: {result.stdout}\nSTDERR: {result.stderr}" + ) + return result + + @staticmethod + def wait_for_clusters_to_start(clusters: list) -> None: + for cluster in clusters: + cluster.wait_for_cluster_to_start() + logger.info("=" * 40) + logger.info("ALL CLUSTERS VERIFIED AND READY") + logger.info("=" * 40) + + @staticmethod + def are_containers_running(docker_compose_file: str | None = None) -> bool: + logger.info("Checking if docker containers are running") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["ps", "--status", "running", "-q"] + result = HBaseDockerClient.__run_subprocess_command(command, "Failed to get docker container status") + return bool(result.stdout.strip()) + + @staticmethod + def set_up_data_store_dir(data_store_root: str) -> None: + command = ["mkdir", "-p", f"{data_store_root}/data-store/hbase", f"{data_store_root}/data-store/run", + f"{data_store_root}/data-store/logs", f"{data_store_root}/data-store/zk"] + HBaseDockerClient.__run_subprocess_command(command, + error_msg=f"Failed to create {data_store_root} and its sub-dirs") + command = ["chmod", "-R", "777", f"{data_store_root}"] + HBaseDockerClient.__run_subprocess_command(command, + error_msg=f"Failed to give {data_store_root} " + f"and its sub-dirs full permissions") + + @staticmethod + def remove_data_store_dir(data_store_root: str, sudo: bool = False) -> None: + command = ["rm", "-rf", data_store_root] + if sudo: + command = ["sudo"] + command + logger.info(f"Deleting HBase data root dir at: {data_store_root}") + HBaseDockerClient.__run_subprocess_command(command, + f"Could not delete data store root: {data_store_root}") + + @staticmethod + def start_or_restart_containers(docker_compose_file: str | None = None, data_store_root: str | None = None) -> None: + if data_store_root: + HBaseDockerClient.set_up_data_store_dir(data_store_root) + + if HBaseDockerClient.are_containers_running(docker_compose_file): + logger.info("Restarting docker containers") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["restart"] + action = "restart" + else: + logger.info("Starting docker containers") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["up", "-d"] + action = "start" + + HBaseDockerClient.__run_subprocess_command(command, f"docker compose {action} failed") + logger.info(f"docker compose {action} completed successfully") + + @staticmethod + def start_service(service_name: str, docker_compose_file: str | None = None) -> None: + logger.info(f"Starting docker compose service: {service_name}") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["up", "-d", service_name] + HBaseDockerClient.__run_subprocess_command(command, f"Failed to start service '{service_name}'") + + @staticmethod + def stop_containers(docker_compose_file: str | None = None, data_store_root: str | None = None, + sudo: bool = False) -> None: + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["down"] + logger.info("Stopping docker containers") + HBaseDockerClient.__run_subprocess_command(command, "stop_containers failed") + logger.info("Successfully stopped docker containers") + + if data_store_root: + HBaseDockerClient.remove_data_store_dir(data_store_root, sudo) diff --git a/dev-support/read-replica/python/src/logger_config.py b/dev-support/read-replica/python/src/logger_config.py new file mode 100644 index 000000000000..e914ebe40929 --- /dev/null +++ b/dev-support/read-replica/python/src/logger_config.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import sys + +from dotenv import load_dotenv +from .environment_loader import get_env + +LOG_FORMAT = '%(asctime)s %(levelname)-5s %(module)s.%(funcName)s(%(lineno)d): %(message)s' + +# Load settings from .env file +load_dotenv() + + +def configure_logging(level=get_env('LOG_LEVEL')): + """ + Centralized logging configuration for HBase testing scripts. + """ + # Convert string level ('DEBUG', 'INFO') to integer level (10, 20) + if isinstance(level, str): + numeric_level = logging.getLevelName(level.upper()) + else: + numeric_level = level + + logging.basicConfig( + format=LOG_FORMAT, + level=numeric_level, + handlers=[ + logging.StreamHandler(sys.stdout) + ] + ) + + # Suppress verbose HTTP connection logs from Python Docker SDK and urllib3 + # when the root logger is set to DEBUG or more verbose. + if numeric_level <= logging.DEBUG: + logging.getLogger("urllib3").setLevel(logging.INFO) + logging.getLogger("docker").setLevel(logging.INFO) + + +def get_logger(name): + """ + Helper to get a logger. This can be used to ensure the config + is applied whenever a logger is requested. + """ + # If the root logger has no handlers, configure it now + if not logging.getLogger().hasHandlers(): + configure_logging() + return logging.getLogger(name) diff --git a/dev-support/read-replica/python/src/utils.py b/dev-support/read-replica/python/src/utils.py new file mode 100644 index 000000000000..5a7635c78089 --- /dev/null +++ b/dev-support/read-replica/python/src/utils.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import os +import time + +from dotenv import load_dotenv + +import python.proto.generated.ActiveClusterSuffix_pb2 as acs + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger + +logger = get_logger(__name__) + + +def log_script_start(file: str, script_logger=None): + (script_logger or logger).info(f"========== START {os.path.basename(file)} ==========") + return time.time() + + +def log_script_end(file: str, script_logger=None, start_time=None): + elapsed = "" + if start_time is not None: + total_seconds = int(time.time() - start_time) + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + elapsed = f" ({hours}h {minutes}m {seconds}s)" + (script_logger or logger).info(f"========== END {os.path.basename(file)}{elapsed} ==========") + + +def add_common_drop_existing_tables_arg(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument('-d', '--drop-existing-tables', action='store_true', + help='Drop all existing tables on the active cluster and ' + 'sync the replica cluster by the refreshing meta') + return parser + + +def add_common_new_containers_arg(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument('-n', '--new-containers', action='store_true', + help='Starts or restarts a clean slate of Docker containers with no pre-existing HBase data') + return parser + + +def load_env_and_set_up_clients(cluster1_name: str = "Cluster 1", + cluster2_name: str = "Cluster 2") -> tuple[HBaseDockerClient, HBaseDockerClient]: + load_dotenv() + container_name = get_env("HBASE_CONTAINER_NAME") + + active_cluster = HBaseDockerClient(container_name=container_name, + local_conf=f"{get_env('ACTIVE_CLUSTER_CONF_DIR')}/hbase-site.xml", + hbase_ui_port=get_env('ACTIVE_CLUSTER_PORT'), + cluster_name=cluster1_name, hbase_host=get_env('HBASE_HOST')) + replica_cluster = HBaseDockerClient(container_name=f'{container_name}-2', + local_conf=f"{get_env('REPLICA_CLUSTER_CONF_DIR')}/hbase-site.xml", + hbase_ui_port=get_env('REPLICA_CLUSTER_PORT'), + cluster_name=cluster2_name, hbase_host=get_env('HBASE_HOST')) + return active_cluster, replica_cluster + + +def run_put_and_get(cluster: HBaseDockerClient, table: str, row: str, cf: str, data: str): + cluster.put(table, row, cf, data) + cluster.assert_table_row_count(table, expected_row_count=1) + return cluster.assert_get_output(table, row, cf, expected_data=data) + + +def assert_crud_operations_work_on_active_cluster(cluster: HBaseDockerClient): + table = 'crud-test-table1' + cf = 'cf' + row = 'r1' + data = '1' + + # Create + cluster.create_table(table, cf) + cluster.assert_table_exists(table) + + # Retrieve + run_put_and_get(cluster, table, row, cf, data) + + # "Update" (there are no true updates in HBase) + data = '2' + run_put_and_get(cluster, table, row, cf, data) + + # Delete + # This row has two versions. This only deletes the first version + cluster.delete(table, row, column=f"{cf}:") + cluster.assert_table_row_count(table, expected_row_count=1) + cluster.assert_get_output(table, row, cf, expected_data='1') + + # Delete the final version + cluster.delete(table, row, column=f"{cf}:") + cluster.assert_table_row_count(table, expected_row_count=0) + + # Drop table + cluster.disable_table(table) + cluster.drop_table(table) + cluster.assert_table_does_not_exist(table) + + +def assert_correct_active_cluster_suffix(cluster: HBaseDockerClient, data_store_root: str): + logger.info(f"Verifying active cluster suffix file matches 'hbase.meta.table.suffix' " + f"in conf file for {cluster.name}") + active_cluster_file = f'{data_store_root}/data-store/hbase/active.cluster.suffix.id' + active_cluster_suffix = acs.ActiveClusterSuffix() + + # The active cluster suffix file may not get created right away + retries = 0 + while not os.path.exists(active_cluster_file): + if retries >= 5: + raise RuntimeError(f"Timed out waiting for active cluster file to exist: {active_cluster_file}") + logger.info(f"Waiting for active cluster file to exist: {active_cluster_file}") + time.sleep(1) + retries += 1 + + # Parse the active cluster suffix protobuf message file + with open(active_cluster_file, 'rb') as f: + data = f.read() + header = b'PBUF' + if data.startswith(header): + active_cluster_suffix.ParseFromString(data[len(header):]) + else: + active_cluster_suffix.ParseFromString(data) + actual_suffix = active_cluster_suffix.suffix + + # Assume the meta table suffix is blank if hbase.meta.table.suffix does not exist in HBase conf + expected_suffix = cluster.get_hbase_conf_property_value('hbase.meta.table.suffix') + if expected_suffix is None: + expected_suffix = '' + + # Verify the active cluster suffix file has the expected meta table suffix + assert actual_suffix == expected_suffix, (f"Expected {cluster.name} to have meta table suffix '{expected_suffix}', " + f"but got '{actual_suffix}' instead") + + +def reset_cluster_setup(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient, + new_containers: bool, docker_compose_file: str, data_store_root: str, sudo: bool = False): + """ + Resets the Read-Replica cluster setup where one cluster is the active cluster (read-write mode) and the other + cluster is the replica cluster (read-only mode). + """ + if new_containers: + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root, + sudo=sudo) + + # If the containers are still running, then we need to run update_all_config in the HBase shell to update + # read-only mode on each cluster. Otherwise, we can just modify the conf files and the containers will be restarted + # in the desired read-only mode. + if new_containers: + run_update_all_config = False + else: + run_update_all_config = True + + # First, make sure both clusters are read-only to prevent an error due to trying to have two active clusters + active_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + replica_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + + # Now activate read-write mode on our active cluster + active_cluster.disable_read_only_mode(run_update_all_config=run_update_all_config) + + if new_containers: + HBaseDockerClient.start_or_restart_containers(docker_compose_file=docker_compose_file, + data_store_root=f'{data_store_root}') + HBaseDockerClient.wait_for_clusters_to_start([active_cluster, replica_cluster]) + + +def reset_docker_container_environment(new_containers: bool = False, + cluster1_name: str = "Cluster 1", + cluster2_name: str = "Cluster 2") -> tuple[HBaseDockerClient, HBaseDockerClient]: + if new_containers: + logger.info("Docker containers will be started/restarted with a fresh data store directory") + + cluster1, cluster2 = load_env_and_set_up_clients(cluster1_name, cluster2_name) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + reset_cluster_setup(active_cluster=cluster1, replica_cluster=cluster2, + new_containers=new_containers, docker_compose_file=docker_compose_file, + data_store_root=data_store_root) + + return cluster1, cluster2 + + +def clean_up_tables(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient) -> None: + """ + Drops all tables on the active cluster and then runs 'refresh_meta' on the + read-replica cluster to remove those tables + """ + tables = active_cluster.list_tables() + if tables: + logger.info(f"Removing all existing tables on {active_cluster.name}: {tables}") + for table in tables: + active_cluster.disable_table(table) + active_cluster.drop_table(table) + logger.info(f"Running 'refresh_meta' and 'refresh_hfiles' on {replica_cluster.name} to sync it with " + f"{active_cluster.name}") + replica_cluster.refresh_meta() + replica_cluster.refresh_hfiles() + + +def swap_cluster_roles(new_active_cluster, new_replica_cluster, run_update_all_config=True): + logger.info(f"Making {new_active_cluster.name} the active cluster and " + f"{new_replica_cluster.name} the replica cluster") + new_replica_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + new_active_cluster.disable_read_only_mode(run_update_all_config=run_update_all_config) + + +def create_table_on_active_cluster(active_cluster: HBaseDockerClient, column_family: str): + """Create a new table on the active cluster and assert it exists""" + tables = active_cluster.list_tables() + new_table = f't{len(tables)+1}' + active_cluster.create_table(new_table, column_family) + active_cluster.assert_table_exists(new_table) + return new_table + + +def add_data_to_each_table_on_active_cluster(active_cluster: HBaseDockerClient, tables: list, column_family: str): + """Add data to each table in the active cluster""" + for i, table in enumerate(tables[::-1], 1): + active_cluster.put(table, f'r{i}', column_family, i) + active_cluster.flush(table) + + +def refresh_replica_and_verify_tables(replica_cluster: HBaseDockerClient, new_table: str, tables: list): + """ + Refresh meta and HFiles on the replica cluster, and verify the new table + exists and each table has the correct number of rows + """ + replica_cluster.refresh_meta_and_hfiles() + replica_cluster.assert_table_exists(new_table) + for i, table in enumerate(tables[::-1], 1): + replica_cluster.assert_table_row_count(table, i) + + +def create_table_and_test_active_and_replica_clusters(active_cluster: HBaseDockerClient, + replica_cluster: HBaseDockerClient, + column_family: str): + """ + Creates a new table and iteratively adds data to each existing table, including the new one. + Also verifies expected behavior for the replica cluster, such as verifying the new table is not + on the replica before refreshing meta, and then verify new table and data existence after + refreshing meta and HFiles. + """ + new_table = create_table_on_active_cluster(active_cluster, column_family) + + # The new table should not exist on the replica cluster before refreshing meta + replica_cluster.assert_table_does_not_exist(new_table) + + tables = active_cluster.list_tables() + # HBase sorts table list by string: ['t1', 't10', 't2, ..., 't9'] + # We want the list sorted by creation time, so we're sorting on the integer: ['t1', 't2, ..., 't9', 't10'] + tables.sort(key=lambda x: int(x[1:])) + add_data_to_each_table_on_active_cluster(active_cluster, tables, column_family) + refresh_replica_and_verify_tables(replica_cluster, new_table, tables) diff --git a/dev-support/read-replica/python/test/__init__.py b/dev-support/read-replica/python/test/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/test/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py b/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py new file mode 100644 index 000000000000..ee48e47adf80 --- /dev/null +++ b/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This script tests bulk-loading data with Read-Replica HBase clusters. +""" +import argparse +import time + +from python.src import get_logger, HBaseDockerClient +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import DockerExecCommandError +from python.src.utils import (add_common_new_containers_arg, swap_cluster_roles, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +class Bulkloader: + def __init__(self, bulkload_script: str): + self.bulkload_script = bulkload_script + + def bulkload_data(self, active_cluster: HBaseDockerClient, table_name: str, column_family: str = 'cf', + num_rows: int = 500, initial_row_num: int = 0): + logger.info(f"Running {self.bulkload_script} to bulkload {num_rows} rows into table '{table_name}' " + f"on {active_cluster.name}, starting with row {initial_row_num}") + active_cluster.run_docker_exec_command( + f"{self.bulkload_script} {table_name} {column_family} -n {num_rows} -i {initial_row_num}") + + +def assert_cannot_bulkload_data_onto_replica(bulkloader: Bulkloader, replica_cluster: HBaseDockerClient): + logger.info(f"Verifying data cannot be loaded onto {replica_cluster.name} because read-only mode is enabled") + try: + bulkloader.bulkload_data(replica_cluster, table_name='replica-blt1', column_family='cf') + raise RuntimeError(f"Expected bulkloading data onto replica cluster {replica_cluster.name} " + f"to result in an error") + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting to bulkload data on to a replica cluster:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Bulkload onto replica cluster {replica_cluster.name} failed as expected") + + +def assert_cannot_split_regions_on_replica(replica_cluster: HBaseDockerClient, table: str): + logger.info(f"Verifying regions cannot be split on {replica_cluster.name} because read-only mode is enabled") + try: + replica_cluster.split(table) + raise RuntimeError(f"Expected region split on replica cluster {replica_cluster.name} to result in an error") + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting to split a region on a replica cluster:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Region splitting on replica cluster {replica_cluster.name} failed as expected") + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + + table1 = 'blt1' + table2 = 'blt2' + table3 = 'blt3' + tables = [table1, table2, table3] + container_utils_dir = get_env("CONTAINER_UTILS_DIR") + + bulkloader = Bulkloader(bulkload_script=f"{container_utils_dir}/bulkload.sh") + + logger.info(f"The active cluster is {cluster1.name} and the replica cluster is {cluster2.name}") + + assert_cannot_bulkload_data_onto_replica(bulkloader, replica_cluster=cluster2) + + # Bulkload data to active cluster and verify the data is there + logger.info(f"Bulkloading data to '{table1}' on the active cluster and verifying the data is there") + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table1) + cluster1.assert_table_exists(table1) + cluster1.assert_table_row_count(table1, expected_row_count=500) + + # Replica cluster should not see bulkloaded data until meta and HFiles have been refreshed + logger.info(f"The replica cluster {cluster2.name} should not see bulkloaded data until " + f"meta and HFiles have been refreshed") + cluster2.assert_table_does_not_exist(table1) + cluster2.refresh_meta_and_hfiles() + cluster2.assert_table_exists(table1) + cluster2.assert_table_row_count(table1, expected_row_count=500) + + # Cluster 1 is now a replica and Cluster 2 is now the active cluster + swap_cluster_roles(new_active_cluster=cluster2, new_replica_cluster=cluster1) + + # Bulkload more data into the existing table on Cluster 2 + logger.info(f"Bulkloading more data into the existing table on {cluster2.name}") + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table1, num_rows=300, initial_row_num=500) + cluster2.assert_table_row_count(table1, expected_row_count=800) + + # Cluster 1 should not see the newly bulkloaded data until its meta and HFiles have been refreshed + logger.info(f"The replica cluster {cluster1.name} should not see bulkloaded data until " + f"meta and HFiles have been refreshed") + cluster1.assert_table_row_count(table1, expected_row_count=500) + cluster1.refresh_meta_and_hfiles() + cluster1.assert_table_row_count(table1, expected_row_count=800) + + # Bulkload data into a new table on Cluster 2 + logger.info(f"Bulkloading data into new table '{table2}' on active cluster {cluster2.name}") + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table2, num_rows=600) + cluster2.assert_table_exists(table2) + cluster2.assert_table_row_count(table2, expected_row_count=600) + + # Cluster 1 should not see this new table until after refreshing meta and HFiles + logger.info(f"The replica cluster {cluster1.name} should not see '{table2}' until after refreshing meta and HFiles") + cluster1.assert_table_does_not_exist(table2) + cluster1.refresh_meta_and_hfiles() + cluster1.assert_table_exists(table2) + cluster1.assert_table_row_count(table2, expected_row_count=600) + cluster1.assert_table_row_count(table1, expected_row_count=800) + + # Cluster 1 is back to being the active cluster and Cluster 2 is once again the replica cluster + swap_cluster_roles(new_active_cluster=cluster1, new_replica_cluster=cluster2) + + # Bulkload data onto both existing tables, and a new third table + logger.info(f"Bulkloading data onto '{table1}' and '{table2}', as well as a new table '{table3}'") + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table1, num_rows=400, initial_row_num=800) + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table2, num_rows=600, initial_row_num=600) + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table3, num_rows=1200) + for table in tables: + cluster1.assert_table_row_count(table, expected_row_count=1200) + + # Cluster 2 should see the old row counts for the existing tables. It won't see the new table + # or the updated row counts until after its meta and HFiles have been refreshed. + logger.info(f"The replica cluster {cluster2.name} should not see '{table3}' or updated values for " + f"'{table1}' and '{table2}' until after refreshing meta and HFiles") + cluster2.assert_table_row_count(table1, expected_row_count=800) + cluster2.assert_table_row_count(table2, expected_row_count=600) + cluster2.assert_table_does_not_exist(table3) + cluster2.refresh_meta_and_hfiles() + for table in tables: + cluster2.assert_table_row_count(table, expected_row_count=1200) + + # Cluster 2 is now the active cluster and Cluster 1 is the replica cluster + swap_cluster_roles(new_active_cluster=cluster2, new_replica_cluster=cluster1) + + # Split regions on two tables on the active cluster + for table in [table1, table2]: + logger.info(f"Splitting table '{table}' on {cluster2.name}") + cluster2.flush_and_split(table) + cluster2.major_compact_and_wait(table) + cluster2.catalogjanitor_run() + time.sleep(5) + cluster2.assert_region_count_for_table(table, expected_region_count=2) + + # Bulkload more rows into each table on Cluster 2 and into a new table + table4 = 'blt4' + logger.info(f"Bulkloading data onto '{table1}', '{table2}', and '{table3}', as well as a new table '{table4}'") + for table in tables: + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table, num_rows=1200, initial_row_num=1200) + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table4, num_rows=2400) + tables.append(table4) + for table in tables: + cluster2.assert_table_row_count(table, expected_row_count=2400) + + # The replica cluster should see the old row counts for the existing tables. It won't see the new table + # or the updated row counts until after its meta and HFiles have been refreshed. + logger.info(f"The replica cluster {cluster1.name} should not see '{table4}' or row counts/region splits for " + f"'{table1}', '{table2}', and '{table3}' until after refreshing meta and HFiles") + for table in tables[:-1]: + cluster1.assert_table_row_count(table, expected_row_count=1200) + cluster1.assert_table_does_not_exist(table4) + + # The replica cluster should not see any region splits until after refreshing meta and HFiles + for table in tables[:-1]: + cluster1.assert_region_count_for_table(table, expected_region_count=1) + + # The replica cluster will now see updated row counts and region splits + cluster1.refresh_meta_and_hfiles() + logger.info(f"Replica cluster {cluster1.name} should now see updated row counts and region splits") + for table, num_regions in zip(tables, [2, 2, 1, 1]): + cluster1.assert_table_row_count(table, expected_row_count=2400) + cluster1.assert_region_count_for_table(table, expected_region_count=num_regions) + + # Make Cluster 1 the active cluster and Cluster 2 the replica cluster + swap_cluster_roles(new_active_cluster=cluster1, new_replica_cluster=cluster2) + + assert_cannot_split_regions_on_replica(replica_cluster=cluster2, table=table3) + + # Split regions on the active cluster. The replica cluster won't see the updated region count until meta and HFiles + # have been refreshed + for table, num_regions in zip(tables, [4, 4, 2, 2]): + cluster1.flush_and_split(table) + cluster1.major_compact(table) + cluster1.catalogjanitor_run() + time.sleep(5) + cluster1.assert_region_count_for_table(table, num_regions) + + # Replica cluster still has old region count + cluster2.assert_region_count_for_table(table, num_regions/2) + + # Update the replica cluster and verify new region count + cluster2.refresh_meta_and_hfiles() + cluster2.assert_region_count_for_table(table, num_regions) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py b/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py new file mode 100644 index 000000000000..1450ba575353 --- /dev/null +++ b/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies a cluster cannot be promoted to an active cluster when another active cluster already exists. + +The test starts with two Read-Replica HBase clusters, where one cluster is the active cluster and the other cluster is +the replica cluster. The test tries to promote the replica cluster to a second active cluster and expects an error to +occur. It then verifies this "second active cluster" is still in read-only mode and that data can still be added to the +actual active cluster. + +This test script verifies the fix for: + +HBASE-30220: A replica cluster can have read-only mode disabled even when another active cluster already exists +https://issues.apache.org/jira/browse/HBASE-30220 + +Before implementing the fix for HBASE-30220, a cluster could be promoted to from a replica cluster to an active cluster +even when another active cluster already existed. +""" +import argparse +from time import sleep + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError +from python.src.logger_config import get_logger +from python.src.utils import (assert_crud_operations_work_on_active_cluster, assert_correct_active_cluster_suffix, + add_common_new_containers_arg, clean_up_tables, + create_table_and_test_active_and_replica_clusters, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + +COLUMN_FAMILY = "cf" +EXPECTED_ERROR_MSG = ("ReadOnlyTransitionException: Cannot disable read-only mode because another active cluster " + "already exists on this storage location. The read-only coprocessors have not been removed.") + + +def assert_error_when_trying_to_have_second_active_cluster(replica_cluster: HBaseDockerClient, expected_error: str): + try: + replica_cluster.disable_read_only_mode() + raise RuntimeError(f"Expected an DockerExecCommandError with the following error message:\n\n" + f"{expected_error}") + except DockerExecCommandError as e: + assert expected_error in str(e), (f"Expected DockerExecCommandError to contain the following message:\n\n" + f"{str(expected_error)}\n\n" + f"Got the following message instead:\n\n{str(e)}") + logger.info(f"Successfully prevented {replica_cluster.name} from becoming a second active cluster") + + +def run_test_iteration(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient, data_root: str): + create_table_and_test_active_and_replica_clusters(active_cluster, replica_cluster, column_family='cf') + assert_error_when_trying_to_have_second_active_cluster(replica_cluster, EXPECTED_ERROR_MSG) + + # Cluster should still be in read-only mode after failed transition from read-only to read-write mode + replica_cluster.assert_read_only_error_occurs('create', 'test_table', COLUMN_FAMILY) + + assert_crud_operations_work_on_active_cluster(active_cluster) + + # Demote active cluster to replica and promote original replica to be the new active cluster + active_cluster.enable_read_only_mode() + replica_cluster.disable_read_only_mode() + active_cluster = replica_cluster + + # Wait for active cluster file to be updated and verify its contents + sleep(3) + assert_correct_active_cluster_suffix(active_cluster, data_root) + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + + assert_correct_active_cluster_suffix(cluster1, data_store_root) + clean_up_tables(active_cluster=cluster1, replica_cluster=cluster2) + + test_iterations = 5 + for i in range(1, test_iterations+1): + logger.info(f"---------- Iteration {i} ----------") + if i % 2 == 1: + run_test_iteration(active_cluster=cluster1, replica_cluster=cluster2, data_root=data_store_root) + else: + run_test_iteration(active_cluster=cluster2, replica_cluster=cluster1, data_root=data_store_root) + logger.info(f"Finished iteration {i} of {test_iterations}") + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_create_drop_behavior.py b/dev-support/read-replica/python/test/test_create_drop_behavior.py new file mode 100644 index 000000000000..e1b03f2cf7ba --- /dev/null +++ b/dev-support/read-replica/python/test/test_create_drop_behavior.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests table creation behavior for read-replica clusters. It creates a table on the active +cluster, and then runs refresh_meta on the replica cluster and verifies the table's existence. +It does a similar process when dropping the table on the active cluster. It also verifies +tables cannot be created/dropped on the replica cluster. +""" +import argparse + +from python.src.logger_config import get_logger +from python.src.utils import (add_common_drop_existing_tables_arg, add_common_new_containers_arg, clean_up_tables, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +def test_table_creation_behavior(active_cluster, replica_cluster, table_name, column_family): + # We should not be able to create a new table on the read-replica cluster + replica_cluster.assert_read_only_error_occurs('create', table_name, column_family) + + active_cluster.create_table(table_name, column_family) + + # Read-Replica cluster should not see the newly created table yet + logger.info(f"Verifying {active_cluster.name} now has table '{table_name}', " + f"while {replica_cluster.name} cluster does not") + active_cluster.assert_table_exists(table_name) + replica_cluster.assert_table_does_not_exist(table_name) + + # Read-Replica cluster should now see the newly created table + replica_cluster.refresh_meta() + logger.info(f"Verifying {replica_cluster.name} has table '{table_name}' after refreshing meta") + replica_cluster.assert_table_exists(table_name) + active_cluster.assert_table_exists(table_name) + + # Cannot drop the table on the Read-Replica cluster. A WriteAttemptedOnReadOnlyClusterException should occur + replica_cluster.disable_table(table_name) + replica_cluster.assert_read_only_error_occurs('drop', table_name, column_family) + # The table should still exist on the read-replica cluster since drops are not allowed + replica_cluster.assert_table_exists(table_name) + + # Drop the table on the active cluster + active_cluster.disable_table(table_name) + active_cluster.drop_table(table_name) + + # The read-replica cluster should still have the table that was dropped on the active + # cluster since 'refresh_meta' has not been run yet. + logger.info(f"Verifying {replica_cluster.name} still has table '{table_name}'") + active_cluster.assert_table_does_not_exist(table_name) + replica_cluster.assert_table_exists(table_name) + + # The read-replica cluster no longer has the dropped table after running 'refresh_meta'. + logger.info(f"Verifying {replica_cluster.name} no longer has table '{table_name}' after " + f"refreshing meta") + replica_cluster.refresh_meta() + replica_cluster.assert_table_does_not_exist(table_name) + + +def run_test(drop_existing_tables: bool = False, new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + active_cluster, replica_cluster = reset_docker_container_environment( + new_containers=new_containers, + cluster1_name="Active Cluster", + cluster2_name="Replica Cluster" + ) + + table_name = "t1" + column_family = "cf" + if drop_existing_tables: + clean_up_tables(active_cluster, replica_cluster) + + test_table_creation_behavior(active_cluster, replica_cluster, table_name, column_family) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_drop_existing_tables_arg(parser) + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + drop_existing_tables=parsed_args.drop_existing_tables, + new_containers=parsed_args.new_containers + ) + + +if __name__ == "__main__": + main() diff --git a/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py b/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py new file mode 100644 index 000000000000..9fbc3bb32668 --- /dev/null +++ b/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies that two clusters cannot both start with read-only mode disabled (both as active clusters) +on the same shared data store. One cluster must fail to start, with the HMaster process not +running, and an error logged to the master log. + +Usage: python3 ./python/scripts/test_dual_active_cluster_startup.py +""" +import argparse +import os +import time + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError, HBaseInitializationError +from python.src.logger_config import get_logger +from python.src.utils import load_env_and_set_up_clients, log_script_start, log_script_end + +logger = get_logger(__name__) + +STARTUP_WAIT_SECONDS = 60 +EXPECTED_ERROR_MSG = "Another cluster is running in active (read-write) mode on this storage location" +HMASTER_STARTUP_MARKER = "STARTING service HMaster" +CLUSTER1_SERVICE_NAME = "hbase" +CLUSTER2_SERVICE_NAME = "hbase2" + + +def is_process_running(cluster: HBaseDockerClient, process_name: str) -> bool: + output = cluster.run_docker_exec_command("jps") + return process_name in output + + +def check_cluster_processes(cluster: HBaseDockerClient) -> bool: + hmaster_running = is_process_running(cluster, "HMaster") + logger.info(f" {cluster.name}: HMaster={'running' if hmaster_running else 'down'}") + return hmaster_running + + +def assert_error_in_master_log(cluster: HBaseDockerClient): + logger.info(f"Checking {cluster.name} master log for expected error message") + awk_cmd = ( + f"awk '/{HMASTER_STARTUP_MARKER}/{{found=0}} " + f"index($0, \"{EXPECTED_ERROR_MSG}\"){{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-master-*.log" + ) + try: + cluster.run_docker_exec_command(awk_cmd) + except DockerExecCommandError: + log_tail = cluster.run_docker_exec_command( + "tail -100 /opt/hbase/logs/hbase-*-master-*.log || true" + ) + raise AssertionError( + f"Expected {cluster.name}'s master log to contain " + f"(after the most recent '{HMASTER_STARTUP_MARKER}'):\n" + f" '{EXPECTED_ERROR_MSG}'\n" + f"but it was not found.\nLog tail:\n{log_tail}" + ) + logger.info(f" [PASS] Found expected error message in {cluster.name}'s master log") + + +def wait_for_active_cluster_file(data_store_root: str, timeout_seconds: int = 30) -> None: + file_path = f"{data_store_root}/data-store/hbase/active.cluster.suffix.id" + logger.info(f"Waiting for active cluster suffix id file: {file_path}") + start = time.time() + while not os.path.exists(file_path): + if time.time() - start > timeout_seconds: + raise HBaseInitializationError(f"Timed out after {timeout_seconds}s waiting for: {file_path}") + time.sleep(1) + logger.info(f"Active cluster suffix id file detected: {file_path}") + + +def run_test(clean_up_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = load_env_and_set_up_clients() + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + test_iterations = 4 + for i in range(1, test_iterations+1): + logger.info(f"---------- Iteration {i} ----------") + + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root) + + # Make both clusters an active cluster (read-only disabled) + cluster1.disable_read_only_mode(run_update_all_config=False) + cluster2.disable_read_only_mode(run_update_all_config=False) + + HBaseDockerClient.set_up_data_store_dir(data_store_root) + + # Alternate which cluster starts first + if i % 2 == 1: + first_cluster, second_cluster = cluster1, cluster2 + first_service, second_service = CLUSTER1_SERVICE_NAME, CLUSTER2_SERVICE_NAME + else: + first_cluster, second_cluster = cluster2, cluster1 + first_service, second_service = CLUSTER2_SERVICE_NAME, CLUSTER1_SERVICE_NAME + + # Start the first cluster and wait for it to claim the active role + logger.info(f"Starting {first_cluster.name} first (service: {first_service})") + HBaseDockerClient.start_service(first_service, docker_compose_file=docker_compose_file) + wait_for_active_cluster_file(data_store_root) + + # Start the second cluster — it should detect the existing active cluster and fail + logger.info(f"Starting {second_cluster.name} second (service: {second_service})") + HBaseDockerClient.start_service(second_service, docker_compose_file=docker_compose_file) + + logger.info(f"Waiting {STARTUP_WAIT_SECONDS}s for {second_cluster.name} to attempt startup...") + time.sleep(STARTUP_WAIT_SECONDS) + + logger.info("Checking HBase processes on both clusters") + first_running = check_cluster_processes(first_cluster) + second_running = check_cluster_processes(second_cluster) + + if not first_running: + raise HBaseInitializationError( + f"Expected {first_cluster.name} (started first) to be running, but HMaster is down" + ) + + assert not second_running, ( + f"Expected {second_cluster.name} (started second) to have failed, " + f"but HMaster is still running" + ) + + logger.info(f"[PASS] {first_cluster.name} is running as the active cluster") + logger.info(f"[PASS] {second_cluster.name} failed to start (HMaster is down)") + + assert_error_in_master_log(second_cluster) + + logger.info(f"Finished iteration {i} of {test_iterations}") + + logger.info("=" * 70) + logger.info("TEST PASSED: All dual active cluster startups were correctly rejected") + logger.info("=" * 70) + + if clean_up_containers: + logger.info("Stopping Docker containers and reverting test environment to having " + "one active cluster and one replica cluster") + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root) + cluster1.disable_read_only_mode(run_update_all_config=False) + cluster2.enable_read_only_mode(run_update_all_config=False) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--clean-up-containers', action='store_true', + help='Stop Docker containers and revert cluster configurations to one ' + 'active cluster and one replica cluster after the test finishes') + parsed_args = parser.parse_args(args) + + run_test( + clean_up_containers=parsed_args.clean_up_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_put_get_delete_behavior.py b/dev-support/read-replica/python/test/test_put_get_delete_behavior.py new file mode 100644 index 000000000000..676b197de767 --- /dev/null +++ b/dev-support/read-replica/python/test/test_put_get_delete_behavior.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies data can be added to/deleted from the active cluster and the read-replica cluster +does not see this data until refresh_hfiles has been run. Also verifies put and delete +operations on the read-replica cluster result in an error. +""" +import argparse + +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError, DockerExecCommandTimeoutError +from python.src.logger_config import get_logger +from python.src.utils import (add_common_drop_existing_tables_arg, add_common_new_containers_arg, clean_up_tables, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +def assert_cannot_flush_table_on_replica(replica_cluster: HBaseDockerClient, table: str, timeout: int = 60): + logger.info(f"Verifying table '{table}' cannot be flushed on {replica_cluster.name} " + f"because read-only mode is enabled") + try: + replica_cluster.flush(table, timeout=timeout) + raise RuntimeError(f"Expected flush on replica cluster {replica_cluster.name} to result in an error") + except DockerExecCommandTimeoutError: + raise RuntimeError( + f"TIMEOUT: flush on replica cluster '{replica_cluster.name}' did not complete within " + f"{timeout} seconds. This may indicate HBASE-30301 has not been fixed on this cluster." + ) + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting a flush on replica cluster {replica_cluster.name}:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Flush for table '{table}' on replica cluster {replica_cluster.name} failed as expected") + + +def test_put_delete_behavior(active_cluster, replica_cluster, table_name, column): + # Add data to the table on the active cluster + logger.info(f"Adding data to '{table_name}' on {active_cluster.name} and verifying it exists") + active_cluster.put(table_name, "row1", column, "value1") + active_cluster.assert_table_row_count(table_name, 1) + active_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify the read-replica cluster does not see this new data + logger.info(f"Verifying '{table_name}' on {replica_cluster.name} still has 0 rows") + replica_cluster.assert_table_row_count(table_name, 0) + + # Flush the table's data on the active cluster + logger.info(f"Flushing '{table_name}' on {active_cluster.name} and refreshing meta and " + f"HFiles on {replica_cluster.name}") + active_cluster.flush(table_name) + + # Refresh meta and HFiles, and verify the read-replica cluster now sees the data + logger.info(f"Refreshing meta and HFiles on {replica_cluster.name}") + replica_cluster.refresh_meta_and_hfiles() + logger.info(f"Verifying '{table_name}' on {replica_cluster.name} has data after refreshing HFiles") + replica_cluster.assert_table_row_count(table_name, 1) + replica_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify replica clusters cannot flush tables + assert_cannot_flush_table_on_replica(replica_cluster, table_name) + + # Verify data cannot be added to the table on the read-replica cluster + logger.info(f"Verifying data cannot be added to '{table_name}' on {replica_cluster.name}") + replica_cluster.assert_read_only_error_occurs('put', table_name, column, 'row2', 'value2') + + # Verify data cannot be deleted from the table on the read-replica cluster + logger.info(f"Verifying data cannot be deleted from '{table_name}' on {replica_cluster.name}") + replica_cluster.assert_read_only_error_occurs('delete', table_name, column, 'row2') + + # Delete data from the active cluster + logger.info(f"Deleting row from '{table_name}' on {active_cluster.name} " + f"and verifying it is gone") + active_cluster.delete(table_name, "row1", column) + active_cluster.flush(table_name) + active_cluster.assert_table_row_count(table_name, 0) + + # Verify deleted data still exists on the read-replica cluster + logger.info(f"Verifying deleted row still exists on {replica_cluster.name}") + replica_cluster.assert_table_row_count(table_name, 1) + replica_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify the read-replica cluster no longer has the data after refreshing HFiles + replica_cluster.refresh_hfiles() + replica_cluster.assert_table_row_count(table_name, 0) + + +def run_test(drop_existing_tables: bool = False, new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + active_cluster, replica_cluster = reset_docker_container_environment( + new_containers=new_containers, + cluster1_name="Active Cluster", + cluster2_name="Replica Cluster" + ) + + table_name = "t1" + column_family = "cf" + column = f"{column_family}:c1" + + if drop_existing_tables: + clean_up_tables(active_cluster, replica_cluster) + + # Create a table on the active cluster and have it appear on the read-replica cluster + active_cluster.create_table(table_name, column_family) + replica_cluster.refresh_meta() + + test_put_delete_behavior(active_cluster, replica_cluster, table_name, column) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_drop_existing_tables_arg(parser) + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + drop_existing_tables=parsed_args.drop_existing_tables, + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_read_only_flag_flipping.py b/dev-support/read-replica/python/test/test_read_only_flag_flipping.py new file mode 100644 index 000000000000..87f20ffc43dc --- /dev/null +++ b/dev-support/read-replica/python/test/test_read_only_flag_flipping.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This test starts with two Read-Replica HBase clusters, where one cluster is the active cluster and the other cluster is +the replica cluster. The test creates a table on the active cluster, adds data to the cluster, and verifies this data +is consistent on the replica cluster after refreshing the meta HFiles. It also verifies write operations cannot be +performed on the replica cluster. Then, the two clusters swap roles, where the active cluster becomes a replica and the +former replica becomes the new active cluster. The previous steps then repeat in an iterative fashion. + +This test script verifies behavior for multiple bug fixes: + +1. HBASE-30090: Table on replica cluster not refreshing after flipping read-only flag twice + https://issues.apache.org/jira/browse/HBASE-30090 + + Before implementing this fix, an existing table on a read-replica cluster was not getting updated after making that + cluster the active cluster and then making it read-only again. + +2. HBASE-30180: Can still add data to read-only region after flipping read-only flag multiple times + https://issues.apache.org/jira/browse/HBASE-30180 + + Before implementing this fix, this cluster setup and series of steps would eventually get to a scenario where data + could be added to a table on cluster with read-only mode disabled. +""" +import argparse + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger +from python.src.utils import (assert_correct_active_cluster_suffix, add_common_new_containers_arg, + clean_up_tables, create_table_and_test_active_and_replica_clusters, + log_script_start, log_script_end, reset_docker_container_environment) + +COLUMN_FAMILY = "cf" +logger = get_logger(__name__) + + +def flip_read_only_flag(new_active_cluster: HBaseDockerClient, + new_replica_cluster: HBaseDockerClient): + # Make cluster read-only and verify it cannot create a table or put data + new_replica_cluster.enable_read_only_mode() + new_replica_cluster.assert_read_only_error_occurs('create', 'testTable', COLUMN_FAMILY) + new_replica_cluster.assert_read_only_error_occurs( + 'put', 't1', COLUMN_FAMILY, row='r2', data='2') + + # Make cluster active + new_active_cluster.disable_read_only_mode() + + +def create_table_and_test_clusters_then_flip_read_only_flag(cluster1, cluster2, data_store_root): + create_table_and_test_active_and_replica_clusters(active_cluster=cluster1, replica_cluster=cluster2, + column_family='cf') + flip_read_only_flag(new_active_cluster=cluster2, new_replica_cluster=cluster1) + assert_correct_active_cluster_suffix(cluster2, data_store_root) + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + + test_iterations = 1 + read_only_flag_flips_per_iteration = 15 + for i in range(1, test_iterations + 1): + logger.info(f"---------- Iteration {i} ----------") + if i > 1: + logger.info(f"Ensuring clusters are in proper modes. " + f"Making both clusters a replica, and then making {cluster1.name} the active cluster") + cluster1.enable_read_only_mode() + cluster2.enable_read_only_mode() + cluster1.disable_read_only_mode() + + # Create table on active cluster + clean_up_tables(cluster1, cluster2) + + # One iteration flips the read-only flag on each cluster and then flips it back. + flip_num = 1 + while flip_num <= read_only_flag_flips_per_iteration: + logger.info(f"*** Testing read-only flag flip number {flip_num} ***") + if flip_num % 2 == 1: + # Cluster 1 is active and Cluster 2 is replica + create_table_and_test_clusters_then_flip_read_only_flag(cluster1, cluster2, data_store_root) + else: + # Cluster 2 is active and Cluster 1 is replica + create_table_and_test_clusters_then_flip_read_only_flag(cluster2, cluster1, data_store_root) + logger.info(f"Finished read-only flag flip {flip_num} of {read_only_flag_flips_per_iteration}") + flip_num += 1 + logger.info(f"Finished iteration {i} of {test_iterations}") + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_read_replica_feature.py b/dev-support/read-replica/python/test/test_read_replica_feature.py new file mode 100644 index 000000000000..e07494414549 --- /dev/null +++ b/dev-support/read-replica/python/test/test_read_replica_feature.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import python.test.test_dual_active_cluster_startup as test_dual_active_cluster_startup +import python.test.test_create_drop_behavior as test_create_drop_behavior +import python.test.test_put_get_delete_behavior as test_put_get_delete_behavior +import python.test.test_read_only_flag_flipping as test_read_only_flag_flipping +import python.test.test_cannot_promote_second_active_cluster as test_cannot_promote_second_active_cluster +import python.test.test_bulkloaded_data_and_region_splits as test_bulkloaded_data_and_region_splits + + +@pytest.mark.flaky(reruns=2, reruns_delay=2) +class TestReadReplica: + def test_dual_active_cluster_startup(self): + test_dual_active_cluster_startup.run_test(clean_up_containers=True) + + def test_create_drop_behavior(self): + test_create_drop_behavior.run_test(drop_existing_tables=True, new_containers=True) + + def test_put_get_delete_behavior(self): + test_put_get_delete_behavior.run_test(drop_existing_tables=True, new_containers=True) + + def test_read_only_flag_flipping(self): + test_read_only_flag_flipping.run_test(new_containers=True) + + def test_cannot_promote_second_active_cluster(self): + test_cannot_promote_second_active_cluster.run_test(new_containers=True) + + def test_bulkloaded_data_and_region_splits(self): + test_bulkloaded_data_and_region_splits.run_test(new_containers=True) diff --git a/dev-support/read-replica/requirements.txt b/dev-support/read-replica/requirements.txt new file mode 100644 index 000000000000..9fc19d5f19ca --- /dev/null +++ b/dev-support/read-replica/requirements.txt @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +certifi==2026.2.25 +charset-normalizer==3.4.7 +docker==7.2.0 +grpcio==1.80.0 +grpcio-tools==1.80.0 +idna==3.11 +iniconfig==2.3.0 +Jinja2==3.1.6 +MarkupSafe==3.0.3 +packaging==26.3 +pluggy==1.6.0 +protobuf==6.33.6 +Pygments==2.21.0 +pytest==9.1.1 +pytest-html==4.2.0 +pytest-metadata==3.1.1 +pytest-rerunfailures==16.6.1 +python-dotenv==1.2.2 +requests==2.33.1 +setuptools==82.0.1 +typing_extensions==4.15.0 +urllib3==2.6.3 diff --git a/dev-support/read-replica/utils/bulkload.sh b/dev-support/read-replica/utils/bulkload.sh new file mode 100755 index 000000000000..01c5d0d96ba7 --- /dev/null +++ b/dev-support/read-replica/utils/bulkload.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +usage() { + echo "Usage: $0 [-n|--num-rows NUM_ROWS] [-i|--initial-row-value INITIAL_ROW_VALUE]" + exit 1 +} + +if [ "$#" -lt 2 ]; then + usage +fi + +TABLE_NAME=$1 +COLUMN_FAMILY=$2 +shift 2 + +NUM_ROWS="" +INITIAL_ROW_VALUE="" + +while [ "$#" -gt 0 ]; do + case "$1" in + -n|--num-rows) + NUM_ROWS="$2" + shift 2 + ;; + -i|--initial-row-value) + INITIAL_ROW_VALUE="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +TSV_GENERATOR_ARGS="" +if [ -n "$NUM_ROWS" ]; then + TSV_GENERATOR_ARGS="$TSV_GENERATOR_ARGS -n $NUM_ROWS" +fi +if [ -n "$INITIAL_ROW_VALUE" ]; then + TSV_GENERATOR_ARGS="$TSV_GENERATOR_ARGS -i $INITIAL_ROW_VALUE" +fi + +# Clean up any existing bulkload directories +rm -rf /tmp/bulkload + +# Re-create the necessary directory structure +mkdir -p /tmp/bulkload/tsvdata + +# Generate TSV data and save to the specified directory +python3 /opt/utils/tsv_generator.py /tmp/bulkload/tsvdata $TSV_GENERATOR_ARGS + +# Import TSV data to create HFiles for bulk loading +hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \ + -Dimporttsv.columns=HBASE_ROW_KEY,$COLUMN_FAMILY:col0,$COLUMN_FAMILY:col1,$COLUMN_FAMILY:col2,$COLUMN_FAMILY:col3,$COLUMN_FAMILY:col4,$COLUMN_FAMILY:col5,$COLUMN_FAMILY:col6,$COLUMN_FAMILY:col7,$COLUMN_FAMILY:col8,$COLUMN_FAMILY:col9 \ + -Dimporttsv.bulk.output=/tmp/bulkload/HFiles \ + $TABLE_NAME /tmp/bulkload/tsvdata/output.tsv + +# Bulk load the generated HFiles into the HBase table +hbase completebulkload /tmp/bulkload/HFiles $TABLE_NAME diff --git a/dev-support/read-replica/utils/tsv_generator.py b/dev-support/read-replica/utils/tsv_generator.py new file mode 100644 index 000000000000..cc858e851670 --- /dev/null +++ b/dev-support/read-replica/utils/tsv_generator.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import os +import random + +NUM_COLUMNS = 10 + + +def generate_data(row_key): + columns = [str(random.randint(1, 100)) for _ in range(NUM_COLUMNS)] + return f"{row_key}\t" + "\t".join(columns) + "\n" + + +def main(output_dir, num_rows, initial_row_value): + tsv_file = os.path.join(output_dir, "output.tsv") + + rows_written = 0 + with open(tsv_file, "w") as f: + for i in range(initial_row_value, num_rows+initial_row_value): + row_key = f"row{i}" + f.write(generate_data(row_key)) + rows_written += 1 + + print(f"TSV file generated at {tsv_file} with {rows_written} rows and {NUM_COLUMNS} columns.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate a TSV file with random data for HBase bulk loading.") + parser.add_argument("output_directory", help="Directory to write the output TSV file") + parser.add_argument("-n", "--num-rows", type=int, default=500, help="Number of rows to generate (default: 500)") + parser.add_argument("-i", "--initial-row-value", type=int, default=0, help="Starting row number (default: 0)") + args = parser.parse_args() + + main(args.output_directory, args.num_rows, args.initial_row_value)