diff --git a/.github/actions/setup-build-cache/action.yml b/.github/actions/setup-build-cache/action.yml new file mode 100644 index 0000000000..6fbd7a4551 --- /dev/null +++ b/.github/actions/setup-build-cache/action.yml @@ -0,0 +1,77 @@ +name: Setup build cache +description: Restore and configure the compiler cache used by CI builds + +inputs: + kind: + description: Cache backend to configure (ccache or bazel) + required: true + cache-key: + description: Cache namespace override for matrix jobs + required: false + +runs: + using: composite + steps: + - name: Restore ccache + if: inputs.kind == 'ccache' + uses: actions/cache@v6 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-ccache-${{ inputs.cache-key || github.job }}--${{ github.sha }} + restore-keys: | + ${{ runner.os }}-ccache-${{ inputs.cache-key || github.job }}-- + + - name: Configure ccache + if: inputs.kind == 'ccache' + shell: bash + run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + brew install ccache + else + sudo apt-get update + sudo apt-get install -y ccache + fi + + mkdir -p "${HOME}/.cache/ccache" "${HOME}/.cache/ccache-bin" + for compiler in cc c++ gcc g++ clang clang++ clang-12 clang++-12; do + ln -sf "$(command -v ccache)" "${HOME}/.cache/ccache-bin/${compiler}" + done + CCACHE_DIR="${HOME}/.cache/ccache" ccache --max-size=2G + CCACHE_DIR="${HOME}/.cache/ccache" ccache --zero-stats + + echo "${HOME}/.cache/ccache-bin" >> "${GITHUB_PATH}" + echo "CCACHE_DIR=${HOME}/.cache/ccache" >> "${GITHUB_ENV}" + echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + echo "CCACHE_COMPILERCHECK=content" >> "${GITHUB_ENV}" + echo "CCACHE_NOHASHDIR=true" >> "${GITHUB_ENV}" + + - name: Restore Bazel repository cache + if: inputs.kind == 'bazel' + uses: actions/cache@v6 + with: + path: | + ~/.cache/bazel-repository + ~/.cache/bazelisk + key: ${{ runner.os }}-bazel-repository-${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock', 'WORKSPACE') }} + restore-keys: | + ${{ runner.os }}-bazel-repository- + + - name: Restore Bazel disk cache + if: inputs.kind == 'bazel' + uses: actions/cache@v6 + with: + path: ~/.cache/bazel-disk + key: ${{ runner.os }}-bazel-disk-${{ github.job }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-bazel-disk-${{ github.job }}- + ${{ runner.os }}-bazel-disk- + + - name: Configure Bazel caches + if: inputs.kind == 'bazel' + shell: bash + run: | + mkdir -p "${HOME}/.cache/bazel-repository" "${HOME}/.cache/bazel-disk" + cat >> "${HOME}/.bazelrc" <- + -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON + -DWITH_RDMA=ON -DWITH_UBRING=ON + -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON + -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON + - name: clang-default + cc: clang + cxx: clang++ + cmake-options: '' + - name: clang-all-options + cc: clang + cxx: clang++ + cmake-options: >- + -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON + -DWITH_RDMA=ON -DWITH_UBRING=ON + -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON + -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache + cache-key: cmake-${{ matrix.name }} - uses: ./.github/actions/install-all-dependencies - - name: gcc with default options - run: | - export CC=gcc && export CXX=g++ - mkdir gcc_build && cd gcc_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - - name: gcc with all options - run: | - export CC=gcc && export CXX=g++ - mkdir gcc_build_all && cd gcc_build_all - cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_UBRING=ON \ - -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON \ - -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - - name: clang with default options + - name: Build + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + CMAKE_OPTIONS: ${{ matrix.cmake-options }} run: | - export CC=clang && export CXX=clang++ - mkdir clang_build && cd clang_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean + read -r -a cmake_options <<< "${CMAKE_OPTIONS}" + cmake -S . -B build -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + "${cmake_options[@]}" + cmake --build build -j ${{env.proc_num}} - - name: clang with all options - run: | - export CC=clang && export CXX=clang++ - mkdir clang_build_all && cd clang_build_all - cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_UBRING=ON \ - -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON \ - -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean + - name: Show ccache statistics + if: always() + run: ccache --show-stats gcc-compile-with-make-protobuf: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - uses: ./.github/actions/install-essential-dependencies - name: protobuf 3.5.1 @@ -115,10 +143,17 @@ jobs: protobuf-install-dir: /protobuf-3.21.12 config-brpc-options: --cc=gcc --cxx=g++ --werror + - name: Show ccache statistics + if: always() + run: ccache --show-stats + gcc-unittest-with-bazel: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel # Install redis-server/mysql-server so the integration tests that fork a # real server (e.g. brpc_redis_unittest) actually run under bazel instead # of skipping. Same shared action the make-based unittest jobs use. @@ -129,6 +164,9 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel - run: sudo apt-get update && sudo apt-get install -y libibverbs-dev - name: root run: | @@ -166,6 +204,9 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - uses: ./.github/actions/install-essential-dependencies - name: protobuf 3.5.1 @@ -192,10 +233,17 @@ jobs: protobuf-install-dir: /protobuf-3.21.12 config-brpc-options: --cc=clang --cxx=clang++ --werror + - name: Show ccache statistics + if: always() + run: ccache --show-stats + clang-unittest-with-bazel: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel # Install redis-server/mysql-server so the forked-server integration tests # actually run under bazel (see gcc-unittest-with-bazel). - uses: ./.github/actions/install-essential-dependencies @@ -210,6 +258,9 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel - run: sudo apt-get update && sudo apt-get install -y libibverbs-dev - name: root run: | @@ -249,6 +300,9 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - uses: ./.github/actions/install-essential-dependencies - uses: ./.github/actions/init-ut-make-config with: @@ -262,11 +316,17 @@ jobs: run: | cd test sh ./run_tests.sh + - name: Show ccache statistics + if: always() + run: ccache --show-stats clang-unittest-asan: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - uses: ./.github/actions/install-essential-dependencies - uses: ./.github/actions/init-ut-make-config with: @@ -284,6 +344,9 @@ jobs: # too slowly, so they flake here (connection refused). Skip just those under ASan; the # redis codec/server tests still run, and the full suite runs in clang-unittest. GTEST_FILTER='-RedisTest.sanity:RedisTest.keys_with_spaces:RedisTest.incr_and_decr:RedisTest.by_components:RedisTest.auth' sh ./run_tests.sh + - name: Show ccache statistics + if: always() + run: ccache --show-stats clang-unittest-bazel-with-babylon-and-new-pb: runs-on: ubuntu-22.04 @@ -295,6 +358,9 @@ jobs: USE_BAZEL_VERSION: "8.3.1" steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel # Install redis-server/mysql-server so the forked-server integration tests # actually run under bazel (see gcc-unittest-with-bazel). - uses: ./.github/actions/install-essential-dependencies diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 36424708ca..9cc89fabc6 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - name: install dependences run: | @@ -41,11 +44,18 @@ jobs: mkdir build && cd build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DWITH_UBRING=ON -DCMAKE_PREFIX_PATH=$(brew --prefix protobuf@21) .. make -j ${{env.proc_num}} && make clean + - name: Show ccache statistics + if: always() + run: ccache --show-stats + compile-with-make-cmake-protobuf29: runs-on: macos-latest # https://github.com/actions/runner-images steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: ccache - name: install dependences run: | @@ -63,8 +73,15 @@ jobs: mkdir build && cd build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DWITH_UBRING=ON -DCMAKE_PREFIX_PATH=$(brew --prefix protobuf@29) .. make -j ${{env.proc_num}} && make clean + - name: Show ccache statistics + if: always() + run: ccache --show-stats + compile-with-bazel: runs-on: macos-latest # https://github.com/actions/runner-images steps: - uses: actions/checkout@v2 + - uses: ./.github/actions/setup-build-cache + with: + kind: bazel - run: bazel build --verbose_failures -- //:brpc -//example/...