_unit_test_coverage.yml

name: Coverage Check
description: "Run FastDeploy Unit Tests and Coverage"

on:
  workflow_call:
    inputs:
      DOCKER_IMAGE:
        description: "Build Images"
        required: true
        type: string
        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310"
      FASTDEPLOY_ARCHIVE_URL:
        description: "URL of the compressed FastDeploy code archive."
        required: true
        type: string
      FASTDEPLOY_WHEEL_URL:
        description: "URL of the FastDeploy Wheel."
        required: true
        type: string
      CACHE_DIR:
        description: "Cache Dir Use"
        required: false
        type: string
        default: ""
      MODEL_CACHE_DIR:
        description: "Cache Dir Use"
        required: false
        type: string
        default: ""
    secrets:
      github-token:
        required: true

jobs:
  check_cov_skip:
    uses: ./.github/workflows/check-bypass.yml
    secrets:
      github-token: ${{ secrets.github-token }}
    with:
      workflow-name: coverage

  run_tests_with_coverage:
    runs-on: [self-hosted, GPU-h1z1-2Cards]
    timeout-minutes: 105
    needs: check_cov_skip
    if: ${{ inputs.FASTDEPLOY_WHEEL_URL != '' && needs.check_cov_skip.outputs.can-skip != 'true' }}
    outputs:
      all_cov_file_url: ${{ steps.cov_upload.outputs.all_cov_file_url }}
      unittest_failed_url: ${{ steps.cov_upload.outputs.unittest_failed_url }}
      unittest_logs_url: ${{ steps.cov_upload.outputs.unittest_logs_url }}
      diff_cov_result_json_url: ${{ steps.cov_upload.outputs.diff_cov_result_json_url }}
    steps:
      - name: Code Prepare
        shell: bash
        env:
          docker_image: ${{ inputs.DOCKER_IMAGE }}
          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
        run: |
          set -x
          REPO="https://github.com/${{ github.repository }}.git"
          FULL_REPO="${{ github.repository }}"
          REPO_NAME="${FULL_REPO##*/}"
          BASE_BRANCH="${{ github.base_ref }}"
          docker pull ${docker_image}
          # Clean the repository directory before starting
          docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
          -e "REPO_NAME=${REPO_NAME}" \
          ${docker_image} /bin/bash -c '
            CLEAN_RETRIES=3
            CLEAN_COUNT=0

            while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do
              echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..."
              rm -rf "${REPO_NAME}"* || true
              sleep 2

              # Check if anything matching ${REPO_NAME}* still exists
              if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then
                echo "All ${REPO_NAME}* removed successfully"
                break
              fi

              CLEAN_COUNT=$((CLEAN_COUNT + 1))
            done

            if ls "${REPO_NAME}"* >/dev/null 2>&1; then
              echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts"
              ls -ld "${REPO_NAME}"*
              echo "Attempting force cleanup with find..."
              find /workspace -mindepth 1 -maxdepth 1 -name "${REPO_NAME}*" -type d -exec chmod -R u+rwx {} \; -exec rm -rf {} + 2>/dev/null || true
              if ls "${REPO_NAME}"* >/dev/null 2>&1; then
                echo "ERROR: Force cleanup still failed"
                exit 1
              else
                echo "Force cleanup succeeded"
              fi
            fi
          '

          wget -q --no-proxy ${fd_archive_url} || {
            echo "ERROR: Failed to download archive from ${fd_archive_url}"
            exit 1
          }

          tar --no-same-owner -xf FastDeploy.tar.gz || {
            echo "ERROR: Failed to extract archive"
            exit 1
          }

          rm -rf FastDeploy.tar.gz
          cd FastDeploy
          git config --global user.name "FastDeployCI"
          git config --global user.email "fastdeploy_ci@example.com"
          git log -n 3 --oneline

      - name: Run FastDeploy Unit Tests and Coverage
        shell: bash
        env:
          docker_image: ${{ inputs.DOCKER_IMAGE }}
          fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
          CACHE_DIR: ${{ inputs.CACHE_DIR }}
          BASE_REF: ${{ github.event.pull_request.base.ref }}
          MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }}
          IS_PR: ${{ github.event_name == 'pull_request' }}
        run: |
          if [[ "$IS_PR" == "true" ]]; then
            echo "Running on PR"
          else
            echo "Not a PR"
          fi
          runner_name="${{ runner.name }}"
          CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
          DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
          DEVICE_PORT=$(echo "$DEVICES" | cut -d',' -f1)

          FLASK_PORT=$((8068 + DEVICE_PORT * 100))
          FD_API_PORT=$((8088 + DEVICE_PORT * 100))
          FD_ENGINE_QUEUE_PORT=$((8058 + DEVICE_PORT * 100))
          FD_METRICS_PORT=$((8078 + DEVICE_PORT * 100))
          FD_CACHE_QUEUE_PORT=$((8098 + DEVICE_PORT * 100))
          FD_ROUTER_PORT=$((8048 + DEVICE_PORT * 100))
          FD_CONNECTOR_PORT=$((8038 + DEVICE_PORT * 100))
          FD_RDMA_PORT=$((8028 + DEVICE_PORT * 100))
          echo "Test ENV Parameter:"
          echo "========================================================="
          echo "FLASK_PORT=${FLASK_PORT}"
          echo "FD_API_PORT=${FD_API_PORT}"
          echo "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}"
          echo "FD_METRICS_PORT=${FD_METRICS_PORT}"
          echo "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}"
          echo "FD_ROUTER_PORT=${FD_ROUTER_PORT}"
          echo "FD_CONNECTOR_PORT=${FD_CONNECTOR_PORT}"
          echo "FD_RDMA_PORT=${FD_RDMA_PORT}"
          echo "DEVICES=${DEVICES}"
          echo "========================================================="

          CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}"
          echo "CACHE_DIR is set to ${CACHE_DIR}"
          if [ ! -f "${CACHE_DIR}/gitconfig" ]; then
            touch "${CACHE_DIR}/gitconfig"
          fi

          PORTS=($FLASK_PORT $FD_API_PORT $FD_ENGINE_QUEUE_PORT $FD_METRICS_PORT $FD_CACHE_QUEUE_PORT)
          LOG_FILE="./port_cleanup_$(date +%Y%m%d_%H%M%S).log"
          echo "==== LOG_FILE is ${LOG_FILE} ===="

          echo "==== PORT CLEAN BEFORE TASK RUN ====" | tee -a $LOG_FILE

          for port in "${PORTS[@]}"; do
              PIDS=$(lsof -t -i :$port || true)
              if [ -n "$PIDS" ]; then
                  echo "Port $port is occupied by PID(s): $PIDS" | tee -a $LOG_FILE
                  echo "$PIDS" | xargs -r kill -9
                  echo "Port $port cleared" | tee -a $LOG_FILE
              else
                  echo "Port $port is free" | tee -a $LOG_FILE
              fi
          done

          echo "==== PORT CLEAN COMPLETE ====" | tee -a $LOG_FILE

          echo "========================================================="
          echo "Ensuring no stale container named ${runner_name} ..."
          if [ "$(docker ps -a -q -f name=${runner_name})" ]; then
            echo "Removing stale container: ${runner_name}"
            docker rm -f ${runner_name} || true
          fi

          export RDMA_DEVICES=$(find /dev/infiniband/uverbs* -maxdepth 1 -not -type d | xargs -I{} echo '--device {}:{}')

          docker run --rm --net=host \
          --sysctl kernel.msgmax=1048576 \
          --sysctl kernel.msgmnb=268435456 \
          --name ${runner_name} \
          --cap-add=SYS_PTRACE --cap-add=IPC_LOCK \
          --shm-size=128G \
          ${RDMA_DEVICES} \
          --device=/dev/infiniband/rdma_cm \
          --ulimit memlock=-1:-1 \
          --ulimit nofile=65536:65536 \
          --ulimit nproc=8192:8192 \
          -v $(pwd):/workspace -w /workspace \
          -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \
          -v "${CACHE_DIR}/.cache:/root/.cache" \
          -v "${CACHE_DIR}/ConfigDir:/root/.config" \
          -v "${MODEL_CACHE_DIR}:/ModelData:ro" \
          -e "MODEL_PATH=/ModelData" \
          -e "FD_API_PORT=${FD_API_PORT}" \
          -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
          -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
          -e "FLASK_PORT=${FLASK_PORT}" \
          -e "FD_CACHE_QUEUE_PORT=${FD_CACHE_QUEUE_PORT}" \
          -e "FD_ROUTER_PORT=${FD_ROUTER_PORT}" \
          -e "FD_CONNECTOR_PORT=${FD_CONNECTOR_PORT}" \
          -e "FD_RDMA_PORT=${FD_RDMA_PORT}" \
          -e "CLEAN_CUDA=1" \
          -e TZ="Asia/Shanghai" \
          -e "fd_wheel_url=${fd_wheel_url}" \
          -e "BASE_REF=${BASE_REF}" \
          -e "IS_PR=${IS_PR}" \
          -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
          --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '

          git config --global --add safe.directory /workspace/FastDeploy
          cd FastDeploy
          git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
          pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

          python -m pip install -r scripts/unittest_requirement.txt
          python -m pip install ${fd_wheel_url}
          rm -rf fastdeploy
          # coverage subprocess use
          python -m pip install ${fd_wheel_url} --no-deps --target=/workspace/FastDeploy
          export PYTHONPATH=/workspace/FastDeploy/
          if [ -d "tests/plugins" ]; then
              cd tests/plugins
              python setup.py install
              cd ../..
          else
              echo "Warning: tests/plugins directory not found, skipping setup.py install"
          fi
          export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
          export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
          TEST_EXIT_CODE=0
          bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
          echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
          coverage combine coveragedata/ || echo "No data to combine"
          coverage report
          coverage xml -o python_coverage_all.xml
          COVERAGE_EXIT_CODE=0
          if [[ "$IS_PR" == "true" ]]; then
            echo "Running diff coverage for PR..."
            diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
            # python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
          else
            echo "Running full coverage"
            coverage report -m > full_coverage_report.txt
            python scripts/generate_full_coverage_csv.py full_coverage_report.txt full_coverage_report.csv
          fi
          echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
          '
          if [ -f FastDeploy/exit_code.env ]; then
            cat FastDeploy/exit_code.env >> $GITHUB_ENV
          fi
      - name: Upload coverage and unit test results to BOS
        id: cov_upload
        shell: bash
        env:
          IS_PR: ${{ github.event_name == 'pull_request' }}
          GITHUB_SHA: ${{ github.sha }}
          BRANCH: ${{ github.ref_name }}
          PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          cd FastDeploy
          python -m pip install -q bce-python-sdk==0.9.29
          wget -q --no-proxy --no-check-certificate \
            https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py \
            -O bos_tools.py
          push_file=$(realpath bos_tools.py)

          if [[ "$IS_PR" == "true" ]]; then
            commit_id=${PR_COMMIT_SHA}
            pr_num=${PR_NUMBER}
            target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_}
          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
            commit_id=${{ github.sha }}
            tag_name=${{ github.ref_name }}
            target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id}/SM${compile_arch//,/_}
            target_path_latest=paddle-github-action/TAG/FastDeploy/${tag_name}/latest/SM${compile_arch//,/_}
            target_path_stripped_latest="${target_path_latest#paddle-github-action/}"
          else
            commit_id=${{ github.sha }}
            branch_name=${{ github.ref_name }}
            target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id}/SM${compile_arch//,/_}
            target_path_latest=paddle-github-action/BRANCH/FastDeploy/${branch_name}/latest/SM${compile_arch//,/_}
            target_path_stripped_latest="${target_path_latest#paddle-github-action/}"
          fi

          target_path_stripped="${target_path#paddle-github-action/}"

          all_coverage_file="python_coverage_all.xml"
          if [ -f ${all_coverage_file} ]; then
            python ${push_file} ${all_coverage_file} ${target_path}/CoverageData
            ALL_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${all_coverage_file}
            echo "all_cov_file_url=${ALL_COV_FILE_URL}" >> $GITHUB_OUTPUT
            echo "all_cov_file_url=${ALL_COV_FILE_URL}" >> $GITHUB_ENV
          fi

          if [[ "$IS_PR" == "true" ]]; then
            diff_cov_result_json="diff_coverage.json"
            if [ -f ${diff_cov_result_json} ]; then
              python ${push_file} ${diff_cov_result_json} ${target_path}/CoverageData
              DIFF_COV_JSON_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_result_json}
              echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT
              echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV
            fi
          fi

          HAS_FAILED_TESTS=false
          unittest_result="failed_tests.log"
          if [ -s ${unittest_result} ]; then
            HAS_FAILED_TESTS=true
            python ${push_file} ${unittest_result} ${target_path}/UnitTestResult
            UNIT_TEST_RESULT_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/UnitTestResult/${unittest_result}
            echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_OUTPUT
            echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_ENV
          fi

          # Only upload logs when tests failed
          unittest_logs_archive="unittest_logs.tar.gz"
          if [ "$HAS_FAILED_TESTS" = true ]; then
            python ${push_file} ${unittest_logs_archive} ${target_path}/UnitTestResult
            UNIT_TEST_LOGS_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/UnitTestResult/${unittest_logs_archive}
            echo "unittest_logs_url=${UNIT_TEST_LOGS_URL}" >> $GITHUB_OUTPUT
            echo "unittest_logs_url=${UNIT_TEST_LOGS_URL}" >> $GITHUB_ENV
          fi

          if [[ "$IS_PR" != "true" ]]; then
            full_cov_file="full_coverage_report.txt"
            full_cov_csv="full_coverage_report.csv"

            if [ -f ${full_cov_file} ]; then
              python ${push_file} ${full_cov_file} ${target_path}/CoverageData
              python ${push_file} ${full_cov_file} ${target_path_latest}/CoverageData
              FULL_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${full_cov_file}
              echo "full_coverage_report_url=${FULL_COV_FILE_URL}" >> $GITHUB_OUTPUT
              echo "full_coverage_report_url=${FULL_COV_FILE_URL}" >> $GITHUB_ENV
            fi

            if [ "$HAS_FAILED_TESTS" = false ] && [ -f ${full_cov_csv} ]; then
              python ${push_file} ${full_cov_csv} ${target_path}/CoverageData
              python ${push_file} ${full_cov_csv} ${target_path_latest}/CoverageData
              FULL_COV_CSV_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${full_cov_csv}
              echo "full_coverage_csv_url=${FULL_COV_CSV_URL}" >> $GITHUB_OUTPUT
              echo "full_coverage_csv_url=${FULL_COV_CSV_URL}" >> $GITHUB_ENV
            fi
          fi
      - name: Check Unit Test Success
        shell: bash
        run: |
          cd FastDeploy
          if [ "$TEST_EXIT_CODE" -eq 8 ]; then
            filename=$(basename "$unittest_failed_url")
            if [ -z "${unittest_failed_url}" ]; then
              echo "No diff unit failed file URL provided."
            else
              rm -rf "${filename}"
              wget -O ${filename} ${unittest_failed_url} || echo "Download unittest file failed, but continuing..."
            fi
            echo "Unit tests failed (exit code 8)"
            if [ -f "${filename}" ];then
              echo "Failed test cases:"
              cat "${filename}"
              echo "unittest_logs_url=${unittest_logs_url}"
            fi
            exit "$TEST_EXIT_CODE"
          fi
          echo "All tests passed"

      - name: Verify Code Coverage Threshold (80%)
        if: ${{ github.event_name == 'pull_request' }}
        shell: bash
        run: |
          cd FastDeploy
          if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then
            echo "Coverage generation failed (exit code 9)"
            filename=$(basename "$diff_cov_result_json_url")
            if [ -z "${diff_cov_result_json_url}" ]; then
              echo "No diff cov result file URL provided."
            else
              rm -rf "${filename}"
              wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..."
            fi
            if [ -f "${filename}" ];then
              echo "GPU Patch Coverage Details:"
              if command -v jq >/dev/null 2>&1; then
                  jq . "${filename}"
              else
                  cat "${filename}"
              fi
            fi
            exit "$COVERAGE_EXIT_CODE"
          fi
          echo "coverage passed"
          exit 0

      - name: Terminate and delete the container
        if: always()
        run: |
          set +e
          docker exec -t ${{ runner.name }} /bin/bash -c 'find /workspace -mindepth 1 -delete'
          docker rm -f ${{ runner.name }}

  diff_coverage_report:
    needs: run_tests_with_coverage
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 15
    env:
      all_cov_file_url: ${{ needs.run_tests_with_coverage.outputs.all_cov_file_url }}
    steps:
      - name: Clone FastDeploy
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Download diff coverage file
        shell: bash
        run: |
          echo "Downloading all coverage file..."
          if ! wget --no-proxy "${all_cov_file_url}" -O python_coverage_all.xml; then
            echo "Download failed, skipping upload."
            exit 0
          fi

          sed -i 's|<source>/workspace/FastDeploy/fastdeploy</source>|<source>fastdeploy</source>|' python_coverage_all.xml

      - name: Upload diff coverage report
        if: always() && hashFiles('python_coverage_all.xml') != ''
        uses: codecov/codecov-action@v6
        with:
          files: ./python_coverage_all.xml
          flags: GPU
          name: python diff coverage
          fail_ci_if_error: false
          verbose: true
          disable_search: true