diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 0000000000..4fc66cab68 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[codespell] +ignore-words-list = thirdparty diff --git a/Jenkinsfile b/Jenkinsfile index b88da9bfce..6e1405145a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,188 +23,164 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '5')) - timeout (time: 20, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') timestamps() checkoutToSubdirectory('src') + // Ensure only one build runs per PR at a time + disableConcurrentBuilds(abortPrevious: true) + } + + parameters { + string(name: 'JIRA_ISSUE_KEY', + defaultValue: '', + description: 'The JIRA issue to comment on. Example: TEZ-1234') } environment { SOURCEDIR = 'src' - // will also need to change notification section below + YETUSDIR = 'yetus' PATCHDIR = 'out' DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile" - YETUS='yetus' - // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION='rel/0.15.1' - } + // Credentials IDs + GITHUB_CRED_ID = 'apache-tez-at-github.com' + JIRA_CRED_ID = 'tez-ci' - parameters { - string(name: 'JIRA_ISSUE_KEY', - defaultValue: '', - description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234') + // Yetus version + YETUS_VERSION = 'rel/0.15.1' } stages { - stage ('install yetus') { + stage('Install Yetus') { steps { - dir("${WORKSPACE}/${YETUS}") { + dir("${WORKSPACE}/${env.YETUSDIR}") { checkout([ $class: 'GitSCM', branches: [[name: "${env.YETUS_VERSION}"]], - userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]] + userRemoteConfigs: [[url: 'https://github.com/apache/yetus']]] ) } } } - stage ('precommit-run') { + stage('Yetus Analysis') { steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-tez-at-github.com', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'tez-ci', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { - sh '''#!/usr/bin/env bash - - set -e - - TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" - - # this must be clean for every run - if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then - rm -rf "${WORKSPACE}/${PATCHDIR}" - fi - mkdir -p "${WORKSPACE}/${PATCHDIR}" - - # if given a JIRA issue, process it. If CHANGE_URL is set - # (e.g., Github Branch Source plugin), process it. - # otherwise exit, because we don't want Hadoop to do a - # full build. We wouldn't normally do this check for smaller - # projects. :) - if [[ -n "${JIRA_ISSUE_KEY}" ]]; then - YETUS_ARGS+=("${JIRA_ISSUE_KEY}") - elif [[ -z "${CHANGE_URL}" ]]; then - echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html" - exit 0 - fi - - YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") - - # where the source is located - YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") - - # our project defaults come from a personality file - YETUS_ARGS+=("--project=tez") - - # lots of different output formats - YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") - YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") - YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") - - # enable writing back to Github - YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}") - - # auto-kill any surefire stragglers during unit test runs - YETUS_ARGS+=("--reapermode=kill") - - # set relatively high limits for ASF machines - # changing these to higher values may cause problems - # with other jobs on systemd-enabled machines - YETUS_ARGS+=("--proclimit=5500") - YETUS_ARGS+=("--dockermemlimit=20g") - - # -1 spotbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--spotbugs-strict-precheck") - # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") - - # URL for user-side presentation in reports and such to our artifacts - # (needs to match the archive bits below) - YETUS_ARGS+=("--build-url-artifacts=artifact/out") - - # plugins to enable - YETUS_ARGS+=("--plugins=all") - - # use Hadoop's bundled shelldocs - YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs") - - # don't let these tests cause -1s because we aren't really paying that - # much attention to them - YETUS_ARGS+=("--tests-filter=checkstyle") - - # run in docker mode and specifically point to our - # Dockerfile since we don't want to use the auto-pulled version. - YETUS_ARGS+=("--docker") - YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") - YETUS_ARGS+=("--mvn-custom-repos") - - # effectively treat dev-suport as a custom maven module - YETUS_ARGS+=("--skip-dirs=dev-support") - - # help keep the ASF boxes clean - YETUS_ARGS+=("--sentinel") - - # test with Java 21 eclipse-temurin docker image path - YETUS_ARGS+=("--java-home=/opt/java/openjdk") - YETUS_ARGS+=("--debug") - - # write Yetus report as GitHub comment (YETUS-1102) - YETUS_ARGS+=("--github-write-comment") - YETUS_ARGS+=("--github-use-emoji-vote") - - "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" - ''' - } + withCredentials([ + usernamePassword(credentialsId: env.GITHUB_CRED_ID, passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: env.JIRA_CRED_ID, passwordVariable: 'JIRA_PASSWORD', usernameVariable: 'JIRA_USER') + ]) { + sh '''#!/usr/bin/env bash + + TESTPATCHBIN="${WORKSPACE}/${YETUSDIR}/precommit/src/main/shell/test-patch.sh" + + rm -rf "${WORKSPACE}/${PATCHDIR}" + mkdir -p "${WORKSPACE}/${PATCHDIR}" + + cd "${WORKSPACE}/${SOURCEDIR}" + # Ensure origin/master is available for diffing + git fetch origin master || true + git diff origin/master...HEAD > "${WORKSPACE}/${PATCHDIR}/local-pr.patch" + cd "${WORKSPACE}" + + YETUS_ARGS=() + + # Uncomment the following line to enable Yetus debugging + # YETUS_ARGS+=("--debug") + + # 1. Target Issue/PR + if [[ -n "${JIRA_ISSUE_KEY}" ]]; then + YETUS_ARGS+=("${JIRA_ISSUE_KEY}") + elif [[ -n "${CHANGE_ID}" ]]; then + echo "Processing PR: ${CHANGE_ID} using local patch" + else + echo "No PR or JIRA key provided. Skipping Yetus." + exit 0 + fi + + # 2. Core Paths & Project + YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") + YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") + YETUS_ARGS+=("--project=tez") + + # 3. Docker & Environment + YETUS_ARGS+=("--docker") + YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--java-home=/opt/java/openjdk") + YETUS_ARGS+=("--build-url-artifacts=artifact/out") + + # 4. Plugins & Personality + # Copy personality to PATCHDIR to avoid deletion by Yetus' robot mode + cp "${WORKSPACE}/${SOURCEDIR}/dev-support/tez-personality.sh" "${WORKSPACE}/${PATCHDIR}/tez-personality.sh" + + YETUS_ARGS+=("--plugins=all") + + YETUS_ARGS+=("--personality=${WORKSPACE}/${PATCHDIR}/tez-personality.sh") + YETUS_ARGS+=("--mvn-custom-repos") + YETUS_ARGS+=("--tests-filter=checkstyle") + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") + YETUS_ARGS+=("--reapermode=kill") + YETUS_ARGS+=("--sentinel") + + # 5. Reporting & GitHub Integration + YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") + YETUS_ARGS+=("--github-write-comment") + YETUS_ARGS+=("--github-use-emoji-vote") + YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") + YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") + YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") + + echo "Running Yetus with local patch overriding GitHub..." + + # Wrap the execution in a timeout to prevent infinite test deadlocks. + # If it hangs, Jenkins will abort this block but still execute the post {} steps to zip the logs. + timeout 3h "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${WORKSPACE}/${PATCHDIR}/local-pr.patch" + ''' + } } } - } post { always { - script { - // Yetus output - archiveArtifacts "${env.PATCHDIR}/**" - // Publish the HTML report so that it can be looked at - // Has to be relative to WORKSPACE. - publishHTML (target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: "${env.PATCHDIR}", - reportFiles: 'report.html', - reportName: 'Yetus Report' - ]) - // Publish JUnit results - try { - junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" - } catch(e) { - echo 'junit processing: ' + e.toString() + script { + sh ''' + # Fix permissions before attempting to read/zip directories created by root Docker processes + echo "Cleaning up workspace permissions to prevent HADOOP-13951 and unzipping errors..." + chmod -R u+rxw "${WORKSPACE}" || true + + # Zip test logs for easier debugging and storage efficiency + cd "${SOURCEDIR}" || exit 0 + find . -type d -name "surefire-reports" | while read -r dir; do + rel_path=$(echo "${dir#./}" | tr '/' '_') + zip -r -q "../${PATCHDIR}/test-logs-${rel_path}.zip" "${dir}" || true + done + ''' + + dir(env.SOURCEDIR) { + // Safely publish junit results + junit testResults: "**/target/surefire-reports/*.xml", allowEmptyResults: true + } + + // Archive Yetus output, reports, and zipped test logs + archiveArtifacts artifacts: "${env.PATCHDIR}/**", allowEmptyArchive: true + + // Publish Yetus HTML report to Jenkins UI sidebar only if it exists + if (fileExists("${env.PATCHDIR}/report.html")) { + publishHTML(target: [ + allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: "${env.PATCHDIR}", + reportFiles: 'report.html', + reportName: 'Yetus Report' + ]) + } } - } } - // Jenkins pipeline jobs fill slaves on PRs without this :( - cleanup() { + cleanup { script { - sh ''' - # See YETUS-764 - if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then - echo "test-patch process appears to still be running: killing" - kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true - sleep 10 - fi - if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then - echo "test-patch container appears to still be running: killing" - docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true - fi - # See HADOOP-13951 - chmod -R u+rxw "${WORKSPACE}" - ''' + // Wipe the workspace to prevent Jenkins agent disk exhaustion deleteDir() } } diff --git a/build-tools/smart-apply-patch.sh b/build-tools/smart-apply-patch.sh deleted file mode 100755 index 2a9dbd24dd..0000000000 --- a/build-tools/smart-apply-patch.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# -# Determine if the patch file is a git diff file with prefixes. -# These files are generated via "git diff" *without* the --no-prefix option. -# -# We can apply these patches more easily because we know that the a/ and b/ -# prefixes in the "diff" lines stands for the project root directory. -# So we don't have to hunt for the project root. -# And of course, we know that the patch file was generated using git, so we -# know git apply can handle it properly. -# -# Arguments: file name. -# Return: 0 if it is a git diff; 1 otherwise. -# -is_git_diff_with_prefix() { - DIFF_TYPE="unknown" - while read -r line; do - if [[ "$line" =~ ^diff\ ]]; then - if [[ "$line" =~ ^diff\ \-\-git ]]; then - DIFF_TYPE="git" - else - return 1 # All diff lines must be diff --git lines. - fi - fi - if [[ "$line" =~ ^\+\+\+\ ]] || - [[ "$line" =~ ^\-\-\-\ ]]; then - if ! [[ "$line" =~ ^....[ab]/ ]]; then - return 1 # All +++ and --- lines must start with a/ or b/. - fi - fi - done < $1 - [ x$DIFF_TYPE == x"git" ] || return 1 - return 0 # return true (= 0 in bash) -} - -PATCH_FILE=$1 -DRY_RUN=$2 -if [ -z "$PATCH_FILE" ]; then - echo usage: $0 patch-file - exit 1 -fi - -PATCH=${PATCH:-patch} # allow overriding patch binary - -# Cleanup handler for temporary files -TOCLEAN="" -cleanup() { - rm $TOCLEAN - exit $1 -} -trap "cleanup 1" HUP INT QUIT TERM - -# Allow passing "-" for stdin patches -if [ "$PATCH_FILE" == "-" ]; then - PATCH_FILE=/tmp/tmp.in.$$ - cat /dev/fd/0 > $PATCH_FILE - TOCLEAN="$TOCLEAN $PATCH_FILE" -fi - -# Special case for git-diff patches without --no-prefix -if is_git_diff_with_prefix "$PATCH_FILE"; then - GIT_FLAGS="--binary -p1 -v" - if [[ -z $DRY_RUN ]]; then - GIT_FLAGS="$GIT_FLAGS --stat --apply " - echo Going to apply git patch with: git apply "${GIT_FLAGS}" - else - GIT_FLAGS="$GIT_FLAGS --check " - fi - git apply ${GIT_FLAGS} "${PATCH_FILE}" - exit $? -fi - -# Come up with a list of changed files into $TMP -TMP=/tmp/tmp.paths.$$ -TOCLEAN="$TOCLEAN $TMP" - -if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then - PLEVEL=0 - #if the patch applied at P0 there is the possability that all we are doing - # is adding new files and they would apply anywhere. So try to guess the - # correct place to put those files. - - TMP2=/tmp/tmp.paths.2.$$ - TOCLEAN="$TOCLEAN $TMP2" - - egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2 - - if [ ! -s $TMP2 ]; then - echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting." - cleanup 1 - fi - - #first off check that all of the files do not exist - FOUND_ANY=0 - for CHECK_FILE in $(cat $TMP2) - do - if [[ -f $CHECK_FILE ]]; then - FOUND_ANY=1 - fi - done - - if [[ "$FOUND_ANY" = "0" ]]; then - #all of the files are new files so we have to guess where the correct place to put it is. - - # if all of the lines start with a/ or b/, then this is a git patch that - # was generated without --no-prefix - if ! grep -qv '^a/\|^b/' $TMP2 ; then - echo Looks like this is a git patch. Stripping a/ and b/ prefixes - echo and incrementing PLEVEL - PLEVEL=$[$PLEVEL + 1] - sed -i -e 's,^[ab]/,,' $TMP2 - fi - fi -elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=1 -elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then - PLEVEL=2 -else - echo "The patch does not appear to apply with p0 to p2"; - cleanup 1; -fi - -# If this is a dry run then exit instead of applying the patch -if [[ -n $DRY_RUN ]]; then - cleanup 0; -fi - -echo Going to apply patch with: $PATCH -p$PLEVEL -$PATCH -p$PLEVEL -E < $PATCH_FILE - -cleanup $? diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh deleted file mode 100755 index feb9644893..0000000000 --- a/build-tools/test-patch.sh +++ /dev/null @@ -1,821 +0,0 @@ -#!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -#set -x - -### Setup some variables. -### BUILD_URL is set by Hudson if it is run by patch process -### Read variables from properties file -bindir=$(dirname $0) - -# Defaults -if [ -z "$MAVEN_HOME" ]; then - MVN=mvn -else - MVN=$MAVEN_HOME/bin/mvn -fi - -PROJECT_NAME=Tez -JENKINS=false -PATCH_DIR=/tmp -BASEDIR=$(pwd) -PS=${PS:-ps} -AWK=${AWK:-awk} -WGET=${WGET:-wget} -GIT=${GIT:-git} -GREP=${GREP:-grep} -PATCH=${PATCH:-patch} -DIFF=${DIFF:-diff} -JIRACLI=${JIRA:-jira} -SED=${SED:-sed} -CURL=${CURL:-curl} -SPOTBUGS_HOME=${SPOTBUGS_HOME} - -############################################################################### -printUsage() { - echo "Usage: $0 [options] patch-file | defect-number" - echo - echo "Where:" - echo " patch-file is a local patch file containing the changes to test" - echo " defect-number is a JIRA defect number (e.g. 'TEZ-1234') to test (Jenkins only)" - echo - echo "Options:" - echo "--patch-dir= The directory for working and output files (default '/tmp')" - echo "--basedir= The directory to apply the patch to (default current directory)" - echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" - echo "--ps-cmd= The 'ps' command to use (default 'ps')" - echo "--awk-cmd= The 'awk' command to use (default 'awk')" - echo "--git-cmd= The 'git' command to use (default 'git')" - echo "--grep-cmd= The 'grep' command to use (default 'grep')" - echo "--patch-cmd= The 'patch' command to use (default 'patch')" - echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--spotbugs-home= Spotbugs home directory (default SPOTBUGS_HOME environment variable)" - echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" - echo "--run-tests Run all tests below the base directory" - echo - echo "Jenkins-only options:" - echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" - echo "--wget-cmd= The 'wget' command to use (default 'wget')" - echo "--jira-cmd= The 'jira' command to use (default 'jira')" - echo "--jira-password= The password for the 'jira' command" -} - -############################################################################### -parseArgs() { - for i in $* - do - case $i in - --jenkins) - JENKINS=true - ;; - --patch-dir=*) - PATCH_DIR=${i#*=} - ;; - --basedir=*) - BASEDIR=${i#*=} - ;; - --mvn-cmd=*) - MVN=${i#*=} - ;; - --ps-cmd=*) - PS=${i#*=} - ;; - --awk-cmd=*) - AWK=${i#*=} - ;; - --wget-cmd=*) - WGET=${i#*=} - ;; - --git-cmd=*) - GIT=${i#*=} - ;; - --grep-cmd=*) - GREP=${i#*=} - ;; - --patch-cmd=*) - PATCH=${i#*=} - ;; - --diff-cmd=*) - DIFF=${i#*=} - ;; - --jira-cmd=*) - JIRACLI=${i#*=} - ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} - ;; - --spotbugs-home=*) - SPOTBUGS_HOME=${i#*=} - ;; - --dirty-workspace) - DIRTY_WORKSPACE=true - ;; - --run-tests) - RUN_TESTS=true - ;; - *) - PATCH_OR_DEFECT=$i - ;; - esac - done - if [ -z "$PATCH_OR_DEFECT" ]; then - printUsage - exit 1 - fi - if [[ $JENKINS == "true" ]] ; then - echo "Running in Jenkins mode" - defect=$PATCH_OR_DEFECT - else - echo "Running in developer mode" - JENKINS=false - ### PATCH_FILE contains the location of the patchfile - PATCH_FILE=$PATCH_OR_DEFECT - if [[ ! -e "$PATCH_FILE" ]] ; then - echo "Unable to locate the patch file $PATCH_FILE" - cleanupAndExit 0 - fi - ### Check if $PATCH_DIR exists. If it does not exist, create a new directory - if [[ ! -e "$PATCH_DIR" ]] ; then - mkdir "$PATCH_DIR" - if [[ $? == 0 ]] ; then - echo "$PATCH_DIR has been created" - else - echo "Unable to create $PATCH_DIR" - cleanupAndExit 0 - fi - fi - ### Obtain the patch filename to append it to the version number - defect=`basename $PATCH_FILE` - fi -} - -############################################################################### -checkout () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Testing patch for ${defect}." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - ### When run by a developer, if the workspace contains modifications, do not continue - ### unless the --dirty-workspace option was set - status=`$GIT status --porcelain` - if [[ $JENKINS == "false" ]] ; then - if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then - echo "ERROR: can't run in a workspace that contains the following modifications" - echo "$status" - cleanupAndExit 1 - fi - echo - else - cd $BASEDIR - $GIT reset --hard - $GIT clean -xdf - $GIT checkout master - $GIT pull --rebase - fi - GIT_REVISION=`git rev-parse --verify --short HEAD` - return $? -} - -############################################################################### -downloadPatch () { - ### Download latest patch file (ignoring .htm and .html) when run from patch process - if [[ $JENKINS == "true" ]] ; then - $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect - if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then - echo "$defect is not \"Patch Available\". Exiting." - cleanupAndExit 0 - fi - relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` - patchURL="http://issues.apache.org${relativePatchURL}" - patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` - echo "$defect patch is being downloaded at `date` from" - echo "$patchURL" - $WGET -q -O $PATCH_DIR/patch $patchURL - VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum} - JIRA_COMMENT="Here are the results of testing the latest attachment - $patchURL - against master revision ${GIT_REVISION}." - - ### Copy the patch file to $PATCH_DIR - else - VERSION=PATCH-${defect} - cp $PATCH_FILE $PATCH_DIR/patch - if [[ $? == 0 ]] ; then - echo "Patch file $PATCH_FILE copied to $PATCH_DIR" - else - echo "Could not copy $PATCH_FILE to $PATCH_DIR" - cleanupAndExit 0 - fi - fi -} - -############################################################################### -verifyPatch () { - # Before building, check to make sure that the patch is valid - $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - else - return 0 - fi -} - -############################################################################### -prebuildWithoutPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Pre-build master to verify master stability and javac warnings" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "Compiling $(pwd)" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/masterJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "master javadoc compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. master compilation may be broken." - return 1 - fi - - return 0 -} - -############################################################################### -### Check for @author tags in the patch -checkAuthor () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are no @author tags in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` - echo "There appear to be $authorTags @author tags in the patch." - if [[ $authorTags != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Tez community has agreed to not allow in code contributions." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 @author{color}. The patch does not contain any @author tags." - return 0 -} - -############################################################################### -### Check for tests in the patch -checkTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are new or changed tests in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` - echo "There appear to be $testReferences test files referenced in the patch." - if [[ $testReferences == 0 ]] ; then - if [[ $JENKINS == "true" ]] ; then - patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` - if [[ $patchIsDoc != 0 ]] ; then - echo "The patch appears to be a documentation patch that doesn't require tests." - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." - return 0 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. - Please justify why no new tests are needed for this patch. - Also please list what manual steps were performed to verify this patch." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." - return 0 -} - -cleanUpXml () { - cd $BASEDIR/conf - for file in `ls *.xml.template` - do - rm -f `basename $file .template` - done - cd $BASEDIR -} - -############################################################################### -### Attempt to apply the patch -applyPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Applying patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - export PATCH - $bindir/smart-apply-patch.sh $PATCH_DIR/patch - if [[ $? != 0 ]] ; then - echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 - fi - return 0 -} - -############################################################################### -calculateJavadocWarnings() { - WARNING_FILE="$1" - RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') -} - -### Check there are no javadoc warnings -checkJavadocWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javadoc warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt" - nummasterJavadocWarnings=$RET - calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" - numPatchJavadocWarnings=$RET - grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" > "$PATCH_DIR/masterJavadocWarningsFiltered.txt" - grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \ - "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ - "$PATCH_DIR/diffJavadocWarnings.txt" - rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - echo "There appear to be $nummasterJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." - if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then - if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages. - See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details." - return 1 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javadoc{color}. There were no new javadoc warning messages." - return 0 -} - -############################################################################### -### Check there are no changes in the number of Javac warnings -checkJavacWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javac warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." - return 2 - fi - ### Compare master and patch javac warning numbers - if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then - $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt > $PATCH_DIR/filteredmasterJavacWarnings.txt - $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt - masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "There appear to be $masterJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." - if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then - if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $masterJavacWarnings warnings)." - - $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt - JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt -$JIRA_COMMENT_FOOTER" - - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." - return 0 -} - -############################################################################### -### Check there are no changes in the number of release audit (RAT) warnings -checkReleaseAuditWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched release audit warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" - $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 - find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt - - ### Compare master and patch release audit warning numbers - if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then - patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` - echo "" - echo "" - echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." - if [[ $patchReleaseAuditWarnings != "" ]] ; then - if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." - $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt - echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt -$JIRA_COMMENT_FOOTER" - return 1 - fi - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." - return 0 -} - - -############################################################################### -### Install the new jars so tests and spotbugs can find all of the updated jars -buildAndInstall () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Installing all of the jars" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" - $MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess - return $? -} - - -############################################################################### -### Check there are no changes in the number of Spotbugs warnings -checkSpotbugsWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched Spotbugs warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - rc=0 - echo " Running spotbugs " - echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1" - $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1 - rc=$? - spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchSpotBugsOutput.txt") - - if [ $rc != 0 ] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs (version ${spotbugs_version}) to fail." - return 1 - fi - - spotbugsWarnings=0 - for file in $(find $BASEDIR -name spotbugsXml.xml) - do - relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/spotbugsXml.xml" ]; then - module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing path - module_suffix=`basename ${module_suffix}` - fi - - cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml - $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml - newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` - echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)" - spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings)) - $SPOTBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html - if [[ $newSpotbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Spotbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html -$JIRA_COMMENT_FOOTER" - fi - done - - if [[ $spotbugsWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 spotbugs{color}. The patch appears to introduce $spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 spotbugs{color}. The patch does not introduce any new Spotbugs (version ${spotbugs_version}) warnings." - return 0 -} - -############################################################################### -### Run the tests -runTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running tests." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - failed_tests="" - failed_test_builds="" - test_timeouts="" - test_logfile=$PATCH_DIR/testrun.txt - echo " Running tests " - echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess" - $MVN clean install -fn > $test_logfile 2>&1 - test_build_result=$? - cat $test_logfile - module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` - if [[ -n "$module_test_timeouts" ]] ; then - test_timeouts="$test_timeouts $module_test_timeouts" - fi - module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "/dev/null - fi -} - -############################################################################### -### Cleanup files -cleanupAndExit () { - local result=$1 - if [[ $JENKINS == "true" ]] ; then - if [ -e "$PATCH_DIR" ] ; then - mv $PATCH_DIR $BASEDIR - fi - fi - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Finished build." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - exit $result -} - -############################################################################### -############################################################################### -############################################################################### - -JIRA_COMMENT="" -JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console - -This message is automatically generated." - -### Check if arguments to the script have been specified properly or not -parseArgs $@ -cd $BASEDIR - -checkout -RESULT=$? -if [[ $JENKINS == "true" ]] ; then - if [[ $RESULT != 0 ]] ; then - exit 100 - fi -fi -downloadPatch -verifyPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -prebuildWithoutPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkAuthor -(( RESULT = RESULT + $? )) - -if [[ $JENKINS == "true" ]] ; then - cleanUpXml -fi -checkTests -(( RESULT = RESULT + $? )) -applyPatch -APPLY_PATCH_RET=$? -(( RESULT = RESULT + $APPLY_PATCH_RET )) -if [[ $APPLY_PATCH_RET != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkJavacWarnings -JAVAC_RET=$? -#2 is returned if the code could not compile -if [[ $JAVAC_RET == 2 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -(( RESULT = RESULT + $JAVAC_RET )) -checkJavadocWarnings -(( RESULT = RESULT + $? )) -buildAndInstall -checkSpotbugsWarnings -(( RESULT = RESULT + $? )) -checkReleaseAuditWarnings -(( RESULT = RESULT + $? )) -### Run tests for Jenkins or if explictly asked for by a developer -if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then - runTests - (( RESULT = RESULT + $? )) -fi -JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ -$JIRA_COMMENT_FOOTER" - -submitJiraComment $RESULT -cleanupAndExit $RESULT diff --git a/dev-support/tez-personality.sh b/dev-support/tez-personality.sh new file mode 100755 index 0000000000..7e204c9eb0 --- /dev/null +++ b/dev-support/tez-personality.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache Yetus personality for Apache Tez + +personality_plugins "all" + +## @description Globals for this personality +## @audience private +## @stability evolving +## @replaceable no +function personality_globals { + export PROJECT_NAME=tez + export BUILDTOOL=maven + export PATCH_BRANCH_DEFAULT=master + export JIRA_ISSUE_RE='^TEZ-[0-9]+$' + export GITHUB_REPO="apache/tez" + + # Increase memory for maven + export MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx4g -XX:+UseG1GC"}" + + # Default Yetus settings + export DOCKER_MEMORY="20g" + export PROC_LIMIT=5500 + + # Force-enable core verification plugins + add_test compile + add_test unit +} + +## @description Module selection +## @audience private +## @stability evolving +## @replaceable no +function personality_modules { + local repostatus=$1 + local testtype=$2 + local extra=() + local MODULES=("${CHANGED_MODULES[@]}") + + yetus_debug "Personality: ${repostatus} ${testtype}" + + clear_personality_queue + + extra=("-Ptools") + + # Apply strict linting profile only during the explicit compile phase. + # Yetus ignores warnings during mvninstall, and unit tests are too slow with strict linting. + if [[ ${testtype} == "compile" ]]; then + extra+=("-Ptest-patch") + fi + + # Execute core compilation, unit tests, and install globally (project root) + if [[ ${testtype} == "unit" || ${testtype} == "compile" || ${testtype} == "mvninstall" ]]; then + yetus_debug "Forcing root module for ${testtype}" + MODULES=(.) + fi + + # Stream unit test logs live to the Jenkins console to debug infinite deadlocks + if [[ ${testtype} == "unit" && ${repostatus} == "patch" ]]; then + yetus_debug "Starting background tail for patch unit tests" + touch "${PATCH_DIR}/patch-unit-root.txt" + tail -f "${PATCH_DIR}/patch-unit-root.txt" & + export TAIL_PID=$! + # Trap to ensure tail is killed when Yetus finishes + trap 'kill ${TAIL_PID} 2>/dev/null || true' EXIT + fi + + if [[ ${testtype} == "spotbugs" ]]; then + extra+=("-Pspotbugs") + fi + + for module in "${MODULES[@]}"; do + personality_enqueue_module "${module}" "${extra[@]}" + done +} + +## @description File filter to determine which tests to run +## @audience private +## @stability evolving +## @replaceable no +function personality_file_filter { + local filename=$1 + + yetus_debug "Tez file filter: ${filename}" + + if [[ ${filename} =~ \.java$ || ${filename} =~ \.proto$ || ${filename} =~ pom\.xml$ ]]; then + add_test javac + add_test spotbugs + add_test checkstyle + add_test javadoc + fi + + if [[ ${filename} =~ findbugs-exclude\.xml$ ]]; then + add_test spotbugs + fi + + if [[ ${filename} =~ \.sh$ ]] || [[ ${filename} =~ Jenkinsfile ]]; then + add_test shellcheck + fi + + if [[ ${filename} =~ \.md$ ]] || [[ ${filename} =~ \.txt$ ]]; then + add_test codespell + fi +} diff --git a/pom.xml b/pom.xml index d576ddbb46..4472bc85c6 100644 --- a/pom.xml +++ b/pom.xml @@ -77,6 +77,7 @@ 1.15.0 32.0.1-jre 3.4.2 + 1.4.0 1.19.4 1.5.4 3.0.0 @@ -101,6 +102,7 @@ ${project.build.directory}/logs 1.7.9 true + 1.5.3 ${scm.url} @@ -773,6 +775,11 @@ protobuf-java ${protobuf.version} + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 + ${hadoop-thirdparty.version} + com.google.guava guava @@ -1100,6 +1107,51 @@ + + com.google.code.maven-replacer-plugin + replacer + ${maven-replacer-plugin.version} + + + replace-generated-sources + generate-sources + + replace + + + ${project.basedir} + + target/generated-sources/**/*.java + + + + ([^\.])com.google.protobuf + $1org.apache.hadoop.thirdparty.protobuf + + + + + + replace-generated-test-sources + generate-sources + + replace + + + ${project.basedir} + + target/generated-test-sources/**/*.java + + + + ([^\.])com.google.protobuf + $1org.apache.hadoop.thirdparty.protobuf + + + + + + diff --git a/tez-api/pom.xml b/tez-api/pom.xml index a2e449f1e6..fc83710737 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -82,8 +82,8 @@ commons-collections4 - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 junit @@ -183,6 +183,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java index dac20edc08..a255c1f633 100644 --- a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -49,8 +50,6 @@ import org.apache.tez.frameworkplugins.FrameworkUtils; import org.apache.tez.frameworkplugins.yarn.YarnClientFrameworkService; -import com.google.protobuf.ServiceException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index 8c16f6be8c..df0b0f9e60 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -80,7 +81,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index fa0ea657f3..fc9b2c07dd 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -56,7 +56,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; @@ -982,7 +982,7 @@ public static DAGClientAMProtocolBlockingPB getAMProxy(final Configuration conf, proxy = userUgi.doAs(new PrivilegedExceptionAction() { @Override public DAGClientAMProtocolBlockingPB run() throws IOException { - RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine2.class); return (DAGClientAMProtocolBlockingPB) RPC.getProxy(DAGClientAMProtocolBlockingPB.class, 0, serviceAddr, conf); } diff --git a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java index ab265f6ea8..d254aab67e 100644 --- a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java +++ b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java @@ -23,13 +23,12 @@ import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.DAGNotRunningException; import org.apache.tez.dag.api.NoCurrentDAGException; import org.apache.tez.dag.api.SessionNotRunning; import org.apache.tez.dag.api.TezException; -import com.google.protobuf.ServiceException; - public final class RPCUtil { private RPCUtil() {} diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java index f376ba2384..d178ffd2ca 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java @@ -44,6 +44,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.client.TezClient; import org.apache.tez.common.security.JobTokenIdentifier; @@ -51,8 +52,6 @@ import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.TezUncheckedException; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java index 9797165145..ff268633f5 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java @@ -28,15 +28,14 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.tez.client.TezClientUtils; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.records.DAGProtos; import org.apache.tez.runtime.api.TaskContext; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; - import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index 1d292e7ad6..fbfe543a78 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -38,6 +38,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ByteString.Output; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -86,8 +88,6 @@ import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.google.protobuf.ByteString.Output; @Private public final class DagTypeConverters { diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index 827d333371..19c55fc9e8 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -50,7 +51,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TryKillDAGRequestProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index 68af185cc3..d1603c16a3 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -55,6 +55,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -96,8 +98,6 @@ import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import com.google.common.collect.Maps; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; import org.hamcrest.CoreMatchers; import org.junit.Assert; diff --git a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java index 507d963b77..83feaf1827 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java @@ -22,11 +22,10 @@ import java.io.IOException; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.SessionNotRunning; import org.apache.tez.dag.api.TezException; -import com.google.protobuf.ServiceException; - import org.junit.Assert; import org.junit.Test; diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index 6e979972a7..c846b199c8 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -40,6 +40,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -74,9 +76,6 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexStatusProto; import org.apache.tez.dag.api.records.DAGProtos.VertexStatusStateProto; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/tez-common/pom.xml b/tez-common/pom.xml index e9d8a907b8..49bfa05b6b 100644 --- a/tez-common/pom.xml +++ b/tez-common/pom.xml @@ -34,8 +34,8 @@ guava - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.apache.hadoop diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java index cb7581981b..525a6b7513 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java @@ -42,6 +42,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.Descriptors; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.log4j.Appender; @@ -64,10 +67,6 @@ import org.apache.tez.serviceplugins.api.TaskAttemptEndReason; import org.apache.tez.util.StopWatch; -import com.google.protobuf.ByteString; -import com.google.protobuf.Descriptors; -import com.google.protobuf.TextFormat; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index 3b9b7cae6e..d5cd7c4dd2 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -30,6 +30,7 @@ import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.client.TezClientUtils; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; @@ -40,8 +41,6 @@ import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor; import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor; -import com.google.protobuf.ByteString; - import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index bf79df2780..0bc12da4bf 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -99,8 +99,8 @@ slf4j-api - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 org.apache.commons @@ -233,6 +233,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java index 56d9f7ad81..d29295064f 100644 --- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java +++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -72,7 +73,6 @@ import org.apache.tez.dag.app.dag.DAG; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java index 4c5a0039e6..fa29d80e39 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java @@ -25,12 +25,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.tez.dag.api.TezConfiguration; @@ -40,8 +41,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.DAGClientAMProtocol; import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider; -import com.google.protobuf.BlockingService; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +127,7 @@ public void setClientAMSecretKey(ByteBuffer key) { private Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, int numHandlers, BlockingService blockingService, String portRangeConfig) throws IOException { - RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine2.class); RPC.Server server = new RPC.Builder(conf).setProtocol(pbProtocol) .setInstance(blockingService).setBindAddress(addr.getHostString()) .setPort(addr.getPort()).setNumHandlers(numHandlers).setVerbose(false) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java index 136d4c80eb..c0838daa8f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java @@ -27,6 +27,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.client.TezAppMasterStatus; import org.apache.tez.dag.api.DagTypeConverters; @@ -54,10 +57,6 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TryKillDAGResponseProto; import org.apache.tez.dag.api.records.DAGProtos.DAGPlan; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - public class DAGClientAMProtocolBlockingPBServerImpl implements DAGClientAMProtocolBlockingPB { DAGClientHandler real; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java index 04812dff17..ca964b758e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.util.functional.FutureIO; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -76,7 +77,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.CodedInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index 392739e292..3f05239ddb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -50,6 +50,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.service.ServiceStateException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; @@ -204,8 +206,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java index ef7f8a0cb8..a04d10ff8f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java @@ -20,8 +20,8 @@ import java.io.IOException; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; public interface HistoryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java index 4b0f314275..9240398cc7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java @@ -20,16 +20,15 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.AMLaunchedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class AMLaunchedEvent implements HistoryEvent { private ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java index cac72fccf7..6c920853e0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java @@ -20,16 +20,15 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.AMStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class AMStartedEvent implements HistoryEvent { private ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java index 5da61e7957..d2bf0558ca 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java @@ -21,14 +21,13 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.VersionInfo; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class AppLaunchedEvent implements HistoryEvent { private ApplicationId applicationId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java index c40ea58a19..fb2e94b18f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -27,10 +30,6 @@ import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.ContainerLaunchedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class ContainerLaunchedEvent implements HistoryEvent { private ContainerId containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java index 85c871f517..3cbb0e5261 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -27,10 +30,6 @@ import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.recovery.records.RecoveryProtos.ContainerStoppedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class ContainerStoppedEvent implements HistoryEvent { private ContainerId containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java index 33511d33df..94f68e7c7c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.io.OutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -29,10 +32,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGCommitStartedEvent implements HistoryEvent, SummaryEvent { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java index f7e3a57f34..06831da127 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java @@ -22,6 +22,10 @@ import java.io.OutputStream; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DagTypeConverters; @@ -37,10 +41,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class DAGFinishedEvent implements HistoryEvent, SummaryEvent, DAGIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java index ef78bc4665..4450952444 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.DAGIDAware; @@ -29,10 +32,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos; import org.apache.tez.dag.recovery.records.RecoveryProtos.DAGInitializedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGInitializedEvent implements HistoryEvent, DAGIDAware { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java index a77098ce60..7e018635ef 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; import java.io.OutputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -29,10 +32,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGKillRequestEvent implements HistoryEvent, SummaryEvent { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java index dcd9103455..4c11f01c4f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java @@ -20,15 +20,14 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.tez.dag.app.dag.DAGState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.TezDAGID; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class DAGRecoveredEvent implements HistoryEvent { private final ApplicationAttemptId applicationAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java index 5aad8bc4b6..1594bf0a73 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.app.dag.DAGState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.recovery.records.RecoveryProtos.DAGStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class DAGStartedEvent implements HistoryEvent, DAGIDAware { private TezDAGID dagID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java index 12ea0af6ca..8e07439e46 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java @@ -23,6 +23,9 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -38,10 +41,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java index 9bac94d9fb..9b089d5cf2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java @@ -23,6 +23,9 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -46,9 +49,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java index 06d1a36e73..5b7ec40ac4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -29,10 +32,6 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskAttemptStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class TaskAttemptStartedEvent implements HistoryEvent, TaskAttemptIDAware { private TezTaskAttemptID taskAttemptId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java index c95afcd8ae..cb600383a1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.common.counters.CounterGroup; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; @@ -31,10 +34,6 @@ import org.apache.tez.dag.records.TezTaskID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskFinishedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java index d9b0f54fe3..88f1598d25 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.oldrecords.TaskState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.TezTaskID; import org.apache.tez.dag.recovery.records.RecoveryProtos.TaskStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class TaskStartedEvent implements HistoryEvent, TaskIDAware { private TezTaskID taskID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java index 8d79706be9..2d1991266d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java @@ -22,6 +22,10 @@ import java.io.OutputStream; import java.nio.charset.Charset; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -30,11 +34,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexCommitStartedProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class VertexCommitStartedEvent implements HistoryEvent, SummaryEvent { private TezVertexID vertexID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index 2362af536c..9931893751 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -22,6 +22,9 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.VertexLocationHint; @@ -35,9 +38,6 @@ import org.apache.tez.runtime.api.InputSpecUpdate; import com.google.common.collect.Maps; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexConfigurationDoneEvent implements HistoryEvent, VertexIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java index 1cd2eb8496..e5cda23a72 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.app.dag.VertexState; import org.apache.tez.dag.app.dag.impl.ServicePluginInfo; @@ -36,10 +39,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexFinishStateProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexFinishedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java index 6d5d2ca324..cc8878dd4a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Collection; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -33,9 +36,6 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexGroupCommitFinishedEvent implements HistoryEvent, SummaryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java index aa40c5d7b8..9e31e3639a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java @@ -22,6 +22,9 @@ import java.io.OutputStream; import java.util.Collection; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.history.SummaryEvent; @@ -33,9 +36,6 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexGroupCommitStartedEvent implements HistoryEvent, SummaryEvent { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java index 81f20f0bdd..4b02b93dfe 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java @@ -23,6 +23,9 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.InputDescriptor; import org.apache.tez.dag.api.InputInitializerDescriptor; @@ -40,9 +43,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; public class VertexInitializedEvent implements HistoryEvent, VertexIDAware { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java index a8bc347f94..e496b7b19e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java @@ -20,6 +20,9 @@ import java.io.IOException; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; import org.apache.tez.dag.app.dag.VertexState; import org.apache.tez.dag.history.HistoryEvent; import org.apache.tez.dag.history.HistoryEventType; @@ -27,10 +30,6 @@ import org.apache.tez.dag.records.VertexIDAware; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexStartedProto; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; - public class VertexStartedEvent implements HistoryEvent, VertexIDAware { private TezVertexID vertexID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java index b412590ed2..90674a2b3e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezConfiguration; @@ -45,7 +46,6 @@ import org.apache.tez.dag.records.TezDAGID; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.CodedOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java index e179b4fd9b..5a241aaf3b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java @@ -18,12 +18,11 @@ package org.apache.tez.dag.utils; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.history.HistoryEventType; import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.dag.recovery.records.RecoveryProtos; -import com.google.protobuf.ByteString; - public final class ProtoUtils { private ProtoUtils() {} diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java index fe33f20f0f..f43a6701e8 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java @@ -53,6 +53,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -87,7 +88,6 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import com.google.protobuf.ByteString; import org.junit.After; import org.junit.Assert; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java index 7a8b39a445..9fdb4f7ef8 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java @@ -46,6 +46,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -157,7 +158,6 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.protobuf.ByteString; import org.junit.After; import org.junit.Assert; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index 833586e57f..7fc8dd396e 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -231,7 +232,6 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.protobuf.ByteString; import org.junit.After; import org.junit.Assert; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index 6b64df2097..29f4d1deb7 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java @@ -32,6 +32,8 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; @@ -73,8 +75,6 @@ import org.apache.tez.runtime.api.impl.TezEvent; import com.google.common.collect.Lists; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; import org.junit.Assert; import org.junit.Test; diff --git a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java index 96da31e141..0d6a3be85a 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java @@ -23,6 +23,7 @@ import java.util.Objects; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.TezUncheckedException; @@ -31,8 +32,6 @@ import org.apache.tez.dag.api.event.VertexStateUpdate; import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index ef1d8c29d1..d5892a04d6 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -38,6 +38,10 @@ com.google.guava guava + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 + org.apache.hadoop hadoop-common @@ -185,6 +189,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java index df7fcbcae7..7305bb08b4 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java @@ -22,6 +22,7 @@ import java.util.concurrent.Executors; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.Message; import org.apache.tez.common.GuavaShim; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; import org.apache.tez.service.impl.TezTestServiceProtocolClientImpl; @@ -36,7 +37,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.Message; public class TezTestServiceCommunicator extends AbstractService { diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java index 8ea112ee59..752d6764ec 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java @@ -18,6 +18,7 @@ import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -34,8 +35,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; -import com.google.protobuf.ByteString; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java index 459e1705db..58aae461ef 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java @@ -24,6 +24,8 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.dag.app.TezTaskCommunicatorImpl; @@ -38,9 +40,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto; import org.apache.tez.util.ProtoConverters; -import com.google.protobuf.ByteString; -import com.google.protobuf.ServiceException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java index f18ad06889..c820e07f07 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java @@ -18,17 +18,16 @@ import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; - public class TezTestServiceProtocolClientImpl implements TezTestServiceProtocolBlockingPB { @@ -75,7 +74,7 @@ public TezTestServiceProtocolBlockingPB getProxy() throws IOException { public TezTestServiceProtocolBlockingPB createProxy() throws IOException { TezTestServiceProtocolBlockingPB p; // TODO Fix security - RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, TezTestServiceProtocolBlockingPB.class, ProtobufRpcEngine2.class); p = (TezTestServiceProtocolBlockingPB) RPC .getProxy(TezTestServiceProtocolBlockingPB.class, 0, serverAddr, conf); return p; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java index a6be6711da..d3e4aa6ff8 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java @@ -20,10 +20,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.thirdparty.protobuf.BlockingService; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.tez.dag.api.TezException; import org.apache.tez.service.ContainerRunner; import org.apache.tez.service.TezTestServiceProtocolBlockingPB; @@ -34,9 +37,6 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.SubmitWorkResponseProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.BlockingService; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,7 +123,7 @@ InetSocketAddress getBindAddress() { private RPC.Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, int numHandlers, BlockingService blockingService) throws IOException { - RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); + RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine2.class); RPC.Server server = new RPC.Builder(conf) .setProtocol(pbProtocol) .setInstance(blockingService) diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml index 3ef921d5a9..f97a34e24f 100644 --- a/tez-mapreduce/pom.xml +++ b/tez-mapreduce/pom.xml @@ -129,8 +129,8 @@ commons-collections4 - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 @@ -165,6 +165,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java index 9abed0d125..59d6c65924 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.split.JobSplitWriter; import org.apache.hadoop.mapreduce.split.TezGroupedSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -74,7 +75,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java index 64d97bf1f1..4d97d5cf38 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.mapreduce.input.base.MRInputBase; @@ -51,7 +52,6 @@ import com.google.common.base.Function; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java index 27a98b2b2d..51d08dea16 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TaskCounter; @@ -70,7 +71,6 @@ import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java index b327b64bcb..b2451b4742 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java @@ -39,6 +39,7 @@ import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.split.TezGroupedSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.DataSourceDescriptor; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.mapreduce.TezTestUtils; @@ -50,8 +51,6 @@ import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent; import org.apache.tez.runtime.api.events.InputDataInformationEvent; -import com.google.protobuf.ByteString; - import org.junit.Test; public class TestMRInputAMSplitGenerator { diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java index 1ad85d1303..22a591602a 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.mapreduce.TezTestUtils; @@ -45,8 +46,6 @@ import org.apache.tez.runtime.api.events.InputDataInformationEvent; import org.apache.tez.runtime.api.events.InputUpdatePayloadEvent; -import com.google.protobuf.ByteString; - import org.junit.Test; public class TestMRInputSplitDistributor { diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java index 92cd8d678c..b96497abb1 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java @@ -41,14 +41,13 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.split.JobSplit; import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.dag.api.DataSourceDescriptor; import org.apache.tez.dag.api.TaskLocationHint; import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto; import org.apache.tez.runtime.api.events.InputDataInformationEvent; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/tez-plugins/tez-aux-services/pom.xml b/tez-plugins/tez-aux-services/pom.xml index d935d8ef74..3f88631ac6 100644 --- a/tez-plugins/tez-aux-services/pom.xml +++ b/tez-plugins/tez-aux-services/pom.xml @@ -100,11 +100,6 @@ org.slf4j slf4j-api - - com.google.protobuf - protobuf-java - provided - org.fusesource.leveldbjni leveldbjni-all diff --git a/tez-plugins/tez-protobuf-history-plugin/pom.xml b/tez-plugins/tez-protobuf-history-plugin/pom.xml index b43ac0a92c..0b33ea45bc 100644 --- a/tez-plugins/tez-protobuf-history-plugin/pom.xml +++ b/tez-plugins/tez-protobuf-history-plugin/pom.xml @@ -37,8 +37,8 @@ hadoop-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 junit @@ -83,6 +83,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java index f5386a81b9..3bfbfd8f3c 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java @@ -34,11 +34,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; import org.apache.hadoop.yarn.util.Clock; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java index 2cac4d8764..de75beb882 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageReader.java @@ -25,9 +25,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile.Reader; - -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; public class ProtoMessageReader implements Closeable { private final Path filePath; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java index 7d20256b67..d2d30fee90 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java @@ -26,12 +26,11 @@ import java.io.OutputStream; import org.apache.hadoop.io.Writable; - -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; +import org.apache.hadoop.thirdparty.protobuf.ExtensionRegistry; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; public class ProtoMessageWritable implements Writable { private T message; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java index 604fd10e55..60a65df190 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWriter.java @@ -27,11 +27,10 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.SequenceFile.Writer; +import org.apache.hadoop.thirdparty.protobuf.MessageLite; +import org.apache.hadoop.thirdparty.protobuf.Parser; import org.apache.tez.common.StreamHelper; -import com.google.protobuf.MessageLite; -import com.google.protobuf.Parser; - public class ProtoMessageWriter implements Closeable { private final Path filePath; private final Writer writer; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java index 8445f0b5ad..fa316b937c 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -64,8 +65,6 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.hadoop.shim.HadoopShim; -import com.google.protobuf.CodedInputStream; - import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java index a8ed9ff185..80a461bc99 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; @@ -55,9 +57,6 @@ import org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig; import org.apache.tez.tests.MiniTezClusterWithTimeline; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index d776cbfd03..13e83f7f80 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -75,8 +75,8 @@ hadoop-yarn-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 com.google.guava @@ -128,6 +128,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java index 77b9cbe8fb..cc1b017872 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.runtime.api.events.CompositeDataMovementEvent; import org.apache.tez.runtime.api.events.CompositeRoutedDataMovementEvent; import org.apache.tez.runtime.api.events.CustomProcessorEvent; @@ -31,7 +32,6 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent; import com.google.common.base.Charsets; -import com.google.protobuf.ByteString; public final class ProtoConverters { diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java index e9d4f1127e..4dc0bc242f 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java @@ -27,6 +27,9 @@ import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.thirdparty.protobuf.AbstractMessage; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.ProtoConverters; import org.apache.tez.common.TezConverterUtils; import org.apache.tez.dag.api.TezUncheckedException; @@ -56,10 +59,6 @@ import org.apache.tez.runtime.internals.api.events.SystemEventProtos.TaskAttemptFailedEventProto; import org.apache.tez.runtime.internals.api.events.SystemEventProtos.TaskAttemptKilledEventProto; -import com.google.protobuf.AbstractMessage; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - public class TezEvent implements Writable { private EventType eventType; diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml index a09311d90a..19685346d9 100644 --- a/tez-runtime-library/pom.xml +++ b/tez-runtime-library/pom.xml @@ -50,8 +50,8 @@ tez-common - com.google.protobuf - protobuf-java + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_25 junit @@ -142,6 +142,10 @@ + + com.google.code.maven-replacer-plugin + replacer + diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java index b158f9f6b5..121fa71fd3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java @@ -22,14 +22,13 @@ import java.util.HashMap; import java.util.Map.Entry; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.FairShuffleEdgeManagerConfigPayloadProto; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.FairShuffleEdgeManagerDestinationTaskPropProto; import org.apache.tez.dag.library.vertexmanager.FairShuffleUserPayloads.RangeProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - /** * Handles edge configuration serialization and de-serialization between diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java index 913bf625bd..585956a36d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java @@ -22,12 +22,11 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand; import org.apache.tez.dag.api.UserPayload; -import com.google.protobuf.InvalidProtocolBufferException; - /** * Edge manager for fair routing. Each destination task has its * DestinationTaskInputsProperty used to decide how to do event routing diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index c4e2fe2a46..2392b8d467 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -32,6 +32,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.EdgeManagerPluginContext; @@ -48,8 +50,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.ShuffleEdgeManagerConfigPayloadProto; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java index b38959b6a3..99dfa9f23a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java @@ -34,6 +34,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; @@ -62,8 +64,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java index 88bfa46a3c..3063ab9a53 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java @@ -27,14 +27,14 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.UserPayload; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; /** * CartesianProductConfig is used to configure both diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java index 6d6af8df80..046bcb6ffd 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java @@ -21,12 +21,12 @@ import javax.annotation.Nullable; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; /** * This EM wrap a real edge manager implementation object. It choose whether it's partitioned or diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java index c80182765a..0a261af726 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.dag.api.EdgeManagerPluginDescriptor; import org.apache.tez.dag.api.EdgeProperty; @@ -41,7 +42,6 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; /** * This VM wrap a real vertex manager implementation object. It choose whether it's partitioned or diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java index 992c302f36..7f7f9d3549 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Queue; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.VertexManagerPluginContext; @@ -44,7 +45,6 @@ import com.google.common.math.LongMath; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java index fdc15dd85e..1da00ca9ac 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java @@ -41,6 +41,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; @@ -67,7 +68,6 @@ import org.apache.tez.util.FastNumberFormat; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; import org.roaringbitmap.RoaringBitmap; import org.slf4j.Logger; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index d46cd86269..bdf0450513 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -26,6 +26,8 @@ import java.util.zip.Inflater; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezUncheckedException; @@ -45,9 +47,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataProto; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index ef977137f4..79dcdd191d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -24,6 +24,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.Inflater; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezUncheckedException; @@ -39,9 +41,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; import org.apache.tez.util.StringInterner; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index 323f81c118..f31bebc7fd 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -51,6 +51,7 @@ import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.CallableWithNdc; import org.apache.tez.common.GuavaShim; import org.apache.tez.common.Preconditions; @@ -84,7 +85,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java index 6dfefcb395..26f1111654 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtils; @@ -60,8 +61,6 @@ import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto; -import com.google.protobuf.ByteString; - import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.roaringbitmap.RoaringBitmap; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java index 629ecebd1f..16d2782699 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java @@ -28,13 +28,12 @@ import java.nio.ByteBuffer; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.dag.api.EdgeManagerPluginContext; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata; import org.apache.tez.dag.api.EdgeManagerPluginOnDemand.EventRouteMetadata; import org.apache.tez.dag.api.UserPayload; -import com.google.protobuf.ByteString; - import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java index 4203c94460..ce20ec847c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java @@ -37,6 +37,8 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.tez.dag.api.EdgeManagerPluginDescriptor; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.VertexManagerPluginContext; @@ -54,8 +56,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java index 1d4c55dcc9..e70faae8e4 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java @@ -32,6 +32,7 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -53,7 +54,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; import org.junit.Assert; import org.junit.Before; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java index 955b1e86a5..f345c70029 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandlerImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezExecutors; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -69,8 +70,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java index 6720c76e97..94a3bd0139 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezExecutors; @@ -43,8 +44,6 @@ import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java index 55505e2ba7..2544eaa390 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java @@ -54,6 +54,7 @@ import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.WritableSerialization; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.runtime.library.common.InputAttemptIdentifier; @@ -67,8 +68,6 @@ import org.apache.tez.runtime.library.utils.BufferUtils; import org.apache.tez.runtime.library.utils.CodecUtils; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Assert; import org.junit.Assume; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java index 5970d8b779..39694bcdb6 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java @@ -47,6 +47,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; @@ -76,8 +77,6 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; import org.apache.tez.util.StringInterner; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index 4f6012a5ab..62854c94aa 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -65,6 +65,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; @@ -96,7 +97,6 @@ import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; import org.junit.After; import org.junit.Before; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java index 8735a9a948..4e9c159c3e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtils; import org.apache.tez.common.counters.TezCounters; @@ -63,8 +64,6 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java index 727407349c..77397da15f 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; import org.apache.tez.common.TezSharedExecutor; import org.apache.tez.common.TezUtils; @@ -73,8 +74,6 @@ import org.apache.tez.runtime.library.testutils.KVDataGen; import org.apache.tez.runtime.library.testutils.KVDataGen.KVPair; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java index 6da45fb896..23dc283d8d 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtilsInternal; @@ -37,8 +38,6 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java index 05849fc700..f91268c10c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezRuntimeFrameworkConfigs; import org.apache.tez.common.TezUtilsInternal; @@ -38,8 +39,6 @@ import org.apache.tez.runtime.library.partitioner.HashPartitioner; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java index 86ebfdf96f..5de6248984 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.runtime.api.Event; @@ -30,8 +31,6 @@ import org.apache.tez.runtime.api.events.CompositeDataMovementEvent; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads; -import com.google.protobuf.ByteString; - import org.junit.Test; // Tests which don't require parameterization diff --git a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java index d895f7e1de..6633e69077 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java +++ b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java @@ -24,6 +24,8 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; +import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; import org.apache.tez.common.Preconditions; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.dag.api.TezReflectionException; @@ -45,9 +47,6 @@ import org.apache.tez.dag.records.TezDAGID; import org.apache.tez.test.RecoveryServiceWithEventHandlingHook.SimpleShutdownCondition.TIMING; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; - import org.slf4j.Logger; import org.slf4j.LoggerFactory;