diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d2108163b8..f3dd903420 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -27,7 +27,7 @@ name: E2E concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: @@ -102,12 +102,13 @@ jobs: name: standalone-image path: /tmp/standalone-image.tar retention-days: 1 - e2e: - name: ${{ matrix.case.name }} + api-e2e: + name: API-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest - timeout-minutes: 180 + timeout-minutes: 120 strategy: + fail-fast: false matrix: case: - name: EnvironmentTest @@ -135,34 +136,98 @@ jobs: - name: VariableManagementTest class: org.apache.streampark.e2e.cases.VariableManagementTest - - name: FlinkOnRemoteClusterDeployTest + # Flink remote cluster (API) + - name: Flink117-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink117OnRemoteClusterDeployTest + - name: Flink118-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink118OnRemoteClusterDeployTest + - name: Flink119-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink119OnRemoteClusterDeployTest + - name: Flink120-RemoteDeploy class: org.apache.streampark.e2e.cases.Flink120OnRemoteClusterDeployTest + - name: Flink200-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink200OnRemoteClusterDeployTest + - name: Flink210-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink210OnRemoteClusterDeployTest + - name: Flink220-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink220OnRemoteClusterDeployTest + - name: Flink230-RemoteDeploy + class: org.apache.streampark.e2e.cases.Flink230OnRemoteClusterDeployTest - # - name: FlinkOnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink120OnYarnClusterDeployTest + # Flink yarn cluster (API) + - name: Flink117-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink117OnYarnClusterDeployTest + - name: Flink118-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink118OnYarnClusterDeployTest + - name: Flink119-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink119OnYarnClusterDeployTest + - name: Flink120-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink120OnYarnClusterDeployTest + - name: Flink200-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink200OnYarnClusterDeployTest + - name: Flink210-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink210OnYarnClusterDeployTest + - name: Flink220-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink220OnYarnClusterDeployTest + - name: Flink230-YarnClusterDeploy + class: org.apache.streampark.e2e.cases.Flink230OnYarnClusterDeployTest - # - name: FlinkSQLOnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL120OnYarnTest + # Flink SQL on Yarn (API) + - name: FlinkSQL117-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL117OnYarnTest + - name: FlinkSQL118-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL118OnYarnTest + - name: FlinkSQL119-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL119OnYarnTest + - name: FlinkSQL120-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL120OnYarnTest + - name: FlinkSQL200-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL200OnYarnTest + - name: FlinkSQL210-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL210OnYarnTest + - name: FlinkSQL220-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL220OnYarnTest + - name: FlinkSQL230-Yarn + class: org.apache.streampark.e2e.cases.FlinkSQL230OnYarnTest - #- name: Flink117OnRemoteClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink117OnRemoteClusterDeployTest - #- name: Flink116OnRemoteClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink116OnRemoteClusterDeployTest + # Flink K8s cluster (API) + - name: Flink117-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink117OnK8sClusterDeployTest + - name: Flink118-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink118OnK8sClusterDeployTest + - name: Flink119-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink119OnK8sClusterDeployTest + - name: Flink120-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink120OnK8sClusterDeployTest + - name: Flink200-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink200OnK8sClusterDeployTest + - name: Flink210-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink210OnK8sClusterDeployTest + - name: Flink220-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink220OnK8sClusterDeployTest + - name: Flink230-K8sClusterDeploy + class: org.apache.streampark.e2e.cases.Flink230OnK8sClusterDeployTest - # - name: Flink116OnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink116OnYarnClusterDeployTest - # - name: Flink117OnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink117OnYarnClusterDeployTest - - # - name: FlinkSQL116OnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL116OnYarnTest - # - name: FlinkSQL117OnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL117OnYarnTest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + # Flink SQL on K8s (API) + - name: FlinkSQL117-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL117OnK8sTest + - name: FlinkSQL118-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL118OnK8sTest + - name: FlinkSQL119-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL119OnK8sTest + - name: FlinkSQL120-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL120OnK8sTest + - name: FlinkSQL200-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL200OnK8sTest + - name: FlinkSQL210-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL210OnK8sTest + - name: FlinkSQL220-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL220OnK8sTest + - name: FlinkSQL230-K8s + class: org.apache.streampark.e2e.cases.FlinkSQL230OnK8sTest steps: - name: Free Disk Space - # refer: https://github.com/jlumbroso/free-disk-space + if: contains(matrix.case.name, 'Flink') uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: dotnet: true @@ -213,19 +278,12 @@ jobs: return 1 } pull_with_retry testcontainers/ryuk:0.7.0 - pull_with_retry selenium/standalone-chrome:4.13.0 || true - - name: Run Test + pull_with_retry rancher/k3s:v1.30.5-k3s1 || true + - name: Run API Test run: | ./mvnw -B -f streampark-e2e/pom.xml -am \ -DfailIfNoTests=false \ -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - uses: actions/upload-artifact@v4 if: always() name: Upload Projects Build Logs @@ -237,7 +295,7 @@ jobs: name: E2E - Result runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ paths-filter, license-header, code-style, build, e2e ] + needs: [ paths-filter, license-header, code-style, build, api-e2e ] if: always() steps: - name: Status @@ -246,7 +304,7 @@ jobs: echo "Skip E2E!" exit 0 fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then + if [[ ${{ needs.api-e2e.result }} != 'success' ]]; then echo "E2E Failed!" exit 1 fi diff --git a/streampark-e2e/pom.xml b/streampark-e2e/pom.xml index a41604694d..f7bf17fccd 100644 --- a/streampark-e2e/pom.xml +++ b/streampark-e2e/pom.xml @@ -41,6 +41,7 @@ 1.7.36 2.17.2 31.0.1-jre + 2.13.5 @@ -127,6 +128,12 @@ ${lombok.version} provided + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + diff --git a/streampark-e2e/scripts/generate-flink-e2e.sh b/streampark-e2e/scripts/generate-flink-e2e.sh new file mode 100755 index 0000000000..98bfe38cf2 --- /dev/null +++ b/streampark-e2e/scripts/generate-flink-e2e.sh @@ -0,0 +1,484 @@ +#!/usr/bin/env bash +# Generates Flink version-specific E2E tests and docker resources. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CASE_DIR="$ROOT/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases" +DOCKER_DIR="$ROOT/streampark-e2e-case/src/test/resources/docker" + +APACHE_LICENSE='/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */' + +YAML_LICENSE='# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#' + +generate_remote_compose() { + local dir="$1" patch="$2" + cat >"$dir/docker-compose.yaml" <"$dir/Dockerfile" <"$dir/docker-compose.yaml" +} + +generate_remote_test() { + local class_prefix="$1" docker_key="$2" patch="$3" + cat >"$CASE_DIR/${class_prefix}OnRemoteClusterDeployTest.java" <addCluster(deployMode) + .jobManagerURL(flinkJobManagerUrl) + .clusterName(flinkClusterName) + .flinkVersion(flinkName) + .submit(); + + Awaitility.await() + .untilAsserted( + () -> assertThat(flinkClustersPage.flinkClusterList) + .as("Flink clusters list should contain newly-created application") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains(flinkClusterName))); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); + + flinkClustersPage.deleteFlinkCluster(flinkClusterName); + + Awaitility.await() + .untilAsserted( + () -> { + browser.navigate().refresh(); + Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); + assertThat(flinkClustersPage.flinkClusterList) + .noneMatch(it -> it.getText().contains(flinkClusterName)); + }); + } +} +EOF +} + +generate_yarn_cluster_test() { + local class_prefix="$1" docker_key="$2" patch="$3" + cat >"$CASE_DIR/${class_prefix}OnYarnClusterDeployTest.java" <addCluster(deployMode) + .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) + .clusterName(flinkClusterName) + .flinkVersion(flinkName) + .submit(); + + Awaitility.await() + .untilAsserted( + () -> assertThat(flinkClustersPage.flinkClusterList) + .as("Flink clusters list should contain newly-created application") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains(flinkClusterName))); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); + + flinkClustersPage.editFlinkCluster(flinkClusterName) + .addCluster(deployMode) + .clusterName(flinkClusterNameEdited) + .submit(); + + Awaitility.await() + .untilAsserted( + () -> assertThat(flinkClustersPage.flinkClusterList) + .as("Flink clusters list should contain edited application") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains(flinkClusterNameEdited))); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); + + flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); + + Awaitility.await() + .untilAsserted( + () -> assertThat(flinkClustersPage.flinkClusterList) + .as("Flink clusters list should contain running application") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains("RUNNING"))); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); + + flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); + + Awaitility.await() + .untilAsserted( + () -> assertThat(flinkClustersPage.flinkClusterList) + .as("Flink clusters list should contain canceled application") + .extracting(WebElement::getText) + .anyMatch(it -> it.contains("CANCELED"))); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); + + flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); + + Awaitility.await() + .untilAsserted( + () -> { + browser.navigate().refresh(); + Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); + assertThat(flinkClustersPage.flinkClusterList) + .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); + }); + } +} +EOF +} + +generate_sql_test() { + local class_prefix="$1" docker_key="$2" patch="$3" app_suffix="$4" + sed \ + -e "s/FlinkSQL120OnYarnTest/FlinkSQL${class_prefix#Flink}OnYarnTest/g" \ + -e "s/flink-1.20-on-yarn/flink-${docker_key}-on-yarn/g" \ + -e "s/flink-1.20.1/flink-${patch}/g" \ + -e "s/flink-120-e2e-test/flink-${app_suffix}-e2e-test/g" \ + -e "s/flink_1.20.1_cluster_e2e/flink_${patch}_cluster_e2e/g" \ + "$CASE_DIR/FlinkSQL120OnYarnTest.java" >"$CASE_DIR/FlinkSQL${class_prefix#Flink}OnYarnTest.java" +} + +generate_k8s_assets() { + local dir="$1" patch="$2" java_tag="$3" + local image="apache/streampark-flink-${patch}-on-k8s:ci" + cat >"$dir/Dockerfile" <"$dir/docker-compose.yaml" +} + +generate_k8s_cluster_test() { + local class_prefix="$1" docker_key="$2" patch="$3" java_tag="$4" + sed \ + -e "s/Flink120OnK8sClusterDeployTest/${class_prefix}OnK8sClusterDeployTest/g" \ + -e "s/flink-1.20-on-k8s/flink-${docker_key}-on-k8s/g" \ + -e "s/flink-1.20.1/flink-${patch}/g" \ + -e "s/flink_1.20.1_k8s_cluster_e2e/flink_${patch}_k8s_cluster_e2e/g" \ + -e "s/flink:1.20.1-scala_2.12-java8/flink:${patch}-scala_2.12-${java_tag}/g" \ + "$CASE_DIR/Flink120OnK8sClusterDeployTest.java" >"$CASE_DIR/${class_prefix}OnK8sClusterDeployTest.java" +} + +generate_k8s_sql_test() { + local class_prefix="$1" docker_key="$2" patch="$3" java_tag="$4" app_suffix="$5" + sed \ + -e "s/FlinkSQL120OnK8sTest/FlinkSQL${class_prefix#Flink}OnK8sTest/g" \ + -e "s/flink-1.20-on-k8s/flink-${docker_key}-on-k8s/g" \ + -e "s/flink-1.20.1/flink-${patch}/g" \ + -e "s/flink_1.20.1_k8s_cluster_e2e/flink_${patch}_k8s_cluster_e2e/g" \ + -e "s/flink120e2etest/flink${app_suffix}e2etest/g" \ + -e "s/flink120session/flink${app_suffix}session/g" \ + -e "s/flink:1.20.1-scala_2.12-java8/flink:${patch}-scala_2.12-${java_tag}/g" \ + "$CASE_DIR/FlinkSQL120OnK8sTest.java" >"$CASE_DIR/FlinkSQL${class_prefix#Flink}OnK8sTest.java" +} + +# docker_key|patch|class_prefix|java_tag|app_suffix +VERSIONS=( + "1.19|1.19.0|Flink119|java8|119" + "2.0|2.0.2|Flink200|java11|200" + "2.1|2.1.2|Flink210|java11|210" + "2.2|2.2.1|Flink220|java11|220" + "2.3|2.3.0|Flink230|java11|230" +) + +for entry in "${VERSIONS[@]}"; do + IFS='|' read -r docker_key patch class_prefix java_tag app_suffix <<<"$entry" + remote_dir="$DOCKER_DIR/flink-${docker_key}-on-remote" + yarn_dir="$DOCKER_DIR/flink-${docker_key}-on-yarn" + k8s_dir="$DOCKER_DIR/flink-${docker_key}-on-k8s" + mkdir -p "$remote_dir" "$yarn_dir" "$k8s_dir" + generate_remote_compose "$remote_dir" "$patch" + generate_yarn_assets "$yarn_dir" "$patch" "$java_tag" + generate_k8s_assets "$k8s_dir" "$patch" "$java_tag" + generate_remote_test "$class_prefix" "$docker_key" "$patch" + generate_yarn_cluster_test "$class_prefix" "$docker_key" "$patch" + generate_k8s_cluster_test "$class_prefix" "$docker_key" "$patch" "$java_tag" + generate_sql_test "$class_prefix" "$docker_key" "$patch" "$app_suffix" + generate_k8s_sql_test "$class_prefix" "$docker_key" "$patch" "$java_tag" "$app_suffix" + echo "Generated Flink ${patch} (${class_prefix})" +done + +# Generate K8s assets for baseline versions managed outside VERSIONS loop +BASELINE_VERSIONS=( + "1.17|1.17.2|Flink117|java8|117" + "1.18|1.18.1|Flink118|java8|118" +) +for entry in "${BASELINE_VERSIONS[@]}"; do + IFS='|' read -r docker_key patch class_prefix java_tag app_suffix <<<"$entry" + k8s_dir="$DOCKER_DIR/flink-${docker_key}-on-k8s" + mkdir -p "$k8s_dir" + generate_k8s_assets "$k8s_dir" "$patch" "$java_tag" + generate_k8s_cluster_test "$class_prefix" "$docker_key" "$patch" "$java_tag" + generate_k8s_sql_test "$class_prefix" "$docker_key" "$patch" "$java_tag" "$app_suffix" + echo "Generated K8s baseline Flink ${patch} (${class_prefix})" +done + +# Remove Flink 1.16 assets +rm -rf "$DOCKER_DIR/flink-1.16-on-remote" "$DOCKER_DIR/flink-1.16-on-yarn" +rm -f "$CASE_DIR"/Flink116*.java "$CASE_DIR"/FlinkSQL116*.java +echo "Removed Flink 1.16 E2E assets" diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java index 87f1f4b17a..d3803433aa 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java @@ -17,133 +17,110 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.alarm.AlarmPage; -import org.apache.streampark.e2e.pages.setting.alarm.AlertTypeDetailForm; -import org.apache.streampark.e2e.pages.setting.alarm.DingTalkAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.EmailAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.LarkAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.WeChatAlertForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class AlarmTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String newEmail = "new@streampark.com"; - private static final String newAlarmName = "new_alarm"; - private static final String editAlarmName = "edit_alarm"; + private static Long alertId; + @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(AlarmPage.class); + api.login(); } @Test @Order(1) public void testCreateAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); - - final String dingTalkURL = ""; - final String dingTalkToken = "dingTalkToken"; - final String dingTalkSecretToken = "dingTalkSecretToken"; - final String dingTalkReceiveUser = "dingTalkUser"; - - final String wechatToken = "wechatToken"; - - final String larkToken = "larkToken"; - final String larkSecretToken = "larkSecretToken"; - - AlertTypeDetailForm alertTypeDetailForm = alarmPage.createAlarm(); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.EMAIL) - .email(newEmail) - .alertName(newAlarmName); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.DINGTALK) - .url(dingTalkURL) - .token(dingTalkToken) - .secretEnable() - .secretToken(dingTalkSecretToken) - .effectToAllUsers() - .receiveUser(dingTalkReceiveUser); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.WECHAT) - .token(wechatToken); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.LARK) - .token(larkToken) - .secretEnable() - .secretToken(larkSecretToken) - .effectToAllUsers() - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as("Alarm list should contain newly-created alarm") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newAlarmName)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.EMAIL.desc)) - .anyMatch(it -> it.contains(newEmail)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.DINGTALK.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.WECHAT.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.LARK.desc)) - .anyMatch(it -> it.contains(dingTalkReceiveUser))); + ApiResponse response = api.postJson("/flink/alert/add", api.writeJson(buildAlertBody(newAlarmName, 23, true))); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postJson("/flink/alert/list", "{}"); + assertThat(list.isSuccess()).isTrue(); + assertThat(list.getData().toString()).contains(newAlarmName); + assertThat(list.getData().toString()).contains(newEmail); } @Test @Order(2) public void testEditAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); - - alarmPage.editAlarm(newAlarmName) - // this step will cancel E-mail type click status. - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.EMAIL) - .alertName(editAlarmName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as("Alarm list should contain edited alarm") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editAlarmName)) - .noneMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.EMAIL.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.DINGTALK.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.WECHAT.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.LARK.desc)) - .noneMatch(it -> it.contains(newEmail))); + alertId = findAlertId(newAlarmName).orElseThrow(); + + ApiResponse response = + api.postJson("/flink/alert/update", api.writeJson(buildAlertBody(editAlarmName, 22, false))); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postJson("/flink/alert/list", "{}"); + assertThat(list.getData().toString()).contains(editAlarmName); + assertThat(list.getData().toString()).doesNotContain(newEmail); } @Test @Order(3) public void testDeleteAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); + ApiResponse response = api.deleteForm("/flink/alert/delete", api.params("id", String.valueOf(alertId))); + assertThat(response.isSuccess()).isTrue(); - alarmPage.deleteAlarm(editAlarmName); + ApiResponse list = api.postJson("/flink/alert/list", "{}"); + assertThat(list.getData().toString()).doesNotContain(editAlarmName); + } + + private static Optional findAlertId(String alertName) { + ApiResponse list = api.postJson("/flink/alert/list", "{}"); + if (list.getData() == null || !list.getData().isArray()) { + return Optional.empty(); + } + for (JsonNode item : list.getData()) { + if (alertName.equals(item.path("alertName").asText())) { + return Optional.of(item.path("id").asLong()); + } + } + return Optional.empty(); + } - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as(String.format("Alarm list shouldn't contain alarm witch named %s", editAlarmName)) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(editAlarmName))); + private static ObjectNode buildAlertBody(String alertName, int alertType, boolean withEmail) { + ObjectNode body = api.objectNode(); + if (alertId != null) { + body.put("id", alertId); + } + body.put("alertName", alertName); + body.put("alertType", alertType); + if (withEmail) { + ObjectNode emailParams = body.putObject("emailParams"); + emailParams.putArray("contacts").add(newEmail); + } + ObjectNode dingTalkParams = body.putObject("dingTalkParams"); + dingTalkParams.put("alertDingURL", ""); + dingTalkParams.put("token", "dingTalkToken"); + dingTalkParams.put("secretEnable", true); + dingTalkParams.put("secretToken", "dingTalkSecretToken"); + dingTalkParams.put("isAtAll", true); + dingTalkParams.putArray("contacts").add("dingTalkUser"); + ObjectNode weComParams = body.putObject("weComParams"); + weComParams.put("token", "wechatToken"); + ObjectNode larkParams = body.putObject("larkParams"); + larkParams.put("token", "larkToken"); + larkParams.put("secretEnable", true); + larkParams.put("secretToken", "larkSecretToken"); + larkParams.put("isAtAll", true); + return body; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java index bf1988167b..f9d0b1c065 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java @@ -17,175 +17,118 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.env.DockerSettingForm; -import org.apache.streampark.e2e.pages.setting.env.EmailSettingForm; -import org.apache.streampark.e2e.pages.setting.env.EnvironmentDetailForm; -import org.apache.streampark.e2e.pages.setting.env.EnvironmentPage; -import org.apache.streampark.e2e.pages.setting.env.IngressSettingForm; -import org.apache.streampark.e2e.pages.setting.env.MavenSettingForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.ui.WebDriverWait; -import org.testcontainers.shaded.org.awaitility.Awaitility; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; +import java.util.LinkedHashMap; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/environment/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/environment/docker-compose.yaml") public class EnvironmentTest { - public static RemoteWebDriver browser; - - // maven - final String mavenFilePath = "/maven/file/path"; - final String mavenCentralRepository = "https://mvnrepository.com/"; - final String mavenAuthUser = "maven_user"; - final String mavenAuthPassword = "maven_password"; - - // ingress - final String ingressDomainAddress = "https://localhost"; - - // docker - final String dockerAddress = "https://hub.docker.com/v2/"; - final String dockerNamespace = "hello"; - final String dockerUser = "docker_user"; - final String dockerPassword = "docker_password"; - - // email - final String emailHost = "smtp.163.com"; - final String editEmailHost = "postfix"; - final String emailPort = "25"; - final String emailAddress = "hello@163.com"; - final String editEmailAddress = "hello@postfix.com"; - final String emailUser = "email_password"; - final String emailPassword = "email_password"; + public static ApiClient api; + + private static final String mavenFilePath = "/maven/file/path"; + private static final String mavenCentralRepository = "https://mvnrepository.com/"; + private static final String mavenAuthUser = "maven_user"; + private static final String mavenAuthPassword = "maven_password"; + private static final String ingressDomainAddress = "https://localhost"; + private static final String dockerAddress = "https://hub.docker.com/v2/"; + private static final String dockerNamespace = "hello"; + private static final String dockerUser = "docker_user"; + private static final String dockerPassword = "docker_password"; + private static final String emailHost = "smtp.163.com"; + private static final String editEmailHost = "postfix"; + private static final String emailPort = "25"; + private static final String emailAddress = "hello@163.com"; + private static final String editEmailAddress = "hello@postfix.com"; + private static final String emailUser = "email_password"; + private static final String emailPassword = "email_password"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(EnvironmentPage.class); + api.login(); } @Test @Order(1) public void testCreateEnvironment() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Maven) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Maven) - .filePath(mavenFilePath) - .centralRepository(mavenCentralRepository) - .authUser(mavenAuthUser) - .authPassword(mavenAuthPassword); - - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Ingress) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Ingress) - .domainAddress(ingressDomainAddress); - - Awaitility.await() - .untilAsserted( - () -> assertThat(environmentPage.settingList) - .as("Setting list should contain newly-created setting") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(mavenFilePath)) - .anyMatch(it -> it.contains(mavenCentralRepository)) - .anyMatch(it -> it.contains(mavenAuthUser)) - .anyMatch(it -> it.contains(ingressDomainAddress))); + updateSetting("streampark.maven.settings", mavenFilePath); + updateSetting("streampark.maven.central.repository", mavenCentralRepository); + updateSetting("streampark.maven.auth.user", mavenAuthUser); + updateSetting("streampark.maven.auth.password", mavenAuthPassword); + updateSetting("ingress.mode.default", ingressDomainAddress); + + ApiResponse list = api.postForm("/setting/all", new LinkedHashMap<>()); + assertThat(list.isSuccess()).isTrue(); + String settings = list.getData().toString(); + assertThat(settings).contains(mavenFilePath); + assertThat(settings).contains(mavenCentralRepository); + assertThat(settings).contains(mavenAuthUser); + assertThat(settings).contains(ingressDomainAddress); } @Test @Order(2) public void testCreateEmailSettingFailedWithAuth() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - EmailSettingForm emailSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .host(emailHost) - .port(emailPort) - .address(emailAddress) - .user(emailUser) - .password(emailPassword) - .ok(); - - String expectedErrorMessage = - "connect to target mail server failed: 535 Error: authentication failed"; - Awaitility.await() - - .untilAsserted( - () -> { - new WebDriverWait(browser, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION); - assertThat(environmentPage.errorMessageList) - .as("Connect failed error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(expectedErrorMessage)); - }); - - WebElementClick(browser, environmentPage.errorMessageConfirmButton); - emailSettingForm.cancel(); + ApiResponse response = api.postForm("/setting/check/email", emailParams(emailHost, emailAddress)); + assertThat(response.isSuccess()).isTrue(); + assertThat(response.getData().path("msg").asText()) + .contains("connect to target mail server failed: 535 Error: authentication failed"); } @Test @Order(3) public void testCreateEmailSettingSuccessful() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - EmailSettingForm emailSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .host(editEmailHost) - .port(emailPort) - .address(editEmailAddress) - .user(emailUser) - .password(emailPassword) - .ok(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(environmentPage.settingList) - .as("Setting list should contain newly-created email setting") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editEmailAddress))); + ApiResponse check = api.postForm("/setting/check/email", emailParams(editEmailHost, editEmailAddress)); + assertThat(check.isSuccess()).isTrue(); + + ApiResponse update = api.postForm("/setting/update/email", emailParams(editEmailHost, editEmailAddress)); + assertThat(update.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/setting/all", new LinkedHashMap<>()); + assertThat(list.getData().toString()).contains(editEmailAddress); } @Test @Order(4) public void testCreateDockerSettingFailed() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - DockerSettingForm dockerSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Docker) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Docker) - .address(dockerAddress) - .namespace(dockerNamespace) - .user(dockerUser) - .password(dockerPassword) - .ok(); - - String expectedErrorMessage = String.format( - "Failed to validate Docker registry, error: Status 500: {\"message\":\"login attempt to %s failed with status: 404 Not Found\"}", - dockerAddress); - Awaitility.await() - - .untilAsserted( - () -> { - new WebDriverWait(browser, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION); - assertThat(environmentPage.errorMessageList) - .as("Failed to validate docker registry error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(expectedErrorMessage)); - }); - - WebElementClick(browser, environmentPage.errorMessageConfirmButton); - dockerSettingForm.cancel(); + Map params = new LinkedHashMap<>(); + params.put("address", dockerAddress); + params.put("namespace", dockerNamespace); + params.put("username", dockerUser); + params.put("password", dockerPassword); + + ApiResponse response = api.postForm("/setting/check/docker", params); + assertThat(response.isSuccess()).isTrue(); + assertThat(response.getData().path("msg").asText()) + .contains( + String.format( + "Failed to validate Docker registry, error: Status 500: {\"message\":\"login attempt to %s failed with status: 404 Not Found\"}", + dockerAddress)); + } + + private static void updateSetting(String key, String value) { + ApiResponse response = + api.postForm("/setting/update", api.params("settingKey", key, "settingValue", value)); + assertThat(response.isSuccess()).isTrue(); + } + + private static Map emailParams(String host, String from) { + Map params = new LinkedHashMap<>(); + params.put("host", host); + params.put("port", emailPort); + params.put("from", from); + params.put("userName", emailUser); + params.put("password", emailPassword); + params.put("ssl", "false"); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java index aba14b87be..6638de0854 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java @@ -17,107 +17,117 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.ExternalLinkPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; +import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class ExternalLinkTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String newLabel = "new_label"; - private static final String editLabel = "edit_label"; - private static final String newName = "new_name"; - private static final String color = "#b54f4f"; - private static final String newLink = "https://grafana/flink-monitoring?var-JobId=var-JobId=1"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(ExternalLinkPage.class); + api.login(); } @Test @Order(1) void testCreateExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - externalLinkPage.createExternalLink(newLabel, newName, color, newLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.externalLinkList) - .as("External link list should contain newly-created link") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newLabel)) - .anyMatch(it -> it.contains(newName)) - .anyMatch(it -> it.contains(newLink))); + ApiResponse response = api.postForm("/flink/externalLink/create", linkParams(newLabel, newName, newLink)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/flink/externalLink/list", new LinkedHashMap<>()); + assertThat(listContainsLink(list, newLabel, newName, newLink)).isTrue(); } @Test @Order(2) void testCreateDuplicateExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - externalLinkPage.createExternalLink(newLabel, newName, color, newLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.errorMessageList) - .as("Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - String.format("The name: %s is already existing.", newName)))); - - externalLinkPage.errorMessageConfirmButton.click(); - externalLinkPage.createExternalLinkForm.buttonCancel.click(); + ApiResponse response = api.postForm("/flink/externalLink/create", linkParams(newLabel, newName, newLink)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains(String.format("The name: %s is already existing.", newName)); } @Test @Order(3) void testEditExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); String editName = "edit_name"; String editLink = "https://grafana/flink-monitoring?var-Job=var-Job=edit"; - externalLinkPage.editExternalLink(newLabel, editLabel, editName, color, editLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.externalLinkList) - .as("External link list should contain edited link") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editLabel)) - .anyMatch(it -> it.contains(editName)) - .anyMatch(it -> it.contains(editLink))); + Long id = findLinkId(newLabel).orElseThrow(); + + Map params = linkParams(editLabel, editName, editLink); + params.put("id", String.valueOf(id)); + + ApiResponse response = api.postForm("/flink/externalLink/update", params); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/flink/externalLink/list", new LinkedHashMap<>()); + assertThat(listContainsLink(list, editLabel, editName, editLink)).isTrue(); } @Test @Order(4) void testDeleteExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); + Long id = findLinkId(editLabel).orElseThrow(); + ApiResponse response = api.deleteForm("/flink/externalLink/delete", api.params("id", String.valueOf(id))); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/flink/externalLink/list", new LinkedHashMap<>()); + assertThat(findLinkId(editLabel)).isEmpty(); + assertThat(list.getData().toString()).doesNotContain(editLabel); + } - externalLinkPage.deleteExternalLink(editLabel); + private static Map linkParams(String label, String name, String linkUrl) { + Map params = new LinkedHashMap<>(); + params.put("badgeLabel", label); + params.put("badgeName", name); + params.put("badgeColor", color); + params.put("linkUrl", linkUrl); + return params; + } + + private static Optional findLinkId(String label) { + ApiResponse list = api.postForm("/flink/externalLink/list", new LinkedHashMap<>()); + if (list.getData() == null || !list.getData().isArray()) { + return Optional.empty(); + } + for (JsonNode item : list.getData()) { + if (label.equals(item.path("badgeLabel").asText())) { + return Optional.of(item.path("id").asLong()); + } + } + return Optional.empty(); + } - Awaitility.await() - .untilAsserted( - () -> { - assertThat(externalLinkPage.externalLinkList) - .noneMatch(it -> it.getText().contains(editLabel)); - }); + private static boolean listContainsLink(ApiResponse list, String label, String name, String linkUrl) { + if (list.getData() == null || !list.getData().isArray()) { + return false; + } + for (JsonNode item : list.getData()) { + if (label.equals(item.path("badgeLabel").asText()) + && name.equals(item.path("badgeName").asText()) + && linkUrl.equals(item.path("linkUrl").asText())) { + return true; + } + } + return false; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java deleted file mode 100644 index b8eb70bb0d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-remote/docker-compose.yaml") -public class Flink116OnRemoteClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/opt/flink/"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java deleted file mode 100644 index b48221907e..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-yarn/docker-compose.yaml") -public class Flink116OnYarnClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/flink-1.16.3"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - private static final String flinkClusterNameEdited = "flink_1.16.3_cluster_e2e_edited"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); - } - - @Test - @Order(3) - public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - } - - @Test - @Order(4) - public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..504e2fd355 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.17-on-k8s/docker-compose.yaml") +public class Flink117OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.17.2"; + + private static final String flinkHome = "/flink-1.17.2"; + + private static final String clusterName = "flink_1.17.2_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_1.17.2_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:1.17.2-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java index fdb0c5d019..1b863a44ad 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java @@ -17,88 +17,60 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.17-on-remote/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.17-on-remote/docker-compose.yaml") public class Flink117OnRemoteClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.17.2"; private static final String flinkHome = "/opt/flink/"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.17.2_cluster_e2e"; - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; + private static final String jobManagerUrl = "http://jobmanager:8081"; - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; + private static long clusterId; @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java index 34e145f9ea..eab4269ba6 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java @@ -17,136 +17,89 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") public class Flink117OnYarnClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.17.2"; private static final String flinkHome = "/flink-1.17.2"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.17.2_cluster_e2e"; - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; + private static final String clusterNameEdited = "flink_1.17.2_cluster_e2e_edited"; - private static final String flinkClusterNameEdited = "flink_1.17.2_cluster_e2e_edited"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; + private static long clusterId; @BeforeAll public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(2) public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); } @Test @Order(3) public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(4) public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..f9ef3fecca --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.18-on-k8s/docker-compose.yaml") +public class Flink118OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.18.1"; + + private static final String flinkHome = "/flink-1.18.1"; + + private static final String clusterName = "flink_1.18.1_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_1.18.1_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:1.18.1-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java index 60f07c97a3..30fcbabc6f 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java @@ -17,88 +17,60 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.18-on-remote/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.18-on-remote/docker-compose.yaml") public class Flink118OnRemoteClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.18.1"; private static final String flinkHome = "/opt/flink/"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.18.1_cluster_e2e"; - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; + private static final String jobManagerUrl = "http://jobmanager:8081"; - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; + private static long clusterId; @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java index cadb7a39d2..75fb1be346 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java @@ -17,136 +17,89 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") public class Flink118OnYarnClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.18.1"; private static final String flinkHome = "/flink-1.18.1"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.18.1_cluster_e2e"; - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; + private static final String clusterNameEdited = "flink_1.18.1_cluster_e2e_edited"; - private static final String flinkClusterNameEdited = "flink_1.18.1_cluster_e2e_edited"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; + private static long clusterId; @BeforeAll public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(2) public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); } @Test @Order(3) public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(4) public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..0e31ac667c --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.19-on-k8s/docker-compose.yaml") +public class Flink119OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.19.0"; + + private static final String flinkHome = "/flink-1.19.0"; + + private static final String clusterName = "flink_1.19.0_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_1.19.0_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:1.19.0-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnRemoteClusterDeployTest.java new file mode 100644 index 0000000000..3a4db6ba9e --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnRemoteClusterDeployTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.19-on-remote/docker-compose.yaml") +public class Flink119OnRemoteClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.19.0"; + + private static final String flinkHome = "/opt/flink/"; + + private static final String clusterName = "flink_1.19.0_cluster_e2e"; + + private static final String jobManagerUrl = "http://jobmanager:8081"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnYarnClusterDeployTest.java new file mode 100644 index 0000000000..a3f44888f9 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink119OnYarnClusterDeployTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.19-on-yarn/docker-compose.yaml") +public class Flink119OnYarnClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.19.0"; + + private static final String flinkHome = "/flink-1.19.0"; + + private static final String clusterName = "flink_1.19.0_cluster_e2e"; + + private static final String clusterNameEdited = "flink_1.19.0_cluster_e2e_edited"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..36b974fcf0 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.20-on-k8s/docker-compose.yaml") +public class Flink120OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.20.1"; + + private static final String flinkHome = "/flink-1.20.1"; + + private static final String clusterName = "flink_1.20.1_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_1.20.1_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:1.20.1-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java index 74c905a8ec..0b4771df20 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java @@ -17,88 +17,60 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.20-on-remote/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.20-on-remote/docker-compose.yaml") public class Flink120OnRemoteClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.20.1"; private static final String flinkHome = "/opt/flink/"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.20.1_cluster_e2e"; - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; + private static final String jobManagerUrl = "http://jobmanager:8081"; - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; + private static long clusterId; @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java index d6d379cc5d..564f6ce888 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java @@ -17,136 +17,89 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") public class Flink120OnYarnClusterDeployTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.20.1"; private static final String flinkHome = "/flink-1.20.1"; - private static final String flinkDescription = "description test"; + private static final String clusterName = "flink_1.20.1_cluster_e2e"; - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; + private static final String clusterNameEdited = "flink_1.20.1_cluster_e2e_edited"; - private static final String flinkClusterNameEdited = "flink_1.20.1_cluster_e2e_edited"; + private static long versionId; - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; + private static long clusterId; @BeforeAll public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); } @Test @Order(1) public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); } @Test @Order(2) public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); } @Test @Order(3) public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(4) public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); } @Test @Order(5) public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..94e38decd1 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.0-on-k8s/docker-compose.yaml") +public class Flink200OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.0.2"; + + private static final String flinkHome = "/flink-2.0.2"; + + private static final String clusterName = "flink_2.0.2_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.0.2_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:2.0.2-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnRemoteClusterDeployTest.java new file mode 100644 index 0000000000..6b9a54f677 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnRemoteClusterDeployTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.0-on-remote/docker-compose.yaml") +public class Flink200OnRemoteClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.0.2"; + + private static final String flinkHome = "/opt/flink/"; + + private static final String clusterName = "flink_2.0.2_cluster_e2e"; + + private static final String jobManagerUrl = "http://jobmanager:8081"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnYarnClusterDeployTest.java new file mode 100644 index 0000000000..d8c359926d --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink200OnYarnClusterDeployTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.0-on-yarn/docker-compose.yaml") +public class Flink200OnYarnClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.0.2"; + + private static final String flinkHome = "/flink-2.0.2"; + + private static final String clusterName = "flink_2.0.2_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.0.2_cluster_e2e_edited"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..4ebc8e75aa --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.1-on-k8s/docker-compose.yaml") +public class Flink210OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.1.2"; + + private static final String flinkHome = "/flink-2.1.2"; + + private static final String clusterName = "flink_2.1.2_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.1.2_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:2.1.2-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnRemoteClusterDeployTest.java new file mode 100644 index 0000000000..8c8f7879a1 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnRemoteClusterDeployTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.1-on-remote/docker-compose.yaml") +public class Flink210OnRemoteClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.1.2"; + + private static final String flinkHome = "/opt/flink/"; + + private static final String clusterName = "flink_2.1.2_cluster_e2e"; + + private static final String jobManagerUrl = "http://jobmanager:8081"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnYarnClusterDeployTest.java new file mode 100644 index 0000000000..2b09e45308 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink210OnYarnClusterDeployTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.1-on-yarn/docker-compose.yaml") +public class Flink210OnYarnClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.1.2"; + + private static final String flinkHome = "/flink-2.1.2"; + + private static final String clusterName = "flink_2.1.2_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.1.2_cluster_e2e_edited"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..31dbbbfd18 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.2-on-k8s/docker-compose.yaml") +public class Flink220OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.2.1"; + + private static final String flinkHome = "/flink-2.2.1"; + + private static final String clusterName = "flink_2.2.1_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.2.1_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:2.2.1-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnRemoteClusterDeployTest.java new file mode 100644 index 0000000000..82a77a81dd --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnRemoteClusterDeployTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.2-on-remote/docker-compose.yaml") +public class Flink220OnRemoteClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.2.1"; + + private static final String flinkHome = "/opt/flink/"; + + private static final String clusterName = "flink_2.2.1_cluster_e2e"; + + private static final String jobManagerUrl = "http://jobmanager:8081"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnYarnClusterDeployTest.java new file mode 100644 index 0000000000..73773002d3 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink220OnYarnClusterDeployTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.2-on-yarn/docker-compose.yaml") +public class Flink220OnYarnClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.2.1"; + + private static final String flinkHome = "/flink-2.2.1"; + + private static final String clusterName = "flink_2.2.1_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.2.1_cluster_e2e_edited"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnK8sClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnK8sClusterDeployTest.java new file mode 100644 index 0000000000..8068718955 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnK8sClusterDeployTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.3-on-k8s/docker-compose.yaml") +public class Flink230OnK8sClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.3.0"; + + private static final String flinkHome = "/flink-2.3.0"; + + private static final String clusterName = "flink_2.3.0_k8s_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.3.0_k8s_cluster_e2e_edited"; + + private static final String flinkImage = "flink:2.3.0-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("id", clusterId); + body.put("flinkImage", flinkImage); + body.put("k8sNamespace", k8sNamespace); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnRemoteClusterDeployTest.java new file mode 100644 index 0000000000..7e1f32043d --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnRemoteClusterDeployTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.3-on-remote/docker-compose.yaml") +public class Flink230OnRemoteClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.3.0"; + + private static final String flinkHome = "/opt/flink/"; + + private static final String clusterName = "flink_2.3.0_cluster_e2e"; + + private static final String jobManagerUrl = "http://jobmanager:8081"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_STANDALONE, versionId); + body.put("address", jobManagerUrl); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnYarnClusterDeployTest.java new file mode 100644 index 0000000000..54fa482373 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink230OnYarnClusterDeployTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.3-on-yarn/docker-compose.yaml") +public class Flink230OnYarnClusterDeployTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.3.0"; + + private static final String flinkHome = "/flink-2.3.0"; + + private static final String clusterName = "flink_2.3.0_cluster_e2e"; + + private static final String clusterNameEdited = "flink_2.3.0_cluster_e2e_edited"; + + private static long versionId; + + private static long clusterId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + } + + @Test + @Order(1) + public void testCreateFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_CHILD_FIRST); + clusterId = flink.createCluster(body); + assertThat(clusterId).isPositive(); + } + + @Test + @Order(2) + public void testEditFlinkCluster() { + ObjectNode body = + flink.baseClusterBody(clusterNameEdited, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("id", clusterId); + flink.updateCluster(body); + assertThat(flink.requireClusterId(clusterNameEdited)).isEqualTo(clusterId); + } + + @Test + @Order(3) + public void testStartFlinkCluster() { + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(4) + public void testStopFlinkCluster() { + flink.shutdownCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_CANCELED, Duration.ofMinutes(15)); + } + + @Test + @Order(5) + public void testDeleteFlinkCluster() { + flink.deleteCluster(clusterId); + assertThatThrownBy(() -> flink.requireClusterId(clusterNameEdited)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java deleted file mode 100644 index a08141a9a9..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.apache.streampark.e2e.pages.common.Constants.TEST_FLINK_SQL; -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-yarn/docker-compose.yaml") -public class FlinkSQL116OnYarnTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/flink-1.16.3"; - - private static final String applicationName = "flink-116-e2e-test"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - flinkClustersPage.startFlinkCluster(flinkClusterName); - - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); - } - - @Test - @Order(1) - void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(2) - void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(3) - void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(4) - @SneakyThrows - void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(9) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(10) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(11) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(12) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(13) - void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnK8sTest.java new file mode 100644 index 0000000000..f3bb40bee7 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.17-on-k8s/docker-compose.yaml") +public class FlinkSQL117OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.17.2"; + + private static final String flinkHome = "/flink-1.17.2"; + + private static final String clusterName = "flink_1.17.2_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink117e2etest"; + + private static final String k8sSessionJobName = "flink117session"; + + private static final String flinkImage = "flink:1.17.2-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java index c2cdaff273..820b3b72b2 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java @@ -17,341 +17,144 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; -import lombok.SneakyThrows; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") public class FlinkSQL117OnYarnTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.17.2"; private static final String flinkHome = "/flink-1.17.2"; - private static final String applicationName = "flink-117-e2e-test"; + private static final String clusterName = "flink_1.17.2_cluster_e2e"; - private static final String flinkDescription = "description test"; + private static final String yarnAppJobName = "flink-117-e2e-test"; - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); + private static final String yarnSessionJobName = "flink-117-session-e2e"; - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); + private static long versionId; - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + private static long clusterId; - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); + private static long yarnAppId; - flinkClustersPage.startFlinkCluster(flinkClusterName); + private static long yarnSessionAppId; - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(1) void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); } @Test @Order(2) void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + flink.releaseApp(yarnAppId); } @Test @Order(3) void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test @Order(4) - @SneakyThrows void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test @Order(5) void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); } @Test @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); } @Test @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); } @Test @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test @Order(9) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test @Order(10) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(11) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(12) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(13) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(14) void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnK8sTest.java new file mode 100644 index 0000000000..4016193c9f --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.18-on-k8s/docker-compose.yaml") +public class FlinkSQL118OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.18.1"; + + private static final String flinkHome = "/flink-1.18.1"; + + private static final String clusterName = "flink_1.18.1_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink118e2etest"; + + private static final String k8sSessionJobName = "flink118session"; + + private static final String flinkImage = "flink:1.18.1-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java index df8e126cd9..4c52cb1836 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java @@ -17,355 +17,144 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; -import lombok.SneakyThrows; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") public class FlinkSQL118OnYarnTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.18.1"; private static final String flinkHome = "/flink-1.18.1"; - private static final String applicationName = "flink-118-e2e-test"; + private static final String clusterName = "flink_1.18.1_cluster_e2e"; - private static final String flinkDescription = "description test"; + private static final String yarnAppJobName = "flink-118-e2e-test"; - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; + private static final String yarnSessionJobName = "flink-118-session-e2e"; - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); + private static long versionId; - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); + private static long clusterId; - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + private static long yarnAppId; - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); + private static long yarnSessionAppId; - flinkClustersPage.startFlinkCluster(flinkClusterName); + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(1) void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); } @Test @Order(2) void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + flink.releaseApp(yarnAppId); } @Test @Order(3) void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test @Order(4) - @SneakyThrows void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test @Order(5) void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); } @Test @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); } @Test @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); } @Test @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test @Order(9) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test @Order(10) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(11) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(12) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(13) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(14) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(15) void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.deleteApplication(applicationName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnK8sTest.java new file mode 100644 index 0000000000..4bc8fadbdb --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.19-on-k8s/docker-compose.yaml") +public class FlinkSQL119OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.19.0"; + + private static final String flinkHome = "/flink-1.19.0"; + + private static final String clusterName = "flink_1.19.0_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink119e2etest"; + + private static final String k8sSessionJobName = "flink119session"; + + private static final String flinkImage = "flink:1.19.0-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnYarnTest.java new file mode 100644 index 0000000000..ce9f9d01ca --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL119OnYarnTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.19-on-yarn/docker-compose.yaml") +public class FlinkSQL119OnYarnTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.19.0"; + + private static final String flinkHome = "/flink-1.19.0"; + + private static final String clusterName = "flink_1.19.0_cluster_e2e"; + + private static final String yarnAppJobName = "flink-119-e2e-test"; + + private static final String yarnSessionJobName = "flink-119-session-e2e"; + + private static long versionId; + + private static long clusterId; + + private static long yarnAppId; + + private static long yarnSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnYarnApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnYarnApplicationMode() { + flink.releaseApp(yarnAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnYarnApplicationMode() { + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnYarnSessionMode() { + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnK8sTest.java new file mode 100644 index 0000000000..383cabcab7 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-1.20-on-k8s/docker-compose.yaml") +public class FlinkSQL120OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-1.20.1"; + + private static final String flinkHome = "/flink-1.20.1"; + + private static final String clusterName = "flink_1.20.1_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink120e2etest"; + + private static final String k8sSessionJobName = "flink120session"; + + private static final String flinkImage = "flink:1.20.1-scala_2.12-java8"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java index 39147c962c..1f5f887779 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java @@ -17,253 +17,144 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; -import lombok.SneakyThrows; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -@StreamPark(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") public class FlinkSQL120OnYarnTest { - public static RemoteWebDriver browser; + public static ApiClient api; + + private static FlinkApiSupport flink; private static final String flinkName = "flink-1.20.1"; private static final String flinkHome = "/flink-1.20.1"; - private static final String applicationName = "flink-120-e2e-test"; + private static final String clusterName = "flink_1.20.1_cluster_e2e"; - private static final String flinkDescription = "description test"; + private static final String yarnAppJobName = "flink-120-e2e-test"; - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); + private static final String yarnSessionJobName = "flink-120-session-e2e"; - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); + private static long versionId; - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); + private static long clusterId; - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); + private static long yarnAppId; - flinkClustersPage.startFlinkCluster(flinkClusterName); + private static long yarnSessionAppId; - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); } @Test @Order(1) void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); } @Test @Order(2) void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + flink.releaseApp(yarnAppId); } @Test @Order(3) void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test @Order(4) - @SneakyThrows void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test @Order(5) void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); } @Test - @Order(11) + @Order(6) void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); } @Test - @Order(12) + @Order(7) void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); + flink.releaseApp(yarnSessionAppId); } @Test - @Order(13) + @Order(8) void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); } @Test - @Order(14) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); } @Test - @Order(15) + @Order(10) void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.deleteApplication(applicationName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnK8sTest.java new file mode 100644 index 0000000000..8a59957bf9 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.0-on-k8s/docker-compose.yaml") +public class FlinkSQL200OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.0.2"; + + private static final String flinkHome = "/flink-2.0.2"; + + private static final String clusterName = "flink_2.0.2_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink200e2etest"; + + private static final String k8sSessionJobName = "flink200session"; + + private static final String flinkImage = "flink:2.0.2-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnYarnTest.java new file mode 100644 index 0000000000..d909bd1f22 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL200OnYarnTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.0-on-yarn/docker-compose.yaml") +public class FlinkSQL200OnYarnTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.0.2"; + + private static final String flinkHome = "/flink-2.0.2"; + + private static final String clusterName = "flink_2.0.2_cluster_e2e"; + + private static final String yarnAppJobName = "flink-200-e2e-test"; + + private static final String yarnSessionJobName = "flink-200-session-e2e"; + + private static long versionId; + + private static long clusterId; + + private static long yarnAppId; + + private static long yarnSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnYarnApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnYarnApplicationMode() { + flink.releaseApp(yarnAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnYarnApplicationMode() { + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnYarnSessionMode() { + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnK8sTest.java new file mode 100644 index 0000000000..0d6dfe9db3 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.1-on-k8s/docker-compose.yaml") +public class FlinkSQL210OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.1.2"; + + private static final String flinkHome = "/flink-2.1.2"; + + private static final String clusterName = "flink_2.1.2_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink210e2etest"; + + private static final String k8sSessionJobName = "flink210session"; + + private static final String flinkImage = "flink:2.1.2-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnYarnTest.java new file mode 100644 index 0000000000..6fa3cc9ef4 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL210OnYarnTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.1-on-yarn/docker-compose.yaml") +public class FlinkSQL210OnYarnTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.1.2"; + + private static final String flinkHome = "/flink-2.1.2"; + + private static final String clusterName = "flink_2.1.2_cluster_e2e"; + + private static final String yarnAppJobName = "flink-210-e2e-test"; + + private static final String yarnSessionJobName = "flink-210-session-e2e"; + + private static long versionId; + + private static long clusterId; + + private static long yarnAppId; + + private static long yarnSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnYarnApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnYarnApplicationMode() { + flink.releaseApp(yarnAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnYarnApplicationMode() { + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnYarnSessionMode() { + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnK8sTest.java new file mode 100644 index 0000000000..d5a94bea3b --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.2-on-k8s/docker-compose.yaml") +public class FlinkSQL220OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.2.1"; + + private static final String flinkHome = "/flink-2.2.1"; + + private static final String clusterName = "flink_2.2.1_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink220e2etest"; + + private static final String k8sSessionJobName = "flink220session"; + + private static final String flinkImage = "flink:2.2.1-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnYarnTest.java new file mode 100644 index 0000000000..bf1c14a7a7 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL220OnYarnTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.2-on-yarn/docker-compose.yaml") +public class FlinkSQL220OnYarnTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.2.1"; + + private static final String flinkHome = "/flink-2.2.1"; + + private static final String clusterName = "flink_2.2.1_cluster_e2e"; + + private static final String yarnAppJobName = "flink-220-e2e-test"; + + private static final String yarnSessionJobName = "flink-220-session-e2e"; + + private static long versionId; + + private static long clusterId; + + private static long yarnAppId; + + private static long yarnSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnYarnApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnYarnApplicationMode() { + flink.releaseApp(yarnAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnYarnApplicationMode() { + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnYarnSessionMode() { + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnK8sTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnK8sTest.java new file mode 100644 index 0000000000..f9c6d46260 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnK8sTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.3-on-k8s/docker-compose.yaml") +public class FlinkSQL230OnK8sTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.3.0"; + + private static final String flinkHome = "/flink-2.3.0"; + + private static final String clusterName = "flink_2.3.0_k8s_cluster_e2e"; + + private static final String k8sAppJobName = "flink230e2etest"; + + private static final String k8sSessionJobName = "flink230session"; + + private static final String flinkImage = "flink:2.3.0-scala_2.12-java11"; + + private static final String k8sNamespace = "default"; + + private static long versionId; + + private static long clusterId; + + private static long k8sAppId; + + private static long k8sSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterBody.put("flinkImage", flinkImage); + clusterBody.put("k8sNamespace", k8sNamespace); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(20)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnKubernetesApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sAppJobName, FlinkApiConstants.DEPLOY_K8S_APPLICATION, versionId); + body.put("k8sNamespace", k8sNamespace); + body.put("flinkImage", flinkImage); + k8sAppId = flink.createApp(body); + assertThat(k8sAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnKubernetesApplicationMode() { + flink.releaseApp(k8sAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnKubernetesApplicationMode() { + flink.startApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sAppId); + flink.waitAppState( + k8sAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnKubernetesApplicationMode() { + flink.deleteApp(k8sAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnKubernetesSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(k8sSessionJobName, FlinkApiConstants.DEPLOY_K8S_SESSION, versionId); + body.put("flinkClusterId", clusterId); + k8sSessionAppId = flink.createApp(body); + assertThat(k8sSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnKubernetesSessionMode() { + flink.releaseApp(k8sSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(25)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnKubernetesSessionMode() { + flink.startApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(25)); + flink.cancelApp(k8sSessionAppId); + flink.waitAppState( + k8sSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(25)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnKubernetesSessionMode() { + flink.deleteApp(k8sSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(k8sSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnYarnTest.java new file mode 100644 index 0000000000..f72f4a501d --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL230OnYarnTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.cases; + +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.flink.FlinkApiConstants; +import org.apache.streampark.e2e.core.api.flink.FlinkApiSupport; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@StreamParkApi(composeFiles = "docker/flink-2.3-on-yarn/docker-compose.yaml") +public class FlinkSQL230OnYarnTest { + + public static ApiClient api; + + private static FlinkApiSupport flink; + + private static final String flinkName = "flink-2.3.0"; + + private static final String flinkHome = "/flink-2.3.0"; + + private static final String clusterName = "flink_2.3.0_cluster_e2e"; + + private static final String yarnAppJobName = "flink-230-e2e-test"; + + private static final String yarnSessionJobName = "flink-230-session-e2e"; + + private static long versionId; + + private static long clusterId; + + private static long yarnAppId; + + private static long yarnSessionAppId; + + @BeforeAll + public static void setup() { + api.login(); + flink = new FlinkApiSupport(api); + flink.createFlinkEnv(flinkName, flinkHome, "description test"); + versionId = flink.requireVersionId(flinkName); + + ObjectNode clusterBody = + flink.baseClusterBody(clusterName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + clusterBody.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + clusterId = flink.createCluster(clusterBody); + flink.startCluster(clusterId); + flink.waitClusterState( + clusterId, state -> state == FlinkApiConstants.CLUSTER_RUNNING, Duration.ofMinutes(15)); + } + + @Test + @Order(1) + void testCreateFlinkApplicationOnYarnApplicationMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnAppJobName, FlinkApiConstants.DEPLOY_YARN_APPLICATION, versionId); + yarnAppId = flink.createApp(body); + assertThat(yarnAppId).isPositive(); + } + + @Test + @Order(2) + void testReleaseFlinkApplicationOnYarnApplicationMode() { + flink.releaseApp(yarnAppId); + } + + @Test + @Order(3) + void testStartFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(4) + void testCancelFlinkApplicationOnYarnApplicationMode() { + flink.startApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnAppId); + flink.waitAppState( + yarnAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(5) + void testDeleteFlinkApplicationOnYarnApplicationMode() { + flink.deleteApp(yarnAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnAppJobName)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + @Order(6) + void testCreateFlinkApplicationOnYarnSessionMode() { + ObjectNode body = + flink.baseSqlAppBody(yarnSessionJobName, FlinkApiConstants.DEPLOY_YARN_SESSION, versionId); + body.put("flinkClusterId", clusterId); + yarnSessionAppId = flink.createApp(body); + assertThat(yarnSessionAppId).isPositive(); + } + + @Test + @Order(7) + void testReleaseFlinkApplicationOnYarnSessionMode() { + flink.releaseApp(yarnSessionAppId); + } + + @Test + @Order(8) + void testStartFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_FINISHED, Duration.ofMinutes(20)); + } + + @Test + @Order(9) + void testCancelFlinkApplicationOnYarnSessionMode() { + flink.startApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_RUNNING, Duration.ofMinutes(20)); + flink.cancelApp(yarnSessionAppId); + flink.waitAppState( + yarnSessionAppId, state -> state == FlinkApiConstants.APP_CANCELED, Duration.ofMinutes(20)); + } + + @Test + @Order(10) + void testDeleteFlinkApplicationOnYarnSessionMode() { + flink.deleteApp(yarnSessionAppId); + assertThatThrownBy(() -> flink.requireAppId(yarnSessionJobName)) + .isInstanceOf(IllegalStateException.class); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java index db60907905..098ad16ac6 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java @@ -17,101 +17,101 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.MemberManagementPage; -import org.apache.streampark.e2e.pages.system.SystemPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class MemberManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String existUserName = "test3"; - private static final String existRole = "developer"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(MemberManagementPage.class); + api.login(); } @Test @Order(1) void testCreateMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - - memberManagementPage.createMember(existUserName, existRole); + ApiResponse response = api.postForm("/member/post", memberParams(existUserName, existRole)); + assertThat(response.isSuccess()).isTrue(); - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.memberList) - .as("Member list should contain newly-created member") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(existUserName))); + ApiResponse list = memberList(); + assertThat(api.pageRecordsContain(list, "userName", existUserName)).isTrue(); } @Test @Order(2) void testCreateDuplicateMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - - memberManagementPage.createMember(existUserName, existRole); - - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.errorMessage) - .as("Member Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .matches(it -> it.contains("please don't add it again."))); - - memberManagementPage.errorMessageConfirmButton.click(); - memberManagementPage.createMemberForm.buttonCancel.click(); + ApiResponse response = api.postForm("/member/post", memberParams(existUserName, existRole)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("please don't add it again."); } @Test @Order(3) void testEditMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); String anotherRole = "team admin"; + ApiResponse list = memberList(); + Long memberId = + api.findInPageRecords(list, "userName", existUserName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); - memberManagementPage.editMember(existUserName, anotherRole); + Map params = memberParams(existUserName, anotherRole); + params.put("id", String.valueOf(memberId)); - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.memberList) - .as("Team list should contain edited team") - .extracting(WebElement::getText) - .anyMatch( - it -> it.contains(existUserName))); + ApiResponse response = api.putForm("/member/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = memberList(); + assertThat(api.pageRecordsContain(list, "userName", existUserName)).isTrue(); } @Test @Order(4) void testDeleteMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); + ApiResponse list = memberList(); + Long memberId = + api.findInPageRecords(list, "userName", existUserName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("id", String.valueOf(memberId)); - memberManagementPage.deleteMember(existUserName); + ApiResponse response = api.deleteForm("/member/delete", params); + assertThat(response.isSuccess()).isTrue(); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); + list = memberList(); + assertThat(api.pageRecordsContain(list, "userName", existUserName)).isFalse(); + } + + private static ApiResponse memberList() { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("pageNum", "1"); + params.put("pageSize", "100"); + return api.postForm("/member/list", params); + } - assertThat(memberManagementPage.memberList) - .noneMatch(it -> it.getText().contains(existUserName)); - }); + private static Map memberParams(String userName, String roleName) { + Long roleId = api.findRoleIdByName(roleName).orElseThrow(); + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("userName", userName); + params.put("roleId", String.valueOf(roleId)); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java index d6b1b82f60..d696a5cd78 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java @@ -17,110 +17,122 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ProjectsPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; +import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.PageFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class ProjectsManagementTest { - private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(15); + private static final Duration PROJECT_BUILD_TIMEOUT = Duration.ofMinutes(15); - public static RemoteWebDriver browser; + public static ApiClient api; private static final String projectName = "e2e_test_project"; - private static final String editedProjectName = "e2e_test_project_edited"; - private static final String url = "https://github.com/apache/streampark-quickstart"; - private static final String branch = "dev"; - private static final String buildArgument = "-pl quickstart-flink/quickstart-apacheflink/apacheflinksql_1.17 -am -Dmaven.test.skip=true"; - private static final String description = "e2e test project description"; + private static Long projectId; + @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(ProjectsPage.class); + api.login(); } @Test @Order(1) void testCreateProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - - projectsPage.createProject(projectName, url, branch, buildArgument, description); + ApiResponse response = api.postForm("/project/create", projectParams(projectName)); + assertThat(response.isSuccess()).isTrue(); - Awaitility.await() - .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain newly-created project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(projectName))); + ApiResponse list = projectList(); + assertThat(api.pageRecordsContain(list, "name", projectName)).isTrue(); } @Test @Order(2) void testEditProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - projectsPage.editProject(projectName, editedProjectName); - Awaitility.await() - .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain edited project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editedProjectName))); + ApiResponse list = projectList(); + projectId = + api.findInPageRecords(list, "name", projectName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = projectParams(editedProjectName); + params.put("id", String.valueOf(projectId)); + + ApiResponse response = api.postForm("/project/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = projectList(); + assertThat(api.pageRecordsContain(list, "name", editedProjectName)).isTrue(); } @Test @Order(3) void testBuildProject() { - ProjectsPage projectsPage = new ProjectsPage(browser); + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("id", String.valueOf(projectId)); - projectsPage.buildProject(editedProjectName); + ApiResponse response = api.postForm("/project/build", params); + assertThat(response.isSuccess()).isTrue(); - Awaitility.await().timeout(PROJECT_BUILD_TIMEOUT_MINUTES) + Awaitility.await() + .atMost(PROJECT_BUILD_TIMEOUT) .untilAsserted( () -> { - browser.navigate().refresh(); - PageFactory.initElements(projectsPage.driver, projectsPage); - assertThat(projectsPage.projectList) - .as("Projects list should contain build success project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESSFUL")); + ApiResponse list = projectList(); + JsonNode record = + api.findInPageRecords(list, "name", editedProjectName).orElseThrow(); + assertThat(record.path("buildState").asInt()).isEqualTo(1); }); - } @Test @Order(4) void testDeleteProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - projectsPage.deleteProject(editedProjectName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(projectsPage.projectList) - .noneMatch(it -> it.getText().contains(editedProjectName)); - }); + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("id", String.valueOf(projectId)); + + ApiResponse response = api.postForm("/project/delete", params); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = projectList(); + assertThat(api.pageRecordsContain(list, "name", editedProjectName)).isFalse(); + } + + private static ApiResponse projectList() { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("pageNum", "1"); + params.put("pageSize", "50"); + return api.postForm("/project/list", params); + } + + private static Map projectParams(String name) { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("name", name); + params.put("url", url); + params.put("refs", branch); + params.put("repository", "1"); + params.put("type", "1"); + params.put("buildArgs", buildArgument); + params.put("description", description); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java index 18f7ace14a..af3c56256d 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java @@ -17,97 +17,84 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.RoleManagementPage; -import org.apache.streampark.e2e.pages.system.SystemPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class RoleManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String newRoleName = "new_role"; - private static final String newDescription = "new_description"; - private static final String existMenuName = "Apache Flink"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(RoleManagementPage.class); + api.login(); } @Test @Order(1) void testCreateRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - roleManagementPage.createRole(newRoleName, newDescription, existMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .as("Role list should contain newly-created role") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newRoleName))); + ApiResponse response = api.postForm("/role/post", roleParams(newRoleName, newDescription, existMenuName)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/role/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "roleName", newRoleName)).isTrue(); } @Test @Order(2) void testCreateDuplicateRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - roleManagementPage.createRole(newRoleName, newDescription, existMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.errorMessageList) - .as("Role Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "Sorry, the role name already exists"))); - - roleManagementPage.createRoleForm.buttonCancel.click(); + ApiResponse response = api.postForm("/role/post", roleParams(newRoleName, newDescription, existMenuName)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("Sorry, the role name already exists"); } + @Test @Order(3) void testEditRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - String newEditDescription = newDescription + "_edit"; - String newEditMenuName = "System"; - roleManagementPage.editRole(newRoleName, newEditDescription, newEditMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .as("Role list should contain edited role") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newEditDescription))); + Long roleId = api.findRoleIdByName(newRoleName).orElseThrow(); + + Map params = roleParams(newRoleName, newEditDescription, "System"); + params.put("roleId", String.valueOf(roleId)); + + ApiResponse response = api.putForm("/role/update", params); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/role/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "description", newEditDescription)).isTrue(); } @Test @Order(4) void testDeleteRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); + Long roleId = api.findRoleIdByName(newRoleName).orElseThrow(); + ApiResponse response = api.deleteForm("/role/delete", api.params("roleId", String.valueOf(roleId))); + assertThat(response.isSuccess()).isTrue(); - roleManagementPage.deleteRole(newRoleName); + ApiResponse list = api.postForm("/role/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "roleName", newRoleName)).isFalse(); + } - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(newRoleName))); + private static Map roleParams(String roleName, String description, String menuTitle) { + String menuId = api.findMenuIdByTitle(menuTitle).orElseThrow(); + Map params = new LinkedHashMap<>(); + params.put("roleName", roleName); + params.put("description", description); + params.put("menuId", menuId); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java index 331851b147..2de97c7c90 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java @@ -17,98 +17,90 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.TeamManagementPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class TeamManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String newTeamName = "test_new_team"; - private static final String newTeamDescription = "test_new_team_description"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(TeamManagementPage.class); + api.login(); } @Test @Order(1) void testCreateTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - teamManagementPage.createTeam(newTeamName, newTeamDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.teamList) - .as("Team list should contain newly-created team") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newTeamName))); + ApiResponse response = api.postForm("/team/post", teamParams(newTeamName, newTeamDescription)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/team/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "teamName", newTeamName)).isTrue(); } @Test @Order(2) void testCreateDuplicateTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - teamManagementPage.createTeam(newTeamName, newTeamDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.errorMessageList) - .as("Team Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("Create team failed."))); - - teamManagementPage.errorMessageConfirmButton.click(); - teamManagementPage.createTeamForm.buttonCancel.click(); + ApiResponse response = api.postForm("/team/post", teamParams(newTeamName, newTeamDescription)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("Create team failed."); } @Test @Order(3) void testEditTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); String editDescription = "edit_" + newTeamDescription; + ApiResponse list = api.postForm("/team/list", api.params("pageNum", "1", "pageSize", "100")); + Long teamId = + api.findInPageRecords(list, "teamName", newTeamName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); - teamManagementPage.editTeam(newTeamName, editDescription); + Map params = teamParams(newTeamName, editDescription); + params.put("id", String.valueOf(teamId)); - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.teamList) - .as("Team list should contain edited team") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editDescription))); + ApiResponse response = api.putForm("/team/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = api.postForm("/team/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "description", editDescription)).isTrue(); } @Test @Order(4) void testDeleteTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); + ApiResponse list = api.postForm("/team/list", api.params("pageNum", "1", "pageSize", "100")); + Long teamId = + api.findInPageRecords(list, "teamName", newTeamName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); - teamManagementPage.deleteTeam(newTeamName); + ApiResponse response = api.deleteForm("/team/delete", api.params("id", String.valueOf(teamId))); + assertThat(response.isSuccess()).isTrue(); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); + list = api.postForm("/team/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "teamName", newTeamName)).isFalse(); + } - assertThat(teamManagementPage.teamList) - .noneMatch(it -> it.getText().contains(newTeamName)); - }); + private static Map teamParams(String teamName, String description) { + Map params = new LinkedHashMap<>(); + params.put("teamName", teamName); + params.put("description", description); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java index 0d1bab7310..61fc017914 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java @@ -17,100 +17,80 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.TokenManagementPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class TokenManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String existUserName = "admin"; - private static final String newTokenDescription = "test_new_token_description"; + private static String createdToken; + private static Long tokenId; + @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(TokenManagementPage.class); + api.login(); } @Test @Order(1) void testCreateToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - tokenManagementPage.createToken(existUserName, newTokenDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(tokenManagementPage.tokenList) - .as("Token list should contain newly-created token") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(existUserName))); + ApiResponse response = api.postForm("/token/create", tokenParams()); + assertThat(response.isSuccess()).isTrue(); + createdToken = response.getData().path("token").asText(); + tokenId = response.getData().path("id").asLong(); + + ApiResponse list = api.postForm("/token/list", api.params("pageNum", "1", "pageSize", "50")); + assertThat(api.pageRecordsContain(list, "username", existUserName)).isTrue(); } @Test @Order(2) void testCopyToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - tokenManagementPage.copyToken(existUserName); - - // put clipboard value into createTokenForm.description - tokenManagementPage.buttonCreateToken.click(); - tokenManagementPage.createTokenForm.inputDescription.sendKeys(Keys.CONTROL, "v"); - String token = tokenManagementPage.createTokenForm.inputDescription.getAttribute("value"); - tokenManagementPage.createTokenForm.buttonCancel.click(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(tokenManagementPage.tokenList) - .as("Clipboard should contain existing token.") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(token))); + ApiResponse list = api.postForm("/token/list", api.params("pageNum", "1", "pageSize", "50")); + String token = + api.findInPageRecords(list, "username", existUserName) + .map(node -> node.path("token").asText()) + .orElseThrow(); + assertThat(token).isEqualTo(createdToken); } @Test @Order(3) void testCreateDuplicateToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - - tokenManagementPage.createToken(existUserName, newTokenDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(browser.findElement(By.tagName("body")).getText()) - .contains(String.format("user %s already has a token", existUserName))); - - tokenManagementPage.createTokenForm.buttonCancel.click(); + ApiResponse response = api.postForm("/token/create", tokenParams()); + assertThat(response.getCode()).isEqualTo(0L); + assertThat(response.getMessage()).contains(String.format("user %s already has a token", existUserName)); } @Test @Order(4) void testDeleteToken() { - final TokenManagementPage teamManagementPage = new TokenManagementPage(browser); - teamManagementPage.deleteToken(existUserName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(teamManagementPage.tokenList) - .noneMatch(it -> it.getText().contains(existUserName)); - }); + ApiResponse response = api.deleteForm("/token/delete", api.params("tokenId", String.valueOf(tokenId))); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/token/list", api.params("pageNum", "1", "pageSize", "50")); + assertThat(api.pageRecordsContain(list, "username", existUserName)).isFalse(); + } + + private static Map tokenParams() { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("userId", String.valueOf(api.getUserId())); + params.put("description", newTokenDescription); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java index 914b88e7b3..91d26a2920 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java @@ -17,125 +17,122 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; -import org.apache.streampark.e2e.pages.resource.UploadsPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class UploadManagementTest { - public static RemoteWebDriver browser; - - private static final String engineType = "Apache Flink"; - - private static final String resourceType = "Jar Library"; + public static ApiClient api; private static final String resourceName = "test-resource"; private static final String mavenPom = - "\n" + - " junit\n" + - " junit\n" + - " 4.13.2\n" + - " test\n" + - ""; + "\n" + + " junit\n" + + " junit\n" + + " 4.13.2\n" + + " test\n" + + ""; private static final String description = "Junit-jar-lib"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(UploadsPage.class); + api.login(); } @Test @Order(1) void testCreateUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - uploadsPage.createUpload(engineType, resourceType, resourceName, mavenPom, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.resourceList) - .as("Resource list should contain newly-created resource") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(resourceName)) - .anyMatch(it -> it.contains(description)) - .anyMatch(it -> it.contains(resourceType)) - .anyMatch(it -> it.contains(engineType))); + ApiResponse response = api.postForm("/resource/add", resourceParams(resourceName, mavenPom, description)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = resourceList(); + assertThat(api.pageRecordsContain(list, "resourceName", resourceName)).isTrue(); + assertThat(api.pageRecordsContain(list, "description", description)).isTrue(); } @Test @Order(2) void testCreateDuplicateUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - - uploadsPage.createUpload(engineType, resourceType, resourceName, mavenPom, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.errorMessageList) - .as("Resource Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - String.format("the resource %s already exists, please check.", resourceName)))); - - uploadsPage.errorMessageConfirmButton.click(); - uploadsPage.createUploadForm.buttonCancel.click(); + ApiResponse response = api.postForm("/resource/add", resourceParams(resourceName, mavenPom, description)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()) + .contains(String.format("the resource %s already exists, please check.", resourceName)); } @Test @Order(3) void testEditUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - browser.navigate().refresh(); - String editDescription = "Kafka-jar-lib"; String editResource = - "\n" + - " org.apache.kafka\n" + - " kafka-clients\n" + - " 3.7.1\n" + - ""; - - uploadsPage.editUpload(engineType, resourceType, resourceName, editResource, - editDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.resourceList) - .as("Resource list should contain edit resource") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(resourceName)) - .anyMatch(it -> it.contains(editDescription)) - .anyMatch(it -> it.contains(resourceType)) - .anyMatch(it -> it.contains(engineType))); + "\n" + + " org.apache.kafka\n" + + " kafka-clients\n" + + " 3.7.1\n" + + ""; + + ApiResponse list = resourceList(); + Long id = + api.findInPageRecords(list, "resourceName", resourceName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = resourceParams(resourceName, editResource, editDescription); + params.put("id", String.valueOf(id)); + + ApiResponse response = api.putForm("/resource/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = resourceList(); + assertThat(api.pageRecordsContain(list, "resourceName", resourceName)).isTrue(); + assertThat(api.pageRecordsContain(list, "description", editDescription)).isTrue(); } @Test @Order(4) void testDeleteUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - uploadsPage.deleteUpload(resourceName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(uploadsPage.resourceList) - .noneMatch(it -> it.getText().contains(resourceName)); - }); + ApiResponse list = resourceList(); + Long id = + api.findInPageRecords(list, "resourceName", resourceName) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("id", String.valueOf(id)); + + ApiResponse response = api.deleteForm("/resource/delete", params); + assertThat(response.isSuccess()).isTrue(); + + list = resourceList(); + assertThat(api.pageRecordsContain(list, "resourceName", resourceName)).isFalse(); + } + + private static ApiResponse resourceList() { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("pageNum", "1"); + params.put("pageSize", "50"); + return api.postForm("/resource/page", params); + } + + private static Map resourceParams(String name, String resource, String resourceDescription) { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("engineType", "FLINK"); + params.put("resourceType", "JAR_LIBRARY"); + params.put("resourceName", name); + params.put("resource", resource); + params.put("description", resourceDescription); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java index e41262db35..1e16b4af2c 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java @@ -17,90 +17,90 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.UserManagementPage; -import org.apache.streampark.e2e.pages.system.entity.UserManagementStatus; -import org.apache.streampark.e2e.pages.system.entity.UserManagementUserType; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class UserManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String password = "streampark"; - private static final String newUserName = "test_new"; - private static final String newUserEmail = "test@email.com"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(UserManagementPage.class); + api.login(); } @Test @Order(1) void testCreateUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - userManagementPage.createUser( - newUserName, "test", password, newUserEmail, UserManagementUserType.ADMIN); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.userList) - .as("User list should contain newly-created user") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newUserName))); + ApiResponse response = api.postForm("/user/post", createUserParams(newUserName, newUserEmail)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/user/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat(api.pageRecordsContain(list, "username", newUserName)).isTrue(); } @Test @Order(2) void testCreateDuplicateUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - userManagementPage.createUser( - newUserName, "test", password, "test@email.com", UserManagementUserType.ADMIN); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.errorMessageList) - .as("User Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "Sorry the username already exists"))); - - userManagementPage.createUserForm.buttonCancel.click(); + ApiResponse response = api.postForm("/user/post", createUserParams(newUserName, newUserEmail)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("Sorry the username already exists"); } @Test @Order(3) void testEditUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); String editEmail = "edit_" + newUserEmail; + ApiResponse list = api.postForm("/user/list", api.params("pageNum", "1", "pageSize", "100")); + Long userId = + api.findInPageRecords(list, "username", newUserName) + .map(node -> node.path("userId").asLong()) + .orElseThrow(); + + Map params = new LinkedHashMap<>(); + params.put("userId", String.valueOf(userId)); + params.put("username", newUserName); + params.put("email", editEmail); + params.put("userType", "ADMIN"); + params.put("status", "LOCKED"); + params.put("sex", "1"); + params.put("nickName", "test"); + + ApiResponse response = api.putForm("/user/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = api.postForm("/user/list", api.params("pageNum", "1", "pageSize", "100")); + assertThat( + api.findInPageRecords(list, "username", newUserName) + .map(node -> node.path("status").asText()) + .orElse("")) + .isEqualTo("LOCKED"); + } - userManagementPage.editUser( - newUserName, editEmail, UserManagementUserType.ADMIN, UserManagementStatus.LOCKED); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.userList) - .as("User list should contain edited user") - .extracting(WebElement::getText) - .anyMatch( - it -> it.contains(UserManagementStatus.LOCKED - .toString().toLowerCase()))); + private static Map createUserParams(String username, String email) { + Map params = new LinkedHashMap<>(); + params.put("username", username); + params.put("nickName", "test"); + params.put("password", password); + params.put("email", email); + params.put("userType", "ADMIN"); + params.put("status", "VALID"); + params.put("sex", "1"); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java index 0ad2fc5fad..9d15e2fbb8 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java @@ -17,101 +17,106 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; -import org.apache.streampark.e2e.pages.resource.VariablesPage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class VariableManagementTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String variableCode = "10000"; - private static final String variableValue = "3306"; - private static final String description = "MySQL default port"; - private static final boolean isNotVisible = true; - @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(VariablesPage.class); + api.login(); } @Test @Order(1) void testCreateVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - variablesPage.createVariable(variableCode, variableValue, description, isNotVisible); - - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .as("Variable list should contain newly-created variable") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(variableCode))); + ApiResponse response = api.postForm("/variable/post", variableParams(variableCode, variableValue, description)); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = variableList(); + assertThat(api.pageRecordsContain(list, "variableCode", variableCode)).isTrue(); } @Test @Order(2) void testCreateDuplicateVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - variablesPage.createVariable(variableCode, variableValue, description, isNotVisible); - - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.errorMessageList) - .as("Variable Code Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "The variable code already exists."))); - - variablesPage.errorMessageConfirmButton.click(); - variablesPage.createVariableForm.buttonCancel.click(); + ApiResponse response = api.postForm("/variable/post", variableParams(variableCode, variableValue, description)); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("The variable code already exists."); } @Test @Order(3) void testEditVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); String editVariableValue = "6379"; String editDescription = "Redis default port"; - variablesPage.editVariable(variableCode, editVariableValue, editDescription, isNotVisible); - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .as("Variable list should contain edited variable") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editVariableValue)) - .anyMatch(it -> it.contains(editDescription))); + ApiResponse list = variableList(); + Long id = + api.findInPageRecords(list, "variableCode", variableCode) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = variableParams(variableCode, editVariableValue, editDescription); + params.put("id", String.valueOf(id)); + + ApiResponse response = api.putForm("/variable/update", params); + assertThat(response.isSuccess()).isTrue(); + + list = variableList(); + assertThat(api.pageRecordsContain(list, "variableValue", editVariableValue)).isTrue(); + assertThat(api.pageRecordsContain(list, "description", editDescription)).isTrue(); } @Test @Order(4) void testDeleteVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); + ApiResponse list = variableList(); + Long id = + api.findInPageRecords(list, "variableCode", variableCode) + .map(node -> node.path("id").asLong()) + .orElseThrow(); + + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("id", String.valueOf(id)); + + ApiResponse response = api.deleteForm("/variable/delete", params); + assertThat(response.isSuccess()).isTrue(); - variablesPage.deleteVariable(variableCode); + list = variableList(); + assertThat(api.pageRecordsContain(list, "variableCode", variableCode)).isFalse(); + } + + private static ApiResponse variableList() { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("pageNum", "1"); + params.put("pageSize", "50"); + return api.postForm("/variable/page", params); + } - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(variableCode))); + private static Map variableParams(String code, String value, String variableDescription) { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("variableCode", code); + params.put("variableValue", value); + params.put("description", variableDescription); + params.put("desensitization", "true"); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java index 25faeb6cb7..778d19bd26 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java @@ -17,97 +17,94 @@ package org.apache.streampark.e2e.cases; -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.YarnQueuePage; +import org.apache.streampark.e2e.core.StreamParkApi; +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; + +import java.util.LinkedHashMap; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") +@StreamParkApi(composeFiles = "docker/basic/docker-compose.yaml") public class YarnQueueTest { - public static RemoteWebDriver browser; + public static ApiClient api; private static final String newQueueLabel = "new_label"; - private static final String editQueueLabel = "edit_label"; - private static final String description = "test_description"; @BeforeAll public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(YarnQueuePage.class); + api.login(); } @Test @Order(1) void testYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - queuePage.createYarnQueue(newQueueLabel, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.yarnQueueList) - .as("Yarn Queue list should contain newly-created item") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newQueueLabel)) - .anyMatch(it -> it.contains(description))); + Map params = queueParams(newQueueLabel, description); + ApiResponse response = api.postForm("/yarn/queue/create", params); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/yarn/queue/list", api.params("pageNum", "1", "pageSize", "50")); + assertThat(api.pageRecordsContain(list, "queueLabel", newQueueLabel)).isTrue(); + assertThat(api.pageRecordsContain(list, "description", description)).isTrue(); } @Test @Order(2) void testCreateDuplicateYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - queuePage.createYarnQueue(newQueueLabel, description); - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.errorMessageList) - .as("Yarn Queue Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("The queue label existed in the current team"))); - - queuePage.createYarnQueueForm.buttonCancel.click(); + Map params = queueParams(newQueueLabel, description); + ApiResponse response = api.postForm("/yarn/queue/create", params); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.getMessage()).contains("The queue label existed in the current team"); } @Test @Order(3) void testEditYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); String editDescription = "edit_" + description; - - queuePage.editYarnQueue(newQueueLabel, editQueueLabel, editDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.yarnQueueList) - .as("Yarn queue list should contain edited yarn queue") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editQueueLabel)) - .anyMatch(it -> it.contains(editDescription))); + Map params = queueParams(editQueueLabel, editDescription); + params.put("id", api.findInPageRecords( + api.postForm("/yarn/queue/list", api.params("pageNum", "1", "pageSize", "50")), + "queueLabel", + newQueueLabel) + .map(node -> String.valueOf(node.path("id").asLong())) + .orElseThrow()); + + ApiResponse response = api.postForm("/yarn/queue/update", params); + assertThat(response.isSuccess()).isTrue(); + + ApiResponse list = api.postForm("/yarn/queue/list", api.params("pageNum", "1", "pageSize", "50")); + assertThat(api.pageRecordsContain(list, "queueLabel", editQueueLabel)).isTrue(); + assertThat(api.pageRecordsContain(list, "description", editDescription)).isTrue(); } @Test @Order(4) void testDeleteYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - - queuePage.deleteYarnQueue(editQueueLabel); - Awaitility.await() - .untilAsserted( - () -> { - assertThat(queuePage.yarnQueueList) - .noneMatch(it -> it.getText().contains(editQueueLabel)); - }); + ApiResponse list = api.postForm("/yarn/queue/list", api.params("pageNum", "1", "pageSize", "50")); + String id = + api.findInPageRecords(list, "queueLabel", editQueueLabel) + .map(node -> String.valueOf(node.path("id").asLong())) + .orElseThrow(); + + ApiResponse response = api.postForm("/yarn/queue/delete", api.params("id", id)); + assertThat(response.isSuccess()).isTrue(); + + list = api.postForm("/yarn/queue/list", api.params("pageNum", "1", "pageSize", "50")); + assertThat(api.pageRecordsContain(list, "queueLabel", editQueueLabel)).isFalse(); + } + + private static Map queueParams(String queueLabel, String queueDescription) { + Map params = new LinkedHashMap<>(api.teamParams()); + params.put("queueLabel", queueLabel); + params.put("description", queueDescription); + return params; } } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java index b9f4ed1e7f..26ea2d2dff 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java @@ -32,6 +32,8 @@ import java.util.List; +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; + @Getter public final class ApplicationForm { @@ -61,6 +63,15 @@ public final class ApplicationForm { @FindBy(xpath = "//div[contains(@codefield, 'yarnSessionClusterId')]//div[contains(@class, 'ant-select-selector')]") public WebElement buttonFlinkClusterDropdown; + @FindBy(xpath = "//div[contains(@codefield, 'k8sSessionClusterId')]//div[contains(@class, 'ant-select-selector')]") + public WebElement buttonK8sSessionClusterDropdown; + + @FindBy(xpath = "//div[contains(@codefield, 'k8sNamespace')]//input") + public WebElement inputK8sNamespace; + + @FindBy(xpath = "//div[contains(@codefield, 'flinkImage')]//input") + public WebElement inputFlinkImage; + @FindBy(className = "ant-select-item-option-content") private List selectFlinkCluster; @@ -241,6 +252,32 @@ public ApplicationForm flinkCluster(String flinkClusterName) { return this; } + @SneakyThrows + public ApplicationForm k8sSessionCluster(String flinkClusterName) { + new Actions(driver).moveToElement(buttonK8sSessionClusterDropdown).build().perform(); + buttonK8sSessionClusterDropdown.click(); + Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); + selectFlinkCluster.stream() + .filter(e -> e.getText().contains(flinkClusterName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException( + String.format("Flink cluster not found: %s", flinkClusterName))) + .click(); + + return this; + } + + public ApplicationForm k8sNamespace(String k8sNamespace) { + WebElementDeleteAndInput(inputK8sNamespace, k8sNamespace); + return this; + } + + public ApplicationForm flinkImage(String flinkImage) { + WebElementDeleteAndInput(inputFlinkImage, flinkImage); + return this; + } + public ApplicationForm submit() { buttonSubmit.click(); diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java index c483ee8b62..23e5fae0e2 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java @@ -65,6 +65,14 @@ public T addCluster(DeployMode deployMode) { String.format("Execution Mode not found: %s", deployMode.desc))) .click(); return (T) new YarnSessionForm(this); + case KUBERNETES_SESSION: + selectDeployMode.stream() + .filter(e -> e.getText().equalsIgnoreCase(DeployMode.KUBERNETES_SESSION.desc)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + String.format("Execution Mode not found: %s", deployMode.desc))) + .click(); + return (T) new KubernetesSessionForm(this); default: throw new UnsupportedOperationException( String.format("Unknown execution mode: %s", deployMode.desc)); diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/KubernetesSessionForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/KubernetesSessionForm.java new file mode 100644 index 0000000000..75901e5637 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/KubernetesSessionForm.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.pages.flink.clusters; + +import lombok.Getter; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.openqa.selenium.support.FindBys; + +import java.util.List; + +import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; + +@Getter +public class KubernetesSessionForm extends CommonForm { + + public WebDriver driver; + + @FindBy(xpath = "//div[contains(@codefield, 'resolveOrder')]//div[contains(@class, 'ant-select-selector')]") + public WebElement buttonResolveOrderDropdown; + + @FindBys({ + @FindBy(css = "[codefield=resolveOrder]"), + @FindBy(className = "ant-select-item-option-content") + }) + private List selectResolveOrder; + + @FindBy(xpath = "//div[contains(@codefield, 'flinkImage')]//input") + public WebElement inputFlinkImage; + + @FindBy(xpath = "//div[contains(@codefield, 'k8sNamespace')]//input") + public WebElement inputK8sNamespace; + + public KubernetesSessionForm(ClusterDetailForm clusterDetailForm) { + super(clusterDetailForm); + this.driver = clusterDetailForm.driver; + } + + public KubernetesSessionForm resolveOrder(ResolveOrder resolveOrder) { + buttonResolveOrderDropdown.click(); + selectResolveOrder.stream() + .filter(e -> e.getText().equalsIgnoreCase(resolveOrder.desc)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(String.format("Resolve Order not found: %s", resolveOrder.desc))) + .click(); + + return this; + } + + public KubernetesSessionForm flinkImage(String flinkImage) { + WebElementDeleteAndInput(inputFlinkImage, flinkImage); + return this; + } + + public KubernetesSessionForm k8sNamespace(String k8sNamespace) { + WebElementDeleteAndInput(inputK8sNamespace, k8sNamespace); + return this; + } + + @Getter + public enum ResolveOrder { + + PARENT_FIRST("parent-first"), + CHILD_FIRST("child-first"); + + private final String desc; + + ResolveOrder(String desc) { + this.desc = desc; + } + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/Dockerfile new file mode 100644 index 0000000000..b9db1ebb7d --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:1.17.2-scala_2.12-java8 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-1.17.2 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..49038299e3 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-1.17.2-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/Dockerfile new file mode 100644 index 0000000000..240a0310cd --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:1.18.1-scala_2.12-java8 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-1.18.1 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..53c85c41e5 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-1.18.1-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/Dockerfile new file mode 100644 index 0000000000..01dd160bda --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:1.19.0-scala_2.12-java8 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-1.19.0 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..e0093b4aae --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-1.19.0-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-remote/Dockerfile similarity index 100% rename from streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile rename to streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-remote/Dockerfile diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-remote/docker-compose.yaml similarity index 98% rename from streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml rename to streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-remote/docker-compose.yaml index be496ff7c0..f4e0bf2bc9 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-remote/docker-compose.yaml @@ -17,7 +17,7 @@ services: jobmanager: - image: flink:1.16.3 + image: flink:1.19.0 command: jobmanager ports: - "8081:8081" @@ -37,7 +37,7 @@ services: retries: 120 taskmanager: - image: flink:1.16.3 + image: flink:1.19.0 depends_on: - jobmanager command: taskmanager diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/Dockerfile similarity index 78% rename from streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile rename to streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/Dockerfile index f123de65d6..110eb91c2c 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/Dockerfile @@ -6,7 +6,7 @@ # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -16,20 +16,17 @@ # FROM apache/streampark:ci as base-image -FROM flink:1.16.3-scala_2.12-java8 as flink-image +FROM flink:1.19.0-scala_2.12-java8 as flink-image FROM eclipse-temurin:11-jdk-jammy as jdk11-image -# Install streampark FROM sbloodys/hadoop:3.3.6 COPY --from=base-image /streampark /streampark RUN sudo chown -R hadoop.hadoop /streampark \ && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml -# Install Flink -COPY --from=flink-image /opt/flink /flink-1.16.3 -RUN sudo chown -R hadoop.hadoop /flink-1.16.3 +COPY --from=flink-image /opt/flink /flink-1.19.0 +RUN sudo chown -R hadoop.hadoop /flink-1.19.0 -# Bundled JDK 11 from Eclipse Temurin for StreamPark Console (avoid CentOS 7 yum mirror retirement) COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 ENV JAVA_HOME=/usr/lib/jvm/jdk11 ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/docker-compose.config similarity index 100% rename from streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config rename to streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/docker-compose.config diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/docker-compose.yaml similarity index 87% rename from streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml rename to streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/docker-compose.yaml index 0412171f46..6e73be2394 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.19-on-yarn/docker-compose.yaml @@ -15,7 +15,7 @@ services: namenode: - image: apache/streampark-flink-1.16.3-on-yarn:ci + image: apache/streampark-flink-1.19.0-on-yarn:ci hostname: namenode command: [ "hdfs", "namenode" ] networks: @@ -42,7 +42,7 @@ services: timeout: 5s retries: 120 datanode: - image: apache/streampark-flink-1.16.3-on-yarn:ci + image: apache/streampark-flink-1.19.0-on-yarn:ci hostname: datanode command: [ "hdfs", "datanode" ] networks: @@ -68,7 +68,7 @@ services: namenode: condition: service_healthy resourcemanager: - image: apache/streampark-flink-1.16.3-on-yarn:ci + image: apache/streampark-flink-1.19.0-on-yarn:ci hostname: resourcemanager command: [ "yarn", "resourcemanager" ] networks: @@ -93,7 +93,7 @@ services: timeout: 5s retries: 120 nodemanager: - image: apache/streampark-flink-1.16.3-on-yarn:ci + image: apache/streampark-flink-1.19.0-on-yarn:ci hostname: nodemanager command: [ "yarn", "nodemanager" ] networks: @@ -119,7 +119,7 @@ services: timeout: 5s retries: 120 streampark: - image: apache/streampark-flink-1.16.3-on-yarn:ci + image: apache/streampark-flink-1.19.0-on-yarn:ci hostname: streampark command: - sh @@ -145,14 +145,14 @@ services: stdin_open: true restart: always depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - resourcemanager: - condition: service_healthy - nodemanager: - condition: service_healthy + namenode: + condition: service_healthy + datanode: + condition: service_healthy + resourcemanager: + condition: service_healthy + nodemanager: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "http://streampark:10000" ] interval: 5s diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/Dockerfile new file mode 100644 index 0000000000..2b42dc0bcf --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:1.20.1-scala_2.12-java8 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-1.20.1 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..16568a1da8 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-1.20.1-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/Dockerfile new file mode 100644 index 0000000000..d65940c174 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.0.2-scala_2.12-java11 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-2.0.2 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..da241c3bc4 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-2.0.2-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/Dockerfile new file mode 100644 index 0000000000..b1e093fd41 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/Dockerfile @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/docker-compose.yaml new file mode 100644 index 0000000000..85f8f93ca6 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-remote/docker-compose.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + jobmanager: + image: flink:2.0.2 + command: jobmanager + ports: + - "8081:8081" + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + taskmanager: + image: flink:2.0.2 + depends_on: + - jobmanager + command: taskmanager + scale: 1 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 2 + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark:ci + command: bash bin/streampark.sh start_docker + build: + context: ./ + dockerfile: ./Dockerfile + ports: + - 10000:10000 + - 10030:10030 + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + - FLINK_JOBMANAGER_URL=http://jobmanager:8081 + privileged: true + restart: unless-stopped + networks: + - e2e + volumes: + - flink_data:/opt/flink + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:10000" ] + interval: 5s + timeout: 5s + retries: 120 +networks: + e2e: +volumes: + flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/Dockerfile new file mode 100644 index 0000000000..b23b63a3c4 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/Dockerfile @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.0.2-scala_2.12-java11 as flink-image +FROM eclipse-temurin:11-jdk-jammy as jdk11-image + +FROM sbloodys/hadoop:3.3.6 +COPY --from=base-image /streampark /streampark +RUN sudo chown -R hadoop.hadoop /streampark \ + && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml + +COPY --from=flink-image /opt/flink /flink-2.0.2 +RUN sudo chown -R hadoop.hadoop /flink-2.0.2 + +COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 +ENV JAVA_HOME=/usr/lib/jvm/jdk11 +ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.config new file mode 100644 index 0000000000..441fa11187 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.config @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_HOME=/opt/hadoop +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.yaml new file mode 100644 index 0000000000..b813efc752 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.0-on-yarn/docker-compose.yaml @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + namenode: + image: apache/streampark-flink-2.0.2-on-yarn:ci + hostname: namenode + command: [ "hdfs", "namenode" ] + networks: + - e2e + build: + context: ./ + ports: + - "19870:9870" + env_file: + - docker-compose.config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://namenode:9870" ] + interval: 5s + timeout: 5s + retries: 120 + datanode: + image: apache/streampark-flink-2.0.2-on-yarn:ci + hostname: datanode + command: [ "hdfs", "datanode" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://datanode:9864" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + namenode: + condition: service_healthy + resourcemanager: + image: apache/streampark-flink-2.0.2-on-yarn:ci + hostname: resourcemanager + command: [ "yarn", "resourcemanager" ] + networks: + - e2e + build: + context: ./ + ports: + - "18088:8088" + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://resourcemanager:8088" ] + interval: 5s + timeout: 5s + retries: 120 + nodemanager: + image: apache/streampark-flink-2.0.2-on-yarn:ci + hostname: nodemanager + command: [ "yarn", "nodemanager" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + resourcemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://nodemanager:8042" ] + interval: 5s + timeout: 5s + retries: 120 + streampark: + image: apache/streampark-flink-2.0.2-on-yarn:ci + hostname: streampark + command: + - sh + - -c + - /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + namenode: + condition: service_healthy + datanode: + condition: service_healthy + resourcemanager: + condition: service_healthy + nodemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/Dockerfile new file mode 100644 index 0000000000..7a08c56093 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.1.2-scala_2.12-java11 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-2.1.2 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..161bd65526 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-2.1.2-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/Dockerfile new file mode 100644 index 0000000000..b1e093fd41 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/Dockerfile @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/docker-compose.yaml new file mode 100644 index 0000000000..a1e9ad3fe7 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-remote/docker-compose.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + jobmanager: + image: flink:2.1.2 + command: jobmanager + ports: + - "8081:8081" + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + taskmanager: + image: flink:2.1.2 + depends_on: + - jobmanager + command: taskmanager + scale: 1 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 2 + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark:ci + command: bash bin/streampark.sh start_docker + build: + context: ./ + dockerfile: ./Dockerfile + ports: + - 10000:10000 + - 10030:10030 + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + - FLINK_JOBMANAGER_URL=http://jobmanager:8081 + privileged: true + restart: unless-stopped + networks: + - e2e + volumes: + - flink_data:/opt/flink + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:10000" ] + interval: 5s + timeout: 5s + retries: 120 +networks: + e2e: +volumes: + flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/Dockerfile new file mode 100644 index 0000000000..861f40797f --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/Dockerfile @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.1.2-scala_2.12-java11 as flink-image +FROM eclipse-temurin:11-jdk-jammy as jdk11-image + +FROM sbloodys/hadoop:3.3.6 +COPY --from=base-image /streampark /streampark +RUN sudo chown -R hadoop.hadoop /streampark \ + && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml + +COPY --from=flink-image /opt/flink /flink-2.1.2 +RUN sudo chown -R hadoop.hadoop /flink-2.1.2 + +COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 +ENV JAVA_HOME=/usr/lib/jvm/jdk11 +ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.config new file mode 100644 index 0000000000..441fa11187 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.config @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_HOME=/opt/hadoop +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.yaml new file mode 100644 index 0000000000..7d4140f33c --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.1-on-yarn/docker-compose.yaml @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + namenode: + image: apache/streampark-flink-2.1.2-on-yarn:ci + hostname: namenode + command: [ "hdfs", "namenode" ] + networks: + - e2e + build: + context: ./ + ports: + - "19870:9870" + env_file: + - docker-compose.config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://namenode:9870" ] + interval: 5s + timeout: 5s + retries: 120 + datanode: + image: apache/streampark-flink-2.1.2-on-yarn:ci + hostname: datanode + command: [ "hdfs", "datanode" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://datanode:9864" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + namenode: + condition: service_healthy + resourcemanager: + image: apache/streampark-flink-2.1.2-on-yarn:ci + hostname: resourcemanager + command: [ "yarn", "resourcemanager" ] + networks: + - e2e + build: + context: ./ + ports: + - "18088:8088" + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://resourcemanager:8088" ] + interval: 5s + timeout: 5s + retries: 120 + nodemanager: + image: apache/streampark-flink-2.1.2-on-yarn:ci + hostname: nodemanager + command: [ "yarn", "nodemanager" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + resourcemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://nodemanager:8042" ] + interval: 5s + timeout: 5s + retries: 120 + streampark: + image: apache/streampark-flink-2.1.2-on-yarn:ci + hostname: streampark + command: + - sh + - -c + - /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + namenode: + condition: service_healthy + datanode: + condition: service_healthy + resourcemanager: + condition: service_healthy + nodemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/Dockerfile new file mode 100644 index 0000000000..6b4763797e --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.2.1-scala_2.12-java11 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-2.2.1 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..af0bf6cb58 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-2.2.1-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/Dockerfile new file mode 100644 index 0000000000..b1e093fd41 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/Dockerfile @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/docker-compose.yaml new file mode 100644 index 0000000000..a7946a9e57 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-remote/docker-compose.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + jobmanager: + image: flink:2.2.1 + command: jobmanager + ports: + - "8081:8081" + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + taskmanager: + image: flink:2.2.1 + depends_on: + - jobmanager + command: taskmanager + scale: 1 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 2 + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark:ci + command: bash bin/streampark.sh start_docker + build: + context: ./ + dockerfile: ./Dockerfile + ports: + - 10000:10000 + - 10030:10030 + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + - FLINK_JOBMANAGER_URL=http://jobmanager:8081 + privileged: true + restart: unless-stopped + networks: + - e2e + volumes: + - flink_data:/opt/flink + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:10000" ] + interval: 5s + timeout: 5s + retries: 120 +networks: + e2e: +volumes: + flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/Dockerfile new file mode 100644 index 0000000000..9b7389f585 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/Dockerfile @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.2.1-scala_2.12-java11 as flink-image +FROM eclipse-temurin:11-jdk-jammy as jdk11-image + +FROM sbloodys/hadoop:3.3.6 +COPY --from=base-image /streampark /streampark +RUN sudo chown -R hadoop.hadoop /streampark \ + && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml + +COPY --from=flink-image /opt/flink /flink-2.2.1 +RUN sudo chown -R hadoop.hadoop /flink-2.2.1 + +COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 +ENV JAVA_HOME=/usr/lib/jvm/jdk11 +ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.config new file mode 100644 index 0000000000..441fa11187 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.config @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_HOME=/opt/hadoop +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.yaml new file mode 100644 index 0000000000..5351bb58f0 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.2-on-yarn/docker-compose.yaml @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + namenode: + image: apache/streampark-flink-2.2.1-on-yarn:ci + hostname: namenode + command: [ "hdfs", "namenode" ] + networks: + - e2e + build: + context: ./ + ports: + - "19870:9870" + env_file: + - docker-compose.config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://namenode:9870" ] + interval: 5s + timeout: 5s + retries: 120 + datanode: + image: apache/streampark-flink-2.2.1-on-yarn:ci + hostname: datanode + command: [ "hdfs", "datanode" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://datanode:9864" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + namenode: + condition: service_healthy + resourcemanager: + image: apache/streampark-flink-2.2.1-on-yarn:ci + hostname: resourcemanager + command: [ "yarn", "resourcemanager" ] + networks: + - e2e + build: + context: ./ + ports: + - "18088:8088" + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://resourcemanager:8088" ] + interval: 5s + timeout: 5s + retries: 120 + nodemanager: + image: apache/streampark-flink-2.2.1-on-yarn:ci + hostname: nodemanager + command: [ "yarn", "nodemanager" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + resourcemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://nodemanager:8042" ] + interval: 5s + timeout: 5s + retries: 120 + streampark: + image: apache/streampark-flink-2.2.1-on-yarn:ci + hostname: streampark + command: + - sh + - -c + - /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + namenode: + condition: service_healthy + datanode: + condition: service_healthy + resourcemanager: + condition: service_healthy + nodemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/Dockerfile new file mode 100644 index 0000000000..312d7095eb --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/Dockerfile @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.3.0-scala_2.12-java11 as flink-image +FROM apache/streampark:ci +COPY --from=flink-image /opt/flink /flink-2.3.0 diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/docker-compose.yaml new file mode 100644 index 0000000000..75d8f34cc3 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-k8s/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + k3s: + image: rancher/k3s:v1.30.5-k3s1 + hostname: k3s + command: server --disable=traefik --tls-san=k3s --write-kubeconfig-mode=666 + privileged: true + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + networks: + - e2e + volumes: + - k3s-server:/var/lib/rancher/k3s + - k3s-kubeconfig:/output + healthcheck: + test: [ "CMD", "kubectl", "get", "nodes" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark-flink-2.3.0-on-k8s:ci + hostname: streampark + command: + - bash + - -c + - | + mkdir -p /root/.kube + sed 's/127.0.0.1/k3s/g; s/localhost/k3s/g' /output/kubeconfig.yaml > /root/.kube/config + chmod 600 /root/.kube/config + /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + volumes: + - k3s-kubeconfig:/output:ro + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + privileged: true + restart: unless-stopped + depends_on: + k3s: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: + +volumes: + k3s-server: + k3s-kubeconfig: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/Dockerfile new file mode 100644 index 0000000000..b1e093fd41 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/Dockerfile @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/docker-compose.yaml new file mode 100644 index 0000000000..956b6f90d1 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-remote/docker-compose.yaml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + jobmanager: + image: flink:2.3.0 + command: jobmanager + ports: + - "8081:8081" + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + taskmanager: + image: flink:2.3.0 + depends_on: + - jobmanager + command: taskmanager + scale: 1 + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 2 + networks: + - e2e + volumes: + - flink_data:/opt/flink + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:8081" ] + interval: 5s + timeout: 5s + retries: 120 + + streampark: + image: apache/streampark:ci + command: bash bin/streampark.sh start_docker + build: + context: ./ + dockerfile: ./Dockerfile + ports: + - 10000:10000 + - 10030:10030 + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + - FLINK_JOBMANAGER_URL=http://jobmanager:8081 + privileged: true + restart: unless-stopped + networks: + - e2e + volumes: + - flink_data:/opt/flink + - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: [ "CMD", "curl", "http://localhost:10000" ] + interval: 5s + timeout: 5s + retries: 120 +networks: + e2e: +volumes: + flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/Dockerfile new file mode 100644 index 0000000000..9f19f65882 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/Dockerfile @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/streampark:ci as base-image +FROM flink:2.3.0-scala_2.12-java11 as flink-image +FROM eclipse-temurin:11-jdk-jammy as jdk11-image + +FROM sbloodys/hadoop:3.3.6 +COPY --from=base-image /streampark /streampark +RUN sudo chown -R hadoop.hadoop /streampark \ + && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml + +COPY --from=flink-image /opt/flink /flink-2.3.0 +RUN sudo chown -R hadoop.hadoop /flink-2.3.0 + +COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 +ENV JAVA_HOME=/usr/lib/jvm/jdk11 +ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.config new file mode 100644 index 0000000000..441fa11187 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.config @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_HOME=/opt/hadoop +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.yaml new file mode 100644 index 0000000000..a7c1343501 --- /dev/null +++ b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-2.3-on-yarn/docker-compose.yaml @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + namenode: + image: apache/streampark-flink-2.3.0-on-yarn:ci + hostname: namenode + command: [ "hdfs", "namenode" ] + networks: + - e2e + build: + context: ./ + ports: + - "19870:9870" + env_file: + - docker-compose.config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://namenode:9870" ] + interval: 5s + timeout: 5s + retries: 120 + datanode: + image: apache/streampark-flink-2.3.0-on-yarn:ci + hostname: datanode + command: [ "hdfs", "datanode" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://datanode:9864" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + namenode: + condition: service_healthy + resourcemanager: + image: apache/streampark-flink-2.3.0-on-yarn:ci + hostname: resourcemanager + command: [ "yarn", "resourcemanager" ] + networks: + - e2e + build: + context: ./ + ports: + - "18088:8088" + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://resourcemanager:8088" ] + interval: 5s + timeout: 5s + retries: 120 + nodemanager: + image: apache/streampark-flink-2.3.0-on-yarn:ci + hostname: nodemanager + command: [ "yarn", "nodemanager" ] + networks: + - e2e + build: + context: ./ + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + resourcemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://nodemanager:8042" ] + interval: 5s + timeout: 5s + retries: 120 + streampark: + image: apache/streampark-flink-2.3.0-on-yarn:ci + hostname: streampark + command: + - sh + - -c + - /streampark/bin/streampark.sh start_docker + networks: + - e2e + build: + context: ./ + ports: + - "20000:10000" + environment: + - SPRING_PROFILES_ACTIVE=h2 + - TZ=Asia/Shanghai + env_file: + - docker-compose.config + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + namenode: + condition: service_healthy + datanode: + condition: service_healthy + resourcemanager: + condition: service_healthy + nodemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://streampark:10000" ] + interval: 5s + timeout: 5s + retries: 120 + +networks: + e2e: diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApi.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApi.java new file mode 100644 index 0000000000..4f031b9219 --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApi.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core; + +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Marks API integration tests that exercise StreamPark REST endpoints without Selenium. */ +@Inherited +@Testcontainers +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@TestMethodOrder(OrderAnnotation.class) +@ExtendWith(StreamParkApiExtension.class) +public @interface StreamParkApi { + + String[] composeFiles(); +} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApiExtension.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApiExtension.java new file mode 100644 index 0000000000..b78fedc0cd --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkApiExtension.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core; + +import org.apache.streampark.e2e.core.api.ApiClient; + +import com.google.common.net.HostAndPort; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Starts StreamPark via docker-compose and injects {@link ApiClient} for REST tests. */ +@Slf4j +final class StreamParkApiExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback { + + private static final int DOCKER_PORT = 10000; + private static final String SERVICE_NAME = "streampark"; + + private final boolean localMode = Objects.equals(System.getProperty("local"), "true"); + private final int localPort = 10001; + + private ComposeContainer compose; + private ApiClient apiClient; + + @Override + public void beforeAll(ExtensionContext context) { + HostAndPort address; + if (localMode) { + address = HostAndPort.fromParts("localhost", localPort); + } else { + compose = createDockerCompose(context); + compose.start(); + address = HostAndPort.fromParts("localhost", compose.getServicePort(SERVICE_NAME, DOCKER_PORT)); + } + apiClient = new ApiClient("http://" + address.getHost() + ":" + address.getPort()); + injectApiClient(context.getRequiredTestClass(), apiClient); + } + + @Override + public void afterAll(ExtensionContext context) { + if (compose != null) { + compose.stop(); + } + } + + @Override + public void beforeEach(ExtensionContext context) { + injectApiClient(context.getRequiredTestClass(), apiClient); + Object instance = context.getRequiredTestInstance(); + Stream.of(instance.getClass().getDeclaredFields()) + .filter(field -> !Modifier.isStatic(field.getModifiers())) + .filter(field -> ApiClient.class.isAssignableFrom(field.getType())) + .forEach(field -> setField(instance, field, apiClient)); + } + + private void injectApiClient(Class clazz, ApiClient client) { + Stream.of(clazz.getDeclaredFields()) + .filter(field -> Modifier.isStatic(field.getModifiers())) + .filter(field -> ApiClient.class.isAssignableFrom(field.getType())) + .forEach(field -> setField(null, field, client)); + } + + private void setField(Object target, Field field, Object value) { + try { + field.setAccessible(true); + field.set(target, value); + } catch (IllegalAccessException e) { + log.error("Failed to inject ApiClient to field: {}", field.getName(), e); + } + } + + private ComposeContainer createDockerCompose(ExtensionContext context) { + final Class clazz = context.getRequiredTestClass(); + final StreamParkApi annotation = clazz.getAnnotation(StreamParkApi.class); + final List files = + Stream.of(annotation.composeFiles()) + .map(path -> StreamParkApi.class.getClassLoader().getResource(path)) + .filter(Objects::nonNull) + .map(URL::getPath) + .map(File::new) + .collect(Collectors.toList()); + + return new ComposeContainer(files) + .withPull(true) + .withTailChildContainers(true) + .withLocalCompose(true) + .withExposedService( + SERVICE_NAME, + DOCKER_PORT, + Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300))) + .withLogConsumer(SERVICE_NAME, outputFrame -> log.info(outputFrame.getUtf8String())) + .waitingFor( + SERVICE_NAME, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); + } +} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiClient.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiClient.java new file mode 100644 index 0000000000..e1ae8bf907 --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiClient.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core.api; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** HTTP client for StreamPark REST API integration tests. */ +@Slf4j +@Getter +public final class ApiClient { + + private static final String ADMIN = "admin"; + private static final String PASSWORD = "streampark"; + + private final String baseUrl; + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + + private String token; + private Long teamId; + private Long userId; + + public ApiClient(String baseUrl) { + this.baseUrl = baseUrl.endsWith("/") ? baseUrl.substring(0, baseUrl.length() - 1) : baseUrl; + this.httpClient = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build(); + this.objectMapper = new ObjectMapper(); + } + + public ApiResponse login() { + Map params = new LinkedHashMap<>(); + params.put("username", ADMIN); + params.put("password", PASSWORD); + ApiResponse response = postForm("/passport/signin", params, false); + if (!response.isSuccess() || response.getData() == null) { + throw new IllegalStateException("Login failed: " + response.text()); + } + JsonNode data = response.getData(); + this.token = data.path("token").asText(null); + JsonNode user = data.path("user"); + if (!user.isMissingNode()) { + this.userId = user.path("userId").isNull() ? null : user.path("userId").asLong(); + this.teamId = + user.path("lastTeamId").isNull() ? null : user.path("lastTeamId").asLong(); + } + if (token == null || teamId == null) { + throw new IllegalStateException("Login response missing token or teamId: " + response.text()); + } + return response; + } + + public ApiResponse postForm(String path, Map params) { + return postForm(path, params, true); + } + + public ApiResponse postForm(String path, Map params, boolean withAuth) { + return exchange("POST", path, "application/x-www-form-urlencoded", encodeForm(params), withAuth); + } + + public ApiResponse postJson(String path, String jsonBody) { + return exchange("POST", path, "application/json", jsonBody, true); + } + + public ApiResponse putForm(String path, Map params) { + return exchange("PUT", path, "application/x-www-form-urlencoded", encodeForm(params), true); + } + + public ApiResponse deleteForm(String path, Map params) { + return exchange("DELETE", path, "application/x-www-form-urlencoded", encodeForm(params), true); + } + + public Map teamParams() { + Map params = new LinkedHashMap<>(); + params.put("teamId", String.valueOf(teamId)); + return params; + } + + public Map params(String... keyValues) { + Map params = new LinkedHashMap<>(); + for (int i = 0; i + 1 < keyValues.length; i += 2) { + params.put(keyValues[i], keyValues[i + 1]); + } + return params; + } + + public ObjectNode objectNode() { + return objectMapper.createObjectNode(); + } + + public String writeJson(ObjectNode node) { + try { + return objectMapper.writeValueAsString(node); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialize JSON", e); + } + } + + public Optional findRecordId(ArrayNode records, String field, String expected) { + for (JsonNode record : records) { + if (expected.equals(record.path(field).asText())) { + return Optional.of(record.path("id").asLong()); + } + } + return Optional.empty(); + } + + public Optional findUserIdByUsername(String username) { + ApiResponse response = postForm("/user/list", params("pageNum", "1", "pageSize", "100")); + return findInPageRecords(response, "username", username).map(node -> node.path("userId").asLong()); + } + + public Optional findRoleIdByName(String roleName) { + ApiResponse response = postForm("/role/list", params("pageNum", "1", "pageSize", "100")); + return findInPageRecords(response, "roleName", roleName).map(node -> node.path("roleId").asLong()); + } + + public Optional findMenuIdByTitle(String title) { + ApiResponse response = postForm("/menu/list", new HashMap<>()); + if (!response.isSuccess() || response.getData() == null) { + return Optional.empty(); + } + return findMenuIdRecursive(response.getData(), title); + } + + public Optional findInPageRecords(ApiResponse response, String field, String expected) { + JsonNode data = response.getData(); + if (data == null || !data.has("records")) { + return Optional.empty(); + } + for (JsonNode record : data.get("records")) { + if (expected.equals(record.path(field).asText())) { + return Optional.of(record); + } + } + return Optional.empty(); + } + + public boolean pageRecordsContain(ApiResponse response, String field, String expected) { + return findInPageRecords(response, field, expected).isPresent(); + } + + public boolean listContains(ApiResponse response, String field, String expected) { + JsonNode data = response.getData(); + if (data == null || !data.isArray()) { + return false; + } + for (JsonNode item : data) { + if (expected.equals(item.path(field).asText())) { + return true; + } + } + return false; + } + + private Optional findMenuIdRecursive(JsonNode menus, String title) { + if (menus == null) { + return Optional.empty(); + } + if (menus.isArray()) { + for (JsonNode menu : menus) { + Optional found = findMenuIdRecursive(menu, title); + if (found.isPresent()) { + return found; + } + } + return Optional.empty(); + } + if (title.equals(menus.path("title").asText()) || title.equals(menus.path("menuName").asText())) { + return Optional.of(String.valueOf(menus.path("menuId").asLong())); + } + Iterator> fields = menus.fields(); + while (fields.hasNext()) { + JsonNode child = fields.next().getValue(); + Optional found = findMenuIdRecursive(child, title); + if (found.isPresent()) { + return found; + } + } + return Optional.empty(); + } + + private ApiResponse exchange( + String method, String path, String contentType, String body, boolean withAuth) { + try { + HttpRequest.Builder builder = + HttpRequest.newBuilder() + .uri(URI.create(baseUrl + path)) + .timeout(Duration.ofMinutes(5)) + .header("Content-Type", contentType); + if (withAuth) { + Objects.requireNonNull(token, "Call login() before authenticated requests"); + builder.header("Authorization", token); + } + builder.method( + method, + body == null + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofString(body)); + HttpResponse httpResponse = + httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString()); + JsonNode root = parseJson(httpResponse.body()); + log.debug("{} {} -> {} {}", method, path, httpResponse.statusCode(), httpResponse.body()); + return new ApiResponse(httpResponse.statusCode(), root); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("HTTP request interrupted: " + path, e); + } catch (IOException e) { + throw new IllegalStateException("HTTP request failed: " + path, e); + } + } + + private JsonNode parseJson(String body) { + if (body == null || body.isBlank()) { + return objectMapper.createObjectNode(); + } + try { + return objectMapper.readTree(body); + } catch (IOException e) { + throw new IllegalStateException("Invalid JSON response: " + body, e); + } + } + + private static String encodeForm(Map params) { + return params.entrySet().stream() + .map( + entry -> urlEncode(entry.getKey()) + "=" + urlEncode(entry.getValue() == null ? "" : entry.getValue())) + .collect(Collectors.joining("&")); + } + + private static String urlEncode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } +} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiResponse.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiResponse.java new file mode 100644 index 0000000000..22e4b6d4a6 --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/ApiResponse.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core.api; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; + +/** Parsed {@code RestResponseBody} envelope from a StreamPark API call. */ +@Getter +public final class ApiResponse { + + private static final long CODE_SUCCESS = 200L; + + private final int httpStatus; + private final JsonNode root; + private final Long code; + private final String message; + private final JsonNode data; + + public ApiResponse(int httpStatus, JsonNode root) { + this.httpStatus = httpStatus; + this.root = root; + this.code = root != null && root.hasNonNull("code") ? root.get("code").asLong() : null; + this.message = root != null && root.has("message") && !root.get("message").isNull() + ? root.get("message").asText() + : null; + this.data = root != null && root.has("data") ? root.get("data") : null; + } + + public boolean isSuccess() { + return code != null && code.longValue() == CODE_SUCCESS; + } + + public String text() { + return root == null ? "" : root.toString(); + } +} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiConstants.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiConstants.java new file mode 100644 index 0000000000..32c08fbb89 --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiConstants.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core.api.flink; + +import lombok.experimental.UtilityClass; + +/** Numeric constants aligned with StreamPark backend enums for API tests. */ +@UtilityClass +public final class FlinkApiConstants { + + public static final int DEPLOY_STANDALONE = 1; + public static final int DEPLOY_YARN_SESSION = 3; + public static final int DEPLOY_YARN_APPLICATION = 4; + public static final int DEPLOY_K8S_SESSION = 5; + public static final int DEPLOY_K8S_APPLICATION = 6; + + public static final int JOB_TYPE_FLINK_SQL = 2; + public static final int APP_TYPE_APACHE_FLINK = 2; + + public static final int RESOLVE_PARENT_FIRST = 0; + public static final int RESOLVE_CHILD_FIRST = 1; + + public static final int CLUSTER_RUNNING = 1; + public static final int CLUSTER_CANCELED = 2; + + public static final int APP_RUNNING = 5; + public static final int APP_CANCELED = 9; + public static final int APP_FINISHED = 10; + + public static final int PIPELINE_SUCCESS = 3; + public static final int RELEASE_DONE = 0; + + /** datagen flink sql for integration tests */ + public static final String TEST_FLINK_SQL = "CREATE TABLE datagen (\n" + + "f_sequence INT,\n" + + "f_random INT,\n" + + "f_random_str STRING,\n" + + "ts AS localtimestamp,\n" + + "WATERMARK FOR ts AS ts\n" + + ") WITH (\n" + + "'connector' = 'datagen',\n" + + "'rows-per-second'='5',\n" + + "'fields.f_sequence.kind'='sequence',\n" + + "'fields.f_sequence.start'='1',\n" + + "'fields.f_sequence.end'='100',\n" + + "'fields.f_random.min'='1',\n" + + "'fields.f_random.max'='100',\n" + + "'fields.f_random_str.length'='10'\n" + + ");\n" + + "\n" + + "CREATE TABLE print_table (\n" + + "f_sequence INT,\n" + + "f_random INT,\n" + + "f_random_str STRING\n" + + ") WITH (\n" + + "'connector' = 'print'\n" + + ");\n" + + "\n" + + "INSERT INTO print_table select f_sequence,f_random,f_random_str from datagen;"; +} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiSupport.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiSupport.java new file mode 100644 index 0000000000..c2b0a642da --- /dev/null +++ b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/api/flink/FlinkApiSupport.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.e2e.core.api.flink; + +import org.apache.streampark.e2e.core.api.ApiClient; +import org.apache.streampark.e2e.core.api.ApiResponse; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.IntPredicate; +import java.util.function.Predicate; + +/** REST helpers for Flink env / cluster / application lifecycle in E2E tests. */ +public final class FlinkApiSupport { + + private static final Duration DEFAULT_POLL = Duration.ofMinutes(15); + private static final Duration POLL_INTERVAL = Duration.ofSeconds(5); + + private final ApiClient api; + + public FlinkApiSupport(ApiClient api) { + this.api = api; + } + + public ApiClient api() { + return api; + } + + public void createFlinkEnv(String flinkName, String flinkHome, String description) { + ObjectNode body = api.objectNode(); + body.put("flinkName", flinkName); + body.put("flinkHome", flinkHome); + body.put("description", description); + ApiResponse response = api.postJson("/flink/env/create", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Create flink env failed: " + response.text()); + } + } + + public long requireVersionId(String flinkName) { + ApiResponse response = api.postForm("/flink/env/list", new LinkedHashMap<>()); + if (!response.isSuccess() || response.getData() == null || !response.getData().isArray()) { + throw new IllegalStateException("List flink env failed: " + response.text()); + } + for (JsonNode env : response.getData()) { + if (flinkName.equals(env.path("flinkName").asText())) { + return env.path("id").asLong(); + } + } + throw new IllegalStateException("Flink env not found: " + flinkName); + } + + public long createCluster(ObjectNode body) { + ApiResponse response = api.postJson("/flink/cluster/create", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Create cluster failed: " + response.text()); + } + String clusterName = body.path("clusterName").asText(); + return requireClusterId(clusterName); + } + + public long requireClusterId(String clusterName) { + ApiResponse response = api.postForm("/flink/cluster/list", new LinkedHashMap<>()); + if (!response.isSuccess() || response.getData() == null || !response.getData().isArray()) { + throw new IllegalStateException("List cluster failed: " + response.text()); + } + for (JsonNode cluster : response.getData()) { + if (clusterName.equals(cluster.path("clusterName").asText())) { + return cluster.path("id").asLong(); + } + } + throw new IllegalStateException("Cluster not found: " + clusterName); + } + + public void updateCluster(ObjectNode body) { + ApiResponse response = api.postJson("/flink/cluster/update", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Update cluster failed: " + response.text()); + } + } + + public void startCluster(long clusterId) { + ApiResponse response = api.postJson("/flink/cluster/start", jsonId(clusterId)); + if (!response.isSuccess()) { + throw new IllegalStateException("Start cluster failed: " + response.text()); + } + } + + public void shutdownCluster(long clusterId) { + ApiResponse response = api.postJson("/flink/cluster/shutdown", jsonId(clusterId)); + if (!response.isSuccess()) { + throw new IllegalStateException("Shutdown cluster failed: " + response.text()); + } + } + + public void deleteCluster(long clusterId) { + ApiResponse response = api.postJson("/flink/cluster/delete", jsonId(clusterId)); + if (!response.isSuccess()) { + throw new IllegalStateException("Delete cluster failed: " + response.text()); + } + } + + public void waitClusterState(long clusterId, IntPredicate stateMatcher, Duration timeout) { + waitUntil( + () -> getClusterState(clusterId), + Optional::isPresent, + cluster -> stateMatcher.test(cluster.get()), + timeout, + "cluster " + clusterId); + } + + public long createApp(ObjectNode body) { + ApiResponse response = api.postJson("/flink/app/create", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Create app failed: " + response.text()); + } + return requireAppId(body.path("jobName").asText()); + } + + public long requireAppId(String jobName) { + ApiResponse response = listApps(jobName); + Optional app = api.findInPageRecords(response, "jobName", jobName); + if (app.isEmpty()) { + throw new IllegalStateException("App not found: " + jobName); + } + return app.get().path("id").asLong(); + } + + public void releaseApp(long appId) { + ObjectNode body = api.objectNode(); + body.put("appId", appId); + body.put("forceBuild", false); + ApiResponse response = api.postJson("/flink/pipe/build", api.writeJson(body)); + if (!response.isSuccess() || !response.getData().asBoolean(false)) { + throw new IllegalStateException("Release app failed: " + response.text()); + } + waitUntil( + () -> getPipelineStatus(appId), + Optional::isPresent, + status -> status.get() == FlinkApiConstants.PIPELINE_SUCCESS, + DEFAULT_POLL, + "pipeline for app " + appId); + } + + public void startApp(long appId) { + ObjectNode body = api.objectNode(); + body.put("id", appId); + body.put("teamId", api.getTeamId()); + body.put("restoreOrTriggerSavepoint", false); + body.put("allowNonRestored", false); + ApiResponse response = api.postJson("/flink/app/start", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Start app failed: " + response.text()); + } + } + + public void cancelApp(long appId) { + ObjectNode body = api.objectNode(); + body.put("id", appId); + body.put("teamId", api.getTeamId()); + body.put("restoreOrTriggerSavepoint", false); + body.put("drain", false); + ApiResponse response = api.postJson("/flink/app/cancel", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Cancel app failed: " + response.text()); + } + } + + public void deleteApp(long appId) { + ObjectNode body = api.objectNode(); + body.put("id", appId); + body.put("teamId", api.getTeamId()); + ApiResponse response = api.postJson("/flink/app/delete", api.writeJson(body)); + if (!response.isSuccess()) { + throw new IllegalStateException("Delete app failed: " + response.text()); + } + } + + public void waitAppState(long appId, IntPredicate stateMatcher, Duration timeout) { + waitUntil( + () -> getAppState(appId), + Optional::isPresent, + state -> stateMatcher.test(state.get()), + timeout, + "app " + appId); + } + + public ObjectNode baseSqlAppBody(String jobName, int deployMode, long versionId) { + ObjectNode body = api.objectNode(); + body.put("teamId", api.getTeamId()); + body.put("jobType", FlinkApiConstants.JOB_TYPE_FLINK_SQL); + body.put("deployMode", deployMode); + body.put("versionId", versionId); + body.put("flinkSql", FlinkApiConstants.TEST_FLINK_SQL); + body.put("appType", FlinkApiConstants.APP_TYPE_APACHE_FLINK); + body.put("jobName", jobName); + body.put("resolveOrder", FlinkApiConstants.RESOLVE_PARENT_FIRST); + return body; + } + + public ObjectNode baseClusterBody(String clusterName, int deployMode, long versionId) { + ObjectNode body = api.objectNode(); + body.put("clusterName", clusterName); + body.put("deployMode", deployMode); + body.put("versionId", versionId); + return body; + } + + private ApiResponse listApps(String jobName) { + Map params = new LinkedHashMap<>(); + params.put("teamId", String.valueOf(api.getTeamId())); + params.put("pageNum", "1"); + params.put("pageSize", "50"); + params.put("jobName", jobName); + return api.postForm("/flink/app/list", params); + } + + private Optional getAppState(long appId) { + ApiResponse response = api.postForm("/flink/app/get", api.params("id", String.valueOf(appId))); + if (!response.isSuccess() || response.getData() == null) { + return Optional.empty(); + } + return Optional.of(response.getData().path("state").asInt(-1)); + } + + private Optional getClusterState(long clusterId) { + ApiResponse response = + api.postForm("/flink/cluster/get", api.params("id", String.valueOf(clusterId))); + if (!response.isSuccess() || response.getData() == null) { + return Optional.empty(); + } + return Optional.of(response.getData().path("clusterState").asInt(-1)); + } + + private Optional getPipelineStatus(long appId) { + Map params = new LinkedHashMap<>(); + params.put("appId", String.valueOf(appId)); + ApiResponse response = api.postForm("/flink/pipe/detail", params); + if (!response.isSuccess() || response.getData() == null) { + return Optional.empty(); + } + JsonNode pipeline = response.getData().path("pipeline"); + if (pipeline.isMissingNode() || pipeline.isNull()) { + return Optional.empty(); + } + return Optional.of(pipeline.path("pipeStatus").asInt(-1)); + } + + private ObjectNode jsonId(long id) { + ObjectNode body = api.objectNode(); + body.put("id", id); + return body; + } + + private static void waitUntil( + java.util.function.Supplier> supplier, + Predicate> present, + Predicate matcher, + Duration timeout, + String label) { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (System.currentTimeMillis() < deadline) { + Optional value = supplier.get(); + if (present.test(value) && matcher.test(value.get())) { + return; + } + sleep(POLL_INTERVAL); + } + throw new IllegalStateException("Timeout waiting for " + label); + } + + private static void sleep(Duration duration) { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting", e); + } + } +}