diff --git a/.dockerignore b/.dockerignore index e8689d0b0..516253961 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,6 @@ !scm-manager/ !scripts/jenkins/ !scripts/scm-manager/ -!scripts/utils.sh !scripts/apply-ng.sh !scripts/downloadHelmCharts.sh !templates/kubernetes/rbac/ diff --git a/scripts/get-remote-url b/scripts/get-remote-url index 53aa83f8d..6a7c42408 100755 --- a/scripts/get-remote-url +++ b/scripts/get-remote-url @@ -1,15 +1,17 @@ #!/usr/bin/env bash set -o errexit -o nounset -o pipefail -BASEDIR=$(dirname $0) -ABSOLUTE_BASEDIR="$(cd ${BASEDIR} && pwd)" +function getExternalIP() { + local serviceName="$1" + local namespace="$2" + local externalIp="" -# Allow for running this script directly via curl without redundant code -if [[ -f ${ABSOLUTE_BASEDIR}/utils.sh ]]; then - source ${ABSOLUTE_BASEDIR}/utils.sh -else - source <(curl -s https://raw.githubusercontent.com/cloudogu/gitops-playground/main/scripts/utils.sh) -fi + while [[ -z "${externalIp}" ]]; do + externalIp=$(kubectl -n "${namespace}" get svc "${serviceName}" -o=jsonpath="{.status.loadBalancer.ingress[0]['hostname','ip']}") + [[ -z "${externalIp}" ]] && sleep 10 + done + echo "${externalIp}" +} serviceName="$1" namespace="$2" @@ -20,4 +22,4 @@ if [[ -z "${ip}" ]]; then exit 1; else echo http://${ip} -fi \ No newline at end of file +fi diff --git a/scripts/jenkins/init-jenkins.sh b/scripts/jenkins/init-jenkins.sh index ee0bea262..e3a3e9619 100755 --- a/scripts/jenkins/init-jenkins.sh +++ b/scripts/jenkins/init-jenkins.sh @@ -3,7 +3,6 @@ set -o errexit -o nounset -o pipefail ABSOLUTE_BASEDIR="$(cd "$(dirname $0)" && pwd)" -source ${ABSOLUTE_BASEDIR}/../utils.sh source ${ABSOLUTE_BASEDIR}/jenkins-REST-client.sh if [[ $TRACE == true ]]; then @@ -20,10 +19,6 @@ if [[ $INSECURE == true ]]; then fi function initJenkins() { - if [[ ${INTERNAL_JENKINS} == true ]]; then - setExternalHostnameIfNecessary "JENKINS" "jenkins" "${NAME_PREFIX}jenkins" - fi - installPlugins } diff --git a/scripts/utils.sh b/scripts/utils.sh deleted file mode 100644 index 0cf55faaf..000000000 --- a/scripts/utils.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Note that this is also used in scripts/get-remote-url -# Move it to get-remote-url, once setExternalHostnameIfNecessary() is no longer needed in init-scmm and init-jenkins.sh -function getExternalIP() { - servicename=$1 - namespace=$2 - - external_ip="" - while [ -z $external_ip ]; do - external_ip=$(kubectl -n ${namespace} get svc ${servicename} -o=jsonpath="{.status.loadBalancer.ingress[0]['hostname','ip']}") - [ -z "$external_ip" ] && sleep 10 - done - echo $external_ip -} - -function setExternalHostnameIfNecessary() { - local variablePrefix="$1" - local serviceName="$2" - local namespace="$3" -} \ No newline at end of file diff --git a/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClient.java b/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClient.java index 62392d2fb..d16d808e2 100644 --- a/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClient.java +++ b/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClient.java @@ -47,7 +47,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.Comparator; @@ -268,68 +267,6 @@ public void createServiceNodePort(String name, String tcp, String nodePort, Stri log.debug("NodePort service {} created/updated successfully", name); } - /** - * Patches the nodePort of a specific port in a service. - * - * @param serviceName name of the service to patch - * @param namespace namespace of the service - * @param portName name of the port entry whose nodePort is replaced - * @param newNodePort new node port value - */ - public void patchServiceNodePort(String serviceName, String namespace, String portName, int newNodePort) { - K8sClientHelper.validateServiceNodePortPatch(serviceName, namespace, portName, newNodePort); - - log.debug("Patching service {} port {} with nodePort {}", serviceName, portName, newNodePort); - - Service service = client.services().inNamespace(namespace).withName(serviceName).get(); - - if (service == null) { - throw new IllegalStateException("Service " + serviceName + NOT_FOUND_IN_NAMESPACE + namespace); - } - - List ports = service.getSpec().getPorts(); - int portIndex = -1; - for (int i = 0; i < ports.size(); i++) { - if (portName.equals(ports.get(i).getName())) { - portIndex = i; - break; - } - } - - if (portIndex == -1) { - throw new IllegalStateException("Port with name " + portName + " not found in service " + serviceName + "."); - } - - // Create JSON patch - List> patch = List.of(Map.of( - "op", - "replace", - "path", - "/spec/ports/" + portIndex + "/nodePort", - "value", - newNodePort - )); - - String patchJson = Serialization.asJson(patch); - PatchContext patchContext = new PatchContext.Builder().withPatchType(io.fabric8.kubernetes.client.dsl.base.PatchType.JSON) - .build(); - - executeWithErrorHandling( - "patch service " + serviceName, () -> { - client.services().inNamespace(namespace).withName(serviceName).patch(patchContext, patchJson); - return null; - } - ); - - log.debug( - "Service {} in namespace {} successfully patched with nodePort {} for port {}.", - serviceName, - namespace, - newNodePort, - portName - ); - } - /** * Creates a namespace (or an OpenShift project) if it does not already exist (idempotent). * @@ -427,27 +364,6 @@ public boolean resourceExists(String resource, String name, String namespace) { } } - /** - * Creates or updates an empty secret in the default namespace (idempotent). - * - * @param type secret type, e.g. {@code generic} - * @param name name of the secret - */ - public void createSecret(String type, String name) { - createSecret(type, name, "", new Tuple[0]); - } - - /** - * Creates or updates an empty secret (idempotent). - * - * @param type secret type, e.g. {@code generic} - * @param name name of the secret - * @param namespace target namespace; empty means the default namespace - */ - public void createSecret(String type, String name, String namespace) { - createSecret(type, name, namespace, new Tuple[0]); - } - /** * Creates or updates a generic secret (idempotent). * @@ -488,18 +404,6 @@ public void createSecret(String type, String name, String namespace, Tuple log.debug("Secret {} created/updated successfully", name); } - /** - * Creates or updates an image pull secret in the default namespace (idempotent). - * - * @param name name of the secret - * @param host registry host the credentials belong to - * @param user registry username - * @param password registry password - */ - public void createImagePullSecret(String name, String host, String user, String password) { - createImagePullSecret(name, "", host, user, password); - } - /** * Creates or updates an image pull secret (idempotent). * @@ -829,53 +733,34 @@ private List loadYamlItems(InputStream stream, String sourceDescrip } /** - * Adds or removes labels on a resource in the default namespace. + * Adds labels to a resource in the default namespace. * * @param resource resource type, e.g. {@code node} - * @param name resource name; {@code --all} applies to all nodes - * @param keyValues labels to set; a key ending in {@code -} removes that label + * @param name resource name + * @param keyValues labels to set */ public void label(String resource, String name, Tuple... keyValues) { label(resource, name, "", keyValues); } /** - * Adds or removes labels on a resource. + * Adds labels to a resource. * * @param resource resource type, e.g. {@code node} - * @param name resource name; {@code --all} applies to all nodes + * @param name resource name * @param namespace namespace of the resource; empty means the default namespace - * @param keyValues labels to set; a key ending in {@code -} removes that label + * @param keyValues labels to set */ public void label(String resource, String name, String namespace, Tuple... keyValues) { if (keyValues == null || keyValues.length == 0) { throw new IllegalArgumentException("Missing key-value-pairs"); } - if ("--all".equals(name)) { - NodeList nodes = client.nodes().list(); - if (nodes != null && nodes.getItems() != null) { - for (Node node : nodes.getItems()) { - label(resource, node.getMetadata().getName(), namespace, keyValues); - } - } - return; - } - log.debug("Labeling {}/{} in namespace {}", resource, name, namespace); Map labelsToAdd = new HashMap<>(); - List labelsToRemove = new ArrayList<>(); - for (Tuple tuple : keyValues) { - String key = String.valueOf(tuple.getFirst()); - String value = String.valueOf(tuple.getSecond()); - - if (key.endsWith("-")) { - labelsToRemove.add(key.substring(0, key.length() - 1)); - } else { - labelsToAdd.put(key, value); - } + labelsToAdd.put(String.valueOf(tuple.getFirst()), String.valueOf(tuple.getSecond())); } executeWithErrorHandling( @@ -886,7 +771,7 @@ public void label(String resource, String name, String namespace, Tuple... name, resolveNamespace(namespace) ); - applyLabelChanges(resourceClient, resource, name, labelsToAdd, labelsToRemove); + applyLabelChanges(resourceClient, resource, name, labelsToAdd, List.of()); return null; } ); @@ -931,30 +816,36 @@ private static void applyLabelChanges( } /** - * Removes the given labels from a resource. + * Removes the given labels from all nodes in the cluster. * - * @param resource resource type, e.g. {@code node} - * @param name resource name; {@code --all} applies to all nodes - * @param namespace namespace of the resource; empty means the default namespace - * @param keys label keys to remove + * @param keys label keys to remove */ - public void labelRemove(String resource, String name, String namespace, String... keys) { - Tuple[] tuples = new Tuple[keys.length]; - for (int i = 0; i < keys.length; i++) { - tuples[i] = new Tuple<>(keys[i] + "-", ""); + public void removeLabelsFromAllNodes(String... keys) { + NodeList nodes = client.nodes().list(); + if (nodes == null || nodes.getItems() == null) { + return; + } + + for (Node node : nodes.getItems()) { + if (node.getMetadata() != null && node.getMetadata().getName() != null) { + removeLabels("node", node.getMetadata().getName(), "", keys); + } } - label(resource, name, namespace, tuples); } - /** - * Patches a resource in the default namespace using the default patch type. - * - * @param resource resource type, e.g. {@code service} - * @param name resource name - * @param yaml patch content as nested map - */ - public void patch(String resource, String name, Map yaml) { - patch(resource, name, "", "", yaml); + private void removeLabels(String resource, String name, String namespace, String... keys) { + executeWithErrorHandling( + "remove labels from " + resource + "/" + name, () -> { + Resource resourceClient = K8sClientHelper.getResourceClient( + client, + resource, + name, + resolveNamespace(namespace) + ); + applyLabelChanges(resourceClient, resource, name, Map.of(), List.of(keys)); + return null; + } + ); } /** @@ -1001,26 +892,6 @@ public void patch(String resource, String name, String namespace, String type, M log.debug("Resource {}/{} patched successfully", resource, name); } - /** - * Deletes resources by label selectors in the default namespace, see {@link #delete(String, - * String, Tuple...)}. - * - * @param resource resource type, e.g. {@code secret} - */ - public void delete(String resource) { - delete(resource, "", new Tuple[0]); - } - - /** - * Deletes resources by label selectors, see {@link #delete(String, String, Tuple...)}. - * - * @param resource resource type, e.g. {@code secret} - * @param namespace namespace to delete in; empty means the default namespace - */ - public void delete(String resource, String namespace) { - delete(resource, namespace, new Tuple[0]); - } - /** * Deletes all resources of a type matching the given label selectors. Failures are logged, not * thrown, since the resources may not exist. @@ -1073,112 +944,40 @@ public void delete(String resource, String namespace, String name) { } /** - * Runs a pod in the default namespace, see {@link #run(String, String, String, Map, String...)}. + * Runs a temporary pod, waits for completion, returns its logs and removes the pod afterwards. + * The pod is always configured with {@code restartPolicy: Never}. * - * @param name name of the pod - * @param image container image to run - * @return a status message or, with {@code --rm}-style params, the pod output + * @param pod pod definition to run + * @return logs written by the completed pod */ - public String run(String name, String image) { - return run(name, image, "", Map.of(), new String[0]); - } - - /** - * Runs a pod, see {@link #run(String, String, String, Map, String...)}. - * - * @param name name of the pod - * @param image container image to run - * @param namespace target namespace; empty means the default namespace - * @return a status message or, with {@code --rm}-style params, the pod output - */ - public String run(String name, String image, String namespace) { - return run(name, image, namespace, Map.of(), new String[0]); - } - - /** - * Runs a pod with pod-spec overrides, see {@link #run(String, String, String, Map, String...)}. - * - * @param name name of the pod - * @param image container image to run - * @param namespace target namespace; empty means the default namespace - * @param overrides pod spec fields to override, analogous to {@code kubectl run --overrides} - * @return a status message or, with {@code --rm}-style params, the pod output - */ - public String run(String name, String image, String namespace, Map overrides) { - return run(name, image, namespace, overrides, new String[0]); - } - - /** - * Runs a pod with kubectl-run-style params, see {@link #run(String, String, String, Map, - * String...)}. - * - * @param name name of the pod - * @param image container image to run - * @param namespace target namespace; empty means the default namespace - * @param params kubectl-run-style flags such as {@code --rm} or {@code --restart=Never} - * @return a status message or, with {@code --rm}-style params, the pod output - */ - public String run(String name, String image, String namespace, String... params) { - return run(name, image, namespace, Map.of(), params); - } - - /** - * Runs a pod, analogous to {@code kubectl run}. - * - * @param name name of the pod - * @param image container image to run - * @param namespace target namespace; empty means the default namespace - * @param overrides pod spec fields to override, analogous to {@code kubectl run --overrides} - * @param params kubectl-run-style flags such as {@code --rm} or {@code --restart=Never} - * @return a status message or, when the params request output collection, the pod output - */ - public String run(String name, String image, String namespace, Map overrides, String... params) { - log.debug("Running pod {} with image {} in namespace {}", name, image, namespace); - String resolvedNamespace = resolveNamespace(namespace); - List runParams = params != null ? Arrays.asList(params) : Collections.emptyList(); - - Pod pod = new PodBuilder().withNewMetadata() - .withName(name) - .withNamespace(resolvedNamespace) - .endMetadata() - .withNewSpec() - .addNewContainer() - .withName(name) - .withImage(image) - .endContainer() - .endSpec() - .build(); - - K8sClientHelper.applyRunParams(pod, runParams); - - if (overrides != null && !overrides.isEmpty()) { - log.debug("Applying overrides: {}", overrides); - pod = K8sClientHelper.applyPodOverrides(pod, overrides); - } - - final Pod finalPod = pod; + public String runTemporaryPod(Pod pod) { + String name = pod.getMetadata().getName(); + String resolvedNamespace = resolveNamespace(pod.getMetadata().getNamespace()); + Pod temporaryPod = new PodBuilder(pod).editMetadata() + .withNamespace(resolvedNamespace) + .endMetadata() + .editOrNewSpec() + .withRestartPolicy("Never") + .endSpec() + .build(); + + log.debug("Running temporary pod {} in namespace {}", name, resolvedNamespace); Pod createdPod = executeWithErrorHandling( - "run pod " + name, () -> client.pods() - .inNamespace(resolvedNamespace) - .resource(finalPod) - .create() + "run temporary pod " + name, () -> client.pods() + .inNamespace(resolvedNamespace) + .resource(temporaryPod) + .create() ); - log.debug("Pod {} created successfully", name); - if (K8sClientHelper.shouldReturnPodOutput(runParams)) { - return K8sClientHelper.collectPodRunOutput( - client, - createdPod.getMetadata() - .getName(), - resolvedNamespace, - K8sClientHelper.shouldRemovePod(runParams), - defaultRetries, - sleepTimeMillis, - this - ); - } - - return "pod/" + createdPod.getMetadata().getName() + " created"; + log.debug("Temporary pod {} created successfully", name); + return K8sClientHelper.collectCompletedPodLogsAndDelete( + client, + createdPod.getMetadata().getName(), + resolvedNamespace, + defaultRetries, + sleepTimeMillis, + this + ); } /** diff --git a/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientHelper.java b/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientHelper.java index 997329781..2f59e4344 100644 --- a/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientHelper.java +++ b/src/main/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientHelper.java @@ -1,7 +1,5 @@ package com.cloudogu.gitops.infrastructure.kubernetes.api; -import com.cloudogu.gitops.utils.MapUtils; -import com.fasterxml.jackson.core.type.TypeReference; import io.fabric8.kubernetes.api.model.APIGroup; import io.fabric8.kubernetes.api.model.APIGroupList; import io.fabric8.kubernetes.api.model.APIResource; @@ -13,15 +11,12 @@ import io.fabric8.kubernetes.client.dsl.base.PatchContext; import io.fabric8.kubernetes.client.dsl.base.PatchType; import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext; -import io.fabric8.kubernetes.client.utils.Serialization; import io.micronaut.core.util.StringUtils; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -29,9 +24,6 @@ @Slf4j class K8sClientHelper { - private static final TypeReference> MAP_TYPE = new TypeReference<>() { - }; - private static final String GROUP_KEY = "group"; private static final String VERSION_KEY = "version"; private static final String KIND_KEY = "kind"; @@ -41,61 +33,10 @@ class K8sClientHelper { private K8sClientHelper() { } - static Pod applyPodOverrides(Pod pod, Map overrides) { - Map podAsMap = Serialization.unmarshal(Serialization.asJson(pod), MAP_TYPE); - Map normalizedOverrides = MapUtils.asStringObjectMap(normalizeOverrideValue(overrides)); - Map mergedPod = MapUtils.deepMerge(normalizedOverrides, podAsMap); - return Serialization.unmarshal(Serialization.asJson(mergedPod), Pod.class); - } - - static Object normalizeOverrideValue(Object value) { - if (value instanceof CharSequence) { - return value.toString(); - } - - if (value instanceof Map) { - Map result = new LinkedHashMap<>(); - ((Map) value).forEach((k, v) -> result.put(k.toString(), normalizeOverrideValue(v))); - return result; - } - - if (value instanceof Collection) { - List result = new ArrayList<>(); - for (Object entry : (Collection) value) { - result.add(normalizeOverrideValue(entry)); - } - return result; - } - - return value; - } - - static void applyRunParams(Pod pod, List params) { - String restartPolicy = null; - for (String param : params) { - if (param.startsWith("--restart=")) { - restartPolicy = param.substring("--restart=".length()); - break; - } - } - if (restartPolicy != null) { - pod.getSpec().setRestartPolicy(restartPolicy); - } - } - - static boolean shouldReturnPodOutput(List params) { - return params.contains("--rm") || params.contains("-i") || params.contains("-it") || params.contains("-ti"); - } - - static boolean shouldRemovePod(List params) { - return params.contains("--rm"); - } - - static String collectPodRunOutput( + static String collectCompletedPodLogsAndDelete( KubernetesClient client, String podName, String namespace, - boolean removePod, int defaultRetries, int sleepTime, K8sClient k8sClient) { @@ -113,9 +54,7 @@ static String collectPodRunOutput( return logOutput; } finally { - if (removePod) { - k8sClient.delete("pod", namespace, podName); - } + k8sClient.delete("pod", namespace, podName); } } @@ -166,12 +105,6 @@ static void validateNamespaceName(String name) { } } - static void validateServiceNodePortPatch(String serviceName, String namespace, String portName, int newNodePort) { - if (StringUtils.isEmpty(serviceName) || StringUtils.isEmpty(namespace) || StringUtils.isEmpty(portName) || newNodePort <= 0) { - throw new IllegalArgumentException("Service name, namespace, port name, and valid nodePort must be provided"); - } - } - static void validateWaitForResourcePhaseParams( String resourceType, String resourceName, diff --git a/src/main/java/com/cloudogu/gitops/tools/core/Jenkins.java b/src/main/java/com/cloudogu/gitops/tools/core/Jenkins.java index 0da18f28f..44f3b97ec 100644 --- a/src/main/java/com/cloudogu/gitops/tools/core/Jenkins.java +++ b/src/main/java/com/cloudogu/gitops/tools/core/Jenkins.java @@ -21,6 +21,8 @@ import com.cloudogu.gitops.utils.NetworkingUtils; import com.cloudogu.gitops.utils.TemplatingEngine; import com.cloudogu.gitops.utils.Tuple; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; import io.micronaut.core.annotation.Order; import io.micronaut.core.util.StringUtils; import jakarta.inject.Singleton; @@ -204,9 +206,9 @@ private void labelJenkinsNode() { // Mark the first node for Jenkins and agents. See jenkins/values.ftl.yaml "agent.workingDir" // for details. // Remove first in case new nodes were added. - k8sClient.labelRemove("node", "--all", "", "node"); + k8sClient.removeLabelsFromAllNodes("node"); - String nodeName = k8sClient.waitForNode().replace("node/", ""); + String nodeName = k8sClient.waitForNode(); k8sClient.label("node", nodeName, new Tuple<>("node", TOOL_NAME)); } @@ -489,16 +491,7 @@ private List getJenkinsOidcBootPlugins() { protected String findDockerGid() { String gid = ""; - String etcGroup = k8sClient.run( - "tmp-docker-gid-grepper-" + RANDOM.nextInt(GID_GREPPER_POD_SUFFIX_BOUND), - "irrelevant" /* Redundant, but mandatory param */, - namespace, - createGidGrepperOverrides(), - "--restart=Never", - "-ti", - "--rm", - "--quiet" - ); + String etcGroup = k8sClient.runTemporaryPod(createGidGrepperPod()); if (etcGroup != null) { String[] lines = etcGroup.split("\n"); @@ -525,26 +518,31 @@ Unable to determine Docker Group ID (GID). Jenkins Agent pods will run as root u } } - Map createGidGrepperOverrides() { - return Map.of( - "spec", Map.of( - "containers", - List.of(Map.of( - "name", - "tmp-docker-gid-grepper", - "image", - toolConfig().server().internalBashImage(), - "args", - List.of("cat", ETC_GROUP_PATH), - "volumeMounts", - List.of(Map.of("name", "group", "mountPath", ETC_GROUP_PATH, "readOnly", true)) - )), - "nodeSelector", - Map.of("node", TOOL_NAME), - "volumes", - List.of(Map.of("name", "group", "hostPath", Map.of("path", ETC_GROUP_PATH))) - ) - ); + private Pod createGidGrepperPod() { + return new PodBuilder().withNewMetadata() + .withName("tmp-docker-gid-grepper-" + RANDOM.nextInt(GID_GREPPER_POD_SUFFIX_BOUND)) + .withNamespace(namespace) + .endMetadata() + .withNewSpec() + .withNodeSelector(Map.of("node", TOOL_NAME)) + .addNewContainer() + .withName("tmp-docker-gid-grepper") + .withImage(toolConfig().server().internalBashImage()) + .withArgs("cat", ETC_GROUP_PATH) + .addNewVolumeMount() + .withName("group") + .withMountPath(ETC_GROUP_PATH) + .withReadOnly(true) + .endVolumeMount() + .endContainer() + .addNewVolume() + .withName("group") + .withNewHostPath() + .withPath(ETC_GROUP_PATH) + .endHostPath() + .endVolume() + .endSpec() + .build(); } } diff --git a/src/test/java/com/cloudogu/gitops/application/content/ContentLoaderTest.java b/src/test/java/com/cloudogu/gitops/application/content/ContentLoaderTest.java index 1af5e2015..53a9cace7 100644 --- a/src/test/java/com/cloudogu/gitops/application/content/ContentLoaderTest.java +++ b/src/test/java/com/cloudogu/gitops/application/content/ContentLoaderTest.java @@ -36,7 +36,6 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.util.SystemReader; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -147,31 +146,30 @@ static void cleanFolders() { } } - @Disabled("TODO: Does not run on Jenkins: Caused by: java.net.UnknownHostException: kubernetes.default.svc: Name or service not known") @Test void deploysImagePullSecrets() { config.getRegistry().setCreateImagePullSecrets(true); config.getContent().setNamespaces(List.of("example-apps-staging", "example-apps-production")); + K8sClient runtimeK8sClient = mock(K8sClient.class); - install(createContent(config), config); + install(createContent(config, runtimeK8sClient), config); - assertRegistrySecrets("reg-user", "reg-pw"); + assertRegistrySecrets(runtimeK8sClient, "reg-user", "reg-pw"); } - @Disabled("TODO: Does not run on Jenkins: Caused by: java.net.UnknownHostException: kubernetes.default.svc: Name or service not known") @Test void deploysImagePullSecretsFromReadOnlyVars() { config.getRegistry().setCreateImagePullSecrets(true); config.getContent().setNamespaces(List.of("example-apps-staging", "example-apps-production")); config.getRegistry().setReadOnlyUsername("other-user"); config.getRegistry().setReadOnlyPassword("other-pw"); + K8sClient runtimeK8sClient = mock(K8sClient.class); - install(createContent(config), config); + install(createContent(config, runtimeK8sClient), config); - assertRegistrySecrets("other-user", "other-pw"); + assertRegistrySecrets(runtimeK8sClient, "other-user", "other-pw"); } - @Disabled("TODO: Does not run on Jenkins: Caused by: java.net.UnknownHostException: kubernetes.default.svc: Name or service not known") @Test void deploysAdditionalImagePullSecretsForProxyRegistry() { config.getRegistry().setCreateImagePullSecrets(true); @@ -180,10 +178,12 @@ void deploysAdditionalImagePullSecretsForProxyRegistry() { config.getRegistry().setProxyUrl("proxy-url"); config.getRegistry().setProxyUsername("proxy-user"); config.getRegistry().setProxyPassword("proxy-pw"); + K8sClient runtimeK8sClient = mock(K8sClient.class); - install(createContent(config), config); + install(createContent(config, runtimeK8sClient), config); - assertRegistrySecrets("reg-user", "reg-pw"); + assertRegistrySecrets(runtimeK8sClient, "reg-user", "reg-pw"); + assertProxyRegistrySecrets(runtimeK8sClient); } @Test @@ -1252,7 +1252,35 @@ private Map parseYaml(String path) throws IOException { return readYaml(new File(path)); } - private void assertRegistrySecrets(String regUser, String regPw) { + private void assertRegistrySecrets(K8sClient runtimeK8sClient, String regUser, String regPw) { + for (String namespace : config.getContent().getNamespaces()) { + verify(runtimeK8sClient).createNamespace(namespace); + verify(runtimeK8sClient).createImagePullSecret( + "registry", + namespace, + "reg-url", + regUser, + regPw + ); + verify(runtimeK8sClient).patch( + "serviceaccount", + "default", + namespace, + Map.of("imagePullSecrets", List.of(Map.of("name", "registry"))) + ); + } + } + + private void assertProxyRegistrySecrets(K8sClient runtimeK8sClient) { + for (String namespace : config.getContent().getNamespaces()) { + verify(runtimeK8sClient).createImagePullSecret( + "proxy-registry", + namespace, + "proxy-url", + "proxy-user", + "proxy-pw" + ); + } } private ContentLoaderForTest createContent(Config contentConfig) { @@ -1268,6 +1296,19 @@ private ContentLoaderForTest createContent(Config contentConfig) { ); } + private ContentLoaderForTest createContent(Config contentConfig, K8sClient runtimeK8sClient) { + return new ContentLoaderForTest( + contentConfig, + runtimeK8sClient, + new CredentialsResolver(runtimeK8sClient), + scmmRepoProvider, + jenkins, + gitHandler, + fileSystemUtils, + deployer + ); + } + private boolean install(ContentLoaderForTest contentLoader, Config contentConfig) { return contentLoader.execute(new ContextBuilder(contentConfig).build(), repositoryWorkspace); } diff --git a/src/test/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientTest.java b/src/test/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientTest.java index ff3f25c55..76a6b9e5e 100644 --- a/src/test/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientTest.java +++ b/src/test/java/com/cloudogu/gitops/infrastructure/kubernetes/api/K8sClientTest.java @@ -409,95 +409,6 @@ void createServiceNodePortCreatesServiceWithoutExplicitNodePort() throws Interru assertThat(ports.get(0).get("nodePort")).isNull(); } - @Test - void patchServiceNodePortUpdatesServicePort() throws InterruptedException { - // Given - var service = new ServiceBuilder() - .withNewMetadata() - .withName("test-service") - .withNamespace("test-ns") - .endMetadata() - .withNewSpec() - .addNewPort() - .withName("http") - .withPort(8080) - .withNodePort(30080) - .endPort() - .endSpec() - .build(); - - server.expect() - .get() - .withPath("/api/v1/namespaces/test-ns/services/test-service") - .andReturn(200, service) - .once(); - - server.expect() - .get() - .withPath("/api/v1/namespaces/test-ns/services/test-service") - .andReturn(200, service) - .once(); - - server.expect() - .patch() - .withPath("/api/v1/namespaces/test-ns/services/test-service") - .andReturn(200, service) - .once(); - - // When - k8sApiClient.patchServiceNodePort("test-service", "test-ns", "http", 30090); - - // Then - var request = server.getLastRequest(); - assertThat(request.getMethod()).isEqualTo("PATCH"); - assertThat(request.getPath()).isEqualTo("/api/v1/namespaces/test-ns/services/test-service"); - assertThat(parseJsonList(request.getUtf8Body())).containsExactly(Map.of( - "op", "replace", - "path", "/spec/ports/0/nodePort", - "value", 30090 - )); - } - - @Test - void patchServiceNodePortThrowsExceptionForInvalidParameters() { - // When/Then - var exception = assertThrows( - IllegalArgumentException.class, - () -> k8sApiClient.patchServiceNodePort("", "test-ns", "http", 30000) - ); - assertThat(exception.getMessage()).contains("Service name"); - } - - @Test - void patchServiceNodePortThrowsExceptionWhenPortNotFound() { - // Given - var service = new ServiceBuilder() - .withNewMetadata() - .withName("test-service") - .withNamespace("test-ns") - .endMetadata() - .withNewSpec() - .addNewPort() - .withName("http") - .withPort(8080) - .endPort() - .endSpec() - .build(); - - server.expect() - .get() - .withPath("/api/v1/namespaces/test-ns/services/test-service") - .andReturn(200, service) - .once(); - - // When/Then - var exception = assertThrows( - RuntimeException.class, - () -> k8sApiClient.patchServiceNodePort("test-service", "test-ns", "https", 30000) - ); - assertThat(exception.getMessage()).contains("Port with name https not found"); - } - // ======================================== // Namespace Operations Tests // ======================================== @@ -1286,29 +1197,46 @@ void labelAddsLabelsToResource() throws InterruptedException { } @Test - void labelRemoveRemovesLabelsFromResource() throws InterruptedException { + void removeLabelsFromAllNodesRemovesLabelsFromEachNode() throws InterruptedException { // Given - var pod = new PodBuilder() + var firstNode = new NodeBuilder() .withNewMetadata() - .withName("test-pod") - .withNamespace("default") - .withLabels(Map.of("app", "myapp", "version", "1.0")) + .withName("node-1") + .withLabels(Map.of("node", "jenkins", "keep", "value")) + .endMetadata() + .build(); + var secondNode = new NodeBuilder() + .withNewMetadata() + .withName("node-2") + .withLabels(Map.of("node", "jenkins", "keep", "value")) .endMetadata() .build(); - server.expect().get().withPath("/api/v1/namespaces/default/pods/test-pod").andReturn(200, pod).once(); - server.expect().get().withPath("/api/v1/namespaces/default/pods/test-pod").andReturn(200, pod).once(); - server.expect().patch().withPath("/api/v1/namespaces/default/pods/test-pod").andReturn(200, pod).once(); + server.expect() + .get() + .withPath("/api/v1/nodes") + .andReturn(200, new NodeListBuilder().withItems(firstNode, secondNode).build()) + .once(); + server.expect().get().withPath("/api/v1/nodes/node-1").andReturn(200, firstNode).times(2); + server.expect().patch().withPath("/api/v1/nodes/node-1").andReturn(200, firstNode).once(); + server.expect().get().withPath("/api/v1/nodes/node-2").andReturn(200, secondNode).times(2); + server.expect().patch().withPath("/api/v1/nodes/node-2").andReturn(200, secondNode).once(); // When - k8sApiClient.labelRemove("pod", "test-pod", "default", "version"); + k8sApiClient.removeLabelsFromAllNodes("node"); // Then - var request = server.getLastRequest(); - assertThat(request.getMethod()).isEqualTo("PATCH"); - assertThat(request.getPath()).isEqualTo("/api/v1/namespaces/default/pods/test-pod"); - assertThat(parseJsonList(request.getUtf8Body())).containsExactly( - Map.of("op", "remove", "path", "/metadata/labels/version") + assertThat(server.getRequestCount()).isEqualTo(7); + assertThat(server.takeRequest().getPath()).isEqualTo("/api/v1/nodes"); + assertThat(server.takeRequest().getPath()).isEqualTo("/api/v1/nodes/node-1"); + assertThat(server.takeRequest().getPath()).isEqualTo("/api/v1/nodes/node-1"); + assertThat(parseJsonList(server.takeRequest().getUtf8Body())).containsExactly( + Map.of("op", "remove", "path", "/metadata/labels/node") + ); + assertThat(server.takeRequest().getPath()).isEqualTo("/api/v1/nodes/node-2"); + assertThat(server.takeRequest().getPath()).isEqualTo("/api/v1/nodes/node-2"); + assertThat(parseJsonList(server.takeRequest().getUtf8Body())).containsExactly( + Map.of("op", "remove", "path", "/metadata/labels/node") ); } @@ -1402,88 +1330,7 @@ void deleteRemovesSpecificResourceByName() throws InterruptedException { } @Test - void runCreatesPodWithImage() { - // Given - server.expect() - .post() - .withPath("/api/v1/namespaces/default/pods") - .andReturn( - 201, new PodBuilder() - .withNewMetadata() - .withName("test-pod") - .endMetadata() - .build() - ) - .once(); - - // When - String result = k8sApiClient.run("test-pod", "nginx:latest", "", Map.of()); - - // Then - assertThat(result).contains("pod/test-pod created"); - } - - @Test - void runAppliesPodOverridesInsteadOfGeneratedParameterValues() throws InterruptedException { - // Given - server.expect() - .post() - .withPath("/api/v1/namespaces/jenkins/pods") - .andReturn( - 201, new PodBuilder() - .withNewMetadata() - .withName("test-pod") - .endMetadata() - .build() - ) - .once(); - - String overrideImage = "bash:42"; - Map overrides = Map.of( - "spec", Map.of( - "containers", List.of(Map.of( - "name", "override-container", - "image", overrideImage, - "args", List.of("cat", "/etc/group"), - "volumeMounts", List.of(Map.of("name", "group", "mountPath", "/etc/group", "readOnly", true)) - )), - "nodeSelector", Map.of("node", "jenkins"), - "volumes", List.of(Map.of("name", "group", "hostPath", Map.of("path", "/etc/group"))) - ) - ); - - // When - k8sApiClient.run("test-pod", "nginx:latest", "jenkins", overrides); - - // Then - Map requestBody = parseJson(server.getLastRequest().getUtf8Body()); - assertThat(((Map) requestBody.get("metadata")).get("name")).isEqualTo("test-pod"); - assertThat(((Map) requestBody.get("metadata")).get("namespace")).isEqualTo("jenkins"); - assertThat(((Map) ((Map) requestBody.get("spec")).get("nodeSelector")).get( - "node")).isEqualTo("jenkins"); - - List> containers = (List>) ((Map) requestBody.get("spec")).get( - "containers"); - assertThat(containers).hasSize(1); - Map container = containers.get(0); - assertThat(container.get("name")).isEqualTo("override-container"); - assertThat(container.get("image")).isEqualTo("bash:42"); - List args = (List) container.get("args"); - assertThat(args).containsExactly("cat", "/etc/group"); - - List> volumeMounts = (List>) container.get("volumeMounts"); - Map volumeMount = volumeMounts.get(0); - assertThat(volumeMount.get("mountPath")).isEqualTo("/etc/group"); - assertThat(volumeMount.get("readOnly")).isEqualTo(true); - - List> volumes = (List>) ((Map) requestBody.get("spec")).get( - "volumes"); - Map volume = volumes.get(0); - assertThat(((Map) volume.get("hostPath")).get("path")).isEqualTo("/etc/group"); - } - - @Test - void runReturnsPodLogsAndRemovesPodForInteractiveRmMode() throws InterruptedException { + void runTemporaryPodReturnsLogsAndRemovesPod() throws InterruptedException { // Given server.expect() .post() @@ -1539,14 +1386,56 @@ void runReturnsPodLogsAndRemovesPodForInteractiveRmMode() throws InterruptedExce .andReturn(200, new StatusBuilder().build()) .once(); + var pod = new PodBuilder().withNewMetadata() + .withName("gid-pod") + .withNamespace("jenkins") + .endMetadata() + .withNewSpec() + .withNodeSelector(Map.of("node", "jenkins")) + .addNewContainer() + .withName("gid-container") + .withImage("bash:42") + .withArgs("cat", "/etc/group") + .addNewVolumeMount() + .withName("group") + .withMountPath("/etc/group") + .withReadOnly(true) + .endVolumeMount() + .endContainer() + .addNewVolume() + .withName("group") + .withNewHostPath() + .withPath("/etc/group") + .endHostPath() + .endVolume() + .endSpec() + .build(); + // When - String result = k8sApiClient.run("gid-pod", "bash:42", "jenkins", "--restart=Never", "-ti", "--rm", "--quiet"); + String result = k8sApiClient.runTemporaryPod(pod); // Then assertThat(result).isEqualTo("root:x:0:\ndocker:x:42:\n"); Map createRequest = parseJson(server.takeRequest().getUtf8Body()); - assertThat(((Map) createRequest.get("spec")).get("restartPolicy")).isEqualTo("Never"); + assertThat(((Map) createRequest.get("metadata")).get("name")).isEqualTo("gid-pod"); + assertThat(((Map) createRequest.get("metadata")).get("namespace")).isEqualTo("jenkins"); + Map spec = (Map) createRequest.get("spec"); + assertThat(spec.get("restartPolicy")).isEqualTo("Never"); + assertThat(((Map) spec.get("nodeSelector")).get("node")).isEqualTo("jenkins"); + + List> containers = (List>) spec.get("containers"); + assertThat(containers).hasSize(1); + Map container = containers.get(0); + assertThat(container.get("name")).isEqualTo("gid-container"); + assertThat(container.get("image")).isEqualTo("bash:42"); + assertThat((List) container.get("args")).containsExactly("cat", "/etc/group"); + + List> volumeMounts = (List>) container.get("volumeMounts"); + assertThat(volumeMounts.get(0)).containsEntry("mountPath", "/etc/group").containsEntry("readOnly", true); + + List> volumes = (List>) spec.get("volumes"); + assertThat(((Map) volumes.get(0).get("hostPath")).get("path")).isEqualTo("/etc/group"); } // ======================================== diff --git a/src/test/java/com/cloudogu/gitops/integration/tools/MonitoringTestIT.java b/src/test/java/com/cloudogu/gitops/integration/tools/MonitoringTestIT.java index f3097339c..c61bbd444 100644 --- a/src/test/java/com/cloudogu/gitops/integration/tools/MonitoringTestIT.java +++ b/src/test/java/com/cloudogu/gitops/integration/tools/MonitoringTestIT.java @@ -1,24 +1,16 @@ package com.cloudogu.gitops.integration.tools; import com.cloudogu.gitops.integration.TestK8sHelper; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import java.util.List; -import static org.assertj.core.api.Assertions.assertThat; - /** * This class checks if Prometheus is started well. - * Prometheus contains own namespace ('monitoring') which owns and 3 Pods: - * - Grafana - * - Operator - * - prometheus-stack + * Prometheus uses its own namespace ('monitoring'). The tests verify that the core monitoring pods + * for Grafana, the Prometheus Operator, and Prometheus are running. */ @EnabledIfSystemProperty(named = "micronaut.environments", matches = "full|full-netpols|operator-full") public class MonitoringTestIT extends KubernetesApiTestSetup { @@ -57,32 +49,8 @@ void ensureOperatorIsStarted() { TestK8sHelper.waitForAllPodsRunningInNamespace(namespace, operatorPod); } - @Disabled("not start on jenkins") @Test void ensureMonitoringIsStarted() { - List pods = listPods(); - assertThat(pods).isNotEmpty(); - - Pod prometheus = null; - for (Pod pod : pods) { - if (pod.getMetadata().getName().contains(prometheusPod)) { - prometheus = pod; - break; - } - } - assertThat(prometheus).isNotNull(); - assertThat(prometheus.getStatus().getPhase()).isEqualTo("Running"); - } - - @Disabled("jenkins got only 2") - @Test - void ensureNamespaceGot3Pods() { - assertThat(listPods()).hasSize(3); - } - - private List listPods() { - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - return client.pods().inNamespace(namespace).list().getItems(); - } + TestK8sHelper.waitForAllPodsRunningInNamespace(namespace, prometheusPod); } } diff --git a/src/test/java/com/cloudogu/gitops/tools/MonitoringTest.java b/src/test/java/com/cloudogu/gitops/tools/MonitoringTest.java index 54d2fec41..9ad2b9339 100644 --- a/src/test/java/com/cloudogu/gitops/tools/MonitoringTest.java +++ b/src/test/java/com/cloudogu/gitops/tools/MonitoringTest.java @@ -306,13 +306,21 @@ void whenExternalMailserverIsSetWithoutUserAndPassword() throws GitAPIException, } @Test - void checkIfKubernetesSecretWillBeCreatedWhenExternalEmailserversCredentialIsSet() throws GitAPIException { + void createsKubernetesSecretWhenExternalMailserverCredentialsAreSet() throws GitAPIException { config.getFeatures().getMail().setActive(true); config.getFeatures().getMail().setSmtpAddress("smtp.example.com"); config.getFeatures().getMail().setSmtpUser("grafana@example.com"); config.getFeatures().getMail().setSmtpPassword("1101ABCabc&/+*~"); install(createStack(scmManagerMock)); + + verify(k8sClient).createSecret( + "generic", + "grafana-email-secret", + "foo-monitoring", + new Tuple<>("user", "grafana@example.com"), + new Tuple<>("password", "1101ABCabc&/+*~") + ); } @Test diff --git a/src/test/java/com/cloudogu/gitops/tools/core/JenkinsTest.java b/src/test/java/com/cloudogu/gitops/tools/core/JenkinsTest.java index 709501d08..1d4c9b181 100644 --- a/src/test/java/com/cloudogu/gitops/tools/core/JenkinsTest.java +++ b/src/test/java/com/cloudogu/gitops/tools/core/JenkinsTest.java @@ -30,6 +30,7 @@ import com.cloudogu.gitops.utils.Tuple; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import io.fabric8.kubernetes.api.model.Pod; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,7 +49,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; @@ -103,9 +103,8 @@ class JenkinsTest { @BeforeEach void setup() { - // waitForInternalNodeIp -> waitForNode() - when(k8sClient.waitForNode()).thenReturn("node/" + expectedNodeName); - when(k8sClient.run(anyString(), anyString(), anyString(), anyMap(), any(String[].class))).thenReturn(""); + when(k8sClient.waitForNode()).thenReturn(expectedNodeName); + when(k8sClient.runTemporaryPod(any(Pod.class))).thenReturn(""); } @Test @@ -122,7 +121,7 @@ void installsJenkins() throws GitAPIException, IOException { config.getJenkins().setInternalBashImage("bash:42"); config.getJenkins().setInternalDockerClientVersion("23"); - when(k8sClient.run(anyString(), anyString(), anyString(), anyMap(), any(String[].class))).thenReturn(""" + when(k8sClient.runTemporaryPod(any(Pod.class))).thenReturn(""" root:x:0: daemon:x:1: docker:x:42:me @@ -147,7 +146,7 @@ void installsJenkins() throws GitAPIException, IOException { verify(repositoryWorkspace).commitAndPushClusterResourcesChanges("Update jenkins GitOps resources"); verify(k8sClient).label("node", expectedNodeName, new Tuple<>("node", "jenkins")); - verify(k8sClient).labelRemove("node", "--all", "", "node"); + verify(k8sClient).removeLabelsFromAllNodes("node"); verify(k8sClient).createSecret( "generic", "jenkins-credentials", @@ -179,20 +178,21 @@ void installsJenkins() throws GitAPIException, IOException { assertThat(agent.get("runAsUser")).isEqualTo(1000); assertThat(agent.get("runAsGroup")).isEqualTo(42); - ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor overridesCaptor = ArgumentCaptor.forClass(Map.class); - verify(k8sClient).run( - nameCaptor.capture(), - anyString(), - eq(jenkins.getNamespace()), - overridesCaptor.capture(), - any(String[].class) - ); - assertThat(nameCaptor.getValue()).startsWith("tmp-docker-gid-grepper-"); - - Map spec = (Map) overridesCaptor.getValue().get("spec"); - List> containers = (List>) spec.get("containers"); - assertThat(containers.get(0).get("image").toString()).isEqualTo("bash:42"); + ArgumentCaptor podCaptor = ArgumentCaptor.forClass(Pod.class); + verify(k8sClient).runTemporaryPod(podCaptor.capture()); + + Pod gidGrepperPod = podCaptor.getValue(); + assertThat(gidGrepperPod.getMetadata().getName()).startsWith("tmp-docker-gid-grepper-"); + assertThat(gidGrepperPod.getMetadata().getNamespace()).isEqualTo(jenkins.getNamespace()); + assertThat(gidGrepperPod.getSpec().getNodeSelector()).containsEntry("node", "jenkins"); + assertThat(gidGrepperPod.getSpec().getContainers()).hasSize(1); + assertThat(gidGrepperPod.getSpec().getContainers().get(0).getImage()).isEqualTo("bash:42"); + assertThat(gidGrepperPod.getSpec().getContainers().get(0).getArgs()).containsExactly("cat", "/etc/group"); + assertThat(gidGrepperPod.getSpec().getContainers().get(0).getVolumeMounts()).hasSize(1); + assertThat(gidGrepperPod.getSpec().getContainers().get(0).getVolumeMounts().get(0).getMountPath()) + .isEqualTo("/etc/group"); + assertThat(gidGrepperPod.getSpec().getVolumes()).hasSize(1); + assertThat(gidGrepperPod.getSpec().getVolumes().get(0).getHostPath().getPath()).isEqualTo("/etc/group"); } @Test @@ -314,7 +314,7 @@ void preparesJenkinsAppContentInClusterResourcesWorkspace() throws GitAPIExcepti @Test void installsJenkinsWithoutDockerGid() throws GitAPIException, IOException { - when(k8sClient.run(anyString(), anyString(), anyString(), anyMap(), any(String[].class))).thenReturn(""" + when(k8sClient.runTemporaryPod(any(Pod.class))).thenReturn(""" root:x:0: daemon:x:1: me:x:1000:""");