diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8770f6..4c929aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ All notable changes to this project will be documented in this file. - Environment variable overrides (`envOverrides`) are now applied after all environment variables set by the operator. In particular, `SPARK_CONF_DIR` of a SparkApplication can now be overridden, whereas previously the operator's values always took precedence ([#753]). +- BREAKING (behaviour): Keys of the `spark-env.sh` `configOverrides` must be valid shell + identifiers (matching `[a-zA-Z_][a-zA-Z0-9_]*`) and are now rejected if they are not ([#761]). ### Fixed @@ -51,6 +53,8 @@ All notable changes to this project will be documented in this file. - The history and connect controllers now watch all resources that they create, the missing RBAC `watch` permissions were added, and all controllers early-exit the reconcile action when the object is marked for deletion ([#757]). +- The `configOverrides` for `spark-env.sh` and `security.properties` of a SparkApplication now take + effect in the submit, driver and executor Pods ([#761]). [#721]: https://github.com/stackabletech/spark-k8s-operator/pull/721 [#727]: https://github.com/stackabletech/spark-k8s-operator/pull/727 @@ -62,6 +66,7 @@ All notable changes to this project will be documented in this file. [#753]: https://github.com/stackabletech/spark-k8s-operator/pull/753 [#754]: https://github.com/stackabletech/spark-k8s-operator/pull/754 [#757]: https://github.com/stackabletech/spark-k8s-operator/pull/757 +[#761]: https://github.com/stackabletech/spark-k8s-operator/pull/761 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc b/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc index e4f5077a..073646eb 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc @@ -74,6 +74,9 @@ NOTE: The given properties are written to `spark-env.sh` in the form `export KEY Make sure to escape the value already in the specification. Be aware that some environment variables may already be set, so prepend or append a reference to them in the value, as it is done in the example. +IMPORTANT: Because the file is sourced by the shell, keys must be valid shell identifiers, matching `[a-zA-Z_][a-zA-Z0-9_]*`. +A key such as `MY-VAR` is rejected, because `export MY-VAR="..."` is not something the shell can execute. + === The security.properties file The `security.properties` file is used to configure JVM security properties. diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index cc5388b8..3264eb7b 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -3,7 +3,7 @@ use stackable_operator::crd::s3; use crate::crd::{ constants::{ JVM_SECURITY_PROPERTIES_FILE, STACKABLE_TLS_STORE_PASSWORD, STACKABLE_TRUST_STORE, - VOLUME_MOUNT_PATH_LOG_CONFIG, + VOLUME_MOUNT_PATH_CONFIG, }, logdir::ResolvedLogDir, tlscerts::tls_secret_names, @@ -25,7 +25,7 @@ pub fn construct_extra_java_options( // kept the implementation as is. We can always re-visit this as needed. let mut jvm_args = vec![format!( - "-Djava.security.properties={VOLUME_MOUNT_PATH_LOG_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" + "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" )]; if tls_secret_names(s3_conn, log_dir).is_some() { @@ -85,11 +85,11 @@ mod tests { assert_eq!( driver_extra_java_options, - "-Djava.security.properties=/stackable/log_config/security.properties" + "-Djava.security.properties=/stackable/spark/conf/security.properties" ); assert_eq!( executor_extra_java_options, - "-Djava.security.properties=/stackable/log_config/security.properties" + "-Djava.security.properties=/stackable/spark/conf/security.properties" ); } @@ -125,7 +125,7 @@ mod tests { assert_eq!( driver_extra_java_options, - "-Djava.security.properties=/stackable/log_config/security.properties -Dhttps.proxyHost=from-driver" + "-Djava.security.properties=/stackable/spark/conf/security.properties -Dhttps.proxyHost=from-driver" ); assert_eq!( executor_extra_java_options, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1b9b824e..387c6004 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -10,7 +10,7 @@ use constants::*; use history::LogFileDirectorySpec; use logdir::ResolvedLogDir; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ builder::pod::volume::{ SecretFormat, SecretOperatorVolumeSourceBuilder, SecretOperatorVolumeSourceBuilderError, @@ -120,6 +120,12 @@ pub enum Error { source: stackable_operator::v2::builder::pod::container::Error, }, + #[snafu(display( + "invalid key [{key}] in the {SPARK_ENV_SH_FILE_NAME} configOverrides: the file is sourced \ + by the shell, so keys must be valid shell identifiers matching [a-zA-Z_][a-zA-Z0-9_]*" + ))] + InvalidSparkEnvShKey { key: String }, + #[snafu(display("failed to configure log directory"))] ConfigureLogDir { source: logdir::Error }, } @@ -127,7 +133,7 @@ pub enum Error { // `_STACKABLE_PRE_HOOK` is evaluated by the entrypoint script (run-spark.sh) in the Spark images // before the actual JVM process is started; the operator uses it to run `containerdebug` in the // background of every `spark` container. -constant!(STACKABLE_PRE_HOOK: EnvVarName = "_STACKABLE_PRE_HOOK"); +constant!(pub STACKABLE_PRE_HOOK: EnvVarName = "_STACKABLE_PRE_HOOK"); constant!(PYTHONPATH: EnvVarName = "PYTHONPATH"); // The environment variable holding the trust store password; its value is the // `STACKABLE_TLS_STORE_PASSWORD` string constant. @@ -1058,18 +1064,28 @@ fn resources_to_executor_props( Ok(()) } -/// Create the content of the file spark-env.sh. -/// The properties are serialized in the form 'export {k}="{v}"', -/// escaping neither the key nor the value. The user is responsible for -/// providing escaped values. -pub fn to_spark_env_sh_string<'a, T>(properties: T) -> String +/// Create the content of the file spark-env.sh, serializing the properties as 'export {k}="{v}"'. +/// The file is sourced by the shell, so keys that are not valid shell identifiers are rejected. +/// Values are left unescaped on purpose, so that they can reference other variables. +pub fn to_spark_env_sh_string<'a, T>(properties: T) -> Result where T: Iterator, { properties - .map(|(k, v)| format!("export {k}=\"{v}\"")) - .collect::>() - .join("\n") + .map(|(k, v)| { + ensure!(is_shell_identifier(k), InvalidSparkEnvShKeySnafu { key: k }); + Ok(format!("export {k}=\"{v}\"")) + }) + .collect::, Error>>() + .map(|lines| lines.join("\n")) +} + +fn is_shell_identifier(key: &str) -> bool { + let mut chars = key.chars(); + chars + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && chars.all(|char| char.is_ascii_alphanumeric() || char == '_') } #[cfg(test)] @@ -1099,6 +1115,49 @@ mod tests { let _ = *STACKABLE_TLS_STORE_PASSWORD_ENV; } + #[rstest] + #[case("SPARK_HISTORY_OPTS")] + #[case("_LEADING_UNDERSCORE")] + #[case("TRAILING_DIGITS_123")] + #[case("lowercase")] + fn to_spark_env_sh_string_accepts_shell_identifiers(#[case] key: &str) { + let overrides = BTreeMap::from([(key.to_string(), "value".to_string())]); + + assert_eq!( + to_spark_env_sh_string(overrides.iter()).expect("the key is a valid shell identifier"), + format!("export {key}=\"value\"") + ); + } + + #[rstest] + #[case("TEST_SPARK-ENV-SH")] + #[case("1_LEADING_DIGIT")] + #[case("WITH SPACE")] + #[case("with.dot")] + #[case("")] + fn to_spark_env_sh_string_rejects_other_keys(#[case] key: &str) { + let overrides = BTreeMap::from([(key.to_string(), "value".to_string())]); + + assert!(matches!( + to_spark_env_sh_string(overrides.iter()), + Err(Error::InvalidSparkEnvShKey { key: invalid }) if invalid == key + )); + } + + /// Values are shell-evaluated by design, so references to other variables must survive. + #[test] + fn to_spark_env_sh_string_leaves_values_untouched() { + let overrides = BTreeMap::from([( + "SPARK_HISTORY_OPTS".to_string(), + "$SPARK_HISTORY_OPTS -Dsome.token=$SAS_TOKEN".to_string(), + )]); + + assert_eq!( + to_spark_env_sh_string(overrides.iter()).expect("the key is a valid shell identifier"), + r#"export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dsome.token=$SAS_TOKEN""# + ); + } + #[test] fn test_default_resource_limits() { let spark_application = serde_yaml::from_str::(indoc! {" diff --git a/rust/operator-binary/src/history/controller/build/resource/config_map.rs b/rust/operator-binary/src/history/controller/build/resource/config_map.rs index 6daef8f7..09e81bf8 100644 --- a/rust/operator-binary/src/history/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/history/controller/build/resource/config_map.rs @@ -48,6 +48,14 @@ pub enum Error { InvalidSparkDefaults { source: stackable_operator::v2::config_file_writer::PropertiesWriterError, }, + + #[snafu(display( + "History server : failed to serialize [{SPARK_ENV_SH_FILE_NAME}] for group {rolegroup}" + ))] + SparkEnvSh { + source: crate::crd::Error, + rolegroup: String, + }, } type Result = std::result::Result; @@ -95,7 +103,10 @@ pub(crate) fn build_config_map( .add_data(SPARK_DEFAULTS_FILE_NAME, spark_defaults) .add_data( SPARK_ENV_SH_FILE_NAME, - to_spark_env_sh_string(rg.config.config_overrides.spark_env_sh.overrides.iter()), + to_spark_env_sh_string(rg.config.config_overrides.spark_env_sh.overrides.iter()) + .with_context(|_| SparkEnvShSnafu { + rolegroup: role_group_name.to_string(), + })?, ) .add_data( JVM_SECURITY_PROPERTIES_FILE, diff --git a/rust/operator-binary/src/spark_k8s_controller/build/mod.rs b/rust/operator-binary/src/spark_k8s_controller/build/mod.rs index 68ceb2dc..88a4983c 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/mod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/mod.rs @@ -203,7 +203,20 @@ pub(crate) fn recommended_labels_for_component_resources( #[cfg(test)] mod tests { + use indoc::indoc; + use stackable_operator::{ + cli::OperatorEnvironmentOptions, + k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}, + }; + use super::*; + use crate::{ + crd::{ + constants::{POD_TEMPLATE_FILE, VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_PATH_CONFIG}, + v1alpha1, + }, + spark_k8s_controller::{dereference::DereferencedSparkApplication, validate::validate}, + }; #[test] fn test_constants() { @@ -214,4 +227,154 @@ mod tests { let _ = *SPARK_JOB_TEMPLATE_COMPONENT_NAME; let _ = *SPARK_SUBMIT_COMPONENT_NAME; } + + /// The Pod specs of the submit Job and of the driver and executor pod templates, each with the + /// name of the resource it was taken from. + fn pod_specs(enable_vector_agent: bool) -> Vec<(String, PodSpec)> { + let yaml = format!( + indoc! {r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + image: oci.example.org/jobs/spark-example:1.0.0 + vectorAggregatorConfigMapName: vector-aggregator-discovery + deps: + requirements: + - tabulate==0.8.9 + packages: + - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.11.0 + driver: + config: + logging: + enableVectorAgent: {enable_vector_agent} + executor: + config: + logging: + enableVectorAgent: {enable_vector_agent} + "#}, + enable_vector_agent = enable_vector_agent + ); + let deserializer = serde_yaml::Deserializer::from_str(&yaml); + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("invalid test SparkApplication YAML"); + + let validated = validate( + DereferencedSparkApplication { + spark_application, + resolved_template_refs: Vec::new(), + s3_connection: None, + log_dir: None, + }, + &OperatorEnvironmentOptions { + operator_namespace: "stackable-operators".to_string(), + operator_service_name: "spark-k8s-operator".to_string(), + image_repository: "oci.example.org/sdp".to_string(), + }, + ) + .expect("the fixture validates"); + + let resources = build(&validated).expect("the resources can be built"); + + let mut pod_specs = vec![( + "spark-submit Job".to_string(), + resources.jobs[0] + .spec + .clone() + .expect("the Job has a spec") + .template + .spec + .expect("the Job has a pod spec"), + )]; + for config_map in &resources.config_maps { + let Some(template) = config_map + .data + .as_ref() + .and_then(|data| data.get(POD_TEMPLATE_FILE)) + else { + continue; + }; + let template: PodTemplateSpec = + serde_yaml::from_str(template).expect("the pod template deserializes"); + pod_specs.push(( + config_map.metadata.name.clone().unwrap_or_default(), + template.spec.expect("the pod template has a spec"), + )); + } + + assert_eq!(pod_specs.len(), 3); + + let vector_containers = pod_specs + .iter() + .flat_map(|(_, pod_spec)| &pod_spec.containers) + .filter(|container| container.name == "vector") + .count(); + assert_eq!(vector_containers, if enable_vector_agent { 2 } else { 0 }); + + pod_specs + } + + #[test] + fn every_declared_volume_is_mounted() { + for enable_vector_agent in [false, true] { + for (name, pod_spec) in pod_specs(enable_vector_agent) { + let PodSpec { + containers, + init_containers, + volumes, + .. + } = pod_spec; + let mounted: Vec<&str> = containers + .iter() + .chain(init_containers.iter().flatten()) + .flat_map(|container| container.volume_mounts.iter().flatten()) + .map(|volume_mount| volume_mount.name.as_str()) + .collect(); + let unmounted: Vec<&str> = volumes + .iter() + .flatten() + .map(|volume| volume.name.as_str()) + .filter(|volume_name| !mounted.contains(volume_name)) + .collect(); + + assert!( + unmounted.is_empty(), + "{name} declares volumes that no container mounts: {unmounted:?}" + ); + } + } + } + + #[test] + fn spark_containers_mount_the_config_volume() { + for enable_vector_agent in [false, true] { + for (name, pod_spec) in pod_specs(enable_vector_agent) { + let spark_container = pod_spec + .containers + .iter() + .find(|container| container.name == "spark" || container.name == "spark-submit") + .unwrap_or_else(|| panic!("{name} has a Spark container")); + let mount_path = spark_container + .volume_mounts + .iter() + .flatten() + .find(|volume_mount| volume_mount.name == VOLUME_MOUNT_NAME_CONFIG.as_ref()) + .map(|volume_mount| volume_mount.mount_path.as_str()); + + assert_eq!( + mount_path, + Some(VOLUME_MOUNT_PATH_CONFIG), + "the Spark container of {name} must mount the config Volume at {VOLUME_MOUNT_PATH_CONFIG}" + ); + } + } + } } diff --git a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs index e7dccd26..dfd08cf6 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs @@ -35,6 +35,7 @@ use stackable_operator::{ use crate::{ crd::{ + STACKABLE_PRE_HOOK, constants::*, roles::{RoleConfig, SparkApplicationRole, SparkContainer}, tlscerts, @@ -241,6 +242,19 @@ pub(crate) fn pod_template( let mut cb = new_container_builder(&SparkContainer::Spark.to_container_name()); let mut env = env.clone(); + + let pre_hook = env + .get(&STACKABLE_PRE_HOOK) + .and_then(|env_var| env_var.value.clone()) + .unwrap_or_default(); + // SPARK_ENV_LOADED keeps Spark from sourcing spark-env.sh again and overwriting the values. + env = env.with_value( + &STACKABLE_PRE_HOOK, + format!( + "{pre_hook} . {VOLUME_MOUNT_PATH_CONFIG}/{SPARK_ENV_SH_FILE_NAME}; export SPARK_ENV_LOADED=1" + ), + ); + if config.logging.enable_vector_agent { env = env.with_value( &STACKABLE_POST_HOOK, @@ -256,7 +270,9 @@ pub(crate) fn pod_template( // variable. let merged_env = spark_application.merged_env(role.clone(), env); - cb.add_volume_mounts(config.volume_mounts(spark_application, s3conn, logdir)) + cb.add_volume_mount(VOLUME_MOUNT_NAME_CONFIG.as_ref(), VOLUME_MOUNT_PATH_CONFIG) + .context(AddVolumeMountSnafu)? + .add_volume_mounts(config.volume_mounts(spark_application, s3conn, logdir)) .context(AddVolumeMountSnafu)? .add_env_vars(merged_env) .resources(config.resources.clone().into()) diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs index f38b1c33..281b7f60 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs @@ -42,6 +42,12 @@ pub enum Error { role: SparkApplicationRole, }, + #[snafu(display("failed to serialize [{SPARK_ENV_SH_FILE_NAME}] for {role}", role = role.as_ref()))] + SparkEnvSh { + source: crate::crd::Error, + role: SparkApplicationRole, + }, + #[snafu(display("failed to build the pod template config map"))] PodTemplateConfigMap { source: stackable_operator::builder::configmap::Error, @@ -131,7 +137,8 @@ pub(crate) fn pod_template_config_map( cm_builder.add_data( SPARK_ENV_SH_FILE_NAME, - to_spark_env_sh_string(config_overrides.spark_env_sh.overrides.iter()), + to_spark_env_sh_string(config_overrides.spark_env_sh.overrides.iter()) + .with_context(|_| SparkEnvShSnafu { role: role.clone() })?, ); let mut jvm_sec_props = default_jvm_security_properties(); @@ -159,7 +166,11 @@ pub(crate) fn submit_job_config_map( cm_builder.add_data( SPARK_ENV_SH_FILE_NAME, - to_spark_env_sh_string(config_overrides.spark_env_sh.overrides.iter()), + to_spark_env_sh_string(config_overrides.spark_env_sh.overrides.iter()).context( + SparkEnvShSnafu { + role: SparkApplicationRole::Submit, + }, + )?, ); let mut jvm_sec_props = default_jvm_security_properties(); diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs index 39820b20..1c1d8df7 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs @@ -64,11 +64,11 @@ pub(crate) fn spark_job( let mut cb = new_container_builder(&SparkContainer::SparkSubmit.to_container_name()); // The SPARK_SUBMIT_OPTS env var is used to configure the JVM settings of the spark-submit job. - // Here we need to point the JVM to our logging configuration and if S3 is used for data or Spark History, - // we also need to tell the JVM where the trust store is located. - // The same properties are also set for the driver and executor pods via the pod template config maps. + // Here we need to point the JVM to the security properties and if S3 is used for data or Spark + // History, we also need to tell the JVM where the trust store is located. The same properties + // are set for driver and executor via `spark.{driver,executor}.extraJavaOptions`. let mut spark_submit_opts_env = vec![format!( - "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}" + "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" )]; if tlscerts::tls_secret_names(s3conn, logdir).is_some() { spark_submit_opts_env.push(format!( @@ -86,7 +86,7 @@ pub(crate) fn spark_job( env.clone() .with_value(&SPARK_SUBMIT_OPTS, spark_submit_opts_env.join(" ")) // TODO: move this to the image - .with_value(&SPARK_CONF_DIR, "/stackable/spark/conf"), + .with_value(&SPARK_CONF_DIR, VOLUME_MOUNT_PATH_CONFIG), ); cb.image_from_product_image(spark_image) @@ -99,6 +99,8 @@ pub(crate) fn spark_job( ]) .args(vec![job_commands.join("\n")]) .resources(job_config.resources.clone().into()) + .add_volume_mount(VOLUME_MOUNT_NAME_CONFIG.as_ref(), VOLUME_MOUNT_PATH_CONFIG) + .context(AddVolumeMountSnafu)? .add_volume_mounts(spark_application.spark_job_volume_mounts(s3conn, logdir)) .context(AddVolumeMountSnafu)? .add_env_vars(merged_env); diff --git a/tests/templates/kuttl/config-overrides/00-patch-ns.yaml.j2 b/tests/templates/kuttl/config-overrides/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/config-overrides/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/config-overrides/01-assert.yaml.j2 b/tests/templates/kuttl/config-overrides/01-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/config-overrides/01-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/config-overrides/01-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/config-overrides/01-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/config-overrides/01-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/config-overrides/10-assert.yaml b/tests/templates/kuttl/config-overrides/10-assert.yaml new file mode 100644 index 00000000..1a62980b --- /dev/null +++ b/tests/templates/kuttl/config-overrides/10-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: spark-config-overrides +status: + phase: Succeeded diff --git a/tests/templates/kuttl/config-overrides/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/config-overrides/10-deploy-spark-app.yaml.j2 new file mode 100644 index 00000000..e7c6b095 --- /dev/null +++ b/tests/templates/kuttl/config-overrides/10-deploy-spark-app.yaml.j2 @@ -0,0 +1,117 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-overrides-log-config +data: + log4j2.properties: |- + appenders = CONSOLE + + appender.CONSOLE.type = Console + appender.CONSOLE.name = CONSOLE + appender.CONSOLE.target = SYSTEM_ERR + appender.CONSOLE.layout.type = PatternLayout + appender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n + + rootLogger.level = INFO + rootLogger.appenderRefs = CONSOLE + rootLogger.appenderRef.CONSOLE.ref = CONSOLE +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: spark-config-overrides +spec: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + sparkImage: +{% if test_scenario['values']['spark'].find(",") > 0 %} + custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['spark'] }}" +{% endif %} + pullPolicy: IfNotPresent + mode: cluster + mainApplicationFile: "local:///stackable/spark/jobs/check-config-overrides.py" + job: + configOverrides: + spark-env.sh: + SPARK_PRINT_LAUNCH_COMMAND: "1" + driver: + configOverrides: + spark-env.sh: + TEST_SPARK_ENV_SH: driver + security.properties: + networkaddress.cache.ttl: "42" + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + spark: + # A custom log configuration points the log config Volume at the ConfigMap above, + # which holds no `security.properties`. + custom: + configMap: config-overrides-log-config + volumeMounts: + - name: script + mountPath: /stackable/spark/jobs + executor: + replicas: 1 + configOverrides: + spark-env.sh: + TEST_SPARK_ENV_SH: executor + security.properties: + networkaddress.cache.ttl: "43" + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + spark: + custom: + configMap: config-overrides-log-config + volumeMounts: + - name: script + mountPath: /stackable/spark/jobs + volumes: + - name: script + configMap: + name: check-config-overrides +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: check-config-overrides +data: + check-config-overrides.py: | + import os + + from pyspark.sql import SparkSession + + spark = SparkSession.builder.appName("check-config-overrides").getOrCreate() + sc = spark.sparkContext + + driver_env = os.environ.get("TEST_SPARK_ENV_SH") + assert driver_env == "driver", driver_env + + ttl = sc._jvm.java.security.Security.getProperty("networkaddress.cache.ttl") + assert ttl == "42", ttl + + executor_env = ( + sc.parallelize(range(4), 4) + .map(lambda _: os.environ.get("TEST_SPARK_ENV_SH")) + .distinct() + .collect() + ) + assert executor_env == ["executor"], executor_env + + + def executor_ttl(_): + prefix = "networkaddress.cache.ttl=" + with open("/stackable/spark/conf/security.properties") as file: + return [line.strip() for line in file if line.startswith(prefix)] + + + executor_ttls = sc.parallelize(range(4), 4).flatMap(executor_ttl).distinct().collect() + assert executor_ttls == ["networkaddress.cache.ttl=43"], executor_ttls diff --git a/tests/templates/kuttl/config-overrides/11-assert.yaml b/tests/templates/kuttl/config-overrides/11-assert.yaml new file mode 100644 index 00000000..7d4c6622 --- /dev/null +++ b/tests/templates/kuttl/config-overrides/11-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: + - script: kubectl -n $NAMESPACE logs job/spark-config-overrides -c spark-submit | grep -q 'Spark Command:' diff --git a/tests/templates/kuttl/overrides/06-assert.yaml b/tests/templates/kuttl/overrides/06-assert.yaml index 04415b86..c84f8474 100644 --- a/tests/templates/kuttl/overrides/06-assert.yaml +++ b/tests/templates/kuttl/overrides/06-assert.yaml @@ -43,6 +43,6 @@ data: test.sparkConf=true spark-env.sh: |- - export TEST_SPARK-ENV-SH_FROM_RG="ROLEGROUP" - export TEST_SPARK-ENV-SH_RG="ROLEGROUP" - export TEST_SPARK-ENV-SH_ROLE="ROLE" + export TEST_SPARK_ENV_SH_FROM_RG="ROLEGROUP" + export TEST_SPARK_ENV_SH_RG="ROLEGROUP" + export TEST_SPARK_ENV_SH_ROLE="ROLE" diff --git a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 index aa705d3f..a50b8fff 100644 --- a/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 +++ b/tests/templates/kuttl/overrides/06-deploy-history-server.yaml.j2 @@ -49,8 +49,8 @@ spec: test.securityProperties.role: role test.securityProperties.fromRg: role spark-env.sh: - TEST_SPARK-ENV-SH_ROLE: ROLE - TEST_SPARK-ENV-SH_FROM_RG: ROLE + TEST_SPARK_ENV_SH_ROLE: ROLE + TEST_SPARK_ENV_SH_FROM_RG: ROLE roleGroups: default: replicas: 1 @@ -64,8 +64,8 @@ spec: test.securityProperties.fromRg: rolegroup test.securityProperties.rg: rolegroup spark-env.sh: - TEST_SPARK-ENV-SH_FROM_RG: ROLEGROUP - TEST_SPARK-ENV-SH_RG: ROLEGROUP + TEST_SPARK_ENV_SH_FROM_RG: ROLEGROUP + TEST_SPARK_ENV_SH_RG: ROLEGROUP podOverrides: spec: containers: diff --git a/tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 index 85080363..52192b2f 100644 --- a/tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/overrides/10-deploy-spark-app.yaml.j2 @@ -53,7 +53,7 @@ spec: security.properties: test.job.securityProperties: test spark-env.sh: - TEST_JOB_SPARK-ENV-SH: TEST + TEST_JOB_SPARK_ENV_SH: TEST podOverrides: spec: serviceAccountName: override-sa @@ -72,7 +72,7 @@ spec: security.properties: test.driver.securityProperties: test spark-env.sh: - TEST_DRIVER_SPARK-ENV-SH: TEST + TEST_DRIVER_SPARK_ENV_SH: TEST podOverrides: spec: serviceAccountName: override-sa @@ -92,7 +92,7 @@ spec: security.properties: test.executor.securityProperties: test spark-env.sh: - TEST_EXECUTOR_SPARK-ENV-SH: TEST + TEST_EXECUTOR_SPARK_ENV_SH: TEST podOverrides: spec: serviceAccountName: override-sa diff --git a/tests/templates/kuttl/overrides/11-assert.yaml b/tests/templates/kuttl/overrides/11-assert.yaml index af5d33b4..b0db099f 100644 --- a/tests/templates/kuttl/overrides/11-assert.yaml +++ b/tests/templates/kuttl/overrides/11-assert.yaml @@ -30,7 +30,7 @@ data: networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=30 test.job.securityProperties=test - spark-env.sh: export TEST_JOB_SPARK-ENV-SH="TEST" + spark-env.sh: export TEST_JOB_SPARK_ENV_SH="TEST" --- apiVersion: v1 kind: ConfigMap @@ -41,7 +41,7 @@ data: networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=30 test.driver.securityProperties=test - spark-env.sh: export TEST_DRIVER_SPARK-ENV-SH="TEST" + spark-env.sh: export TEST_DRIVER_SPARK_ENV_SH="TEST" --- apiVersion: v1 kind: ConfigMap @@ -52,4 +52,4 @@ data: networkaddress.cache.negative.ttl=0 networkaddress.cache.ttl=30 test.executor.securityProperties=test - spark-env.sh: export TEST_EXECUTOR_SPARK-ENV-SH="TEST" + spark-env.sh: export TEST_EXECUTOR_SPARK_ENV_SH="TEST" diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json index 5fb909b5..d3b356ec 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json index c82835cd..7d15d473 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json index 976109ef..ffeb4e18 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json @@ -3,7 +3,7 @@ "containers": [ { "args": [ - "containerdebug --output=/stackable/log/containerdebug-state.json --loop & /stackable/spark/bin/spark-submit --verbose --master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} --deploy-mode cluster --name pyspark-pi --conf spark.kubernetes.driver.podTemplateFile=/stackable/spark/driver-pod-templates/template.yaml --conf spark.kubernetes.executor.podTemplateFile=/stackable/spark/executor-pod-templates/template.yaml --conf spark.kubernetes.driver.podTemplateContainerName=spark --conf spark.kubernetes.executor.podTemplateContainerName=spark --conf spark.kubernetes.namespace=__NAMESPACE__ --conf spark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.driver.extraClassPath=/stackable/spark/extra-jars/* --conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.executor.extraClassPath=/stackable/spark/extra-jars/* --conf spark.driver.extraJavaOptions=\"-Djava.security.properties=/stackable/log_config/security.properties\" --conf spark.executor.extraJavaOptions=\"-Djava.security.properties=/stackable/log_config/security.properties\" --conf spark.metrics.conf.\\*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet --conf spark.metrics.conf.\\*.sink.prometheusServlet.path=/metrics/prometheus --conf spark.ui.prometheus.enabled=true --conf spark.sql.streaming.metricsEnabled=true --conf \"spark.driver.cores=2\" --conf \"spark.driver.memory=640m\" --conf \"spark.executor.cores=2\" --conf \"spark.executor.instances=1\" --conf \"spark.executor.memory=640m\" --conf \"spark.kubernetes.driver.limit.cores=2\" --conf \"spark.kubernetes.driver.request.cores=1\" --conf \"spark.kubernetes.executor.limit.cores=2\" --conf \"spark.kubernetes.executor.request.cores=1\" --conf \"spark.kubernetes.memoryOverheadFactor=0.0\" local:///stackable/spark/examples/src/main/python/pi.py" + "containerdebug --output=/stackable/log/containerdebug-state.json --loop & /stackable/spark/bin/spark-submit --verbose --master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} --deploy-mode cluster --name pyspark-pi --conf spark.kubernetes.driver.podTemplateFile=/stackable/spark/driver-pod-templates/template.yaml --conf spark.kubernetes.executor.podTemplateFile=/stackable/spark/executor-pod-templates/template.yaml --conf spark.kubernetes.driver.podTemplateContainerName=spark --conf spark.kubernetes.executor.podTemplateContainerName=spark --conf spark.kubernetes.namespace=__NAMESPACE__ --conf spark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.driver.extraClassPath=/stackable/spark/extra-jars/* --conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.executor.extraClassPath=/stackable/spark/extra-jars/* --conf spark.driver.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.executor.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.metrics.conf.\\*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet --conf spark.metrics.conf.\\*.sink.prometheusServlet.path=/metrics/prometheus --conf spark.ui.prometheus.enabled=true --conf spark.sql.streaming.metricsEnabled=true --conf \"spark.driver.cores=2\" --conf \"spark.driver.memory=640m\" --conf \"spark.executor.cores=2\" --conf \"spark.executor.instances=1\" --conf \"spark.executor.memory=640m\" --conf \"spark.kubernetes.driver.limit.cores=2\" --conf \"spark.kubernetes.driver.request.cores=1\" --conf \"spark.kubernetes.executor.limit.cores=2\" --conf \"spark.kubernetes.executor.request.cores=1\" --conf \"spark.kubernetes.memoryOverheadFactor=0.0\" local:///stackable/spark/examples/src/main/python/pi.py" ], "command": [ "/bin/bash", @@ -23,7 +23,7 @@ }, { "name": "SPARK_SUBMIT_OPTS", - "value": "-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties" + "value": "-Djava.security.properties=/stackable/spark/conf/security.properties" }, { "name": "_STACKABLE_PRE_HOOK", @@ -46,6 +46,10 @@ "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [ + { + "mountPath": "/stackable/spark/conf", + "name": "config" + }, { "mountPath": "/stackable/spark/driver-pod-templates", "name": "driver-pod-template" @@ -65,7 +69,8 @@ "restartPolicy": "Never", "schedulerName": "default-scheduler", "securityContext": { - "fsGroup": 1000 + "fsGroup": 1000, + "runAsNonRoot": true }, "serviceAccount": "pyspark-pi", "serviceAccountName": "pyspark-pi", diff --git a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json index ac0b2aef..8a4df2d3 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json @@ -2,5 +2,5 @@ "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-3.5.8.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" } diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 3a367337..c2570180 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -80,6 +80,10 @@ tests: dimensions: - spark - openshift + - name: config-overrides + dimensions: + - spark + - openshift - name: spark-history-server dimensions: - spark