diff --git a/Cargo.lock b/Cargo.lock index 1bbbcdd63..a4cc94218 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -337,6 +337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", + "regex-automata", "serde_core", ] @@ -404,9 +405,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -481,6 +482,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "config-utils" +version = "0.5.0" +dependencies = [ + "clap", + "rstest", + "similar-asserts", + "snafu 0.9.2", + "tempfile", +] + [[package]] name = "console" version = "0.16.4" @@ -1082,7 +1094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1800,7 +1812,7 @@ dependencies = [ "console", "globset", "once_cell", - "similar", + "similar 2.7.0", "tempfile", "walkdir", ] @@ -3259,7 +3271,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3316,7 +3328,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3663,6 +3675,26 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "similar" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f66ca1f7aca2474dc10c942eb22feffc897735f54cd1db90138c2fddb490987" +dependencies = [ + "bstr", + "unicode-segmentation", +] + +[[package]] +name = "similar-asserts" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997e6ca38e97437973fc9f7f50a50d1274cacd874341a4960fea90067291038c" +dependencies = [ + "console", + "similar 3.2.0", +] + [[package]] name = "slab" version = "0.4.12" @@ -4100,7 +4132,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4876,7 +4908,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1f7a04ed2..07311a6e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,11 +84,13 @@ serde_yaml = "0.9.34" # This is the last available version, see https://github.c # digest 0.11 and signature 3.0, which rsa 0.9 does not support. sha2 = { version = "0.10.9", features = ["oid"] } signature = "2.2.0" +similar-asserts = "2.0.0" snafu = "0.9.2" stackable-operator-derive = { path = "stackable-operator-derive" } strum = { version = "0.28.0", features = ["derive"] } syn = "3.0.3" sysinfo = "0.39.6" +tempfile = "3.27.0" time = { version = "0.3.55" } tokio = { version = "1.53.1", features = ["macros", "rt-multi-thread", "fs"] } # We use ring instead of aws-lc-rs, as this currently fails to build in "make run-dev" diff --git a/crates/config-utils/.gitattributes b/crates/config-utils/.gitattributes new file mode 100644 index 000000000..5de03bbf4 --- /dev/null +++ b/crates/config-utils/.gitattributes @@ -0,0 +1,4 @@ +tests/resources/**/*.properties.expected linguist-language=properties +tests/resources/**/*.properties.in linguist-language=properties +tests/resources/**/*.xml.expected linguist-language=xml +tests/resources/**/*.xml.in linguist-language=xml diff --git a/crates/config-utils/.gitignore b/crates/config-utils/.gitignore new file mode 100644 index 000000000..f31e01c84 --- /dev/null +++ b/crates/config-utils/.gitignore @@ -0,0 +1,8 @@ +/target + +# The following files are mentioned as an example in the README, so let's exclude them +example.xml +example-password + +tests/resources/properties/*.properties +tests/resources/xml/*.xml diff --git a/crates/config-utils/.pre-commit-config.yaml b/crates/config-utils/.pre-commit-config.yaml new file mode 100644 index 000000000..c8cb7f1bb --- /dev/null +++ b/crates/config-utils/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +# We ideally want something like: https://github.com/j178/prek/issues/1869 +--- +default_language_version: + node: system + +repos: + - repo: local + hooks: + - id: cargo-test-no-default-features + name: cargo-test-no-default-features + language: system + entry: cargo test --no-default-features --package config-utils + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: .*\.rs$|Cargo\.toml + + - id: cargo-test-all-features + name: cargo-test-no-default-features + language: system + entry: cargo test --all-features --package config-utils + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: .*\.rs$|Cargo\.toml diff --git a/crates/config-utils/CHANGELOG.md b/crates/config-utils/CHANGELOG.md new file mode 100644 index 000000000..c258f751e --- /dev/null +++ b/crates/config-utils/CHANGELOG.md @@ -0,0 +1,34 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +## [0.5.0] - 2026-06-30 + +### Changed + +- Bump dependencies ([#15]). + +[#15]: https://github.com/stackabletech/config-utils/pull/15 + +## [0.4.0] - 2026-05-21 + +### Changed + +- Bump `clap` to 4.6 and Rust to 1.95.0 ([#13]). +- Replace `lazy_static` with `std::sync::LazyLock` ([#13]). + +[#13]: https://github.com/stackabletech/config-utils/pull/13 + +## [0.2.0] - 2024-06-25 + +### Added + +- Support escaped command invocations ([#8]). + +[#8]: https://github.com/stackabletech/config-utils/pull/8 + +## [0.1.0] - 2024-06-14 + +Initial release diff --git a/crates/config-utils/Cargo.toml b/crates/config-utils/Cargo.toml new file mode 100644 index 000000000..52810c32b --- /dev/null +++ b/crates/config-utils/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "config-utils" +version = "0.5.0" +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +# We try hard to have as less dependencies as possible! +clap.workspace = true +snafu.workspace = true + +[dev-dependencies] +rstest.workspace = true +similar-asserts.workspace = true +tempfile.workspace = true diff --git a/crates/config-utils/LICENSE b/crates/config-utils/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/crates/config-utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/config-utils/README.md b/crates/config-utils/README.md new file mode 100644 index 000000000..a35c3670e --- /dev/null +++ b/crates/config-utils/README.md @@ -0,0 +1,55 @@ +# config-utils + +This utility currently only supports filling your config with contents from environmental variables or files (called templating). + +## Templating + +> **_TIP:_** The concept was heavily inspired by [Druids configuration interpolation](https://druid.apache.org/docs/latest/configuration/#configuration-interpolation). + +Imagine the following `example.xml`: + +```xml +cat > example.xml << 'EOF' + + ${env:EXAMPLE_USERNAME} + ${file:UTF-8:example-password} + +EOF +``` + +and the following `example-password`: + +```bash +echo 'example-password <123>!' > example-password +``` + +You can run the following command to replace both placeholders: + +```bash +export EXAMPLE_USERNAME=my-user + +config-utils template example.xml +``` + +Afterwards the XML looks like + +```xml + + my-user + example-password <123>! + +``` + +`config-utils` did the following steps to achieve the result: + +1. Use the file extension to determine the file type (XML in this case). You can also specify the file type manually as a CLI argument. +2. Read the env var `EXAMPLE_USERNAME`, xml-escape it and insert it +3. Read the contents of the file `example-password`, xml-escape it and insert it + +Please note that `config-utils` also supports nested templating, so the name of the file to read can come from an env var (or even another file as well). +This looks something like `${env:${env:ENV_TEST_PASSWORD_ENV_NAME}}` + +## Currently supported file formats + +1. `.properties` files +2. XML files diff --git a/crates/config-utils/src/cli_args.rs b/crates/config-utils/src/cli_args.rs new file mode 100644 index 000000000..c8b3b14a6 --- /dev/null +++ b/crates/config-utils/src/cli_args.rs @@ -0,0 +1,16 @@ +use clap::{Parser, Subcommand}; + +use crate::template::cli_args::TemplateCommand; + +/// Utility that helps you handling config files. +#[derive(Debug, Parser)] +#[command(version, about, long_about = None)] +pub struct Args { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command { + Template(TemplateCommand), +} diff --git a/crates/config-utils/src/file_types/mod.rs b/crates/config-utils/src/file_types/mod.rs new file mode 100644 index 000000000..aaac84bfe --- /dev/null +++ b/crates/config-utils/src/file_types/mod.rs @@ -0,0 +1,35 @@ +use std::{collections::HashMap, sync::LazyLock}; + +use clap::ValueEnum; +use properties::PropertiesEscaper; +use xml::XmlEscaper; + +mod properties; +mod xml; + +// Yes, we could use `strum` for that, but we try to keep the dependencies minimal. +pub static KNOWN_FILE_TYPES: LazyLock> = LazyLock::new(|| { + let mut types = HashMap::new(); + types.insert("properties".to_owned(), FileType::Properties); + types.insert("xml".to_owned(), FileType::Xml); + types +}); + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum FileType { + Properties, + Xml, +} + +pub trait Escape { + fn escape(line: String) -> String; +} + +impl FileType { + pub fn escape(&self, line: String) -> String { + match self { + FileType::Properties => PropertiesEscaper::escape(line), + FileType::Xml => XmlEscaper::escape(line), + } + } +} diff --git a/crates/config-utils/src/file_types/properties.rs b/crates/config-utils/src/file_types/properties.rs new file mode 100644 index 000000000..0772a8045 --- /dev/null +++ b/crates/config-utils/src/file_types/properties.rs @@ -0,0 +1,45 @@ +use super::Escape; + +pub struct PropertiesEscaper; + +// https://docs.oracle.com/javase/6/docs/api/java/util/Properties.html#load(java.io.Reader) +impl Escape for PropertiesEscaper { + fn escape(line: String) -> String { + // Copied from https://github.com/adamcrume/java-properties/blob/0335bfb733444e0b9326405bc7845be449bec1f3/src/lib.rs#L809 + let mut escaped = String::new(); + for c in line.chars() { + match c { + '\\' => escaped.push_str("\\\\"), + ' ' => escaped.push_str("\\ "), + '\t' => escaped.push_str("\\t"), + '\r' => escaped.push_str("\\r"), + '\n' => escaped.push_str("\\n"), + '\x0c' => escaped.push_str("\\f"), + ':' => escaped.push_str("\\:"), + '=' => escaped.push_str("\\="), + '!' => escaped.push_str("\\!"), + '#' => escaped.push_str("\\#"), + _ if c < ' ' => escaped.push_str(&format!("\\u{:x}", c as u16)), + _ => escaped.push(c), + } + } + + escaped + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("foo", "foo")] + #[case("foo bar", "foo\\ bar")] + #[case(" bar", "\\ bar")] + #[case("foo<>'\"&\r\nbar", "foo<>'\"&\\r\\nbar")] + fn test_xml_escaping(#[case] input: String, #[case] expected: String) { + assert_eq!(PropertiesEscaper::escape(input), expected); + } +} diff --git a/crates/config-utils/src/file_types/xml.rs b/crates/config-utils/src/file_types/xml.rs new file mode 100644 index 000000000..5ba6f9921 --- /dev/null +++ b/crates/config-utils/src/file_types/xml.rs @@ -0,0 +1,39 @@ +use super::Escape; + +pub struct XmlEscaper; + +impl Escape for XmlEscaper { + fn escape(line: String) -> String { + let mut escaped = String::new(); + for c in line.chars() { + match c { + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + '&' => escaped.push_str("&"), + '\n' => escaped.push_str(" "), + '\r' => escaped.push_str(" "), + _ => escaped.push(c), + } + } + + escaped + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("foo", "foo")] + #[case("foo bar", "foo bar")] + #[case(" bar", "<foo> bar")] + #[case("foo<>'\"&\r\nbar", "foo<>'"& bar")] + fn test_xml_escaping(#[case] input: String, #[case] expected: String) { + assert_eq!(XmlEscaper::escape(input), expected); + } +} diff --git a/crates/config-utils/src/lib.rs b/crates/config-utils/src/lib.rs new file mode 100644 index 000000000..412e320ac --- /dev/null +++ b/crates/config-utils/src/lib.rs @@ -0,0 +1,10 @@ +pub mod file_types; +pub mod template; + +// It could be the case that the colon (:) in the start pattern has been escaped, as e.g. the product-config crate does +pub const ENV_VAR_START_PATTERNS: [&str; 2] = ["${env:", "${env\\:"]; +pub const ENV_VAR_END_PATTERN: &str = "}"; + +// It could be the case that the colon (:) in the start pattern has been escaped, as e.g. the product-config crate does +pub const FILE_START_PATTERNS: [&str; 2] = ["${file:UTF-8:", "${file\\:UTF-8\\:"]; +pub const FILE_END_PATTERN: &str = "}"; diff --git a/crates/config-utils/src/main.rs b/crates/config-utils/src/main.rs new file mode 100644 index 000000000..5c7961540 --- /dev/null +++ b/crates/config-utils/src/main.rs @@ -0,0 +1,31 @@ +use clap::Parser; +use config_utils::template::{self, cli_args::TemplateCommand, template}; +use snafu::{ResultExt, Snafu}; + +use crate::cli_args::{Args, Command}; +mod cli_args; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Failed to template file"))] + TemplateFile { source: template::Error }, +} + +type Result = std::result::Result; + +#[snafu::report] +fn main() -> Result<()> { + let args = Args::parse(); + + match args.command { + Command::Template(TemplateCommand { + file, + file_type, + dont_escape, + }) => { + template(&file, file_type.as_ref(), !dont_escape).context(TemplateFileSnafu)?; + } + } + + Ok(()) +} diff --git a/crates/config-utils/src/template/cli_args.rs b/crates/config-utils/src/template/cli_args.rs new file mode 100644 index 000000000..7075018d9 --- /dev/null +++ b/crates/config-utils/src/template/cli_args.rs @@ -0,0 +1,23 @@ +use std::path::PathBuf; + +use clap::Parser; + +use crate::file_types::FileType; + +/// Fill out variables in config files from either env variables or files directly. +#[derive(Debug, Parser)] +pub struct TemplateCommand { + /// The path to the file that should be templated + pub file: PathBuf, + + /// The optional file type of the file to be templated. If this is not specified this utility will try to infer + /// the type based on the file name. + #[arg(value_enum)] + pub file_type: Option, + + /// By default inserted values are automatically escaped according to the deteced file format. You can disable + /// this, e.g. when you need to insert XML tags (as they otherwise would be escaped). + /// NOTE: Please make sure to correctly escape the inserted text on your own! + #[clap(long)] + pub dont_escape: bool, +} diff --git a/crates/config-utils/src/template/mod.rs b/crates/config-utils/src/template/mod.rs new file mode 100644 index 000000000..ee65b2ca3 --- /dev/null +++ b/crates/config-utils/src/template/mod.rs @@ -0,0 +1,245 @@ +use std::{ + env, + fs::{self, File}, + io::{BufRead, BufReader, Write}, + path::PathBuf, +}; + +use snafu::{OptionExt, ResultExt, Snafu}; + +use crate::{ + ENV_VAR_END_PATTERN, ENV_VAR_START_PATTERNS, FILE_END_PATTERN, FILE_START_PATTERNS, + file_types::{FileType, KNOWN_FILE_TYPES}, +}; + +pub mod cli_args; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Could not read file {file_name:?}"))] + ReadFile { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Failed to get file extension from file {file_name:?}"))] + GetFileExtension { file_name: PathBuf }, + + #[snafu(display("Failed to convert file name {file_name:?} to string"))] + ConvertFileNameToString { file_name: PathBuf }, + + #[snafu(display( + "The extension {extension} is not known, can not determine file type. Please specify the file type manually." + ))] + ExtensionUnkown { extension: String }, + + #[snafu(display("Failed to create temporary file {tmp_file_name:?}"))] + CreateTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + }, + + #[snafu(display("Failed to read line from file {file_name:?}"))] + ReadLine { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Failed to write to temporary file {tmp_file_name:?}"))] + WriteToTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + }, + + #[snafu(display( + "Failed to rename temporary file {tmp_file_name:?} to destination file {destination_file_name:?}" + ))] + RenameTemporaryFile { + source: std::io::Error, + tmp_file_name: PathBuf, + destination_file_name: PathBuf, + }, + + #[snafu(display( + "Could not find the end pattern {end_pattern:?} in expression {expression:?}" + ))] + FindEndPatten { + end_pattern: String, + expression: String, + }, + + #[snafu(display("Could not read file {file_name:?} for templating"))] + ReadFileForTemplating { + source: std::io::Error, + file_name: PathBuf, + }, + + #[snafu(display("Could not read env var {env_var_name:?} for templating"))] + ReadEnvVarForTemplating { + source: std::env::VarError, + env_var_name: String, + }, +} + +type Result = std::result::Result; + +pub fn template(file_name: &PathBuf, file_type: Option<&FileType>, escape: bool) -> Result<()> { + let file_type = match file_type { + Some(file_type) => file_type, + None => { + let extension = file_name + .extension() + .context(GetFileExtensionSnafu { file_name })? + .to_str() + .context(GetFileExtensionSnafu { file_name })?; + + KNOWN_FILE_TYPES + .get(extension) + .context(ExtensionUnkownSnafu { extension })? + } + }; + + let file = File::open(file_name).context(ReadFileSnafu { file_name })?; + let buf_reader = BufReader::new(file); + + let tmp_file_name = PathBuf::from(format!( + "{}.tmp_config_utils", + file_name + .to_str() + .context(ConvertFileNameToStringSnafu { file_name })? + )); + let mut temp_file = File::create(&tmp_file_name).context(CreateTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + + for line in buf_reader.lines() { + let mut line = line.context(ReadLineSnafu { file_name })?; + + run_all_replacements_on_line(&mut line, file_type, escape)?; + + temp_file + .write_all(line.as_bytes()) + .context(WriteToTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + temp_file + .write_all(b"\n") + .context(WriteToTemporaryFileSnafu { + tmp_file_name: tmp_file_name.clone(), + })?; + } + + fs::rename(&tmp_file_name, file_name).context(RenameTemporaryFileSnafu { + tmp_file_name, + destination_file_name: file_name, + })?; + + Ok(()) +} + +fn run_all_replacements_on_line( + line: &mut String, + file_type: &FileType, + escape: bool, +) -> Result<()> { + loop { + #[allow(clippy::type_complexity)] // It's only used in a single place + let mut replacements: Vec<(&str, &str, fn(&str) -> Result)> = Vec::new(); + + for start_pattern in ENV_VAR_START_PATTERNS { + replacements.push(( + start_pattern, + ENV_VAR_END_PATTERN, + replacement_action_for_env_var, + )); + } + for start_pattern in FILE_START_PATTERNS { + replacements.push((start_pattern, FILE_END_PATTERN, replacement_action_for_file)); + } + + let mut changed = false; + for (start_pattern, end_pattern, replacement_action) in replacements { + changed |= replace_thingy_in_line( + line, + start_pattern, + end_pattern, + replacement_action, + file_type, + escape, + )?; + } + + if !changed { + break; + } + } + + Ok(()) +} + +fn replacement_action_for_file(file_name: &str) -> Result { + let file_content = + fs::read_to_string(file_name).context(ReadFileForTemplatingSnafu { file_name })?; + let file_content = file_content.trim_end_matches('\n'); + + Ok(file_content.to_owned()) +} + +fn replacement_action_for_env_var(env_var_name: &str) -> Result { + let env_var_content = + env::var(env_var_name).context(ReadEnvVarForTemplatingSnafu { env_var_name })?; + + Ok(env_var_content) +} + +/// * `line` is the current line [`String`] that should be templated. +/// * `start_pattern` must be the start pattern, e.g `${env:`. +/// * `end_pattern` must be the end pattern, `}` in the most cases. +/// * `replacement_action` must be a function that is called and get passed the [`&str`] content between the start and end +/// pattern. This can e.g. be the name of the env var or file name to read. +/// +/// Returns wether the `line` was modified. +fn replace_thingy_in_line( + line: &mut String, + start_pattern: &str, + end_pattern: &str, + replacement_action: fn(&str) -> Result, + file_type: &FileType, + escape: bool, +) -> Result { + // We need to go back to forth to not destroy stuff while iterating. + // Also this is needed to correctly handle nested cases. + let matches = line + .rmatch_indices(start_pattern) + .map(|(index, _)| index) + .collect::>(); + + if matches.is_empty() { + // Nothing to do + return Ok(false); + } + + for index in matches { + debug_assert_eq!(&line[index..index + start_pattern.len()], start_pattern); + let (parameter, _) = line[index + start_pattern.len()..] + .split_once(end_pattern) + .context(FindEndPattenSnafu { + // FIXME: Truncate string to not bloat error message + expression: &line[index..], + end_pattern, + })?; + + let mut new_content = replacement_action(parameter)?; + if escape { + new_content = file_type.escape(new_content); + } + + line.replace_range( + index..index + start_pattern.len() + parameter.len() + end_pattern.len(), + &new_content, + ); + } + + // We modified stuff + Ok(true) +} diff --git a/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected new file mode 100644 index 000000000..153bbf387 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.expected @@ -0,0 +1,3 @@ +# We must also recognize commands where the colon was escaped +test.from.env=foo +test.from.file=42 diff --git a/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in new file mode 100644 index 000000000..078eb87b0 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_escaped_command.properties.in @@ -0,0 +1,3 @@ +# We must also recognize commands where the colon was escaped +test.from.env=${env\:ENV_TEST} +test.from.file=${file\:UTF-8\:${env\:${env\:FILE_TEST_42_FILE_ENV_NAME}}} diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.expected b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected new file mode 100644 index 000000000..9633d5823 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.expected @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=foo +meine.suß.Pröpertie=42 +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_env.properties.in b/crates/config-utils/tests/resources/properties/security_from_env.properties.in new file mode 100644 index 000000000..2b5643c64 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_env.properties.in @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${env:ENV_TEST} +meine.suß.Pröpertie=42 +example-password=${env:ENV_TEST_PASSWORD} diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected b/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected new file mode 100644 index 000000000..acf8cbfc2 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_nested.properties.expected @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=42 +meine.suß.Pröpertie=42 +example-password=admin-pw\=\ withSpace$%"'\ &&}\ § diff --git a/crates/config-utils/tests/resources/properties/security_from_nested.properties.in b/crates/config-utils/tests/resources/properties/security_from_nested.properties.in new file mode 100644 index 000000000..de07da7b7 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_from_nested.properties.in @@ -0,0 +1,4 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${file:UTF-8:${env:${env:FILE_TEST_42_FILE_ENV_NAME}}} +meine.suß.Pröpertie=42 +example-password=${env:${env:ENV_TEST_PASSWORD_ENV_NAME}} diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.expected b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected new file mode 100644 index 000000000..be2a47474 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.expected @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${FOO},${dont:replace} +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/properties/security_untouched.properties.in b/crates/config-utils/tests/resources/properties/security_untouched.properties.in new file mode 100644 index 000000000..be2a47474 --- /dev/null +++ b/crates/config-utils/tests/resources/properties/security_untouched.properties.in @@ -0,0 +1,3 @@ +networkaddress.cache.negative.ttl=0 +networkaddress.cache.ttl=${FOO},${dont:replace} +meine.suß.Pröpertie=42,3 diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected new file mode 100644 index 000000000..59705090d --- /dev/null +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.expected @@ -0,0 +1,32 @@ + + + + login-identity-provider + org.apache.nifi.ldap.LdapProvider + LDAPS + + example user + admin-pw= withSpace$%"' &&} § + + THROW + 10 secs + 10 secs + + ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 + ou=my users,dc=example,dc=org + uid={0} + + NONE + /stackable/server_tls/keystore.p12 + secret + PKCS12 + /stackable/server_tls/truststore.p12 + secret + PKCS12 + TLSv1.2 + true + + USE_DN + 7 days + + diff --git a/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in new file mode 100644 index 000000000..0b204cb6b --- /dev/null +++ b/crates/config-utils/tests/resources/xml/nifi_ldap.xml.in @@ -0,0 +1,32 @@ + + + + login-identity-provider + org.apache.nifi.ldap.LdapProvider + LDAPS + + ${env:ENV_TEST_USERNAME} + ${env:ENV_TEST_PASSWORD} + + THROW + 10 secs + 10 secs + + ldaps://openldap.kuttl-test-tidy-asp.svc.cluster.local:1636 + ou=my users,dc=example,dc=org + uid={0} + + NONE + /stackable/server_tls/keystore.p12 + secret + PKCS12 + /stackable/server_tls/truststore.p12 + secret + PKCS12 + TLSv1.2 + true + + USE_DN + 7 days + + diff --git a/crates/config-utils/tests/templating.rs b/crates/config-utils/tests/templating.rs new file mode 100644 index 000000000..4262952e0 --- /dev/null +++ b/crates/config-utils/tests/templating.rs @@ -0,0 +1,50 @@ +use std::{ + env, + fs::{self, File}, + io::Write, + path::{Path, PathBuf}, +}; + +use config_utils::template::template; +use rstest::rstest; +use tempfile::tempdir; + +#[rstest] +fn test_file_templating(#[files("tests/resources/**/*.in")] test_file_in: PathBuf) { + let example_dir = create_example_files(); + set_example_envs(&example_dir); + + let test_file = test_file_in.with_extension(""); + let test_file_expected = test_file_in.with_extension("expected"); + + fs::copy(&test_file_in, &test_file).unwrap(); + template(&test_file, None, true).unwrap(); + + let actual = fs::read_to_string(test_file).unwrap(); + let expected = fs::read_to_string(test_file_expected).unwrap(); + + similar_asserts::assert_eq!(actual, expected); +} + +fn set_example_envs(example_dir: &Path) { + // SAFETY: We only use a single thread to set this env vars + unsafe { + env::set_var("ENV_TEST", "foo"); + env::set_var("ENV_TEST_USERNAME", "example user"); + env::set_var("ENV_TEST_PASSWORD", "admin-pw= withSpace$%\"' &&} §"); + env::set_var("ENV_TEST_PASSWORD_ENV_NAME", "ENV_TEST_PASSWORD"); + + env::set_var("FILE_TEST_42_FILE", example_dir.join("42")); + env::set_var("FILE_TEST_42_FILE_ENV_NAME", "FILE_TEST_42_FILE"); + } +} + +/// Returns the directory where the files reside +fn create_example_files() -> PathBuf { + let dir = tempdir().expect("Failed to create temp dir").keep(); + + let mut file = File::create(dir.join("42")).unwrap(); + file.write_all(b"42").unwrap(); + + dir +}