From acc438afe967daac4c6476e2eb53de0042473edf Mon Sep 17 00:00:00 2001
From: Simon Gerber <simon.gerber@vshn.ch>
Date: Thu, 28 May 2026 10:58:32 +0200
Subject: [PATCH] Introduce new parameter `--processes` for `catalog compile`

This allows us to better tune the number of processes spawned by Kapitan
when running in CI (from some local experiments, peak memory working set
size is more or less directly correlated to number of Kapitan worker
processes). Unfortunately, extending our `cpu_count()` logic to account
for K8s CPU requests/limits is non-trivial, so we just allow users to
override the detected number of CPUs.
---
 commodore/cli/catalog.py                   | 12 ++++++++++++
 commodore/config.py                        | 10 ++++++++++
 commodore/helpers.py                       |  6 +++++-
 docs/modules/ROOT/pages/reference/cli.adoc |  6 ++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/commodore/cli/catalog.py b/commodore/cli/catalog.py
index f7dbdb881..0b922e229 100644
--- a/commodore/cli/catalog.py
+++ b/commodore/cli/catalog.py
@@ -164,6 +164,16 @@ def _complete_clusters(ctx: click.Context, _, incomplete: str) -> list[str]:
     + "uncommitted changes in tracked files count as local changes."
     + "The parameter has no effect if `--local` is given.",
 )
+@click.option(
+    "--processes",
+    type=int,
+    default=0,
+    show_default=True,
+    help="Control the number of worker processes that are spawned when compiling the catalog. "
+    + "A value of `0` will set the number of worker processes to the number of CPUs available "
+    + "on the system. Note that this parameter doesn't adjust the number of threads used by "
+    + "reclass-rs.",
+)
 @options.verbosity
 @options.pass_config
 # pylint: disable=too-many-arguments
@@ -187,6 +197,7 @@ def compile_catalog(
     migration,
     dynamic_fact: str,
     force: bool,
+    processes: int,
 ):
     config.update_verbosity(verbose)
     config.api_url = api_url
@@ -204,6 +215,7 @@ def compile_catalog(
     config.fetch_dependencies = fetch_dependencies
     config.dynamic_facts = parse_dynamic_facts_from_cli(dynamic_fact)
     config.force = not config.local and force
+    config.processes = processes
 
     if config.push and (
         config.global_repo_revision_override or config.tenant_repo_revision_override
diff --git a/commodore/config.py b/commodore/config.py
index bb328b599..b1b5a2030 100644
--- a/commodore/config.py
+++ b/commodore/config.py
@@ -103,6 +103,7 @@ class Config:
     _request_timeout: int
     _managed_tools: dict[str, str]
     _api_token: Optional[str]
+    _processes: int
 
     oidc_client: Optional[str]
     oidc_discovery_url: Optional[str]
@@ -144,6 +145,7 @@ def __init__(
         self._github_token = None
         self._request_timeout = 5
         self._managed_tools = {}
+        self._processes = 0
 
     @property
     def verbose(self):
@@ -327,6 +329,14 @@ def managed_tools(self) -> dict[str, str]:
     def managed_tools(self, managed_tools: dict[str, str]):
         self._managed_tools = managed_tools
 
+    @property
+    def processes(self) -> int:
+        return self._processes
+
+    @processes.setter
+    def processes(self, processes: int):
+        self._processes = processes
+
     @property
     def inventory(self):
         return self._inventory
diff --git a/commodore/helpers.py b/commodore/helpers.py
index 4b081dc8e..aabb1b8a3 100644
--- a/commodore/helpers.py
+++ b/commodore/helpers.py
@@ -251,6 +251,10 @@ def kapitan_compile(
     refController = RefController(config.refs_dir)
     if fake_refs:
         refController.register_backend(FakeVaultBackend())
+    processes: Optional[int] = config.processes
+    if config.processes == 0:
+        processes = cpu_count()
+
     click.secho("Compiling catalog...", bold=True)
     # workaround the non-modifiable Namespace() default value for cached.args
     cached.args.inventory_backend = "reclass-rs"
@@ -261,7 +265,7 @@ def kapitan_compile(
     cached.args.verbose = config.trace
     cached.args.output_path = output_dir
     cached.args.targets = targets
-    cached.args.parallelism = cpu_count()
+    cached.args.parallelism = processes
     cached.args.labels = None
     cached.args.prune = False
     cached.args.indent = 2
diff --git a/docs/modules/ROOT/pages/reference/cli.adoc b/docs/modules/ROOT/pages/reference/cli.adoc
index e82688b3f..15c7816aa 100644
--- a/docs/modules/ROOT/pages/reference/cli.adoc
+++ b/docs/modules/ROOT/pages/reference/cli.adoc
@@ -129,6 +129,12 @@ With `--no-force`, local changes in global, tenant, or dependency checkouts are
 Specifying `--force` has no effect if `--local` is given, and is silently ignored.
 Defaults to `--no-force`.
 
+*--processes*::
+  Control the number of worker processes that are spawned when compiling the catalog.
+  A value of `0` will set the number of worker processes to the number of CPUs available on the system.
+  Note that this parameter doesn't adjust the number of threads used by reclass-rs.
+  Defaults to `0`.
+
 *--help*::
   Show catalog clean usage and options then exit.