[Bug] Fix config verification after SOL introduced

.github/workflows/ci.yml

@@ -105,8 +105,6 @@ jobs:
 
           python -m venv .smoke-venv
           .smoke-venv/bin/python -m pip install .
-          .smoke-venv/bin/python -m pip uninstall --yes cloudai
-          .smoke-venv/bin/python -m pip install --editable .
 
       - name: Test commands
         run: |
@@ -115,9 +113,6 @@ jobs:
 
           .smoke-venv/bin/cloudai --help
 
-          # this checks that all TOMLs are valid, Test Scenarios are checked for _all_ tests
-          .smoke-venv/bin/cloudai verify-configs conf/
-
           # this checks that all TOMLs are valid, Test Scenarios are checked _only_ the tests in the specified directory
           .smoke-venv/bin/cloudai verify-configs --tests-dir conf/common/test conf/common
           .smoke-venv/bin/cloudai verify-configs --tests-dir conf/release/spcx/l40s/test conf/release/spcx/l40s

src/cloudai/cli/cli.py

@@ -268,7 +268,7 @@ def generate_report(system_cfg: Path, tests_dir: Path, scenario_cfg: Path, hook_
 def verify_configs(configs_dir: Path, tests_dir: Path):
     """Verify the configuration TOML files."""
     args = argparse.Namespace(configs_dir=configs_dir, tests_dir=tests_dir)
-    handle_verify_all_configs(args)
+    raise click.exceptions.Exit(1 if handle_verify_all_configs(args) else 0)
 
 
 @main.command()

src/cloudai/cli/handlers.py

@@ -451,13 +451,15 @@ def verify_system_configs(system_tomls: List[Path]) -> int:
                 with _ensure_kube_config_exists(system_toml, content):
                     Parser.parse_system(system_toml)
             except Exception as e:
-                logging.debug(f"Failed to parse system config {system_toml}: {e}", exc_info=True)
+                logging.error(f"Failed to verify system config {system_toml}: {e}")
+                logging.debug("", exc_info=True)
                 nfailed += 1
         else:
             try:
                 Parser.parse_system(system_toml)
             except Exception as e:
-                logging.debug(f"Failed to parse system config {system_toml}: {e}", exc_info=True)
+                logging.error(f"Failed to verify system config {system_toml}: {e}")
+                logging.debug("", exc_info=True)
                 nfailed += 1
 
     if nfailed:
@@ -477,7 +479,9 @@ def verify_test_configs(test_tomls: List[Path]) -> int:
             with test_toml.open() as fh:
                 tp.current_file = test_toml
                 tp.load_test_definition(load_test_toml_file(fh, test_toml))
-        except Exception:
+        except Exception as e:
+            logging.error(f"Failed to verify Test: {test_toml}: {e}")
+            logging.debug("", exc_info=True)
             nfailed += 1
 
     if nfailed:
@@ -491,7 +495,7 @@ def verify_test_configs(test_tomls: List[Path]) -> int:
 def verify_test_scenarios(
     scenario_tomls: List[Path], test_tomls: list[Path], hook_tomls: List[Path], hook_test_tomls: list[Path]
 ) -> int:
-    system = Mock(spec=System)
+    system = Mock(spec=System, sol={})
     nfailed = 0
     for scenario_file in scenario_tomls:
         logging.debug(f"Verifying Test Scenario: {scenario_file}...")
@@ -501,7 +505,9 @@ def verify_test_scenarios(
             hooks = Parser.parse_hooks(hook_tomls, system, {t.name: t for t in hook_tests})
             scenario = Parser.parse_test_scenario(scenario_file, system, {t.name: t for t in tests}, hooks)
             validate_domain_randomization_active(scenario)
-        except Exception:
+        except Exception as e:
+            logging.error(f"Failed to verify Test Scenario: {scenario_file}: {e}")
+            logging.debug("", exc_info=True)
             nfailed += 1
 
     if nfailed: