From 04473b47071b91c8ed84acbd1a2467b0d2779e5a Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 02:01:23 +0200 Subject: [PATCH 01/19] fix: ADDON-90004 add safe log-rendering helpers to observability module Add three private helper functions (_sanitize_for_log, _safe_exception_repr, _safe_exception_str) to safely render exception messages and type names before logging them. These functions strip CR/LF, replace unpaired Unicode surrogates, truncate long text, and provide fallbacks for rendering failures. Co-Authored-By: Claude --- solnlib/observability.py | 37 ++++++++++ tests/unit/test_observability.py | 121 +++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index 204f8cb8..ab9fd322 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -73,6 +73,43 @@ ATTR_MODINPUT_NAME = "splunk.modinput.name" +def _sanitize_for_log(text) -> str: + """Return one bounded, UTF-8-safe physical log-line fragment.""" + try: + # Calling the base implementation directly neutralizes overridden + # methods on a str subclass and returns a genuine plain str. + text = str.__str__(text) if isinstance(text, str) else str(text) + text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ") + text = text.encode("utf-8", errors="replace").decode("utf-8") + if len(text) > 500: + text = text[:500] + "...(truncated)" + return text + except BaseException: + return "" + + +def _safe_exception_repr(error: BaseException) -> str: + try: + return _sanitize_for_log(repr(error)) + except BaseException: + pass + try: + return _sanitize_for_log(f"{type(error).__name__} (repr unavailable)") + except BaseException: + return "" + + +def _safe_exception_str(error: BaseException) -> str: + try: + return _sanitize_for_log(str(error)) + except BaseException: + pass + try: + return _sanitize_for_log(f"{type(error).__name__} (details unavailable)") + except BaseException: + return "" + + class LoggerMetricExporter(MetricExporter): """An OpenTelemetry ``MetricExporter`` that logs every data point. diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index eb803d32..14bcbdab 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -14,6 +14,7 @@ # limitations under the License. # +import io import logging import sys from unittest.mock import MagicMock, patch @@ -30,6 +31,126 @@ def logger(): return MagicMock(spec=logging.Logger) +@pytest.fixture +def real_logger(): + """A real Logger + StreamHandler over a UTF-8 TextIOWrapper. + + MagicMock does not execute lazy `%`-formatting and StringIO does not + perform UTF-8 encoding, so tests that must prove formatting/encoding + never raises need a real logger. + """ + stream = io.TextIOWrapper(io.BytesIO(), encoding="utf-8", errors="strict") + handler = logging.StreamHandler(stream) + test_logger = logging.getLogger("test.solnlib.observability.real") + test_logger.setLevel(logging.DEBUG) + test_logger.handlers = [handler] + test_logger.propagate = False + yield test_logger, stream + test_logger.handlers = [] + + +# --------------------------------------------------------------------------- +# Safe log rendering +# --------------------------------------------------------------------------- + + +class _RaisesOnStr: + def __str__(self): + raise RuntimeError("boom") + + +class _BadExceptionRepr(Exception): + def __repr__(self): + raise RuntimeError("bad repr") + + def __str__(self): + raise RuntimeError("bad str") + + +class _WeirdStr(str): + def __str__(self): + return "overridden!" + + +class TestSafeRendering: + @pytest.mark.parametrize( + "value, expected", + [ + ("plain safe text", "plain safe text"), + ("line1\r\nline2", "line1 line2"), + ("line1\nline2", "line1 line2"), + ("line1\rline2", "line1 line2"), + ("\ud800", "?"), + ], + ) + def test_sanitize_for_log_plain_cases(self, value, expected): + from solnlib.observability import _sanitize_for_log + + assert _sanitize_for_log(value) == expected + + def test_sanitize_for_log_truncates_long_text(self): + from solnlib.observability import _sanitize_for_log + + result = _sanitize_for_log("x" * 600) + assert result == "x" * 500 + "...(truncated)" + + def test_sanitize_for_log_non_string_raising_str(self): + from solnlib.observability import _sanitize_for_log + + assert _sanitize_for_log(_RaisesOnStr()) == "" + + def test_sanitize_for_log_str_subclass_returns_plain_str(self): + from solnlib.observability import _sanitize_for_log + + result = _sanitize_for_log(_WeirdStr("hello")) + assert result == "hello" + assert type(result) is str + + def test_safe_exception_repr_normal_exception(self): + from solnlib.observability import _safe_exception_repr + + error = ValueError("bad value") + assert _safe_exception_repr(error) == repr(error) + + def test_safe_exception_repr_falls_back_when_repr_raises(self): + from solnlib.observability import _safe_exception_repr + + result = _safe_exception_repr(_BadExceptionRepr("x")) + assert result == "_BadExceptionRepr (repr unavailable)" + + def test_safe_exception_str_normal_exception(self): + from solnlib.observability import _safe_exception_str + + error = ValueError("bad value") + assert _safe_exception_str(error) == str(error) + + def test_safe_exception_str_falls_back_when_str_raises(self): + from solnlib.observability import _safe_exception_str + + result = _safe_exception_str(_BadExceptionRepr("x")) + assert result == "_BadExceptionRepr (details unavailable)" + + def test_safe_exception_str_end_to_end_through_real_logger( + self, real_logger, capsys + ): + from solnlib.observability import _safe_exception_str + + test_logger, stream = real_logger + error = ValueError("multi\r\nline\r\nmessage") + test_logger.info("boom: %s", _safe_exception_str(error)) + stream.flush() + stream.seek(0) + output = stream.read() + assert "\n" not in output.strip("\n") + assert "boom: multi line message" in output + + # logging.Handler.handleError() writes "--- Logging error ---" to the + # real sys.stderr directly, never to the handler's own stream, so a + # formatting/encoding failure must be detected there, not in `stream`. + captured = capsys.readouterr() + assert "--- Logging error ---" not in captured.err + + # --------------------------------------------------------------------------- # LoggerMetricExporter # --------------------------------------------------------------------------- From 4d40269a0c23d3c14c7612b469d10161827234ed Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 02:22:01 +0200 Subject: [PATCH 02/19] fix: ADDON-90004 cap solnlib.observability log calls at INFO --- solnlib/observability.py | 53 +++++--- tests/unit/test_observability.py | 215 ++++++++++++++++++++++++++++++- 2 files changed, 243 insertions(+), 25 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index ab9fd322..96abf461 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -203,8 +203,10 @@ def export( metric_count, ) return MetricExportResult.SUCCESS - except Exception as e: - self._logger.error("Failed to export metrics: %s", e, exc_info=True) + except Exception as error: + self._logger.info( + "Failed to export metrics: %s", _safe_exception_str(error) + ) return MetricExportResult.FAILURE def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: @@ -364,8 +366,11 @@ def __init__( ta_version, modinput_type, ) - except Exception as e: - self._logger.warning("Failed to initialise ObservabilityService: %s", e) + except Exception as error: + self._logger.info( + "Failed to initialise ObservabilityService: %s", + _safe_exception_str(error), + ) def _read_ta_info(self) -> tuple[Optional[str], Optional[str]]: """Read the add-on name and version from ``app.conf``. @@ -382,8 +387,11 @@ def _read_ta_info(self) -> tuple[Optional[str], Optional[str]]: ) ta_version = scoped_stanzas.get("launcher", {}).get("version") or None return ta_name, ta_version - except Exception as e: - self._logger.warning("Failed to read TA info from app.conf: %s", e) + except Exception as error: + self._logger.info( + "Failed to read TA info from app.conf: %s", + _safe_exception_str(error), + ) return None, None def _get_ipc_broker_port(self) -> Optional[int]: @@ -396,9 +404,10 @@ def _get_ipc_broker_port(self) -> Optional[int]: try: stanzas = get_conf_stanzas("server") return int(stanzas["ipc_broker"]["port"]) - except Exception as e: - self._logger.warning( - "Failed to read IPC broker port from server.conf: %s", e + except Exception as error: + self._logger.info( + "Failed to read IPC broker port from server.conf: %s", + _safe_exception_str(error), ) return None @@ -413,7 +422,7 @@ def _discover_otlp_port_via_ipc_broker(self) -> Optional[str]: """ ipc_broker_port = self._get_ipc_broker_port() if ipc_broker_port is None: - self._logger.warning("IPC broker port not found in server.conf") + self._logger.info("IPC broker port not found in server.conf") return None url = ( @@ -434,15 +443,18 @@ def _discover_otlp_port_via_ipc_broker(self) -> Optional[str]: data = json.loads(resp.read().decode()) if not data.get("success"): - self._logger.warning( + self._logger.info( "IPC broker discovery returned unsuccessful response: %s", data ) return None port = str(data["port"]) self._logger.info("Discovered OTLP port via IPC broker: %s", port) return port - except Exception as e: - self._logger.warning("IPC broker OTLP port discovery failed: %s", e) + except Exception as error: + self._logger.info( + "IPC broker OTLP port discovery failed: %s", + _safe_exception_str(error), + ) return None def _resolve_otlp_port(self) -> Optional[str]: @@ -503,7 +515,7 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: ) if not otel_port: - self._logger.warning( + self._logger.info( "OTLP port could not be determined from env or IPC broker, " "OTLP export disabled" ) @@ -518,7 +530,7 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: ) if not os.path.exists(cert_file): - self._logger.error( + self._logger.info( "OTel Collector certificate not found at %s, OTLP export disabled", cert_file, ) @@ -539,9 +551,10 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: self._logger.info("OTLP gRPC exporter configured with TLS for %s", endpoint) return exporter - except Exception as e: - self._logger.warning( - "Failed to configure OTLP exporter: %s", e, exc_info=True + except Exception as error: + self._logger.info( + "Failed to configure OTLP exporter: %s", + _safe_exception_str(error), ) return None @@ -602,8 +615,8 @@ def flush(self, timeout_millis: float = 30_000) -> None: return try: self._provider.force_flush(timeout_millis=int(timeout_millis)) - except Exception as e: - self._logger.warning("Failed to flush metrics: %s", e) + except Exception as error: + self._logger.info("Failed to flush metrics: %s", _safe_exception_str(error)) class StanzaObservabilityRecorder: diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 14bcbdab..a546d991 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -276,7 +276,8 @@ def test_export_returns_failure_on_exception(self, logger): result = exporter.export(metrics_data) # Assert assert result == MetricExportResult.FAILURE - logger.error.assert_called() + logger.info.assert_called() + logger.error.assert_not_called() def test_shutdown_does_not_raise(self, logger): # Arrange / Act / Assert @@ -329,7 +330,7 @@ def test_extra_exporter_is_added(self, logger, monkeypatch): _make_service(logger, monkeypatch, extra_exporters=[extra]) # No assertion needed beyond not raising; the exporter is wrapped internally - def test_missing_ta_name_logs_warning(self, logger, monkeypatch): + def test_missing_ta_name_logs_info(self, logger, monkeypatch): # Arrange monkeypatch.setattr( "solnlib.observability.ObservabilityService._create_otlp_exporter", @@ -343,7 +344,8 @@ def test_missing_ta_name_logs_warning(self, logger, monkeypatch): svc = ObservabilityService(modinput_type="test-input", logger=logger) # Assert assert svc._meter is None - logger.warning.assert_called() + logger.info.assert_called() + logger.warning.assert_not_called() def test_register_instrument_returns_none_when_meter_missing( self, logger, monkeypatch @@ -617,7 +619,7 @@ def test_flush_is_noop_when_provider_missing(self, logger, monkeypatch): # Act / Assert — must not raise svc.flush() - def test_flush_logs_warning_on_exception(self, logger, monkeypatch): + def test_flush_logs_info_on_exception(self, logger, monkeypatch): # Arrange svc = _make_service(logger, monkeypatch) mock_provider = MagicMock() @@ -626,7 +628,8 @@ def test_flush_logs_warning_on_exception(self, logger, monkeypatch): # Act svc.flush() # Assert - logger.warning.assert_called() + logger.info.assert_called() + logger.warning.assert_not_called() def test_module_importable_without_grpc(self, monkeypatch): # Arrange @@ -677,6 +680,208 @@ def mock_import(name, *args, **kwargs): ] = otlp_mod for k, v in saved_obs_mods.items(): sys.modules[k] = v + # `import solnlib.observability` also rebinds the `observability` + # attribute on the `solnlib` package module; pytest's monkeypatch + # dotted-path resolver reads that attribute in preference to + # sys.modules, so it must be restored too or later + # monkeypatch.setattr("solnlib.observability....") calls in the + # same test run silently patch the discarded reimported module. + if "solnlib.observability" in saved_obs_mods: + import solnlib + + solnlib.observability = saved_obs_mods["solnlib.observability"] + + +# --------------------------------------------------------------------------- +# Log-level downgrade (WARNING/ERROR -> INFO) +# --------------------------------------------------------------------------- + + +class TestLogLevelDowngrade: + def test_logger_metric_exporter_export_exception_uses_safe_str( + self, logger, monkeypatch + ): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + metrics_data = MagicMock() + metrics_data.resource_metrics.__iter__ = MagicMock( + side_effect=RuntimeError("boom") + ) + exporter = LoggerMetricExporter(logger) + exporter.export(metrics_data) + logger.info.assert_called_once() + args, kwargs = logger.info.call_args + assert "SAFE" in args + assert "exc_info" not in kwargs + + def test_init_exception_uses_safe_str(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._read_ta_info", + MagicMock(side_effect=RuntimeError("boom")), + ) + ObservabilityService(modinput_type="test-input", logger=logger) + assert any("SAFE" in call.args for call in logger.info.call_args_list) + logger.warning.assert_not_called() + + def test_read_ta_info_exception_uses_safe_str(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + monkeypatch.setattr( + "solnlib.observability.get_conf_stanzas", + MagicMock(side_effect=RuntimeError("boom")), + ) + svc = ObservabilityService(modinput_type="test-input", logger=logger) + assert svc._meter is None + assert any("SAFE" in call.args for call in logger.info.call_args_list) + logger.warning.assert_not_called() + + def test_get_ipc_broker_port_exception_uses_safe_str(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + monkeypatch.setattr( + "solnlib.observability.get_conf_stanzas", + MagicMock(side_effect=RuntimeError("boom")), + ) + svc = _make_service(logger, monkeypatch) + logger.reset_mock() + assert svc._get_ipc_broker_port() is None + assert any("SAFE" in call.args for call in logger.info.call_args_list) + logger.warning.assert_not_called() + + def test_discover_otlp_port_missing_broker_port_is_info(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._get_ipc_broker_port", + lambda self: None, + ) + svc = _make_service(logger, monkeypatch) + logger.reset_mock() + assert svc._discover_otlp_port_via_ipc_broker() is None + logger.info.assert_called() + logger.warning.assert_not_called() + + def test_discover_otlp_port_unsuccessful_response_is_info( + self, logger, monkeypatch + ): + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._get_ipc_broker_port", + lambda self: 8088, + ) + mock_resp = MagicMock() + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + mock_resp.read.return_value = b'{"success": false}' + monkeypatch.setattr("urllib.request.urlopen", lambda *a, **kw: mock_resp) + svc = _make_service(logger, monkeypatch) + logger.reset_mock() + assert svc._discover_otlp_port_via_ipc_broker() is None + logger.info.assert_called() + logger.warning.assert_not_called() + + def test_discover_otlp_port_exception_uses_safe_str(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._get_ipc_broker_port", + lambda self: 8088, + ) + monkeypatch.setattr( + "urllib.request.urlopen", + MagicMock(side_effect=RuntimeError("boom")), + ) + svc = _make_service(logger, monkeypatch) + logger.reset_mock() + assert svc._discover_otlp_port_via_ipc_broker() is None + assert any("SAFE" in call.args for call in logger.info.call_args_list) + logger.warning.assert_not_called() + + def test_create_otlp_exporter_missing_port_is_info(self, logger, monkeypatch): + monkeypatch.delenv("SPOTLIGHT_OTEL_RECEIVER_PORT", raising=False) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._discover_otlp_port_via_ipc_broker", + lambda self: None, + ) + # Create service WITHOUT patching _create_otlp_exporter + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + logger.reset_mock() + assert ObservabilityService._create_otlp_exporter(svc) is None + logger.info.assert_called() + logger.warning.assert_not_called() + + def test_create_otlp_exporter_missing_cert_is_info( + self, logger, monkeypatch, tmp_path + ): + monkeypatch.setenv("SPOTLIGHT_OTEL_RECEIVER_PORT", "4317") + monkeypatch.setenv("SPLUNK_HOME", str(tmp_path)) + # Create service WITHOUT patching _create_otlp_exporter + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + logger.reset_mock() + assert ObservabilityService._create_otlp_exporter(svc) is None + logger.info.assert_called() + logger.error.assert_not_called() + + def test_create_otlp_exporter_exception_uses_safe_str_no_exc_info( + self, logger, monkeypatch + ): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + monkeypatch.setenv("SPOTLIGHT_OTEL_RECEIVER_PORT", "4317") + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._resolve_otlp_port", + MagicMock(side_effect=RuntimeError("boom")), + ) + # Create service WITHOUT patching _create_otlp_exporter + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + logger.reset_mock() + assert ObservabilityService._create_otlp_exporter(svc) is None + logger.info.assert_called_once() + args, kwargs = logger.info.call_args + assert "SAFE" in args + assert "exc_info" not in kwargs + logger.warning.assert_not_called() + + def test_flush_exception_uses_safe_str(self, logger, monkeypatch): + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", lambda error: "SAFE" + ) + svc = _make_service(logger, monkeypatch) + mock_provider = MagicMock() + mock_provider.force_flush.side_effect = RuntimeError("boom") + svc._provider = mock_provider + logger.reset_mock() + svc.flush() + assert any("SAFE" in call.args for call in logger.info.call_args_list) + logger.warning.assert_not_called() # --------------------------------------------------------------------------- From 2c18948d40bf7a2d8a3c8b577d07e9a5420a452c Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 09:52:14 +0200 Subject: [PATCH 03/19] fix: ADDON-90004 add downgrade-to-INFO logging filter Implements Task 3 of the ADDON-90004 plan: adds _DowngradeToInfoFilter class that downgrades WARNING/ERROR/CRITICAL records to INFO, plus tuples of logger names for OTel SDK internals that will be attached in Tasks 4-5. Co-Authored-By: Claude --- solnlib/observability.py | 26 ++++++++++++++ tests/unit/test_observability.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index 96abf461..6d6b60b8 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -110,6 +110,32 @@ def _safe_exception_str(error: BaseException) -> str: return "" +class _DowngradeToInfoFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + if record.levelno > logging.INFO: + record.levelno = logging.INFO + record.levelname = "INFO" + return True + + +_downgrade_to_info_filter = _DowngradeToInfoFilter() + +_OTLP_LOGGERS = ( + "opentelemetry.exporter.otlp.proto.grpc.exporter", + "opentelemetry.util.re", + "opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder", + "opentelemetry.exporter.otlp.proto.common._internal", +) + +_METRICS_SDK_LOGGERS = ( + "opentelemetry.sdk.metrics._internal.export", + "opentelemetry.sdk.metrics._internal", + "opentelemetry.sdk.metrics._internal.instrument", + "opentelemetry.metrics._internal", + "opentelemetry.attributes", +) + + class LoggerMetricExporter(MetricExporter): """An OpenTelemetry ``MetricExporter`` that logs every data point. diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index a546d991..5ab0e5be 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -692,6 +692,65 @@ def mock_import(name, *args, **kwargs): solnlib.observability = saved_obs_mods["solnlib.observability"] +# --------------------------------------------------------------------------- +# _DowngradeToInfoFilter +# --------------------------------------------------------------------------- + + +class TestDowngradeToInfoFilter: + def test_downgrades_error_and_critical_to_info(self): + from solnlib.observability import _downgrade_to_info_filter + + for level in (logging.ERROR, logging.CRITICAL): + record = logging.LogRecord( + "x", level, __file__, 1, "msg %s", ("arg",), None + ) + assert _downgrade_to_info_filter.filter(record) is True + assert record.levelno == logging.INFO + assert record.levelname == "INFO" + assert record.msg == "msg %s" + assert record.args == ("arg",) + + def test_preserves_debug_and_info(self): + from solnlib.observability import _downgrade_to_info_filter + + for level in (logging.DEBUG, logging.INFO): + record = logging.LogRecord( + "x", level, __file__, 1, "msg %s", ("arg",), None + ) + _downgrade_to_info_filter.filter(record) + assert record.levelno == level + assert record.msg == "msg %s" + assert record.args == ("arg",) + + def test_repeated_attachment_does_not_duplicate(self): + from solnlib.observability import _downgrade_to_info_filter + + test_logger = logging.getLogger("test.solnlib.observability.dedup") + test_logger.filters = [] + test_logger.addFilter(_downgrade_to_info_filter) + test_logger.addFilter(_downgrade_to_info_filter) + assert test_logger.filters.count(_downgrade_to_info_filter) == 1 + test_logger.filters = [] + + def test_otlp_and_metrics_sdk_logger_names_are_defined(self): + from solnlib.observability import _METRICS_SDK_LOGGERS, _OTLP_LOGGERS + + assert _OTLP_LOGGERS == ( + "opentelemetry.exporter.otlp.proto.grpc.exporter", + "opentelemetry.util.re", + "opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder", + "opentelemetry.exporter.otlp.proto.common._internal", + ) + assert _METRICS_SDK_LOGGERS == ( + "opentelemetry.sdk.metrics._internal.export", + "opentelemetry.sdk.metrics._internal", + "opentelemetry.sdk.metrics._internal.instrument", + "opentelemetry.metrics._internal", + "opentelemetry.attributes", + ) + + # --------------------------------------------------------------------------- # Log-level downgrade (WARNING/ERROR -> INFO) # --------------------------------------------------------------------------- From c8bbc5ddbe519a2f5962c5468e6ee0febf310fbf Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 10:07:32 +0200 Subject: [PATCH 04/19] fix: ADDON-90004 downgrade metrics SDK logger noise to INFO Attach the _downgrade_to_info_filter to the 5 metrics-SDK loggers (opentelemetry.sdk.metrics.*) inside ObservabilityService.__init__. This suppresses WARNING/ERROR level log entries from the SDK and converts them to INFO, reducing observability noise. Co-Authored-By: Claude --- solnlib/observability.py | 3 ++ tests/unit/test_observability.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index 6d6b60b8..bfba9261 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -340,6 +340,9 @@ def __init__( self._meter: Optional[Meter] = None self._provider: Optional[MeterProvider] = None + for logger_name in _METRICS_SDK_LOGGERS: + logging.getLogger(logger_name).addFilter(_downgrade_to_info_filter) + try: if ta_name is None or ta_version is None: _ta_name, _ta_version = self._read_ta_info() diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 5ab0e5be..78c8810d 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -14,6 +14,7 @@ # limitations under the License. # +import contextlib import io import logging import sys @@ -49,6 +50,22 @@ def real_logger(): test_logger.handlers = [] +@contextlib.contextmanager +def _clean_logger_filters(*logger_names): + """Snapshot each logger's filters, clear them for a clean-slate + precondition, then restore the exact original list afterward — + regardless of what the wrapped code attaches or removes.""" + loggers = [logging.getLogger(name) for name in logger_names] + original = [list(lg.filters) for lg in loggers] + for lg in loggers: + lg.filters = [] + try: + yield loggers + finally: + for lg, filters in zip(loggers, original): + lg.filters = filters + + # --------------------------------------------------------------------------- # Safe log rendering # --------------------------------------------------------------------------- @@ -631,6 +648,42 @@ def test_flush_logs_info_on_exception(self, logger, monkeypatch): logger.info.assert_called() logger.warning.assert_not_called() + def test_init_attaches_filter_to_metrics_sdk_loggers(self, logger, monkeypatch): + from solnlib.observability import ( + _downgrade_to_info_filter, + _METRICS_SDK_LOGGERS, + ) + + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + with _clean_logger_filters(*_METRICS_SDK_LOGGERS) as loggers: + ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + for lg in loggers: + assert _downgrade_to_info_filter in lg.filters + + def test_init_does_not_attach_filter_to_unrelated_logger(self, logger, monkeypatch): + from solnlib.observability import _downgrade_to_info_filter + + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + unrelated = logging.getLogger("opentelemetry.sdk.resources") + assert _downgrade_to_info_filter not in unrelated.filters + def test_module_importable_without_grpc(self, monkeypatch): # Arrange import builtins From fb967ab17b6c3c2b8473840a9a673bae4e973f92 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 10:22:39 +0200 Subject: [PATCH 05/19] fix: ADDON-90004 suppress gRPC C-core stderr and downgrade OTLP logger noise - Set GRPC_VERBOSITY=NONE at start of _create_otlp_exporter to suppress C-core handshake diagnostics to stderr unless explicitly overridden by caller - Attach _downgrade_to_info_filter to 4 _OTLP_LOGGERS before OTLPMetricExporter construction to downgrade WARNING/ERROR logs from opentelemetry exporter and utilities to INFO level, matching the filtering already applied to the 5 metrics SDK loggers during ObservabilityService.__init__ - Add comprehensive tests: - test_create_otlp_exporter_sets_grpc_verbosity_when_absent: verifies NONE is set - test_create_otlp_exporter_respects_existing_grpc_verbosity: verifies caller's value is preserved - test_create_otlp_exporter_attaches_filter_to_otlp_loggers: verifies filters are present on the logger objects - test_create_otlp_exporter_does_not_attach_filter_on_missing_port: verifies filters only attached on the success path - test_filters_are_attached_before_construction_logs_occur: regression guard ensuring filter attachment happens before OTLPMetricExporter() runs, using real grpc + malformed env vars to trigger actual WARNING logs - TestGrpcTlsHandshakeStderrSuppression integration test: verifies that gRPC C-core stderr is actually suppressed with GRPC_VERBOSITY=NONE --- solnlib/observability.py | 5 + tests/unit/test_observability.py | 293 +++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index bfba9261..576b7317 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -528,6 +528,8 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: - Any other exception occurs during exporter construction (including a missing ``grpcio`` package). """ + os.environ.setdefault("GRPC_VERBOSITY", "NONE") + try: import grpc from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( @@ -565,6 +567,9 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: ) return None + for logger_name in _OTLP_LOGGERS: + logging.getLogger(logger_name).addFilter(_downgrade_to_info_filter) + with open(cert_file, "rb") as f: server_cert = f.read() diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 78c8810d..f40a1fc7 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -17,6 +17,9 @@ import contextlib import io import logging +import os +import shutil +import subprocess import sys from unittest.mock import MagicMock, patch @@ -27,6 +30,37 @@ from solnlib.observability import LoggerMetricExporter, ObservabilityService +_GRPC_TLS_HANDSHAKE_SCRIPT = """ +import sys +import grpc +from concurrent import futures + +server_key_path, server_crt_path, wrong_root_path = sys.argv[1:4] + +with open(server_key_path, "rb") as f: + server_key = f.read() +with open(server_crt_path, "rb") as f: + server_cert = f.read() +with open(wrong_root_path, "rb") as f: + wrong_root = f.read() + +server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) +server_creds = grpc.ssl_server_credentials([(server_key, server_cert)]) +port = server.add_secure_port("127.0.0.1:0", server_creds) +server.start() + +client_creds = grpc.ssl_channel_credentials(root_certificates=wrong_root) +channel = grpc.secure_channel(f"127.0.0.1:{port}", client_creds) +try: + grpc.channel_ready_future(channel).result(timeout=3) +except Exception: + pass +finally: + channel.close() + server.stop(0) +""" + + @pytest.fixture def logger(): return MagicMock(spec=logging.Logger) @@ -744,6 +778,185 @@ def mock_import(name, *args, **kwargs): solnlib.observability = saved_obs_mods["solnlib.observability"] + def test_create_otlp_exporter_sets_grpc_verbosity_when_absent( + self, logger, monkeypatch + ): + monkeypatch.delenv("GRPC_VERBOSITY", raising=False) + monkeypatch.delenv("SPOTLIGHT_OTEL_RECEIVER_PORT", raising=False) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._discover_otlp_port_via_ipc_broker", + lambda self: None, + ) + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + ObservabilityService._create_otlp_exporter(svc) + assert os.environ["GRPC_VERBOSITY"] == "NONE" + + def test_create_otlp_exporter_respects_existing_grpc_verbosity( + self, logger, monkeypatch + ): + monkeypatch.setenv("GRPC_VERBOSITY", "DEBUG") + monkeypatch.delenv("SPOTLIGHT_OTEL_RECEIVER_PORT", raising=False) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._discover_otlp_port_via_ipc_broker", + lambda self: None, + ) + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + ObservabilityService._create_otlp_exporter(svc) + assert os.environ["GRPC_VERBOSITY"] == "DEBUG" + + def test_create_otlp_exporter_attaches_filter_to_otlp_loggers( + self, logger, monkeypatch, tmp_path + ): + from solnlib.observability import _downgrade_to_info_filter, _OTLP_LOGGERS + + monkeypatch.setenv("SPOTLIGHT_OTEL_RECEIVER_PORT", "4317") + monkeypatch.setenv("SPLUNK_HOME", str(tmp_path)) + cert_path = tmp_path / "var/packages/data/spotlight-collector" + cert_path.mkdir(parents=True) + (cert_path / "server.crt").write_bytes(b"fake-cert") + mock_grpc = MagicMock() + mock_grpc.ssl_channel_credentials = MagicMock(return_value=MagicMock()) + monkeypatch.setitem(sys.modules, "grpc", mock_grpc) + mock_otlp_module = MagicMock() + mock_otlp_module.OTLPMetricExporter = MagicMock(return_value=MagicMock()) + monkeypatch.setitem( + sys.modules, + "opentelemetry.exporter.otlp.proto.grpc.metric_exporter", + mock_otlp_module, + ) + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + with _clean_logger_filters(*_OTLP_LOGGERS) as loggers: + ObservabilityService._create_otlp_exporter(svc) + for lg in loggers: + assert _downgrade_to_info_filter in lg.filters + + def test_create_otlp_exporter_does_not_attach_filter_on_missing_port( + self, logger, monkeypatch + ): + from solnlib.observability import _downgrade_to_info_filter, _OTLP_LOGGERS + + monkeypatch.delenv("SPOTLIGHT_OTEL_RECEIVER_PORT", raising=False) + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._discover_otlp_port_via_ipc_broker", + lambda self: None, + ) + svc = _make_service(logger, monkeypatch) + with _clean_logger_filters(*_OTLP_LOGGERS) as loggers: + assert ObservabilityService._create_otlp_exporter(svc) is None + for lg in loggers: + assert _downgrade_to_info_filter not in lg.filters + + def test_filters_are_attached_before_construction_logs_occur( + self, logger, monkeypatch, tmp_path + ): + """Regression guard for the required ordering: the filter must be on + the logger *before* OTLPMetricExporter() runs, not merely present by + the time _create_otlp_exporter() returns. Checking `.filters` after + the call (as the two tests above do) cannot tell "attached early" + apart from "attached late" — both leave the filter present at the + end. This test instead attaches a capturing Handler to each OTLP + logger *before* calling _create_otlp_exporter(), forces two of the + four loggers to actually emit a WARNING during real OTLPMetricExporter + construction (malformed OTEL_EXPORTER_OTLP_METRICS_HEADERS and + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE — verified against + opentelemetry-exporter-otlp-proto-grpc 1.39.1 to log through + `opentelemetry.util.re` and + `opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder` + respectively), and asserts every record the handler observed was + already at INFO by the time it reached the handler. Filters run + before handlers in Logger.handle(), so this only passes if the + filter was attached before that specific log call — i.e. before + OTLPMetricExporter() ran. Uses the real grpc/OTLPMetricExporter + (no sys.modules mocking): constructing ssl_channel_credentials and + OTLPMetricExporter does not perform network I/O, so no real + collector is needed, and a syntactically-invalid cert file is + accepted at construction time (verified empirically). + + Calls _create_otlp_exporter() on a bare instance built via + ObservabilityService.__new__() rather than going through the full + constructor. The full constructor would wrap the real returned + exporter in a real PeriodicExportingMetricReader — in OTel 1.39.1 + that reader defaults export_interval_millis to 60_000 and spawns a + genuine daemon thread (MeterProvider also registers an atexit + shutdown hook), and nothing in this test would ever join or shut + that thread down. _create_otlp_exporter only reads self._logger + (SPOTLIGHT_OTEL_RECEIVER_PORT is set below, so _resolve_otlp_port() + never touches the IPC-broker path, which is the only other place + that reads self attributes), so a bare instance with just _logger + set is sufficient. The real exporter this returns still opens a + gRPC channel object, so it is explicitly shut down in finally.""" + from solnlib.observability import _OTLP_LOGGERS + + monkeypatch.setenv("SPOTLIGHT_OTEL_RECEIVER_PORT", "4317") + monkeypatch.setenv("SPLUNK_HOME", str(tmp_path)) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "not-a-valid-header-no-equals-sign" + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "not-a-real-preference", + ) + cert_path = tmp_path / "var/packages/data/spotlight-collector" + cert_path.mkdir(parents=True) + (cert_path / "server.crt").write_bytes(b"fake-cert") + + svc = ObservabilityService.__new__(ObservabilityService) + svc._logger = logger + + captured = [] # list of (logger_name, levelno) tuples + + class _CapturingHandler(logging.Handler): + def emit(self, record): + captured.append((record.name, record.levelno)) + + capturing_handler = _CapturingHandler(level=logging.NOTSET) + + with _clean_logger_filters(*_OTLP_LOGGERS) as loggers: + original_levels = [lg.level for lg in loggers] + for lg in loggers: + lg.addHandler(capturing_handler) + lg.setLevel(logging.NOTSET) + exporter = None + try: + exporter = svc._create_otlp_exporter() + finally: + for lg, level in zip(loggers, original_levels): + lg.removeHandler(capturing_handler) + lg.setLevel(level) + if exporter is not None: + exporter.shutdown() + + captured_logger_names = {name for name, _ in captured} + assert "opentelemetry.util.re" in captured_logger_names, ( + "expected the malformed-headers trigger to still log through " + "opentelemetry.util.re — if this fails, the trigger stopped " + "working and the test no longer proves anything" + ) + assert ( + "opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder" + in captured_logger_names + ), ( + "expected the malformed-temporality-preference trigger to still " + "log through the metrics encoder — if this fails, the trigger " + "stopped working and the test no longer proves anything" + ) + assert all(levelno <= logging.INFO for _, levelno in captured) + # --------------------------------------------------------------------------- # _DowngradeToInfoFilter @@ -1165,3 +1378,83 @@ def test_register_instrument_returns_none_when_service_not_initialised( result = rec.register_instrument(lambda meter: meter.create_counter("x")) assert result is None + + +class TestGrpcTlsHandshakeStderrSuppression: + """Integration check: a real TLS handshake failure logs a gRPC C-core + diagnostic line straight to stderr unless GRPC_VERBOSITY=NONE is set + before grpc initializes. Requires the system `openssl` binary.""" + + @staticmethod + def _generate_self_signed_cert(directory, name, subject): + key_path = directory / f"{name}.key" + crt_path = directory / f"{name}.crt" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + str(key_path), + "-out", + str(crt_path), + "-days", + "1", + "-nodes", + "-subj", + subject, + ], + check=True, + capture_output=True, + ) + return key_path, crt_path + + @pytest.mark.skipif( + shutil.which("openssl") is None, reason="requires system openssl binary" + ) + def test_grpc_verbosity_none_suppresses_tls_handshake_stderr(self, tmp_path): + server_key, server_crt = self._generate_self_signed_cert( + tmp_path, "server", "/CN=localhost" + ) + _, wrong_root_crt = self._generate_self_signed_cert( + tmp_path, "wrong_root", "/CN=wrong-root" + ) + + control_env = dict(os.environ) + control_env.pop("GRPC_VERBOSITY", None) + control = subprocess.run( + [ + sys.executable, + "-c", + _GRPC_TLS_HANDSHAKE_SCRIPT, + str(server_key), + str(server_crt), + str(wrong_root_crt), + ], + env=control_env, + capture_output=True, + text=True, + timeout=15, + ) + + suppressed_env = dict(os.environ) + suppressed_env["GRPC_VERBOSITY"] = "NONE" + suppressed = subprocess.run( + [ + sys.executable, + "-c", + _GRPC_TLS_HANDSHAKE_SCRIPT, + str(server_key), + str(server_crt), + str(wrong_root_crt), + ], + env=suppressed_env, + capture_output=True, + text=True, + timeout=15, + ) + + assert "Handshake failed" in control.stderr + assert "Handshake failed" not in suppressed.stderr From cfb5696cec9c00ecc40ccbed2e8a4dbe614147c6 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 11:14:17 +0200 Subject: [PATCH 06/19] fix: ADDON-90004 add circuit breaker for the OTLP exporter Add _CircuitBreakerExporter class that wraps an inner MetricExporter and stops calling it after 3 consecutive failed exports (via MetricExportResult.FAILURE or exception) within the same process. Once tripped, the exporter silently returns SUCCESS without calling the inner exporter or logging further messages. This eliminates WARNING/ERROR log noise from OTLP export failures while still recording that the 3rd failure occurred. Co-Authored-By: Claude --- solnlib/observability.py | 59 ++++++++ tests/unit/test_observability.py | 224 ++++++++++++++++++++++++++++++- 2 files changed, 282 insertions(+), 1 deletion(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 576b7317..b5aa32dc 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -244,6 +244,65 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool: return True +class _CircuitBreakerExporter(MetricExporter): + """Wraps the internally constructed OTLP exporter and stops calling it + after 3 consecutive failed exports in this process.""" + + _MAX_CONSECUTIVE_FAILURES = 3 + + def __init__(self, inner: MetricExporter, logger: _Logger) -> None: + super().__init__( + preferred_temporality=inner._preferred_temporality, + preferred_aggregation=inner._preferred_aggregation, + ) + self._inner = inner + self._logger = logger + self._consecutive_failures = 0 + self._tripped = False + self._shutdown_called = False + + def export( + self, + metrics_data: MetricsData, + timeout_millis: float = 10_000, + **kwargs, + ) -> MetricExportResult: + if self._tripped: + return MetricExportResult.SUCCESS + + try: + result = self._inner.export( + metrics_data, timeout_millis=timeout_millis, **kwargs + ) + except Exception as error: + self._logger.info( + "OTLP export raised an exception: %s", + _safe_exception_repr(error), + ) + result = MetricExportResult.FAILURE + + if result == MetricExportResult.SUCCESS: + self._consecutive_failures = 0 + return result + + self._consecutive_failures += 1 + if self._consecutive_failures >= self._MAX_CONSECUTIVE_FAILURES: + self._tripped = True + self._logger.info("OTLP export disabled after 3 consecutive failures") + return result + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + if self._tripped: + return True + return self._inner.force_flush(timeout_millis=timeout_millis) + + def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: + if self._shutdown_called: + return + self._shutdown_called = True + self._inner.shutdown(timeout_millis=timeout_millis, **kwargs) + + class ObservabilityService: """OpenTelemetry observability service for a Splunk modular input. diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index f40a1fc7..060e8d5a 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -17,6 +17,7 @@ import contextlib import io import logging +import math import os import shutil import subprocess @@ -25,7 +26,12 @@ import pytest from opentelemetry.sdk.metrics import Counter, Histogram -from opentelemetry.sdk.metrics.export import AggregationTemporality, MetricExportResult +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + MetricExporter, + MetricExportResult, + PeriodicExportingMetricReader, +) from solnlib.observability import LoggerMetricExporter, ObservabilityService @@ -339,6 +345,222 @@ def test_force_flush_returns_true(self, logger): assert LoggerMetricExporter(logger).force_flush() is True +# --------------------------------------------------------------------------- +# _CircuitBreakerExporter +# --------------------------------------------------------------------------- + + +class _FakeInnerExporter(MetricExporter): + """Real MetricExporter subclass for deterministic circuit-breaker tests.""" + + def __init__(self, temporality=None, aggregation=None): + super().__init__( + preferred_temporality=temporality + or { + Counter: AggregationTemporality.DELTA, + Histogram: AggregationTemporality.DELTA, + }, + preferred_aggregation=aggregation or {}, + ) + self.results = [] # queue of MetricExportResult values or Exception instances + self.export_calls = [] + self.force_flush_calls = [] + self.shutdown_calls = [] + + def export(self, metrics_data, timeout_millis=10_000, **kwargs): + self.export_calls.append((metrics_data, timeout_millis, kwargs)) + outcome = self.results.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def force_flush(self, timeout_millis=10_000): + self.force_flush_calls.append(timeout_millis) + return True + + def shutdown(self, timeout_millis=30_000, **kwargs): + self.shutdown_calls.append((timeout_millis, kwargs)) + + +class TestCircuitBreakerExporter: + def test_preserves_preferred_temporality_and_aggregation(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + temporality = {Counter: AggregationTemporality.DELTA} + aggregation = {"some": "aggregation"} + inner = _FakeInnerExporter(temporality, aggregation) + wrapper = _CircuitBreakerExporter(inner, logger) + assert wrapper._preferred_temporality == temporality + assert wrapper._preferred_aggregation == aggregation + + def test_real_reader_reads_temporality_from_wrapper(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + temporality = { + Counter: AggregationTemporality.DELTA, + Histogram: AggregationTemporality.DELTA, + } + inner = _FakeInnerExporter(temporality) + wrapper = _CircuitBreakerExporter(inner, logger) + reader = PeriodicExportingMetricReader( + wrapper, export_interval_millis=math.inf + ) + try: + assert reader._preferred_temporality == temporality + finally: + reader.shutdown() + + def test_export_forwards_timeout_and_kwargs(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.append(MetricExportResult.SUCCESS) + wrapper = _CircuitBreakerExporter(inner, logger) + metrics_data = MagicMock() + wrapper.export(metrics_data, timeout_millis=1234, extra_kwarg="sentinel") + assert inner.export_calls == [(metrics_data, 1234, {"extra_kwarg": "sentinel"})] + + def test_force_flush_forwards_timeout_before_trip(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + wrapper = _CircuitBreakerExporter(inner, logger) + assert wrapper.force_flush(timeout_millis=5000) is True + assert inner.force_flush_calls == [5000] + + def test_force_flush_is_noop_after_trip(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend([MetricExportResult.FAILURE] * 3) + wrapper = _CircuitBreakerExporter(inner, logger) + for _ in range(3): + wrapper.export(MagicMock()) + assert wrapper.force_flush(timeout_millis=5000) is True + assert inner.force_flush_calls == [] + + def test_shutdown_forwards_timeout_millis_and_kwargs(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + wrapper = _CircuitBreakerExporter(inner, logger) + wrapper.shutdown(timeout_millis=9999, extra_kwarg="sentinel") + assert inner.shutdown_calls == [(9999, {"extra_kwarg": "sentinel"})] + + def test_shutdown_accepts_timeout_kwarg_from_real_reader(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + wrapper = _CircuitBreakerExporter(inner, logger) + wrapper.shutdown(timeout=1.5) + assert inner.shutdown_calls == [(30_000, {"timeout": 1.5})] + + def test_shutdown_delegates_exactly_once_including_after_trip(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend([MetricExportResult.FAILURE] * 3) + wrapper = _CircuitBreakerExporter(inner, logger) + for _ in range(3): + wrapper.export(MagicMock()) + wrapper.shutdown() + wrapper.shutdown() + assert len(inner.shutdown_calls) == 1 + + def test_export_returns_failure_and_increments_state_on_failure_result( + self, logger + ): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.append(MetricExportResult.FAILURE) + wrapper = _CircuitBreakerExporter(inner, logger) + result = wrapper.export(MagicMock()) + assert result == MetricExportResult.FAILURE + assert wrapper._consecutive_failures == 1 + logger.info.assert_not_called() + + def test_export_catches_exception_logs_and_returns_failure(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.append(RuntimeError("boom")) + wrapper = _CircuitBreakerExporter(inner, logger) + result = wrapper.export(MagicMock()) + assert result == MetricExportResult.FAILURE + assert wrapper._consecutive_failures == 1 + logger.info.assert_called_once() + + def test_export_resets_failure_count_after_success(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend( + [MetricExportResult.FAILURE, MetricExportResult.SUCCESS] + ) + wrapper = _CircuitBreakerExporter(inner, logger) + wrapper.export(MagicMock()) + assert wrapper._consecutive_failures == 1 + wrapper.export(MagicMock()) + assert wrapper._consecutive_failures == 0 + + @pytest.mark.parametrize( + "outcomes", + [ + [MetricExportResult.FAILURE] * 3, + [RuntimeError("boom")] * 3, + [MetricExportResult.FAILURE, RuntimeError("boom"), MetricExportResult.FAILURE], + ], + ) + def test_export_trips_on_third_consecutive_failure(self, logger, outcomes): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend(outcomes) + wrapper = _CircuitBreakerExporter(inner, logger) + for _ in range(3): + wrapper.export(MagicMock()) + assert wrapper._tripped is True + + def test_export_third_attempt_returns_actual_failure_not_synthetic_success( + self, logger + ): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend([MetricExportResult.FAILURE] * 3) + wrapper = _CircuitBreakerExporter(inner, logger) + results = [wrapper.export(MagicMock()) for _ in range(3)] + assert results == [MetricExportResult.FAILURE] * 3 + + def test_export_after_trip_never_calls_inner_and_no_further_logs(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend([MetricExportResult.FAILURE] * 3) + wrapper = _CircuitBreakerExporter(inner, logger) + for _ in range(3): + wrapper.export(MagicMock()) + logger.reset_mock() + result = wrapper.export(MagicMock()) + assert result == MetricExportResult.SUCCESS + assert len(inner.export_calls) == 3 + logger.info.assert_not_called() + + def test_three_consecutive_exceptions_emit_three_info_plus_one_trip_log( + self, logger + ): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.extend([RuntimeError("boom")] * 3) + wrapper = _CircuitBreakerExporter(inner, logger) + for _ in range(3): + wrapper.export(MagicMock()) + assert logger.info.call_count == 4 + assert "3 consecutive failures" in logger.info.call_args_list[-1].args[0] + + # --------------------------------------------------------------------------- # ObservabilityService # --------------------------------------------------------------------------- From 5c1133302c761bbca7784f4abbcc6e711fc41fa3 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 11:39:12 +0200 Subject: [PATCH 07/19] fix: ADDON-90004 synchronize circuit breaker state against concurrent access Add threading.Lock() to _CircuitBreakerExporter to prevent unsynchronized mutations of _tripped, _consecutive_failures, and _shutdown_called while concurrent threads may be accessing these fields. Wrap entire method bodies (export, force_flush, shutdown) to ensure atomicity of state checks and delegations to the inner exporter. Add deterministic concurrency test proving export() and shutdown() are now mutually exclusive: export blocks on a threading.Event, and shutdown() is demonstrated to block while the export holds the lock, then proceed normally after release. Co-Authored-By: Claude --- solnlib/observability.py | 58 +++++++++++++++++--------------- tests/unit/test_observability.py | 36 ++++++++++++++++++++ 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index b5aa32dc..2287496f 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -260,6 +260,7 @@ def __init__(self, inner: MetricExporter, logger: _Logger) -> None: self._consecutive_failures = 0 self._tripped = False self._shutdown_called = False + self._lock = threading.Lock() def export( self, @@ -267,40 +268,43 @@ def export( timeout_millis: float = 10_000, **kwargs, ) -> MetricExportResult: - if self._tripped: - return MetricExportResult.SUCCESS + with self._lock: + if self._tripped: + return MetricExportResult.SUCCESS - try: - result = self._inner.export( - metrics_data, timeout_millis=timeout_millis, **kwargs - ) - except Exception as error: - self._logger.info( - "OTLP export raised an exception: %s", - _safe_exception_repr(error), - ) - result = MetricExportResult.FAILURE + try: + result = self._inner.export( + metrics_data, timeout_millis=timeout_millis, **kwargs + ) + except Exception as error: + self._logger.info( + "OTLP export raised an exception: %s", + _safe_exception_repr(error), + ) + result = MetricExportResult.FAILURE - if result == MetricExportResult.SUCCESS: - self._consecutive_failures = 0 - return result + if result == MetricExportResult.SUCCESS: + self._consecutive_failures = 0 + return result - self._consecutive_failures += 1 - if self._consecutive_failures >= self._MAX_CONSECUTIVE_FAILURES: - self._tripped = True - self._logger.info("OTLP export disabled after 3 consecutive failures") - return result + self._consecutive_failures += 1 + if self._consecutive_failures >= self._MAX_CONSECUTIVE_FAILURES: + self._tripped = True + self._logger.info("OTLP export disabled after 3 consecutive failures") + return result def force_flush(self, timeout_millis: float = 10_000) -> bool: - if self._tripped: - return True - return self._inner.force_flush(timeout_millis=timeout_millis) + with self._lock: + if self._tripped: + return True + return self._inner.force_flush(timeout_millis=timeout_millis) def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: - if self._shutdown_called: - return - self._shutdown_called = True - self._inner.shutdown(timeout_millis=timeout_millis, **kwargs) + with self._lock: + if self._shutdown_called: + return + self._shutdown_called = True + self._inner.shutdown(timeout_millis=timeout_millis, **kwargs) class ObservabilityService: diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 060e8d5a..77190034 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -22,6 +22,8 @@ import shutil import subprocess import sys +import threading +import time from unittest.mock import MagicMock, patch import pytest @@ -560,6 +562,40 @@ def test_three_consecutive_exceptions_emit_three_info_plus_one_trip_log( assert logger.info.call_count == 4 assert "3 consecutive failures" in logger.info.call_args_list[-1].args[0] + def test_export_and_shutdown_are_mutually_exclusive(self, logger): + from solnlib.observability import _CircuitBreakerExporter + + inner = _FakeInnerExporter() + inner.results.append(MetricExportResult.SUCCESS) + export_started = threading.Event() + release_export = threading.Event() + real_export = inner.export + + def blocking_export(metrics_data, timeout_millis=10_000, **kwargs): + export_started.set() + release_export.wait(timeout=5) + return real_export(metrics_data, timeout_millis=timeout_millis, **kwargs) + + inner.export = blocking_export + wrapper = _CircuitBreakerExporter(inner, logger) + + export_thread = threading.Thread(target=wrapper.export, args=(MagicMock(),)) + export_thread.start() + assert export_started.wait(timeout=5) + + shutdown_thread = threading.Thread(target=wrapper.shutdown) + shutdown_thread.start() + time.sleep(0.2) + assert wrapper._shutdown_called is False + assert inner.shutdown_calls == [] + + release_export.set() + export_thread.join(timeout=5) + shutdown_thread.join(timeout=5) + + assert wrapper._shutdown_called is True + assert len(inner.shutdown_calls) == 1 + # --------------------------------------------------------------------------- # ObservabilityService From bdc76b3d614ec4f0b39aa9564b3250a6110481d4 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 12:11:56 +0200 Subject: [PATCH 08/19] fix: ADDON-90004 wrap the OTLP exporter in the circuit breaker Modified _create_otlp_exporter() to return the OTLP exporter wrapped in _CircuitBreakerExporter instead of returning it directly. This enables the circuit breaker to stop calling the inner exporter after 3 consecutive failed exports, eliminating WARNING/ERROR log noise from solnlib.observability when the Spotlight collector becomes unavailable. Updated test_create_otlp_exporter_returns_exporter_when_cert_present to verify the wrapper is returned. Added test_create_otlp_exporter_wrapper_forwards_shutdown to verify the wrapper properly delegates shutdown calls to the inner exporter. Co-Authored-By: Claude --- solnlib/observability.py | 5 +++-- tests/unit/test_observability.py | 34 +++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 2287496f..b2422ed6 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -584,7 +584,8 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: ``AggregationTemporality.DELTA`` so that each export interval reports only the change since the previous interval. - Returns the configured exporter, or ``None`` when: + Returns the configured exporter wrapped in ``_CircuitBreakerExporter``, + or ``None`` when: - The OTLP port cannot be resolved (see :meth:`_resolve_otlp_port`). - The certificate file does not exist. @@ -646,7 +647,7 @@ def _create_otlp_exporter(self) -> Optional[MetricExporter]: }, ) self._logger.info("OTLP gRPC exporter configured with TLS for %s", endpoint) - return exporter + return _CircuitBreakerExporter(exporter, self._logger) except Exception as error: self._logger.info( diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 77190034..99951b53 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -870,7 +870,39 @@ def test_create_otlp_exporter_returns_exporter_when_cert_present( # Act result = ObservabilityService._create_otlp_exporter(svc) # Assert - assert result is mock_exporter + from solnlib.observability import _CircuitBreakerExporter + + assert isinstance(result, _CircuitBreakerExporter) + assert result._inner is mock_exporter + + def test_create_otlp_exporter_wrapper_forwards_shutdown( + self, logger, monkeypatch, tmp_path + ): + monkeypatch.setenv("SPOTLIGHT_OTEL_RECEIVER_PORT", "4317") + monkeypatch.setenv("SPLUNK_HOME", str(tmp_path)) + cert_path = tmp_path / "var/packages/data/spotlight-collector" + cert_path.mkdir(parents=True) + (cert_path / "server.crt").write_bytes(b"fake-cert") + mock_grpc = MagicMock() + mock_grpc.ssl_channel_credentials = MagicMock(return_value=MagicMock()) + monkeypatch.setitem(sys.modules, "grpc", mock_grpc) + mock_otlp_module = MagicMock() + mock_exporter = MagicMock() + mock_otlp_module.OTLPMetricExporter = MagicMock(return_value=mock_exporter) + monkeypatch.setitem( + sys.modules, + "opentelemetry.exporter.otlp.proto.grpc.metric_exporter", + mock_otlp_module, + ) + svc = ObservabilityService( + modinput_type="test-input", + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + result = ObservabilityService._create_otlp_exporter(svc) + result.shutdown(timeout_millis=1234) + mock_exporter.shutdown.assert_called_once_with(timeout_millis=1234) def test_create_otlp_exporter_uses_delta_temporality( self, logger, monkeypatch, tmp_path From 80606b8591043aa8c3062ff9a981bb063b7dc479 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 12:39:55 +0200 Subject: [PATCH 09/19] fix: ADDON-90004 validate event_count/byte_count before recording Add input validation for event_count and byte_count arguments to StanzaObservabilityRecorder.record() to reject invalid values (negative numbers, non-ints, out-of-int64-range values) and log them at INFO level instead of raising or silently forwarding to the OTel SDK. Co-Authored-By: Claude --- solnlib/observability.py | 40 +++++++++++++++- tests/unit/test_observability.py | 79 ++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index b2422ed6..5133a33e 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -717,6 +717,34 @@ def flush(self, timeout_millis: float = 30_000) -> None: self._logger.info("Failed to flush metrics: %s", _safe_exception_str(error)) +_INT64_MIN = -(2**63) +_INT64_MAX = 2**63 - 1 + + +def _is_encodable_str(value: str) -> bool: + # Caller guarantees type(value) is str. + try: + value.encode("utf-8") + return True + except UnicodeEncodeError: + return False + + +def _is_safe_identifier_str(value: str) -> bool: + return _is_encodable_str(value) and "\n" not in value and "\r" not in value + + +def _count_error(value) -> Optional[str]: + """Return None for a valid event/byte count, else a safe reason string.""" + if type(value) is not int: + return f"expected int, got {_sanitize_for_log(type(value).__name__)}" + if value < 0: + return "count is negative" + if value > _INT64_MAX: + return f"count exceeds int64 range ({value.bit_length()} bits)" + return None + + class StanzaObservabilityRecorder: """Stanza-scoped observability recorder backed by a shared ``ObservabilityService``. @@ -894,9 +922,17 @@ def record( """ attrs = dict(extra_attrs) if extra_attrs else {} attrs[ATTR_MODINPUT_NAME] = self._stanza_name - if self._service.event_count_counter: + + event_count_error = _count_error(event_count) + if event_count_error is not None: + self._service._logger.info("Skipping invalid event_count: %s", event_count_error) + elif self._service.event_count_counter: self._service.event_count_counter.add(event_count, attributes=attrs) - if self._service.event_bytes_counter: + + byte_count_error = _count_error(byte_count) + if byte_count_error is not None: + self._service._logger.info("Skipping invalid byte_count: %s", byte_count_error) + elif self._service.event_bytes_counter: self._service.event_bytes_counter.add(byte_count, attributes=attrs) def flush(self) -> None: diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 99951b53..599c354a 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -1499,6 +1499,49 @@ def test_flush_exception_uses_safe_str(self, logger, monkeypatch): logger.warning.assert_not_called() +# --------------------------------------------------------------------------- +# Recorder input validation +# --------------------------------------------------------------------------- + + +class TestCountValidation: + @pytest.mark.parametrize("value", [0, 1, 2**63 - 1]) + def test_count_error_accepts_valid_values(self, value): + from solnlib.observability import _count_error + + assert _count_error(value) is None + + @pytest.mark.parametrize( + "value", + [-1, 2**63, True, False, 1.0, 2**100, -(2**100)], + ) + def test_count_error_rejects_invalid_values(self, value): + from solnlib.observability import _count_error + + assert _count_error(value) is not None + + def test_count_error_message_does_not_render_raw_oversized_int(self): + from solnlib.observability import _count_error + + huge = 2**200 + error = _count_error(huge) + assert str(huge) not in error + assert str(huge.bit_length()) in error + + def test_count_error_type_name_is_sanitized(self): + # type(value).__name__ is derived from a caller-supplied object's + # class and is not guaranteed safe: type("bad\r\nname", (), {}) is + # a real, constructible class whose __name__ contains raw CR/LF and + # can be arbitrarily long. It must go through _sanitize_for_log the + # same as any other value that reaches lazy log formatting. + from solnlib.observability import _count_error + + evil_type = type("bad\r\nname", (), {}) + error = _count_error(evil_type()) + assert "\n" not in error + assert "\r" not in error + + # --------------------------------------------------------------------------- # StanzaObservabilityRecorder # --------------------------------------------------------------------------- @@ -1669,6 +1712,42 @@ def test_register_instrument_returns_none_when_service_not_initialised( assert result is None + def test_record_skips_invalid_event_count_keeps_valid_byte_count( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + mock_count = MagicMock() + mock_bytes = MagicMock() + rec._service.event_count_counter = mock_count + rec._service.event_bytes_counter = mock_bytes + mock_count.reset_mock() + mock_bytes.reset_mock() + + rec.record(-1, 1024) + + mock_count.add.assert_not_called() + mock_bytes.add.assert_called_once_with( + 1024, attributes={"splunk.modinput.name": "my:stanza"} + ) + + def test_record_skips_invalid_byte_count_keeps_valid_event_count( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + mock_count = MagicMock() + mock_bytes = MagicMock() + rec._service.event_count_counter = mock_count + rec._service.event_bytes_counter = mock_bytes + mock_count.reset_mock() + mock_bytes.reset_mock() + + rec.record(5, True) + + mock_count.add.assert_called_once_with( + 5, attributes={"splunk.modinput.name": "my:stanza"} + ) + mock_bytes.add.assert_not_called() + class TestGrpcTlsHandshakeStderrSuppression: """Integration check: a real TLS handshake failure logs a gRPC C-core From 3b54dc1d346d44d3dd5d05e459e81832fe4eafd0 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 13:06:29 +0200 Subject: [PATCH 10/19] fix: ADDON-90004 validate extra_attrs keys and values before recording Add _attr_key_error and _attr_value_error helper functions to validate OpenTelemetry attribute keys and values per OTEL spec (string keys, bool/int/float/string values). Modify StanzaObservabilityRecorder.record to validate and filter extra_attrs, dropping invalid entries and preventing override of splunk.modinput.name. Co-Authored-By: Claude --- solnlib/observability.py | 54 +++++++++++++- tests/unit/test_observability.py | 122 +++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 5133a33e..3b9c91e8 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -745,6 +745,35 @@ def _count_error(value) -> Optional[str]: return None +def _attr_key_error(key) -> Optional[str]: + """Return None for a valid attribute key, else a safe reason string.""" + if type(key) is not str: + return f"expected str key, got {_sanitize_for_log(type(key).__name__)}" + if not key: + return "key is empty" + if not _is_safe_identifier_str(key): + return "key is not UTF-8 encodable or contains CR/LF" + return None + + +def _attr_value_error(value) -> Optional[str]: + """Return None for a valid attribute value, else a safe reason string.""" + value_type = type(value) + if value_type is bool: + return None + if value_type is int: + if _INT64_MIN <= value <= _INT64_MAX: + return None + return f"int value out of int64 range ({value.bit_length()} bits)" + if value_type is float: + return None + if value_type is str: + if _is_encodable_str(value): + return None + return "value is not UTF-8 encodable" + return f"unsupported value type: {_sanitize_for_log(value_type.__name__)}" + + class StanzaObservabilityRecorder: """Stanza-scoped observability recorder backed by a shared ``ObservabilityService``. @@ -920,7 +949,30 @@ def record( extra_attrs={"my_ta.partition": partition_id}, ) """ - attrs = dict(extra_attrs) if extra_attrs else {} + attrs = {} + if extra_attrs is not None: + if type(extra_attrs) is not dict: + self._service._logger.info( + "Ignoring extra_attrs: expected dict or None, got %s", + _sanitize_for_log(type(extra_attrs).__name__), + ) + else: + for key, value in extra_attrs.items(): + key_error = _attr_key_error(key) + if key_error is not None: + self._service._logger.info( + "Ignoring invalid attribute: %s", key_error + ) + continue + value_error = _attr_value_error(value) + if value_error is not None: + self._service._logger.info( + "Ignoring invalid value for attribute %r: %s", + key, + value_error, + ) + continue + attrs[key] = value attrs[ATTR_MODINPUT_NAME] = self._stanza_name event_count_error = _count_error(event_count) diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 599c354a..689f8861 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -1748,6 +1748,128 @@ def test_record_skips_invalid_byte_count_keeps_valid_event_count( ) mock_bytes.add.assert_not_called() + def test_record_rejects_non_dict_extra_attrs( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + mock_count = MagicMock() + rec._service.event_count_counter = mock_count + mock_count.reset_mock() + + rec.record(1, 1, extra_attrs=[("a", "b")]) + + mock_count.add.assert_called_once_with( + 1, attributes={"splunk.modinput.name": "my:stanza"} + ) + + def test_record_drops_invalid_entries_keeps_valid_ones( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + mock_count = MagicMock() + rec._service.event_count_counter = mock_count + mock_count.reset_mock() + + rec.record( + 1, + 1, + extra_attrs={ + "good.key": "good value", + "bad.value": [1, 2, 3], + 123: "bad key", + }, + ) + + mock_count.add.assert_called_once_with( + 1, + attributes={ + "splunk.modinput.name": "my:stanza", + "good.key": "good value", + }, + ) + + def test_record_extra_attrs_cannot_override_modinput_name( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + mock_count = MagicMock() + rec._service.event_count_counter = mock_count + mock_count.reset_mock() + + rec.record(1, 1, extra_attrs={"splunk.modinput.name": "hijacked"}) + + mock_count.add.assert_called_once_with( + 1, attributes={"splunk.modinput.name": "my:stanza"} + ) + + def test_record_non_dict_extra_attrs_log_sanitizes_type_name( + self, monkeypatch, clear_stanza_recorder_cache + ): + rec = self._make_recorder(monkeypatch) + evil_type = type("bad\r\nname", (), {}) + + rec.record(1, 1, extra_attrs=evil_type()) + + for call in rec._service._logger.info.call_args_list: + for arg in call.args: + assert "\n" not in str(arg) + assert "\r" not in str(arg) + + +class TestAttrValidation: + def test_attr_key_error_accepts_valid_key(self): + from solnlib.observability import _attr_key_error + + assert _attr_key_error("valid.key") is None + + @pytest.mark.parametrize( + "key", + [123, b"bytes", None, "", "bad\nkey", "bad\rkey", "\ud800"], + ) + def test_attr_key_error_rejects_invalid_key(self, key): + from solnlib.observability import _attr_key_error + + assert _attr_key_error(key) is not None + + @pytest.mark.parametrize( + "value", + [True, False, 0, 2**63 - 1, -(2**63), 1.5, float("nan"), float("inf"), "ok"], + ) + def test_attr_value_error_accepts_valid_values(self, value): + from solnlib.observability import _attr_value_error + + assert _attr_value_error(value) is None + + def test_attr_value_error_accepts_crlf_in_string_value(self): + from solnlib.observability import _attr_value_error + + assert _attr_value_error("has\r\nnewline") is None + + @pytest.mark.parametrize( + "value", + [2**63, -(2**63) - 1, [1, 2], (1, 2), {"a": 1}, None, "\ud800"], + ) + def test_attr_value_error_rejects_invalid_values(self, value): + from solnlib.observability import _attr_value_error + + assert _attr_value_error(value) is not None + + def test_attr_key_error_type_name_is_sanitized(self): + from solnlib.observability import _attr_key_error + + evil_type = type("bad\r\nname", (), {}) + error = _attr_key_error(evil_type()) + assert "\n" not in error + assert "\r" not in error + + def test_attr_value_error_type_name_is_sanitized(self): + from solnlib.observability import _attr_value_error + + evil_type = type("bad\r\nname", (), {}) + error = _attr_value_error(evil_type()) + assert "\n" not in error + assert "\r" not in error + class TestGrpcTlsHandshakeStderrSuppression: """Integration check: a real TLS handshake failure logs a gRPC C-core From 0eef64d5a073d15b8625984ef160020333ab3ed0 Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 13:41:49 +0200 Subject: [PATCH 11/19] fix: ADDON-90004 validate recorder identity strings, raise TypeError Add validation in StanzaObservabilityRecorder.__init__ to reject modinput_type and stanza_name that are not strings or contain CR/LF. Raise TypeError before cache lookup to prevent invalid entries. Replace all 5 self._service._logger.info(...) calls in the record() method with self._logger.info(...) so the recorder logs on its own behalf rather than reaching into the service's logger. Co-Authored-By: Claude --- solnlib/observability.py | 21 +++++++-- tests/unit/test_observability.py | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 3b9c91e8..e498e986 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -859,6 +859,17 @@ def __init__( ``"my_stanza"``). Attached as ``"splunk.modinput.name"`` on every recorded data point. """ + self._logger = logger + if type(modinput_type) is not str or not _is_safe_identifier_str(modinput_type): + raise TypeError( + "modinput_type must be a str without CR/LF, got " + f"{_sanitize_for_log(type(modinput_type).__name__)}" + ) + if type(stanza_name) is not str or not _is_safe_identifier_str(stanza_name): + raise TypeError( + "stanza_name must be a str without CR/LF, got " + f"{_sanitize_for_log(type(stanza_name).__name__)}" + ) self._stanza_name = stanza_name self._service = self._get_or_create_service(modinput_type, logger) self._emit_zero_baseline() @@ -952,7 +963,7 @@ def record( attrs = {} if extra_attrs is not None: if type(extra_attrs) is not dict: - self._service._logger.info( + self._logger.info( "Ignoring extra_attrs: expected dict or None, got %s", _sanitize_for_log(type(extra_attrs).__name__), ) @@ -960,13 +971,13 @@ def record( for key, value in extra_attrs.items(): key_error = _attr_key_error(key) if key_error is not None: - self._service._logger.info( + self._logger.info( "Ignoring invalid attribute: %s", key_error ) continue value_error = _attr_value_error(value) if value_error is not None: - self._service._logger.info( + self._logger.info( "Ignoring invalid value for attribute %r: %s", key, value_error, @@ -977,13 +988,13 @@ def record( event_count_error = _count_error(event_count) if event_count_error is not None: - self._service._logger.info("Skipping invalid event_count: %s", event_count_error) + self._logger.info("Skipping invalid event_count: %s", event_count_error) elif self._service.event_count_counter: self._service.event_count_counter.add(event_count, attributes=attrs) byte_count_error = _count_error(byte_count) if byte_count_error is not None: - self._service._logger.info("Skipping invalid byte_count: %s", byte_count_error) + self._logger.info("Skipping invalid byte_count: %s", byte_count_error) elif self._service.event_bytes_counter: self._service.event_bytes_counter.add(byte_count, attributes=attrs) diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 689f8861..ece55383 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -1815,6 +1815,87 @@ def test_record_non_dict_extra_attrs_log_sanitizes_type_name( assert "\n" not in str(arg) assert "\r" not in str(arg) + @pytest.mark.parametrize( + "modinput_type, stanza_name", + [ + (123, "ok"), + ("ok", 123), + (None, "ok"), + ("ok", "bad\nname"), + ("ok", "bad\rname"), + ("\ud800", "ok"), + ], + ) + def test_init_rejects_invalid_identity_values( + self, monkeypatch, clear_stanza_recorder_cache, modinput_type, stanza_name + ): + from solnlib.observability import StanzaObservabilityRecorder + + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + with pytest.raises(TypeError): + StanzaObservabilityRecorder( + modinput_type, MagicMock(spec=logging.Logger), stanza_name + ) + + def test_init_error_message_omits_raw_value( + self, monkeypatch, clear_stanza_recorder_cache + ): + from solnlib.observability import StanzaObservabilityRecorder + + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + secret_value = "TOP-SECRET-SHOULD-NOT-APPEAR" + with pytest.raises(TypeError) as exc_info: + StanzaObservabilityRecorder(secret_value + "\n", MagicMock(spec=logging.Logger), "ok") + assert secret_value not in str(exc_info.value) + + def test_init_accepts_empty_identity_strings( + self, monkeypatch, clear_stanza_recorder_cache + ): + from solnlib.observability import StanzaObservabilityRecorder + + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + rec = StanzaObservabilityRecorder("", MagicMock(spec=logging.Logger), "") + assert rec._stanza_name == "" + + def test_init_rejects_invalid_identity_before_cache_lookup( + self, monkeypatch, clear_stanza_recorder_cache + ): + from solnlib.observability import StanzaObservabilityRecorder + + with pytest.raises(TypeError): + StanzaObservabilityRecorder( + "bad\nmodinput", MagicMock(spec=logging.Logger), "ok" + ) + assert "bad\nmodinput" not in StanzaObservabilityRecorder._instances + + def test_init_type_error_message_sanitizes_evil_type_name( + self, monkeypatch, clear_stanza_recorder_cache + ): + # A non-str modinput_type/stanza_name is reported by class name, and + # that class name is caller-controlled: type("bad\r\nname", (), {}) + # is a real class whose __name__ contains raw CR/LF. This TypeError + # propagates to the caller (unlike the internally-caught validation + # in ObservabilityService), so its message must be pre-sanitized — + # solnlib does not control what the caller does with it afterward. + from solnlib.observability import StanzaObservabilityRecorder + + evil_type = type("bad\r\nname", (), {}) + with pytest.raises(TypeError) as exc_info: + StanzaObservabilityRecorder( + evil_type(), MagicMock(spec=logging.Logger), "ok" + ) + assert "\n" not in str(exc_info.value) + assert "\r" not in str(exc_info.value) + class TestAttrValidation: def test_attr_key_error_accepts_valid_key(self): From 8761cf7817adba373f07b6140923f959f66dab8b Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 13:56:47 +0200 Subject: [PATCH 12/19] fix: ADDON-90004 validate modinput_type for direct ObservabilityService callers Co-Authored-By: Claude --- solnlib/observability.py | 8 ++++ tests/unit/test_observability.py | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index e498e986..a201938b 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -407,6 +407,14 @@ def __init__( logging.getLogger(logger_name).addFilter(_downgrade_to_info_filter) try: + if type(modinput_type) is not str or not _is_safe_identifier_str( + modinput_type + ): + raise ValueError( + "modinput_type must be a str without CR/LF, got " + f"{_sanitize_for_log(type(modinput_type).__name__)}" + ) + if ta_name is None or ta_version is None: _ta_name, _ta_version = self._read_ta_info() ta_name = ta_name or _ta_name diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index ece55383..d1ead6c9 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -1247,6 +1247,72 @@ def emit(self, record): ) assert all(levelno <= logging.INFO for _, levelno in captured) + @pytest.mark.parametrize( + "modinput_type", + [123, None, "bad\nvalue", "bad\rvalue", "\ud800"], + ) + def test_init_direct_call_degrades_gracefully_on_invalid_modinput_type( + self, logger, monkeypatch, modinput_type + ): + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + svc = ObservabilityService( + modinput_type=modinput_type, + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + assert svc._meter is None + assert svc.event_count_counter is None + assert svc.event_bytes_counter is None + logger.info.assert_called() + + def test_init_valid_modinput_type_still_initialises(self, logger, monkeypatch): + svc = _make_service(logger, monkeypatch) + assert svc._meter is not None + + def test_init_invalid_modinput_type_error_sanitizes_evil_type_name( + self, logger, monkeypatch + ): + # Same class-name exposure as StanzaObservabilityRecorder's TypeError + # (Task 10): fix at the raise site for consistency. Checking the + # final logged output is not sufficient proof here — this ValueError + # is always caught internally and passed through _safe_exception_str, + # which re-sanitizes whatever it's given. A test that only inspects + # logger.info.call_args_list would still pass even if the raise-site + # fix were reverted, since the downstream re-sanitization masks the + # regression. Capture the exception object handed to + # _safe_exception_str instead, and assert its own stored message + # (args[0]) is already clean, proving the local fix independent of + # that downstream safety net. + monkeypatch.setattr( + "solnlib.observability.ObservabilityService._create_otlp_exporter", + lambda self: None, + ) + captured_errors = [] + + def _capturing_safe_exception_str(error): + captured_errors.append(error) + return "SAFE" + + monkeypatch.setattr( + "solnlib.observability._safe_exception_str", + _capturing_safe_exception_str, + ) + evil_type = type("bad\r\nname", (), {}) + ObservabilityService( + modinput_type=evil_type(), + logger=logger, + ta_name="my_ta", + ta_version="1.0.0", + ) + assert captured_errors, "expected the ValueError to reach _safe_exception_str" + raw_message = captured_errors[0].args[0] + assert "\n" not in raw_message + assert "\r" not in raw_message + # --------------------------------------------------------------------------- # _DowngradeToInfoFilter From bf0f800d2df3efaa3e6e4434880451864fbdb80f Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 14:10:39 +0200 Subject: [PATCH 13/19] test: ADDON-90004 assert no WARNING/ERROR logs on invalid modinput_type Code-quality review of the modinput_type graceful-degrade test flagged that it only checked logger.info was called, not that warning/error were absent -- the actual contract this ticket is about. Co-Authored-By: Claude --- tests/unit/test_observability.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index d1ead6c9..61891076 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -1268,6 +1268,8 @@ def test_init_direct_call_degrades_gracefully_on_invalid_modinput_type( assert svc.event_count_counter is None assert svc.event_bytes_counter is None logger.info.assert_called() + logger.warning.assert_not_called() + logger.error.assert_not_called() def test_init_valid_modinput_type_still_initialises(self, logger, monkeypatch): svc = _make_service(logger, monkeypatch) From 81a843d2d1fab10af7e3f68b32e82df0fa4f234a Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 14:12:24 +0200 Subject: [PATCH 14/19] docs: ADDON-90004 fix ObservabilityService docstring to match INFO log policy --- solnlib/observability.py | 2 +- tests/unit/test_observability.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index a201938b..0be540ff 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -312,7 +312,7 @@ class ObservabilityService: Sets up a ``MeterProvider`` with two built-in event counters and, when the Spotlight collector is reachable, an OTLP gRPC exporter. - Initialisation failures are caught and logged as warnings so that a + Initialisation failures are caught and logged at INFO so that a missing or misconfigured observability stack never breaks the add-on. **Resource attributes** (fixed for the lifetime of the process): diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 61891076..628fc93f 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -619,6 +619,9 @@ def _make_service(logger, monkeypatch, extra_exporters=None): class TestObservabilityService: + def test_class_docstring_does_not_mention_warnings(self): + assert "warning" not in ObservabilityService.__doc__.lower() + def test_counters_are_created(self, logger, monkeypatch): # Arrange / Act svc = _make_service(logger, monkeypatch) From 529925580b39f6ec3b34321f0a387ce3db10f2bd Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 14:29:53 +0200 Subject: [PATCH 15/19] style: ADDON-90004 apply black formatting to observability changes --- solnlib/observability.py | 4 +--- tests/unit/test_observability.py | 30 +++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 0be540ff..040fe888 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -979,9 +979,7 @@ def record( for key, value in extra_attrs.items(): key_error = _attr_key_error(key) if key_error is not None: - self._logger.info( - "Ignoring invalid attribute: %s", key_error - ) + self._logger.info("Ignoring invalid attribute: %s", key_error) continue value_error = _attr_value_error(value) if value_error is not None: diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 628fc93f..79aec955 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -404,9 +404,7 @@ def test_real_reader_reads_temporality_from_wrapper(self, logger): } inner = _FakeInnerExporter(temporality) wrapper = _CircuitBreakerExporter(inner, logger) - reader = PeriodicExportingMetricReader( - wrapper, export_interval_millis=math.inf - ) + reader = PeriodicExportingMetricReader(wrapper, export_interval_millis=math.inf) try: assert reader._preferred_temporality == temporality finally: @@ -497,9 +495,7 @@ def test_export_resets_failure_count_after_success(self, logger): from solnlib.observability import _CircuitBreakerExporter inner = _FakeInnerExporter() - inner.results.extend( - [MetricExportResult.FAILURE, MetricExportResult.SUCCESS] - ) + inner.results.extend([MetricExportResult.FAILURE, MetricExportResult.SUCCESS]) wrapper = _CircuitBreakerExporter(inner, logger) wrapper.export(MagicMock()) assert wrapper._consecutive_failures == 1 @@ -511,7 +507,11 @@ def test_export_resets_failure_count_after_success(self, logger): [ [MetricExportResult.FAILURE] * 3, [RuntimeError("boom")] * 3, - [MetricExportResult.FAILURE, RuntimeError("boom"), MetricExportResult.FAILURE], + [ + MetricExportResult.FAILURE, + RuntimeError("boom"), + MetricExportResult.FAILURE, + ], ], ) def test_export_trips_on_third_consecutive_failure(self, logger, outcomes): @@ -1922,7 +1922,9 @@ def test_init_error_message_omits_raw_value( ) secret_value = "TOP-SECRET-SHOULD-NOT-APPEAR" with pytest.raises(TypeError) as exc_info: - StanzaObservabilityRecorder(secret_value + "\n", MagicMock(spec=logging.Logger), "ok") + StanzaObservabilityRecorder( + secret_value + "\n", MagicMock(spec=logging.Logger), "ok" + ) assert secret_value not in str(exc_info.value) def test_init_accepts_empty_identity_strings( @@ -1985,7 +1987,17 @@ def test_attr_key_error_rejects_invalid_key(self, key): @pytest.mark.parametrize( "value", - [True, False, 0, 2**63 - 1, -(2**63), 1.5, float("nan"), float("inf"), "ok"], + [ + True, + False, + 0, + 2**63 - 1, + -(2**63), + 1.5, + float("nan"), + float("inf"), + "ok", + ], ) def test_attr_value_error_accepts_valid_values(self, value): from solnlib.observability import _attr_value_error From 572db4804c719822160b2c1c39d916c263ebd95f Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 16:25:01 +0200 Subject: [PATCH 16/19] fix: ADDON-90004 don't hold breaker lock across blocking export/shutdown calls _CircuitBreakerExporter.export()/force_flush()/shutdown() held the state lock across the blocking calls into the inner OTLP exporter, so shutdown() could be blocked for the full duration of an in-flight export instead of promptly reaching the inner exporter's own shutdown-triggered retry interruption. --- solnlib/observability.py | 28 ++++++++++++++++------------ tests/unit/test_observability.py | 14 ++++++-------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/solnlib/observability.py b/solnlib/observability.py index 040fe888..8c0ef9d0 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -272,17 +272,21 @@ def export( if self._tripped: return MetricExportResult.SUCCESS - try: - result = self._inner.export( - metrics_data, timeout_millis=timeout_millis, **kwargs - ) - except Exception as error: - self._logger.info( - "OTLP export raised an exception: %s", - _safe_exception_repr(error), - ) - result = MetricExportResult.FAILURE + # Never hold _lock across this blocking call: shutdown() must be able + # to reach the inner exporter and interrupt an in-flight retry even + # while an export is still outstanding on another thread. + try: + result = self._inner.export( + metrics_data, timeout_millis=timeout_millis, **kwargs + ) + except Exception as error: + self._logger.info( + "OTLP export raised an exception: %s", + _safe_exception_repr(error), + ) + result = MetricExportResult.FAILURE + with self._lock: if result == MetricExportResult.SUCCESS: self._consecutive_failures = 0 return result @@ -297,14 +301,14 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool: with self._lock: if self._tripped: return True - return self._inner.force_flush(timeout_millis=timeout_millis) + return self._inner.force_flush(timeout_millis=timeout_millis) def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: with self._lock: if self._shutdown_called: return self._shutdown_called = True - self._inner.shutdown(timeout_millis=timeout_millis, **kwargs) + self._inner.shutdown(timeout_millis=timeout_millis, **kwargs) class ObservabilityService: diff --git a/tests/unit/test_observability.py b/tests/unit/test_observability.py index 79aec955..84c907f9 100644 --- a/tests/unit/test_observability.py +++ b/tests/unit/test_observability.py @@ -23,7 +23,6 @@ import subprocess import sys import threading -import time from unittest.mock import MagicMock, patch import pytest @@ -562,7 +561,7 @@ def test_three_consecutive_exceptions_emit_three_info_plus_one_trip_log( assert logger.info.call_count == 4 assert "3 consecutive failures" in logger.info.call_args_list[-1].args[0] - def test_export_and_shutdown_are_mutually_exclusive(self, logger): + def test_shutdown_is_not_blocked_by_in_flight_export(self, logger): from solnlib.observability import _CircuitBreakerExporter inner = _FakeInnerExporter() @@ -585,17 +584,16 @@ def blocking_export(metrics_data, timeout_millis=10_000, **kwargs): shutdown_thread = threading.Thread(target=wrapper.shutdown) shutdown_thread.start() - time.sleep(0.2) - assert wrapper._shutdown_called is False - assert inner.shutdown_calls == [] - - release_export.set() - export_thread.join(timeout=5) shutdown_thread.join(timeout=5) + # shutdown() must complete without waiting for the in-flight export to + # finish, so the real gRPC exporter can interrupt its retry backoff. assert wrapper._shutdown_called is True assert len(inner.shutdown_calls) == 1 + release_export.set() + export_thread.join(timeout=5) + # --------------------------------------------------------------------------- # ObservabilityService From b43e4f3aea2470c3d984d90fe7469fd8ac9be91e Mon Sep 17 00:00:00 2001 From: Wojciech Tobis Date: Wed, 19 Aug 2026 19:01:41 +0200 Subject: [PATCH 17/19] docs: ADDON-90004 document process-wide OTel log policy --- solnlib/observability.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solnlib/observability.py b/solnlib/observability.py index 8c0ef9d0..5c3eae7f 100644 --- a/solnlib/observability.py +++ b/solnlib/observability.py @@ -120,6 +120,10 @@ def filter(self, record: logging.LogRecord) -> bool: _downgrade_to_info_filter = _DowngradeToInfoFilter() +# Intentional process-wide policy: observability is auxiliary functionality, so +# diagnostics emitted by the OpenTelemetry SDK and exporters are capped at INFO +# across all signal types (metrics, logs, and traces). This preserves diagnostic +# messages without surfacing observability failures as add-on WARNING/ERROR events. _OTLP_LOGGERS = ( "opentelemetry.exporter.otlp.proto.grpc.exporter", "opentelemetry.util.re", From 9fff4cc013556d5a56262639467c036fbfb4ef9f Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:30:17 +0000 Subject: [PATCH 18/19] chore(release): 8.1.0-beta.2 # [8.1.0-beta.2](https://github.com/splunk/addonfactory-solutions-library-python/compare/v8.1.0-beta.1...v8.1.0-beta.2) (2026-08-20) ### Bug Fixes * ADDON-90004 add circuit breaker for the OTLP exporter ([cfb5696](https://github.com/splunk/addonfactory-solutions-library-python/commit/cfb5696cec9c00ecc40ccbed2e8a4dbe614147c6)) * ADDON-90004 add downgrade-to-INFO logging filter ([2c18948](https://github.com/splunk/addonfactory-solutions-library-python/commit/2c18948d40bf7a2d8a3c8b577d07e9a5420a452c)) * ADDON-90004 add safe log-rendering helpers to observability module ([04473b4](https://github.com/splunk/addonfactory-solutions-library-python/commit/04473b47071b91c8ed84acbd1a2467b0d2779e5a)) * ADDON-90004 cap solnlib.observability log calls at INFO ([4d40269](https://github.com/splunk/addonfactory-solutions-library-python/commit/4d40269a0c23d3c14c7612b469d10161827234ed)) * ADDON-90004 change observability log level ([#461](https://github.com/splunk/addonfactory-solutions-library-python/issues/461)) ([5a1fa77](https://github.com/splunk/addonfactory-solutions-library-python/commit/5a1fa7784f16c098022bb79c87e45cce07ef8c66)) * ADDON-90004 don't hold breaker lock across blocking export/shutdown calls ([572db48](https://github.com/splunk/addonfactory-solutions-library-python/commit/572db4804c719822160b2c1c39d916c263ebd95f)) * ADDON-90004 downgrade metrics SDK logger noise to INFO ([c8bbc5d](https://github.com/splunk/addonfactory-solutions-library-python/commit/c8bbc5ddbe519a2f5962c5468e6ee0febf310fbf)) * ADDON-90004 suppress gRPC C-core stderr and downgrade OTLP logger noise ([fb967ab](https://github.com/splunk/addonfactory-solutions-library-python/commit/fb967ab17b6c3c2b8473840a9a673bae4e973f92)) * ADDON-90004 synchronize circuit breaker state against concurrent access ([5c11333](https://github.com/splunk/addonfactory-solutions-library-python/commit/5c1133302c761bbca7784f4abbcc6e711fc41fa3)) * ADDON-90004 validate event_count/byte_count before recording ([80606b8](https://github.com/splunk/addonfactory-solutions-library-python/commit/80606b8591043aa8c3062ff9a981bb063b7dc479)) * ADDON-90004 validate extra_attrs keys and values before recording ([3b54dc1](https://github.com/splunk/addonfactory-solutions-library-python/commit/3b54dc1d346d44d3dd5d05e459e81832fe4eafd0)) * ADDON-90004 validate modinput_type for direct ObservabilityService callers ([8761cf7](https://github.com/splunk/addonfactory-solutions-library-python/commit/8761cf7817adba373f07b6140923f959f66dab8b)) * ADDON-90004 validate recorder identity strings, raise TypeError ([0eef64d](https://github.com/splunk/addonfactory-solutions-library-python/commit/0eef64d5a073d15b8625984ef160020333ab3ed0)) * ADDON-90004 wrap the OTLP exporter in the circuit breaker ([bdc76b3](https://github.com/splunk/addonfactory-solutions-library-python/commit/bdc76b3d614ec4f0b39aa9564b3250a6110481d4)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae76b0b0..16173d54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "8.1.0-beta.1" +version = "8.1.0-beta.2" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 07e94e29..6404fbbd 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -55,4 +55,4 @@ "utils", ] -__version__ = "8.1.0-beta.1" +__version__ = "8.1.0-beta.2" From 3e3333e137d36410012010d1f73fc154839b8671 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:08:16 +0000 Subject: [PATCH 19/19] chore(release): 8.1.1-beta.1 ## [8.1.1-beta.1](https://github.com/splunk/addonfactory-solutions-library-python/compare/v8.1.0...v8.1.1-beta.1) (2026-08-20) ### Bug Fixes * ADDON-90004 add circuit breaker for the OTLP exporter ([cfb5696](https://github.com/splunk/addonfactory-solutions-library-python/commit/cfb5696cec9c00ecc40ccbed2e8a4dbe614147c6)) * ADDON-90004 add downgrade-to-INFO logging filter ([2c18948](https://github.com/splunk/addonfactory-solutions-library-python/commit/2c18948d40bf7a2d8a3c8b577d07e9a5420a452c)) * ADDON-90004 add safe log-rendering helpers to observability module ([04473b4](https://github.com/splunk/addonfactory-solutions-library-python/commit/04473b47071b91c8ed84acbd1a2467b0d2779e5a)) * ADDON-90004 cap solnlib.observability log calls at INFO ([4d40269](https://github.com/splunk/addonfactory-solutions-library-python/commit/4d40269a0c23d3c14c7612b469d10161827234ed)) * ADDON-90004 change observability log level ([#461](https://github.com/splunk/addonfactory-solutions-library-python/issues/461)) ([5a1fa77](https://github.com/splunk/addonfactory-solutions-library-python/commit/5a1fa7784f16c098022bb79c87e45cce07ef8c66)) * ADDON-90004 don't hold breaker lock across blocking export/shutdown calls ([572db48](https://github.com/splunk/addonfactory-solutions-library-python/commit/572db4804c719822160b2c1c39d916c263ebd95f)) * ADDON-90004 downgrade metrics SDK logger noise to INFO ([c8bbc5d](https://github.com/splunk/addonfactory-solutions-library-python/commit/c8bbc5ddbe519a2f5962c5468e6ee0febf310fbf)) * ADDON-90004 suppress gRPC C-core stderr and downgrade OTLP logger noise ([fb967ab](https://github.com/splunk/addonfactory-solutions-library-python/commit/fb967ab17b6c3c2b8473840a9a673bae4e973f92)) * ADDON-90004 synchronize circuit breaker state against concurrent access ([5c11333](https://github.com/splunk/addonfactory-solutions-library-python/commit/5c1133302c761bbca7784f4abbcc6e711fc41fa3)) * ADDON-90004 validate event_count/byte_count before recording ([80606b8](https://github.com/splunk/addonfactory-solutions-library-python/commit/80606b8591043aa8c3062ff9a981bb063b7dc479)) * ADDON-90004 validate extra_attrs keys and values before recording ([3b54dc1](https://github.com/splunk/addonfactory-solutions-library-python/commit/3b54dc1d346d44d3dd5d05e459e81832fe4eafd0)) * ADDON-90004 validate modinput_type for direct ObservabilityService callers ([8761cf7](https://github.com/splunk/addonfactory-solutions-library-python/commit/8761cf7817adba373f07b6140923f959f66dab8b)) * ADDON-90004 validate recorder identity strings, raise TypeError ([0eef64d](https://github.com/splunk/addonfactory-solutions-library-python/commit/0eef64d5a073d15b8625984ef160020333ab3ed0)) * ADDON-90004 wrap the OTLP exporter in the circuit breaker ([bdc76b3](https://github.com/splunk/addonfactory-solutions-library-python/commit/bdc76b3d614ec4f0b39aa9564b3250a6110481d4)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 16173d54..d11945e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "8.1.0-beta.2" +version = "8.1.1-beta.1" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 6404fbbd..00a25025 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -55,4 +55,4 @@ "utils", ] -__version__ = "8.1.0-beta.2" +__version__ = "8.1.1-beta.1"