From 2081cc72926d89dbb59421614f866a61de5ea816 Mon Sep 17 00:00:00 2001 From: Paul Cruse III Date: Tue, 16 Jun 2026 16:58:56 -0500 Subject: [PATCH 1/5] logger: add pre-print callback hook + PII redaction filter Add an extensible callback mechanism to CommonLogger: register_callback / reset_callbacks run each log record through registered callables before it is printed (record = {level, time, trace, log}). The hook is generic (redaction, enrichment, routing). Add redaction_filter(keys, patterns, redact_with='[REDACTED]') built on that hook: redacts matching field names (case-insensitive, any depth) and regex value matches. PII redaction with sensible defaults is registered by default so logs are scrubbed before they reach stdout/CloudWatch; opt out with ACAI_LOG_REDACTION=off. --- acai_aws/common/logger/__init__.py | 23 +++++++++ acai_aws/common/logger/common_logger.py | 68 ++++++++++++++++++------- acai_aws/common/logger/defaults.py | 45 ++++++++++++++++ acai_aws/common/logger/redaction.py | 42 +++++++++++++++ tests/acai_aws/common/test_logger.py | 47 +++++++++++++++++ tests/acai_aws/common/test_redaction.py | 62 ++++++++++++++++++++++ 6 files changed, 268 insertions(+), 19 deletions(-) create mode 100644 acai_aws/common/logger/defaults.py create mode 100644 acai_aws/common/logger/redaction.py create mode 100644 tests/acai_aws/common/test_redaction.py diff --git a/acai_aws/common/logger/__init__.py b/acai_aws/common/logger/__init__.py index 0e9efb4..8791dc0 100644 --- a/acai_aws/common/logger/__init__.py +++ b/acai_aws/common/logger/__init__.py @@ -1,6 +1,29 @@ import logging from acai_aws.common.logger.common_logger import CommonLogger +from acai_aws.common.logger.redaction import redaction_filter +from acai_aws.common.logger.defaults import ( + DEFAULT_PII_KEYS, + DEFAULT_PII_PATTERNS, + redaction_enabled, + register_default_redaction, +) + +__all__ = [ + 'log', + 'CommonLogger', + 'redaction_filter', + 'DEFAULT_PII_KEYS', + 'DEFAULT_PII_PATTERNS', + 'redaction_enabled', + 'register_default_redaction', +] + +# Register PII redaction by default so every consumer scrubs sensitive fields +# before they are printed (and therefore before they reach CloudWatch). Opt out +# with ACAI_LOG_REDACTION=off, or extend with CommonLogger.register_callback. +if redaction_enabled(): + register_default_redaction() def log(**kwargs): diff --git a/acai_aws/common/logger/common_logger.py b/acai_aws/common/logger/common_logger.py index e0ca352..54817ce 100644 --- a/acai_aws/common/logger/common_logger.py +++ b/acai_aws/common/logger/common_logger.py @@ -6,6 +6,15 @@ class CommonLogger: + """Structured logger that emits a JSON, pretty JSON, or inline record to stdout. + + Before a record is printed it is passed through any registered callbacks + (see ``register_callback``). Callbacks are an extension point used for + redaction, enrichment, or routing; they receive the assembled record dict + and return the record to emit. + """ + + _callbacks = [] def __init__(self): self.__json = jsonpickle @@ -26,14 +35,41 @@ def __init__(self): if self.__format not in ['JSON', 'PRETTY', 'INLINE']: raise ValueError(f'LOG_FORMAT ENV must be either `JSON`, `PRETTY`, or `INLINE`, recieved: {self.__format}') + @classmethod + def register_callback(cls, callback): + """Register a callable run on every log record before it is printed. + + The callback signature is ``callback(record: dict) -> dict`` where + ``record`` holds ``level``, ``time``, ``trace``, and ``log``. The + returned record is what gets emitted, so a callback may mutate or + replace any part of it. Callbacks run in registration order. + """ + cls._callbacks.append(callback) + + @classmethod + def reset_callbacks(cls): + """Remove all registered callbacks.""" + cls._callbacks = [] + def log(self, **kwargs): - default_log = {'level': kwargs.get('level', 'INFO'), 'log': kwargs.get('log', {})} - if self.__should_log(default_log['level']) and self.__format == 'JSON': - self.__log_json(**kwargs) - elif self.__should_log(default_log['level']) and self.__format == 'PRETTY': - self.__log_json(pretty=True, **kwargs) - elif self.__should_log(default_log['level']) and self.__format == 'INLINE': - self.__log_inline(**kwargs) + """Build a log record, run it through registered callbacks, and emit it.""" + level = kwargs.get('level', 'INFO') + if not self.__should_log(level): + return + record = { + 'level': level, + 'time': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'trace': [trace.strip() for trace in self.__get_traceback().split('\n') if trace], + 'log': kwargs.get('log', {}) + } + for callback in CommonLogger._callbacks: + record = callback(record) + if self.__format == 'JSON': + self.__log_json(record) + elif self.__format == 'PRETTY': + self.__log_json(record, pretty=True) + elif self.__format == 'INLINE': + self.__log_inline(record) def __get_traceback(self): trace = traceback.format_exc() @@ -46,21 +82,15 @@ def __should_log(self, level): log_level_setting = self.log_levels[self.__log_level] return current_log_level >= log_level_setting - def __log_json(self, pretty=False, **kwargs): - print(self.__json.encode({ - 'level': kwargs['level'], - 'time': datetime.datetime.now(datetime.timezone.utc).isoformat(), - 'trace': [trace.strip() for trace in self.__get_traceback().split('\n') if trace], - 'log': kwargs['log'] - }, indent=4 if pretty else None)) + def __log_json(self, record, pretty=False): + print(self.__json.encode(record, indent=4 if pretty else None)) - def __log_inline(self, **kwargs): - timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat() - trace = self.__get_traceback().strip().replace('\n', ' | ') - log_value = kwargs['log'] + def __log_inline(self, record): + log_value = record['log'] if not isinstance(log_value, str): log_value = str(log_value) - inline_message = f"{kwargs['level']}|time={timestamp} log={log_value}" + inline_message = f"{record['level']}|time={record['time']} log={log_value}" + trace = ' | '.join(record['trace']) if trace: inline_message = f"{inline_message} trace={trace}" print(inline_message) diff --git a/acai_aws/common/logger/defaults.py b/acai_aws/common/logger/defaults.py new file mode 100644 index 0000000..90ea067 --- /dev/null +++ b/acai_aws/common/logger/defaults.py @@ -0,0 +1,45 @@ +import os + +from acai_aws.common.logger.common_logger import CommonLogger +from acai_aws.common.logger.redaction import redaction_filter + +# Field names redacted by the default PII filter (case-insensitive, any depth). +DEFAULT_PII_KEYS = [ + 'first_name', + 'last_name', + 'worker_first_name', + 'worker_last_name', + 'email', + 'worker_email', + 'phone', + 'worker_phone', + 'ssn', + 'social_security_number', + 'fein', + 'ein', +] + +# Value patterns redacted by the default PII filter. +DEFAULT_PII_PATTERNS = [ + r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}', # email + r'\b\d{3}-\d{2}-\d{4}\b', # US SSN + r'\b\d{2}-\d{7}\b', # US EIN / FEIN + r'\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}', # US phone +] + + +def redaction_enabled(): + """Return whether default PII redaction should be registered. + + Controlled by the ``ACAI_LOG_REDACTION`` environment variable; redaction is + on unless it is set to ``off``, ``false``, ``0``, or ``disabled``. + """ + setting = os.getenv('ACAI_LOG_REDACTION', 'on').strip().lower() + return setting not in ('off', 'false', '0', 'disabled') + + +def register_default_redaction(): + """Register the default PII redaction filter on ``CommonLogger``.""" + CommonLogger.register_callback( + redaction_filter(keys=DEFAULT_PII_KEYS, patterns=DEFAULT_PII_PATTERNS) + ) diff --git a/acai_aws/common/logger/redaction.py b/acai_aws/common/logger/redaction.py new file mode 100644 index 0000000..66b83b2 --- /dev/null +++ b/acai_aws/common/logger/redaction.py @@ -0,0 +1,42 @@ +import re + +DEFAULT_REDACTION = '[REDACTED]' + + +def redaction_filter(keys=None, patterns=None, redact_with=DEFAULT_REDACTION): + """Build a logger callback that redacts sensitive data from the log payload. + + ``keys`` are field names redacted wherever they appear in the ``log`` + payload (case-insensitive, at any nesting depth). ``patterns`` are regular + expression strings; every match found in a string value is replaced. + ``redact_with`` is the replacement text (default ``[REDACTED]``); pass + ``redact_with=''`` or any custom string to override it. + + Returns a callback compatible with ``CommonLogger.register_callback``; the + custom filter therefore rides on the generic callback hook rather than + special-casing the logger. + """ + lowered_keys = {str(key).lower() for key in (keys or [])} + compiled_patterns = [re.compile(pattern) for pattern in (patterns or [])] + + def scrub(value): + if isinstance(value, dict): + return { + key: redact_with if str(key).lower() in lowered_keys else scrub(item) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [scrub(item) for item in value] + if isinstance(value, str): + redacted = value + for pattern in compiled_patterns: + redacted = pattern.sub(redact_with, redacted) + return redacted + return value + + def callback(record): + if isinstance(record, dict) and 'log' in record: + record['log'] = scrub(record['log']) + return record + + return callback diff --git a/tests/acai_aws/common/test_logger.py b/tests/acai_aws/common/test_logger.py index f0c2008..791e10d 100644 --- a/tests/acai_aws/common/test_logger.py +++ b/tests/acai_aws/common/test_logger.py @@ -6,6 +6,8 @@ from acai_aws.common import logger from acai_aws.common.logger.decorator import log +from acai_aws.common.logger.common_logger import CommonLogger +from acai_aws.common.logger.redaction import redaction_filter def some_log_condition(*args, **_): if args[0] == 1: @@ -164,3 +166,48 @@ def test_logger_accepts_critical_level(self): @mock.patch.dict(os.environ, {'RUN_MODE': 'SEE-LOGS', 'LOG_STAGE_VARIABLE': 'STAGE', 'STAGE': 'local', 'LOG_LEVEL': 'ERROR', 'LOG_FORMAT': 'BAD'}) def test_logger_handles_bad_format(self): logger.log(level='INFO', log={'INFO': 'ignore'}) + + +class LoggerCallbackTest(TestCase): + + def setUp(self): + self._saved_callbacks = list(CommonLogger._callbacks) + CommonLogger.reset_callbacks() + + def tearDown(self): + CommonLogger._callbacks = self._saved_callbacks + + @mock.patch.dict(os.environ, {'LOG_FORMAT': 'JSON', 'LOG_LEVEL': 'INFO'}) + def test_callback_runs_before_print(self): + CommonLogger.register_callback(lambda record: {**record, 'log': {'replaced': True}}) + buffer = io.StringIO() + with redirect_stdout(buffer): + logger.log(level='INFO', log={'email': 'ada@example.com'}) + parsed = json.loads(buffer.getvalue().strip()) + self.assertEqual({'replaced': True}, parsed['log']) + + @mock.patch.dict(os.environ, {'LOG_FORMAT': 'JSON', 'LOG_LEVEL': 'INFO'}) + def test_redaction_filter_runs_through_logger(self): + CommonLogger.register_callback(redaction_filter(keys=['email'])) + buffer = io.StringIO() + with redirect_stdout(buffer): + logger.log(level='INFO', log={'email': 'ada@example.com', 'id': 1}) + parsed = json.loads(buffer.getvalue().strip()) + self.assertEqual('[REDACTED]', parsed['log']['email']) + self.assertEqual(1, parsed['log']['id']) + + @mock.patch.dict(os.environ, {'LOG_FORMAT': 'JSON', 'LOG_LEVEL': 'INFO'}) + def test_callbacks_run_in_registration_order(self): + CommonLogger.register_callback(lambda record: {**record, 'log': {'step': 'one'}}) + CommonLogger.register_callback(lambda record: {**record, 'log': {'step': 'two'}}) + buffer = io.StringIO() + with redirect_stdout(buffer): + logger.log(level='INFO', log={'step': 'zero'}) + parsed = json.loads(buffer.getvalue().strip()) + self.assertEqual({'step': 'two'}, parsed['log']) + + def test_reset_callbacks_clears_registry(self): + CommonLogger.register_callback(lambda record: record) + self.assertEqual(1, len(CommonLogger._callbacks)) + CommonLogger.reset_callbacks() + self.assertEqual(0, len(CommonLogger._callbacks)) diff --git a/tests/acai_aws/common/test_redaction.py b/tests/acai_aws/common/test_redaction.py new file mode 100644 index 0000000..35eb303 --- /dev/null +++ b/tests/acai_aws/common/test_redaction.py @@ -0,0 +1,62 @@ +from unittest import TestCase + +from acai_aws.common.logger.redaction import redaction_filter +from acai_aws.common.logger.defaults import DEFAULT_PII_KEYS, DEFAULT_PII_PATTERNS + + +class RedactionFilterTest(TestCase): + + def test_redacts_matching_keys(self): + callback = redaction_filter(keys=['email', 'last_name']) + record = callback({'log': {'first_name': 'Ada', 'last_name': 'Lovelace', 'email': 'ada@example.com'}}) + self.assertEqual('Ada', record['log']['first_name']) + self.assertEqual('[REDACTED]', record['log']['last_name']) + self.assertEqual('[REDACTED]', record['log']['email']) + + def test_key_match_is_case_insensitive(self): + callback = redaction_filter(keys=['email']) + record = callback({'log': {'Email': 'ada@example.com'}}) + self.assertEqual('[REDACTED]', record['log']['Email']) + + def test_redacts_nested_keys_in_dicts_and_lists(self): + callback = redaction_filter(keys=['ssn']) + payload = {'worker': {'ssn': '123-45-6789', 'id': 5}, 'items': [{'ssn': '111-22-3333'}]} + record = callback({'log': payload}) + self.assertEqual('[REDACTED]', record['log']['worker']['ssn']) + self.assertEqual(5, record['log']['worker']['id']) + self.assertEqual('[REDACTED]', record['log']['items'][0]['ssn']) + + def test_redacts_value_patterns(self): + callback = redaction_filter(patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) + record = callback({'log': {'note': 'ssn is 123-45-6789 today'}}) + self.assertEqual('ssn is [REDACTED] today', record['log']['note']) + + def test_custom_redact_with(self): + callback = redaction_filter(keys=['email'], redact_with='') + record = callback({'log': {'email': 'ada@example.com'}}) + self.assertEqual('', record['log']['email']) + + def test_leaves_non_matching_data_untouched(self): + callback = redaction_filter(keys=['email'], patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) + record = callback({'log': {'count': 3, 'ok': True, 'name': 'service-payroll'}}) + self.assertEqual({'count': 3, 'ok': True, 'name': 'service-payroll'}, record['log']) + + def test_handles_record_without_log_key(self): + callback = redaction_filter(keys=['email']) + record = callback({'level': 'INFO'}) + self.assertEqual({'level': 'INFO'}, record) + + def test_default_pii_keys_and_patterns(self): + callback = redaction_filter(keys=DEFAULT_PII_KEYS, patterns=DEFAULT_PII_PATTERNS) + record = callback({'log': { + 'first_name': 'Ada', + 'worker_email': 'ada@example.com', + 'ein': '12-3456789', + 'message': 'call 585-555-1234 or email ada@example.com', + }}) + log = record['log'] + self.assertEqual('[REDACTED]', log['first_name']) + self.assertEqual('[REDACTED]', log['worker_email']) + self.assertEqual('[REDACTED]', log['ein']) + self.assertNotIn('585-555-1234', log['message']) + self.assertNotIn('ada@example.com', log['message']) From b693f9c3b2f350bfcad8c0951f400ee103c6670c Mon Sep 17 00:00:00 2001 From: Paul Cruse III Date: Tue, 16 Jun 2026 17:06:37 -0500 Subject: [PATCH 2/5] refactor logger redaction to OOP-first per python standards Replace the module-level redaction_filter function and the defaults module (module-level functions + vars) with a single RedactionFilter class: keys/patterns/redact_with via **kwargs, callable so an instance is the callback, defaults as class constants, register_default classmethod for the env-gated wiring. Drop descriptive what-docstrings from CommonLogger. --- acai_aws/common/logger/__init__.py | 24 +------- acai_aws/common/logger/common_logger.py | 16 ----- acai_aws/common/logger/defaults.py | 45 -------------- acai_aws/common/logger/redaction.py | 82 ++++++++++++++++--------- tests/acai_aws/common/test_logger.py | 4 +- tests/acai_aws/common/test_redaction.py | 37 ++++++----- 6 files changed, 76 insertions(+), 132 deletions(-) delete mode 100644 acai_aws/common/logger/defaults.py diff --git a/acai_aws/common/logger/__init__.py b/acai_aws/common/logger/__init__.py index 8791dc0..82c4848 100644 --- a/acai_aws/common/logger/__init__.py +++ b/acai_aws/common/logger/__init__.py @@ -1,29 +1,11 @@ import logging from acai_aws.common.logger.common_logger import CommonLogger -from acai_aws.common.logger.redaction import redaction_filter -from acai_aws.common.logger.defaults import ( - DEFAULT_PII_KEYS, - DEFAULT_PII_PATTERNS, - redaction_enabled, - register_default_redaction, -) +from acai_aws.common.logger.redaction import RedactionFilter -__all__ = [ - 'log', - 'CommonLogger', - 'redaction_filter', - 'DEFAULT_PII_KEYS', - 'DEFAULT_PII_PATTERNS', - 'redaction_enabled', - 'register_default_redaction', -] +__all__ = ['log', 'CommonLogger', 'RedactionFilter'] -# Register PII redaction by default so every consumer scrubs sensitive fields -# before they are printed (and therefore before they reach CloudWatch). Opt out -# with ACAI_LOG_REDACTION=off, or extend with CommonLogger.register_callback. -if redaction_enabled(): - register_default_redaction() +RedactionFilter.register_default(logger_class=CommonLogger) def log(**kwargs): diff --git a/acai_aws/common/logger/common_logger.py b/acai_aws/common/logger/common_logger.py index 54817ce..252308a 100644 --- a/acai_aws/common/logger/common_logger.py +++ b/acai_aws/common/logger/common_logger.py @@ -6,13 +6,6 @@ class CommonLogger: - """Structured logger that emits a JSON, pretty JSON, or inline record to stdout. - - Before a record is printed it is passed through any registered callbacks - (see ``register_callback``). Callbacks are an extension point used for - redaction, enrichment, or routing; they receive the assembled record dict - and return the record to emit. - """ _callbacks = [] @@ -37,22 +30,13 @@ def __init__(self): @classmethod def register_callback(cls, callback): - """Register a callable run on every log record before it is printed. - - The callback signature is ``callback(record: dict) -> dict`` where - ``record`` holds ``level``, ``time``, ``trace``, and ``log``. The - returned record is what gets emitted, so a callback may mutate or - replace any part of it. Callbacks run in registration order. - """ cls._callbacks.append(callback) @classmethod def reset_callbacks(cls): - """Remove all registered callbacks.""" cls._callbacks = [] def log(self, **kwargs): - """Build a log record, run it through registered callbacks, and emit it.""" level = kwargs.get('level', 'INFO') if not self.__should_log(level): return diff --git a/acai_aws/common/logger/defaults.py b/acai_aws/common/logger/defaults.py deleted file mode 100644 index 90ea067..0000000 --- a/acai_aws/common/logger/defaults.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -from acai_aws.common.logger.common_logger import CommonLogger -from acai_aws.common.logger.redaction import redaction_filter - -# Field names redacted by the default PII filter (case-insensitive, any depth). -DEFAULT_PII_KEYS = [ - 'first_name', - 'last_name', - 'worker_first_name', - 'worker_last_name', - 'email', - 'worker_email', - 'phone', - 'worker_phone', - 'ssn', - 'social_security_number', - 'fein', - 'ein', -] - -# Value patterns redacted by the default PII filter. -DEFAULT_PII_PATTERNS = [ - r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}', # email - r'\b\d{3}-\d{2}-\d{4}\b', # US SSN - r'\b\d{2}-\d{7}\b', # US EIN / FEIN - r'\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}', # US phone -] - - -def redaction_enabled(): - """Return whether default PII redaction should be registered. - - Controlled by the ``ACAI_LOG_REDACTION`` environment variable; redaction is - on unless it is set to ``off``, ``false``, ``0``, or ``disabled``. - """ - setting = os.getenv('ACAI_LOG_REDACTION', 'on').strip().lower() - return setting not in ('off', 'false', '0', 'disabled') - - -def register_default_redaction(): - """Register the default PII redaction filter on ``CommonLogger``.""" - CommonLogger.register_callback( - redaction_filter(keys=DEFAULT_PII_KEYS, patterns=DEFAULT_PII_PATTERNS) - ) diff --git a/acai_aws/common/logger/redaction.py b/acai_aws/common/logger/redaction.py index 66b83b2..17a9764 100644 --- a/acai_aws/common/logger/redaction.py +++ b/acai_aws/common/logger/redaction.py @@ -1,42 +1,66 @@ +import os import re -DEFAULT_REDACTION = '[REDACTED]' +class RedactionFilter: -def redaction_filter(keys=None, patterns=None, redact_with=DEFAULT_REDACTION): - """Build a logger callback that redacts sensitive data from the log payload. + DEFAULT_REDACTION = '[REDACTED]' + REDACTION_ENV = 'ACAI_LOG_REDACTION' + DISABLED_VALUES = ('off', 'false', '0', 'disabled') - ``keys`` are field names redacted wherever they appear in the ``log`` - payload (case-insensitive, at any nesting depth). ``patterns`` are regular - expression strings; every match found in a string value is replaced. - ``redact_with`` is the replacement text (default ``[REDACTED]``); pass - ``redact_with=''`` or any custom string to override it. + DEFAULT_KEYS = ( + 'first_name', + 'last_name', + 'worker_first_name', + 'worker_last_name', + 'email', + 'worker_email', + 'phone', + 'worker_phone', + 'ssn', + 'social_security_number', + 'fein', + 'ein', + ) - Returns a callback compatible with ``CommonLogger.register_callback``; the - custom filter therefore rides on the generic callback hook rather than - special-casing the logger. - """ - lowered_keys = {str(key).lower() for key in (keys or [])} - compiled_patterns = [re.compile(pattern) for pattern in (patterns or [])] + DEFAULT_PATTERNS = ( + r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}', + r'\b\d{3}-\d{2}-\d{4}\b', + r'\b\d{2}-\d{7}\b', + r'\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}', + ) - def scrub(value): + def __init__(self, **kwargs): + self._keys = {str(key).lower() for key in kwargs.get('keys') or []} + self._patterns = [re.compile(pattern) for pattern in kwargs.get('patterns') or []] + self._redact_with = kwargs.get('redact_with', self.DEFAULT_REDACTION) + + @classmethod + def register_default(cls, logger_class): + if os.environ.get(cls.REDACTION_ENV, 'on').strip().lower() in cls.DISABLED_VALUES: + return + logger_class.register_callback(cls(keys=cls.DEFAULT_KEYS, patterns=cls.DEFAULT_PATTERNS)) + + def __call__(self, record): + if isinstance(record, dict) and 'log' in record: + record['log'] = self._scrub(record['log']) + return record + + def _scrub(self, value): if isinstance(value, dict): - return { - key: redact_with if str(key).lower() in lowered_keys else scrub(item) - for key, item in value.items() - } + return {key: self._scrub_field(key, item) for key, item in value.items()} if isinstance(value, (list, tuple)): - return [scrub(item) for item in value] + return [self._scrub(item) for item in value] if isinstance(value, str): - redacted = value - for pattern in compiled_patterns: - redacted = pattern.sub(redact_with, redacted) - return redacted + return self._scrub_text(value) return value - def callback(record): - if isinstance(record, dict) and 'log' in record: - record['log'] = scrub(record['log']) - return record + def _scrub_field(self, key, value): + if str(key).lower() in self._keys: + return self._redact_with + return self._scrub(value) - return callback + def _scrub_text(self, value): + for pattern in self._patterns: + value = pattern.sub(self._redact_with, value) + return value diff --git a/tests/acai_aws/common/test_logger.py b/tests/acai_aws/common/test_logger.py index 791e10d..60c8e13 100644 --- a/tests/acai_aws/common/test_logger.py +++ b/tests/acai_aws/common/test_logger.py @@ -7,7 +7,7 @@ from acai_aws.common import logger from acai_aws.common.logger.decorator import log from acai_aws.common.logger.common_logger import CommonLogger -from acai_aws.common.logger.redaction import redaction_filter +from acai_aws.common.logger.redaction import RedactionFilter def some_log_condition(*args, **_): if args[0] == 1: @@ -188,7 +188,7 @@ def test_callback_runs_before_print(self): @mock.patch.dict(os.environ, {'LOG_FORMAT': 'JSON', 'LOG_LEVEL': 'INFO'}) def test_redaction_filter_runs_through_logger(self): - CommonLogger.register_callback(redaction_filter(keys=['email'])) + CommonLogger.register_callback(RedactionFilter(keys=['email'])) buffer = io.StringIO() with redirect_stdout(buffer): logger.log(level='INFO', log={'email': 'ada@example.com', 'id': 1}) diff --git a/tests/acai_aws/common/test_redaction.py b/tests/acai_aws/common/test_redaction.py index 35eb303..79a3b2c 100644 --- a/tests/acai_aws/common/test_redaction.py +++ b/tests/acai_aws/common/test_redaction.py @@ -1,54 +1,53 @@ from unittest import TestCase -from acai_aws.common.logger.redaction import redaction_filter -from acai_aws.common.logger.defaults import DEFAULT_PII_KEYS, DEFAULT_PII_PATTERNS +from acai_aws.common.logger.redaction import RedactionFilter class RedactionFilterTest(TestCase): def test_redacts_matching_keys(self): - callback = redaction_filter(keys=['email', 'last_name']) - record = callback({'log': {'first_name': 'Ada', 'last_name': 'Lovelace', 'email': 'ada@example.com'}}) + redact = RedactionFilter(keys=['email', 'last_name']) + record = redact({'log': {'first_name': 'Ada', 'last_name': 'Lovelace', 'email': 'ada@example.com'}}) self.assertEqual('Ada', record['log']['first_name']) self.assertEqual('[REDACTED]', record['log']['last_name']) self.assertEqual('[REDACTED]', record['log']['email']) def test_key_match_is_case_insensitive(self): - callback = redaction_filter(keys=['email']) - record = callback({'log': {'Email': 'ada@example.com'}}) + redact = RedactionFilter(keys=['email']) + record = redact({'log': {'Email': 'ada@example.com'}}) self.assertEqual('[REDACTED]', record['log']['Email']) def test_redacts_nested_keys_in_dicts_and_lists(self): - callback = redaction_filter(keys=['ssn']) + redact = RedactionFilter(keys=['ssn']) payload = {'worker': {'ssn': '123-45-6789', 'id': 5}, 'items': [{'ssn': '111-22-3333'}]} - record = callback({'log': payload}) + record = redact({'log': payload}) self.assertEqual('[REDACTED]', record['log']['worker']['ssn']) self.assertEqual(5, record['log']['worker']['id']) self.assertEqual('[REDACTED]', record['log']['items'][0]['ssn']) def test_redacts_value_patterns(self): - callback = redaction_filter(patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) - record = callback({'log': {'note': 'ssn is 123-45-6789 today'}}) + redact = RedactionFilter(patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) + record = redact({'log': {'note': 'ssn is 123-45-6789 today'}}) self.assertEqual('ssn is [REDACTED] today', record['log']['note']) def test_custom_redact_with(self): - callback = redaction_filter(keys=['email'], redact_with='') - record = callback({'log': {'email': 'ada@example.com'}}) + redact = RedactionFilter(keys=['email'], redact_with='') + record = redact({'log': {'email': 'ada@example.com'}}) self.assertEqual('', record['log']['email']) def test_leaves_non_matching_data_untouched(self): - callback = redaction_filter(keys=['email'], patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) - record = callback({'log': {'count': 3, 'ok': True, 'name': 'service-payroll'}}) + redact = RedactionFilter(keys=['email'], patterns=[r'\b\d{3}-\d{2}-\d{4}\b']) + record = redact({'log': {'count': 3, 'ok': True, 'name': 'service-payroll'}}) self.assertEqual({'count': 3, 'ok': True, 'name': 'service-payroll'}, record['log']) def test_handles_record_without_log_key(self): - callback = redaction_filter(keys=['email']) - record = callback({'level': 'INFO'}) + redact = RedactionFilter(keys=['email']) + record = redact({'level': 'INFO'}) self.assertEqual({'level': 'INFO'}, record) - def test_default_pii_keys_and_patterns(self): - callback = redaction_filter(keys=DEFAULT_PII_KEYS, patterns=DEFAULT_PII_PATTERNS) - record = callback({'log': { + def test_default_keys_and_patterns(self): + redact = RedactionFilter(keys=RedactionFilter.DEFAULT_KEYS, patterns=RedactionFilter.DEFAULT_PATTERNS) + record = redact({'log': { 'first_name': 'Ada', 'worker_email': 'ada@example.com', 'ein': '12-3456789', From 2a504e1b011c4bf0ddfb052352aff16c390e6c43 Mon Sep 17 00:00:00 2001 From: Paul Cruse III Date: Tue, 16 Jun 2026 17:10:44 -0500 Subject: [PATCH 3/5] docs: document logger callbacks + RedactionFilter Add a Logging & Redaction section to the README (callback hook + custom filter usage, default-on PII redaction, ACAI_LOG_REDACTION opt-out) and extend the .agents AGENTS.md logger guidance with the same. --- .agents/AGENTS.md | 9 +++++++++ README.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index b38f0a9..3d11ff2 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -82,6 +82,15 @@ Tips when editing event modules: - `acai_aws.common.logger` logs structured JSON by default. Switching `LOG_FORMAT=INLINE` helps during local dev while `LOG_FORMAT=JSON` keeps CloudWatch-friendly output. `LOG_LEVEL` gates log emission (`INFO`, `DEBUG`, `WARN`, `ERROR`). - The `@log` decorator wraps any function, optionally gating logs with a boolean `condition`. Maintain argument pass-through so debugging remains straightforward. - Error traces should include the stack plus the high-level message; tests assert the JSON keys stay consistent (`level`, `time`, `error_trace`, `log`). +- Records pass through callbacks registered with `CommonLogger.register_callback(callback)` before they print; each callback receives the `{level, time, trace, log}` record and returns the record to emit (redaction, enrichment, routing). `CommonLogger.reset_callbacks()` clears them. +- `RedactionFilter(keys=[...], patterns=[...], redact_with='[REDACTED]')` is a callback that scrubs matching field names (case-insensitive, any depth) and regex value matches. A default filter covering common PII (names, email, phone, SSN, EIN) is auto-registered at import; disable with `ACAI_LOG_REDACTION=off`. + +```python +from acai_aws.common.logger.common_logger import CommonLogger +from acai_aws.common.logger.redaction import RedactionFilter + +CommonLogger.register_callback(RedactionFilter(keys=['account_number'])) +``` ## Agent Checklist 1. **Understand the route or event** you’re touching—confirm how the filesystem maps to the API or which event module processes the payload. diff --git a/README.md b/README.md index d1e25d5..85f1fc6 100644 --- a/README.md +++ b/README.md @@ -440,11 +440,55 @@ The old `raise_body_error=True` and `raise_operation_error=True` kwargs still wo --- +## 🪵 Logging & Redaction + +`acai_aws.common.logger` emits structured records to stdout. `LOG_FORMAT` selects `JSON` (default), `PRETTY`, or `INLINE`; `LOG_LEVEL` (`DEBUG`, `INFO`, `WARN`, `ERROR`) gates emission. + +```python +from acai_aws.common import logger + +logger.log(level='INFO', log={'message': 'worker created', 'worker_id': worker_id}) +``` + +### Callbacks + +Every record (`{level, time, trace, log}`) passes through registered callbacks before it is printed. A callback receives the record and returns the record to emit, so it can redact, enrich, or route. Callbacks run in registration order. + +```python +from acai_aws.common.logger.common_logger import CommonLogger + + +def add_service(record): + record['log']['service'] = 'payroll' + return record + + +CommonLogger.register_callback(add_service) +``` + +### Redaction + +`RedactionFilter` is a callback that scrubs sensitive data: field names (case-insensitive, at any depth) and regex matches in string values. `redact_with` overrides the `[REDACTED]` default. + +```python +from acai_aws.common.logger.common_logger import CommonLogger +from acai_aws.common.logger.redaction import RedactionFilter + +CommonLogger.register_callback(RedactionFilter(keys=['account_number'], redact_with='***')) + +logger.log(level='INFO', log={'account_number': '987654321', 'amount': 100}) +# emitted log -> {'account_number': '***', 'amount': 100} +``` + +A default `RedactionFilter` covering common PII (names, email, phone, SSN, EIN) is registered automatically, so PII is scrubbed before it reaches stdout. Disable it with `ACAI_LOG_REDACTION=off`. + +--- + ## 🧰 Tooling & Development Experience - **OpenAPI Generator** – CLI (`python -m acai_aws.apigateway generate-openapi`) scans handlers and updates schema docs - **Request/Response Helpers** – Access JSON, GraphQL, form, XML, or raw bodies via `Request.json`, `Request.form`, etc. -- **Logging** – Configurable JSON/inline logging via `acai_aws.common.logger` +- **Logging** – Structured JSON/inline logging via `acai_aws.common.logger`, with pluggable pre-print callbacks and built-in PII redaction - **Validation** – JSON Schema (Draft 7) and Pydantic support with helpful error messages --- From 3501c494ca11acb3a90dd9ee4ac00b4d7f766a3d Mon Sep 17 00:00:00 2001 From: Paul Cruse III Date: Tue, 16 Jun 2026 17:13:55 -0500 Subject: [PATCH 4/5] logger: drop default-on redaction, make RedactionFilter opt-in Remove the auto-registered default PII filter, the DEFAULT_KEYS / DEFAULT_PATTERNS lists, the register_default classmethod, and the ACAI_LOG_REDACTION env switch. RedactionFilter is now purely opt-in: callers register it with their own keys/patterns. Docs updated to match. --- .agents/AGENTS.md | 2 +- README.md | 4 ++-- acai_aws/common/logger/__init__.py | 2 -- acai_aws/common/logger/redaction.py | 31 ------------------------- tests/acai_aws/common/test_redaction.py | 17 +++++++------- 5 files changed, 12 insertions(+), 44 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 3d11ff2..648f8b7 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -83,7 +83,7 @@ Tips when editing event modules: - The `@log` decorator wraps any function, optionally gating logs with a boolean `condition`. Maintain argument pass-through so debugging remains straightforward. - Error traces should include the stack plus the high-level message; tests assert the JSON keys stay consistent (`level`, `time`, `error_trace`, `log`). - Records pass through callbacks registered with `CommonLogger.register_callback(callback)` before they print; each callback receives the `{level, time, trace, log}` record and returns the record to emit (redaction, enrichment, routing). `CommonLogger.reset_callbacks()` clears them. -- `RedactionFilter(keys=[...], patterns=[...], redact_with='[REDACTED]')` is a callback that scrubs matching field names (case-insensitive, any depth) and regex value matches. A default filter covering common PII (names, email, phone, SSN, EIN) is auto-registered at import; disable with `ACAI_LOG_REDACTION=off`. +- `RedactionFilter(keys=[...], patterns=[...], redact_with='[REDACTED]')` is a callback that scrubs matching field names (case-insensitive, any depth) and regex value matches. It is opt-in: register it with `CommonLogger.register_callback(...)` to apply it. ```python from acai_aws.common.logger.common_logger import CommonLogger diff --git a/README.md b/README.md index 85f1fc6..47753d2 100644 --- a/README.md +++ b/README.md @@ -480,7 +480,7 @@ logger.log(level='INFO', log={'account_number': '987654321', 'amount': 100}) # emitted log -> {'account_number': '***', 'amount': 100} ``` -A default `RedactionFilter` covering common PII (names, email, phone, SSN, EIN) is registered automatically, so PII is scrubbed before it reaches stdout. Disable it with `ACAI_LOG_REDACTION=off`. +Filters are opt-in: register the ones you want at startup and they apply to every subsequent log. --- @@ -488,7 +488,7 @@ A default `RedactionFilter` covering common PII (names, email, phone, SSN, EIN) - **OpenAPI Generator** – CLI (`python -m acai_aws.apigateway generate-openapi`) scans handlers and updates schema docs - **Request/Response Helpers** – Access JSON, GraphQL, form, XML, or raw bodies via `Request.json`, `Request.form`, etc. -- **Logging** – Structured JSON/inline logging via `acai_aws.common.logger`, with pluggable pre-print callbacks and built-in PII redaction +- **Logging** – Structured JSON/inline logging via `acai_aws.common.logger`, with pluggable pre-print callbacks and an opt-in PII redaction filter - **Validation** – JSON Schema (Draft 7) and Pydantic support with helpful error messages --- diff --git a/acai_aws/common/logger/__init__.py b/acai_aws/common/logger/__init__.py index 82c4848..717f813 100644 --- a/acai_aws/common/logger/__init__.py +++ b/acai_aws/common/logger/__init__.py @@ -5,8 +5,6 @@ __all__ = ['log', 'CommonLogger', 'RedactionFilter'] -RedactionFilter.register_default(logger_class=CommonLogger) - def log(**kwargs): try: diff --git a/acai_aws/common/logger/redaction.py b/acai_aws/common/logger/redaction.py index 17a9764..f7a57e1 100644 --- a/acai_aws/common/logger/redaction.py +++ b/acai_aws/common/logger/redaction.py @@ -1,46 +1,15 @@ -import os import re class RedactionFilter: DEFAULT_REDACTION = '[REDACTED]' - REDACTION_ENV = 'ACAI_LOG_REDACTION' - DISABLED_VALUES = ('off', 'false', '0', 'disabled') - - DEFAULT_KEYS = ( - 'first_name', - 'last_name', - 'worker_first_name', - 'worker_last_name', - 'email', - 'worker_email', - 'phone', - 'worker_phone', - 'ssn', - 'social_security_number', - 'fein', - 'ein', - ) - - DEFAULT_PATTERNS = ( - r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}', - r'\b\d{3}-\d{2}-\d{4}\b', - r'\b\d{2}-\d{7}\b', - r'\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}', - ) def __init__(self, **kwargs): self._keys = {str(key).lower() for key in kwargs.get('keys') or []} self._patterns = [re.compile(pattern) for pattern in kwargs.get('patterns') or []] self._redact_with = kwargs.get('redact_with', self.DEFAULT_REDACTION) - @classmethod - def register_default(cls, logger_class): - if os.environ.get(cls.REDACTION_ENV, 'on').strip().lower() in cls.DISABLED_VALUES: - return - logger_class.register_callback(cls(keys=cls.DEFAULT_KEYS, patterns=cls.DEFAULT_PATTERNS)) - def __call__(self, record): if isinstance(record, dict) and 'log' in record: record['log'] = self._scrub(record['log']) diff --git a/tests/acai_aws/common/test_redaction.py b/tests/acai_aws/common/test_redaction.py index 79a3b2c..bda613a 100644 --- a/tests/acai_aws/common/test_redaction.py +++ b/tests/acai_aws/common/test_redaction.py @@ -45,17 +45,18 @@ def test_handles_record_without_log_key(self): record = redact({'level': 'INFO'}) self.assertEqual({'level': 'INFO'}, record) - def test_default_keys_and_patterns(self): - redact = RedactionFilter(keys=RedactionFilter.DEFAULT_KEYS, patterns=RedactionFilter.DEFAULT_PATTERNS) + def test_keys_and_patterns_together(self): + redact = RedactionFilter( + keys=['first_name', 'email'], + patterns=[r'\b\d{3}-\d{2}-\d{4}\b', r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'], + ) record = redact({'log': { 'first_name': 'Ada', - 'worker_email': 'ada@example.com', - 'ein': '12-3456789', - 'message': 'call 585-555-1234 or email ada@example.com', + 'email': 'ada@example.com', + 'message': 'ssn 123-45-6789 mailto ada@example.com', }}) log = record['log'] self.assertEqual('[REDACTED]', log['first_name']) - self.assertEqual('[REDACTED]', log['worker_email']) - self.assertEqual('[REDACTED]', log['ein']) - self.assertNotIn('585-555-1234', log['message']) + self.assertEqual('[REDACTED]', log['email']) + self.assertNotIn('123-45-6789', log['message']) self.assertNotIn('ada@example.com', log['message']) From 5885e6a6deb4696759088256b8a55d1e1fdfeeb9 Mon Sep 17 00:00:00 2001 From: Paul Cruse III Date: Tue, 16 Jun 2026 17:15:20 -0500 Subject: [PATCH 5/5] docs: show patterns key in RedactionFilter README example --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 47753d2..457330c 100644 --- a/README.md +++ b/README.md @@ -474,10 +474,14 @@ CommonLogger.register_callback(add_service) from acai_aws.common.logger.common_logger import CommonLogger from acai_aws.common.logger.redaction import RedactionFilter -CommonLogger.register_callback(RedactionFilter(keys=['account_number'], redact_with='***')) - -logger.log(level='INFO', log={'account_number': '987654321', 'amount': 100}) -# emitted log -> {'account_number': '***', 'amount': 100} +CommonLogger.register_callback(RedactionFilter( + keys=['account_number'], + patterns=[r'\b\d{3}-\d{2}-\d{4}\b'], + redact_with='***', +)) + +logger.log(level='INFO', log={'account_number': '987654321', 'note': 'ssn 123-45-6789'}) +# emitted log -> {'account_number': '***', 'note': 'ssn ***'} ``` Filters are opt-in: register the ones you want at startup and they apply to every subsequent log.