From a7b7c5363119f9ce4126d56d23a668465b4d4a06 Mon Sep 17 00:00:00 2001 From: Mohit Anand Date: Tue, 17 Nov 2026 06:12:10 +0000 Subject: [PATCH] fix(logger): emit buffered logs when trace ID is unavailable - Emit log records immediately when no X-Ray trace ID is available, preventing messages produced during initialization or outside a traced Lambda invocation from being silently dropped. - Preserve existing invocation-scoped buffering when a trace ID is available, including buffer isolation and overflow handling. - Update the logger documentation and example to explain the immediate-emission behavior. - Add functional tests covering immediate emission through the Powertools Logger and external loggers configured with buffering, including verification that emitted records are not duplicated when the buffer is subsequently flushed. --- .../references/logger.md | 6 ++ aws_lambda_powertools/logging/logger.py | 61 +++++++++++-------- docs/core/logger.md | 2 + .../getting_started_with_buffering_logs.py | 4 ++ .../test_logger_utils.py | 36 +++++++++++ .../test_powertools_logger_buffer.py | 32 ++++++++++ 6 files changed, 115 insertions(+), 26 deletions(-) diff --git a/.agents/skills/building-lambda-functions-with-powertools-python/references/logger.md b/.agents/skills/building-lambda-functions-with-powertools-python/references/logger.md index 1f1ecb3d3fc..c9e97522f2c 100644 --- a/.agents/skills/building-lambda-functions-with-powertools-python/references/logger.md +++ b/.agents/skills/building-lambda-functions-with-powertools-python/references/logger.md @@ -1064,6 +1064,8 @@ The following environment variables are available to configure Logger at a globa Log buffering enables you to buffer logs for a specific request or invocation. Enable log buffering by passing `logger_buffer` when initializing a Logger instance. You can buffer logs at the `WARNING`, `INFO` or `DEBUG` level, and flush them automatically on error or manually as needed. +When an X-Ray trace ID is unavailable, log records at the configured buffering level are emitted immediately instead of being buffered. This ensures logs generated during initialization or outside a Lambda invocation are not lost. + > **Tip: This is useful when you want to reduce the number of log messages emitted while still having detailed logs when needed, such as when troubleshooting issues.** **getting_started_with_buffering_logs.py** @@ -1076,6 +1078,10 @@ from aws_lambda_powertools.utilities.typing import LambdaContext logger_buffer_config = LoggerBufferConfig(max_bytes=20480, flush_on_error_log=True) logger = Logger(level="INFO", buffer_config=logger_buffer_config) +# This initialization-phase log is emitted immediately because no +# X-Ray trace ID is available yet. +logger.debug("Logger initialized") + def lambda_handler(event: dict, context: LambdaContext): logger.debug("a debug log") # this is buffered logger.info("an info log") # this is not buffered diff --git a/aws_lambda_powertools/logging/logger.py b/aws_lambda_powertools/logging/logger.py index 5d2a515d516..631acb20512 100644 --- a/aws_lambda_powertools/logging/logger.py +++ b/aws_lambda_powertools/logging/logger.py @@ -1162,36 +1162,45 @@ def _add_log_record_to_buffer( between different tracer contexts. """ - # Determine tracer ID, defaulting to first invoke marker + # Get the current X-Ray trace ID tracer_id = get_tracer_id() - if tracer_id and self._buffer_config: - if not self._buffer_cache.get(tracer_id): - # Detect new Lambda invocation context and reset buffer to maintain log isolation - # Ensures logs from previous invocations do not leak into current execution - # Prevent memory excessive usage - self._buffer_cache.clear() - - log_record: dict[str, Any] = _create_buffer_record( - level=level, - msg=msg, - args=args, - exc_info=exc_info, - stack_info=stack_info, - extra=extra, + if not self._buffer_config: + return + + log_record: dict[str, Any] = _create_buffer_record( + level=level, + msg=msg, + args=args, + exc_info=exc_info, + stack_info=stack_info, + extra=extra, + ) + + if not tracer_id: + # Without a tracer ID, buffer isolation is not possible. + # Emit immediately to avoid losing the log record. + self._create_and_flush_log_record(log_record) + return + + if not self._buffer_cache.get(tracer_id): + # Detect new Lambda invocation context and reset buffer to maintain log isolation + # Ensures logs from previous invocations do not leak into current execution + # Prevent memory excessive usage + self._buffer_cache.clear() + + try: + self._buffer_cache.add(tracer_id, log_record) + except BufferError: + warnings.warn( + message="Cannot add item to the buffer. " + f"Item size exceeds total cache size {self._buffer_config.max_bytes} bytes", + category=PowertoolsUserWarning, + stacklevel=2, ) - try: - self._buffer_cache.add(tracer_id, log_record) - except BufferError: - warnings.warn( - message="Cannot add item to the buffer. " - f"Item size exceeds total cache size {self._buffer_config.max_bytes} bytes", - category=PowertoolsUserWarning, - stacklevel=2, - ) - # flush this log to avoid data loss - self._create_and_flush_log_record(log_record) + # flush this log to avoid data loss + self._create_and_flush_log_record(log_record) def flush_buffer(self) -> None: """ diff --git a/docs/core/logger.md b/docs/core/logger.md index 1f60b9ff124..996a6fdcb6b 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -556,6 +556,8 @@ The following environment variables are available to configure Logger at a globa Log buffering enables you to buffer logs for a specific request or invocation. Enable log buffering by passing `logger_buffer` when initializing a Logger instance. You can buffer logs at the `WARNING`, `INFO` or `DEBUG` level, and flush them automatically on error or manually as needed. +When an X-Ray trace ID is unavailable, log records at the configured buffering level are emitted immediately instead of being buffered. This ensures logs generated during initialization or outside a Lambda invocation are not lost. + !!! tip "This is useful when you want to reduce the number of log messages emitted while still having detailed logs when needed, such as when troubleshooting issues." === "getting_started_with_buffering_logs.py" diff --git a/examples/logger/src/getting_started_with_buffering_logs.py b/examples/logger/src/getting_started_with_buffering_logs.py index 8e210662aa0..5fb633a1a81 100644 --- a/examples/logger/src/getting_started_with_buffering_logs.py +++ b/examples/logger/src/getting_started_with_buffering_logs.py @@ -5,6 +5,10 @@ logger_buffer_config = LoggerBufferConfig(max_bytes=20480, flush_on_error_log=True) logger = Logger(level="INFO", buffer_config=logger_buffer_config) +# This initialization-phase log is emitted immediately because no +# X-Ray trace ID is available yet. +logger.debug("Logger initialized") + def lambda_handler(event: dict, context: LambdaContext): logger.debug("a debug log") # this is buffered diff --git a/tests/functional/logger/required_dependencies/test_logger_utils.py b/tests/functional/logger/required_dependencies/test_logger_utils.py index 24a082729df..c92166ddadb 100644 --- a/tests/functional/logger/required_dependencies/test_logger_utils.py +++ b/tests/functional/logger/required_dependencies/test_logger_utils.py @@ -356,6 +356,42 @@ def test_copy_config_with_buffering_uses_buffering_handler(stdout, logger, log_l assert "Test message from external logger" in output +def test_copy_config_with_buffering_logs_immediately_without_tracer_id(stdout, logger, log_level): + # GIVEN an external logger and Powertools logger with buffer_config + ext_logger = logger() + buffer_config = LoggerBufferConfig(max_bytes=10240, buffer_at_verbosity="WARNING") + powertools_logger = Logger( + service=service_name(), + level=log_level.INFO.value, + stream=stdout, + buffer_config=buffer_config, + ) + + # WHEN configuration is copied with include_buffering=True + utils.copy_config_to_registered_loggers( + source_logger=powertools_logger, + include_buffering=True, + include={ext_logger.name}, + ) + + # AND the external logger emits a message without a tracer ID + with patch("aws_lambda_powertools.logging.logger.get_tracer_id", return_value=None): + ext_logger.info("Test message from external logger") + + # THEN the message is emitted immediately + log = capture_logging_output(stdout) + assert log["message"] == "Test message from external logger" + + # AND a subsequent flush with a tracer ID does not emit it again + with patch( + "aws_lambda_powertools.logging.logger.get_tracer_id", + return_value="1-67c39786-5908a82a246fb67f3089263f", + ): + powertools_logger.flush_buffer() + + assert len(stdout.getvalue().strip().splitlines()) == 1 + + def test_copy_config_buffering_without_config_uses_normal_handlers(stdout, logger, log_level): # GIVEN an external logger and Powertools logger WITHOUT buffer_config ext_logger = logger() diff --git a/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py b/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py index c25ad5e9747..0809025dfcb 100644 --- a/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py +++ b/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py @@ -584,3 +584,35 @@ def test_flush_buffer_preserves_percent_style_formatting(stdout, service_name, m # THEN The flushed log should contain the interpolated message log = capture_multiple_logging_statements_output(stdout) assert log[0]["message"] == "user=alice count=42" + + +def test_logger_buffer_logs_immediately_without_trace_id(stdout, service_name, monkeypatch): + monkeypatch.delenv(constants.XRAY_TRACE_ID_ENV, raising=False) + + logger_buffer_config = LoggerBufferConfig( + max_bytes=10240, + buffer_at_verbosity="DEBUG", + ) + logger = Logger( + level="DEBUG", + service=service_name, + stream=stdout, + buffer_config=logger_buffer_config, + ) + + logger.debug("This message should be logged immediately") + + log = capture_multiple_logging_statements_output(stdout) + + assert len(log) == 1 + assert log[0]["message"] == "This message should be logged immediately" + + # Simulate the trace ID becoming available when the Lambda invocation starts + monkeypatch.setenv( + constants.XRAY_TRACE_ID_ENV, + "1-67c39786-5908a82a246fb67f3089263f", + ) + + logger.flush_buffer() + + assert len(capture_multiple_logging_statements_output(stdout)) == 1