diff --git a/aws_lambda_powertools/logging/buffer/functions.py b/aws_lambda_powertools/logging/buffer/functions.py index f21d2429fbb..490a77328e9 100644 --- a/aws_lambda_powertools/logging/buffer/functions.py +++ b/aws_lambda_powertools/logging/buffer/functions.py @@ -1,12 +1,45 @@ from __future__ import annotations +import io import sys import time +import traceback from typing import TYPE_CHECKING, Any if TYPE_CHECKING: import logging from collections.abc import Mapping + from types import FrameType + + +def _get_caller_frame(stacklevel: int) -> FrameType: + """ + Walk up the stack from _create_buffer_record's caller, honouring stacklevel. + + The walk starts at _add_log_record_to_buffer and counts user frames above + the logging call, mirroring logging.Logger.findCaller: the default + stacklevel of 2 reports the direct caller of the Powertools log method, + and each extra level reports one more caller up the stack. Stops at the + outermost frame if stacklevel walks past the top of the stack. + """ + frame = sys._getframe(2) # _add_log_record_to_buffer + for _ in range(stacklevel): + next_frame = frame.f_back + if next_frame is None: + break + frame = next_frame + return frame + + +def _format_stack(frame: FrameType) -> str: + """ + Format the stack starting at frame, the same way logging.Logger.findCaller does. + """ + with io.StringIO() as sio: + sio.write("Stack (most recent call last):\n") + traceback.print_stack(frame, file=sio) + stack = sio.getvalue() + return stack[:-1] if stack.endswith("\n") else stack def _create_buffer_record( @@ -16,6 +49,12 @@ def _create_buffer_record( exc_info: logging._ExcInfoType = None, stack_info: bool = False, extra: Mapping[str, object] | None = None, + stacklevel: int = 2, + pathname: str | None = None, + lineno: int | None = None, + func_name: str | None = None, + logger_name: str | None = None, + sinfo: str | None = None, ) -> dict[str, Any]: """ Create a structured log record for buffering to save in buffer. @@ -31,10 +70,30 @@ def _create_buffer_record( exc_info : logging._ExcInfoType, optional Exception information to be included in the log record. If None, no exception details will be captured. + Only True is resolved to the current exception; a tuple or an + exception instance passed by the caller is kept as-is. stack_info : bool, default False Flag to include stack trace information in the log record. extra : Mapping[str, object], optional Additional context or metadata to be attached to the log record. + stacklevel : int, default 2 + Number of user frames above the logging call to report as the + record's location. The default reports the direct caller. + pathname : str, optional + Source filename of the record. Resolved from the caller frame + honouring stacklevel when not provided. + lineno : int, optional + Source line number of the record. Resolved from the caller frame + honouring stacklevel when not provided. + func_name : str, optional + Source function name of the record. Resolved from the caller frame + honouring stacklevel when not provided. + logger_name : str, optional + Name of the logger the record came from. Records from third-party + loggers carry their own logger name. + sinfo : str, optional + Pre-formatted stack information. Computed from the caller frame + when stack_info is True and no sinfo is provided. Returns ------- @@ -45,30 +104,42 @@ def _create_buffer_record( - Captures caller frame information for precise log source tracking - Automatically handles exception context """ - # Retrieve the caller's frame information to capture precise log context - # Uses inspect.stack() with index 3 to get the original caller's details - caller_frame = sys._getframe(3) + # Records from third-party loggers arrive with their origin already known; + # use it as-is instead of resolving a frame from inside the logging internals. + if pathname is None and lineno is None and func_name is None: + caller_frame = _get_caller_frame(stacklevel) + pathname = caller_frame.f_code.co_filename + lineno = caller_frame.f_lineno + func_name = caller_frame.f_code.co_name + + if stack_info and sinfo is None: + sinfo = _format_stack(caller_frame) # Get the current timestamp timestamp = time.time() - # Dynamically replace exc_info with current system exception information - # This ensures the most recent exception is captured if available - if exc_info: + # Only True resolves to the current exception, mirroring logging.Logger._log. + # A tuple or an exception instance passed by the caller is kept as-is, + # so an exception saved outside its except block still flushes with it. + if exc_info is True: exc_info = sys.exc_info() + elif isinstance(exc_info, BaseException): + exc_info = (type(exc_info), exc_info, exc_info.__traceback__) - # Construct and return the og record dictionary + # Construct and return the log record dictionary return { "level": level, "msg": msg, "args": args, - "filename": caller_frame.f_code.co_filename, - "line": caller_frame.f_lineno, - "function": caller_frame.f_code.co_name, + "pathname": pathname, + "lineno": lineno, + "funcName": func_name, + "name": logger_name, "extra": extra, "timestamp": timestamp, "exc_info": exc_info, "stack_info": stack_info, + "sinfo": sinfo, } diff --git a/aws_lambda_powertools/logging/buffer/handler.py b/aws_lambda_powertools/logging/buffer/handler.py index a70c845c71a..f3bb9d2d503 100644 --- a/aws_lambda_powertools/logging/buffer/handler.py +++ b/aws_lambda_powertools/logging/buffer/handler.py @@ -63,10 +63,17 @@ def emit(self, record: logging.LogRecord) -> None: self.source_logger._logger.handle(record) return + # The record already knows its origin; carry its pathname, line, + # function, logger name and stack info into the buffer instead of + # resolving a frame from inside the logging internals. self.source_logger._add_log_record_to_buffer( level=record.levelno, msg=record.msg, args=record.args, exc_info=record.exc_info, - stack_info=False, + pathname=record.pathname, + lineno=record.lineno, + func_name=record.funcName, + logger_name=record.name, + sinfo=record.stack_info, ) diff --git a/aws_lambda_powertools/logging/logger.py b/aws_lambda_powertools/logging/logger.py index 5d2a515d516..40bdf3848e5 100644 --- a/aws_lambda_powertools/logging/logger.py +++ b/aws_lambda_powertools/logging/logger.py @@ -608,6 +608,7 @@ def debug( exc_info=exc_info, stack_info=stack_info, extra=extra, + stacklevel=stacklevel, ) def info( @@ -662,6 +663,7 @@ def info( exc_info=exc_info, stack_info=stack_info, extra=extra, + stacklevel=stacklevel, ) def warning( @@ -716,6 +718,7 @@ def warning( exc_info=exc_info, stack_info=stack_info, extra=extra, + stacklevel=stacklevel, ) def error( @@ -1098,12 +1101,14 @@ def _create_and_flush_log_record(self, log_line: dict) -> None: log_line : dict[str, Any] Dictionary containing log record details with keys: - 'level': Logging level - - 'filename': Source filename - - 'line': Line number + - 'pathname': Source filename + - 'lineno': Source line number - 'msg': Log message - - 'function': Source function name + - 'funcName': Source function name + - 'name': Name of the logger the record came from - 'extra': Additional context - 'timestamp': Original log creation time + - 'sinfo': Pre-formatted stack information, if any Notes ----- @@ -1111,17 +1116,24 @@ def _create_and_flush_log_record(self, log_line: dict) -> None: Preserves original timestamp and source information. """ record = self._logger.makeRecord( - name=self.name, + name=log_line["name"], level=log_line["level"], - fn=log_line["filename"], - lno=log_line["line"], + fn=log_line["pathname"], + lno=log_line["lineno"], msg=log_line["msg"], args=log_line.get("args", ()), exc_info=log_line["exc_info"], - func=log_line["function"], + func=log_line["funcName"], extra=log_line["extra"], + sinfo=log_line.get("sinfo"), ) + # Restore the creation timestamp, including the milliseconds and the + # time relative to process start, so the record prints as if it had + # never been buffered. makeRecord stamps relativeCreated from the + # flush time, so shift it back by the time spent in the buffer. + record.relativeCreated = record.relativeCreated - (record.created - log_line["timestamp"]) * 1000 record.created = log_line["timestamp"] + record.msecs = (log_line["timestamp"] - int(log_line["timestamp"])) * 1000 self._logger.handle(record) def _add_log_record_to_buffer( @@ -1132,6 +1144,12 @@ def _add_log_record_to_buffer( exc_info: logging._ExcInfoType = None, stack_info: bool = False, extra: Mapping[str, object] | None = None, + stacklevel: int = 2, + pathname: str | None = None, + lineno: int | None = None, + func_name: str | None = None, + logger_name: str | None = None, + sinfo: str | None = None, ) -> None: """ Add log record to buffer with intelligent tracer ID handling. @@ -1145,11 +1163,27 @@ def _add_log_record_to_buffer( args : object Additional arguments for the log message. exc_info : logging._ExcInfoType, optional - Exception information for the log record. + Exception information for the record. stack_info : bool, optional Whether to include stack information. extra : Mapping[str, object], optional Additional contextual information for the log record. + stacklevel : int, optional + Number of user frames above the logging call to report as the + record's location. The default reports the direct caller. + pathname : str, optional + Source filename of the record. Resolved from the caller frame + honouring stacklevel when not provided. + lineno : int, optional + Source line number of the record. Resolved from the caller frame + honouring stacklevel when not provided. + func_name : str, optional + Source function name of the record. Resolved from the caller frame + honouring stacklevel when not provided. + logger_name : str, optional + Name of the logger the record came from. Defaults to this logger. + sinfo : str, optional + Pre-formatted stack information for the record. Raises ------ @@ -1179,6 +1213,12 @@ def _add_log_record_to_buffer( exc_info=exc_info, stack_info=stack_info, extra=extra, + stacklevel=stacklevel, + pathname=pathname, + lineno=lineno, + func_name=func_name, + logger_name=logger_name if logger_name is not None else self.name, + sinfo=sinfo, ) try: self._buffer_cache.add(tracer_id, log_record) diff --git a/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py b/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py index c25ad5e9747..fc7ef4162df 100644 --- a/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py +++ b/tests/functional/logger/required_dependencies/test_powertools_logger_buffer.py @@ -2,10 +2,14 @@ from __future__ import annotations +import datetime as dt import io import json +import logging import random import string +import sys +import time import warnings from collections import namedtuple @@ -13,6 +17,7 @@ from aws_lambda_powertools import Logger from aws_lambda_powertools.logging.buffer import LoggerBufferConfig +from aws_lambda_powertools.logging.utils import copy_config_to_registered_loggers from aws_lambda_powertools.shared import constants from aws_lambda_powertools.warnings import PowertoolsUserWarning @@ -584,3 +589,94 @@ def test_flush_buffer_preserves_percent_style_formatting(stdout, service_name, m # THEN The flushed log should contain the interpolated message log = capture_multiple_logging_statements_output(stdout) assert log[0]["message"] == "user=alice count=42" + + +def test_buffer_honours_stacklevel_when_flushed(stdout, service_name, monkeypatch): + monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f") + + # GIVEN A logger configured with a buffer + logger_buffer_config = LoggerBufferConfig(max_bytes=10240) + logger = Logger(level="DEBUG", service=service_name, stream=stdout, buffer_config=logger_buffer_config) + + def business(): + def wrapper(): + logger.debug("direct caller", stacklevel=2) + logger.debug("caller of the wrapper", stacklevel=3) + + wrapper() + + # WHEN Logging from a nested function with different stacklevels and flushing the buffer + business() + logger.flush_buffer() + + # THEN Each flushed record reports the same location the unbuffered logger would + logs = capture_multiple_logging_statements_output(stdout) + assert "wrapper" in logs[0]["location"] + assert "business" in logs[1]["location"] + + +def test_buffer_preserves_third_party_logger_name_location_and_stack_info(stdout, service_name, monkeypatch): + monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f") + + # GIVEN A logger configured with a buffer, shared with a third-party logger + logger_buffer_config = LoggerBufferConfig(max_bytes=10240) + logger = Logger(level="DEBUG", service=service_name, stream=stdout, buffer_config=logger_buffer_config) + logger.append_keys(stack_info="%(stack_info)s") + external_logger = logging.getLogger("external_library") + external_logger.setLevel(logging.DEBUG) + copy_config_to_registered_loggers(logger, include_buffering=True) + + def library_function(): + external_logger.debug("from a third-party library", stack_info=True) + + # WHEN The third-party logger logs and the buffer is flushed + library_function() + logger.flush_buffer() + + # THEN The location and name are the library's, and the stack info is kept + log = capture_logging_output(stdout) + assert "library_function" in log["location"] + assert log["name"] == "external_library" + assert "Stack (most recent call last)" in log["stack_info"] + + +def test_buffer_preserves_provided_exc_info(stdout, service_name, monkeypatch): + monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f") + + # GIVEN A logger configured with a buffer + logger_buffer_config = LoggerBufferConfig(max_bytes=10240) + logger = Logger(level="DEBUG", service=service_name, stream=stdout, buffer_config=logger_buffer_config) + + # WHEN An exception is saved outside its except block and logged later + try: + raise ValueError("saved") + except ValueError: + saved_exc_info = sys.exc_info() + + logger.debug("later", exc_info=saved_exc_info) + logger.flush_buffer() + + # THEN The flushed record still carries the exception + log = capture_logging_output(stdout) + assert "ValueError: saved" in log["exception"] + assert log["exception_name"] == "ValueError" + + +def test_buffer_preserves_original_timestamp(stdout, service_name, monkeypatch): + monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f") + + # GIVEN A logger configured with a buffer + logger_buffer_config = LoggerBufferConfig(max_bytes=10240) + logger = Logger(level="DEBUG", service=service_name, stream=stdout, buffer_config=logger_buffer_config) + + # WHEN Logging a message, waiting, and then flushing the buffer + before = dt.datetime.now(dt.timezone.utc) + logger.debug("this log line will be flushed") + after = dt.datetime.now(dt.timezone.utc) + time.sleep(0.4) + logger.flush_buffer() + + # THEN The timestamp carries the creation milliseconds, not the flush time + log = capture_logging_output(stdout) + logged_at = dt.datetime.strptime(log["timestamp"], "%Y-%m-%d %H:%M:%S,%f%z") + assert int(before.timestamp() * 1000) - 1 <= int(logged_at.timestamp() * 1000) <= int(after.timestamp() * 1000)