From 12ee8d2698f6412d817590da3ad87b0e2ee81f2b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 25 Jul 2023 14:39:50 +0200 Subject: [PATCH 1/5] fix(parameters): make cache aware of single vs multiple calls Signed-off-by: heitorlessa --- aws_lambda_powertools/utilities/parameters/base.py | 2 +- aws_lambda_powertools/utilities/parameters/types.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 710634636d0..411a2520ae9 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -27,7 +27,7 @@ from aws_lambda_powertools.shared import constants, user_agent from aws_lambda_powertools.shared.functions import resolve_max_age -from aws_lambda_powertools.utilities.parameters.types import TransformOptions +from aws_lambda_powertools.utilities.parameters.types import RecursiveOptions, TransformOptions from .exceptions import GetParameterError, TransformParameterError diff --git a/aws_lambda_powertools/utilities/parameters/types.py b/aws_lambda_powertools/utilities/parameters/types.py index faa06cee89e..a916f1a344d 100644 --- a/aws_lambda_powertools/utilities/parameters/types.py +++ b/aws_lambda_powertools/utilities/parameters/types.py @@ -1,3 +1,4 @@ from aws_lambda_powertools.shared.types import Literal TransformOptions = Literal["json", "binary", "auto", None] +RecursiveOptions = Literal[True, False] From f58f8d7236bdc8a45633048d2db3e91edad7fa8d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 25 Jul 2023 15:16:51 +0200 Subject: [PATCH 2/5] chore: cleanup, add test for single and nested Signed-off-by: heitorlessa --- aws_lambda_powertools/utilities/parameters/base.py | 2 +- aws_lambda_powertools/utilities/parameters/types.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 411a2520ae9..710634636d0 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -27,7 +27,7 @@ from aws_lambda_powertools.shared import constants, user_agent from aws_lambda_powertools.shared.functions import resolve_max_age -from aws_lambda_powertools.utilities.parameters.types import RecursiveOptions, TransformOptions +from aws_lambda_powertools.utilities.parameters.types import TransformOptions from .exceptions import GetParameterError, TransformParameterError diff --git a/aws_lambda_powertools/utilities/parameters/types.py b/aws_lambda_powertools/utilities/parameters/types.py index a916f1a344d..faa06cee89e 100644 --- a/aws_lambda_powertools/utilities/parameters/types.py +++ b/aws_lambda_powertools/utilities/parameters/types.py @@ -1,4 +1,3 @@ from aws_lambda_powertools.shared.types import Literal TransformOptions = Literal["json", "binary", "auto", None] -RecursiveOptions = Literal[True, False] From 110da283f26d257d8bfb4437da8f5b5dda50ceda Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 Dec 2023 17:15:57 +0100 Subject: [PATCH 3/5] fix(logger): utf-8 encoding json Signed-off-by: heitorlessa --- aws_lambda_powertools/logging/formatter.py | 1 + tests/functional/test_logger.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/aws_lambda_powertools/logging/formatter.py b/aws_lambda_powertools/logging/formatter.py index 22419b160d1..ff0f05f9ff5 100644 --- a/aws_lambda_powertools/logging/formatter.py +++ b/aws_lambda_powertools/logging/formatter.py @@ -133,6 +133,7 @@ def __init__( default=self.json_default, separators=(",", ":"), indent=self.json_indent, + ensure_ascii=False, ) self.datefmt = datefmt diff --git a/tests/functional/test_logger.py b/tests/functional/test_logger.py index dbe2ed1917f..50e32f1f974 100644 --- a/tests/functional/test_logger.py +++ b/tests/functional/test_logger.py @@ -1120,3 +1120,21 @@ def filter(self, record): log = capture_multiple_logging_statements_output(stdout) assert log[0]["api_key"] == "REDACTED" assert log[1]["api_key"] != "REDACTED" + + +def test_logger_json_unicode(stdout, service_name): + # GIVEN Logger is initialized + # WHEN a UTF-8 string is logged + logger = Logger(service=service_name, stream=stdout) + + japanese_encoded_string = "\u30b9\u30b3\u30d3\u30eb\u30c7\u30e2\uff12" + japanese_string = "スコビルデモ２" + japanese_field = "47業レルし化" + + logger.info(japanese_encoded_string, custom_field=japanese_field) + + # THEN string should be JSON stringified without data loss + log = capture_logging_output(stdout) + + assert log["message"] == japanese_string + assert log["custom_field"] == japanese_field From bdc94f740ababaa5e4c701b29efda4405aded436 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 11 Dec 2023 14:06:58 +0100 Subject: [PATCH 4/5] chore: add all non_ascii compatible with 3.7+ --- tests/functional/test_logger.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/functional/test_logger.py b/tests/functional/test_logger.py index 50e32f1f974..fc6e5b98ee8 100644 --- a/tests/functional/test_logger.py +++ b/tests/functional/test_logger.py @@ -1124,17 +1124,19 @@ def filter(self, record): def test_logger_json_unicode(stdout, service_name): # GIVEN Logger is initialized - # WHEN a UTF-8 string is logged logger = Logger(service=service_name, stream=stdout) - japanese_encoded_string = "\u30b9\u30b3\u30d3\u30eb\u30c7\u30e2\uff12" - japanese_string = "スコビルデモ２" + # WHEN all non-ascii chars are logged as messages + # AND non-ascii is also used as ephemeral fields + # latest: https://www.unicode.org/versions/Unicode15.1.0/#Summary + non_ascii_chars = [chr(i) for i in range(128, 111_411_1)] japanese_field = "47業レルし化" + japanese_string = "スコビルデモ２" - logger.info(japanese_encoded_string, custom_field=japanese_field) + logger.info(non_ascii_chars, **{japanese_field: japanese_string}) - # THEN string should be JSON stringified without data loss + # THEN JSON logs should not try to escape them log = capture_logging_output(stdout) - assert log["message"] == japanese_string - assert log["custom_field"] == japanese_field + assert log["message"] == non_ascii_chars + assert log[japanese_field] == japanese_string From e2f0d57efb6293d577024d93d760fc3ca7938393 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 11 Dec 2023 14:24:55 +0100 Subject: [PATCH 5/5] chore: mention issue for future debuggability --- aws_lambda_powertools/logging/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/logging/formatter.py b/aws_lambda_powertools/logging/formatter.py index ff0f05f9ff5..8b34b326435 100644 --- a/aws_lambda_powertools/logging/formatter.py +++ b/aws_lambda_powertools/logging/formatter.py @@ -133,7 +133,7 @@ def __init__( default=self.json_default, separators=(",", ":"), indent=self.json_indent, - ensure_ascii=False, + ensure_ascii=False, # see #3474 ) self.datefmt = datefmt