From bd823d28dd9c74095cd7a1b66065da5c194e2ef3 Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Wed, 22 Jun 2022 07:52:03 -0700 Subject: [PATCH 1/6] Support 30 dimensions per dimension set --- aws_embedded_metrics/constants.py | 2 +- aws_embedded_metrics/exceptions.py | 17 ++++++++++ .../logger/metrics_context.py | 10 ++++++ .../serializers/log_serializer.py | 7 ++++- tests/logger/test_metrics_context.py | 28 +++++++++++++++++ tests/serializer/test_log_serializer.py | 31 ------------------- 6 files changed, 62 insertions(+), 33 deletions(-) create mode 100644 aws_embedded_metrics/exceptions.py diff --git a/aws_embedded_metrics/constants.py b/aws_embedded_metrics/constants.py index 6f99bfa..6cab4a3 100644 --- a/aws_embedded_metrics/constants.py +++ b/aws_embedded_metrics/constants.py @@ -12,6 +12,6 @@ # limitations under the License. DEFAULT_NAMESPACE = "aws-embedded-metrics" -MAX_DIMENSIONS = 9 +MAX_DIMENSIONS = 30 MAX_METRICS_PER_EVENT = 100 MAX_DATAPOINTS_PER_METRIC = 100 diff --git a/aws_embedded_metrics/exceptions.py b/aws_embedded_metrics/exceptions.py new file mode 100644 index 0000000..60839d4 --- /dev/null +++ b/aws_embedded_metrics/exceptions.py @@ -0,0 +1,17 @@ +# Copyright 2022 Amazon.com, Inc. or its affiliates. +# Licensed under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class DimensionsExceededError(Exception): + def __init__(self, message: str) -> None: + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 11a84ca..a8b3fd2 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -14,6 +14,8 @@ from aws_embedded_metrics import constants, utils from aws_embedded_metrics.config import get_config +from aws_embedded_metrics.constants import MAX_DIMENSIONS +from aws_embedded_metrics.exceptions import DimensionsExceededError from aws_embedded_metrics.logger.metric import Metric from typing import List, Dict, Any @@ -66,6 +68,9 @@ def put_dimensions(self, dimensions: Dict[str, str]) -> None: # TODO add ability to define failure strategy return + if len(dimensions) > MAX_DIMENSIONS: + raise DimensionsExceededError(f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}") + self.dimensions.append(dimensions) def set_dimensions(self, dimensionSets: List[Dict[str, str]]) -> None: @@ -78,6 +83,11 @@ def set_dimensions(self, dimensionSets: List[Dict[str, str]]) -> None: ``` """ self.should_use_default_dimensions = False + + for dimensionSet in dimensionSets: + if len(dimensionSet) > MAX_DIMENSIONS: + raise DimensionsExceededError(f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}") + self.dimensions = dimensionSets def set_default_dimensions(self, default_dimensions: Dict) -> None: diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index d5d7ec8..01750a8 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -17,6 +17,7 @@ from aws_embedded_metrics.constants import ( MAX_DIMENSIONS, MAX_METRICS_PER_EVENT, MAX_DATAPOINTS_PER_METRIC ) +from aws_embedded_metrics.exceptions import DimensionsExceededError import json from typing import Any, Dict, List @@ -31,7 +32,11 @@ def serialize(context: MetricsContext) -> List[str]: for dimension_set in context.get_dimensions(): keys = list(dimension_set.keys()) - dimension_keys.append(keys[0:MAX_DIMENSIONS]) + if len(keys) > MAX_DIMENSIONS: + err_msg = (f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}. " + f"Account for default dimensions if not using set_dimensions.") + raise DimensionsExceededError(err_msg) + dimension_keys.append(keys) dimensions_properties = {**dimensions_properties, **dimension_set} def create_body() -> Dict[str, Any]: diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index fde3d82..4fd8b79 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,8 +1,10 @@ from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.constants import DEFAULT_NAMESPACE +from aws_embedded_metrics.exceptions import DimensionsExceededError from importlib import reload from faker import Faker +import pytest fake = Faker() @@ -264,3 +266,29 @@ def test_create_copy_with_context_does_not_repeat_dimensions(): # assert assert len(new_context.get_dimensions()) == 1 + + +def test_cannot_set_more_than_30_dimensions(): + context = MetricsContext() + dimensions_to_add = 32 + dimension_set = generate_dimension_set(dimensions_to_add) + + with pytest.raises(DimensionsExceededError): + context.set_dimensions([dimension_set]) + + +def test_cannot_put_more_than_30_dimensions(): + context = MetricsContext() + dimensions_to_add = 32 + dimension_set = generate_dimension_set(dimensions_to_add) + + with pytest.raises(DimensionsExceededError): + context.put_dimensions(dimension_set) + + +def generate_dimension_set(dimensions_to_add): + dimension_set = {} + for i in range(0, dimensions_to_add): + dimension_set[f"{i}"] = fake.word() + + return dimension_set diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 314adb9..91f2474 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -30,37 +30,6 @@ def test_serialize_dimensions(): assert_json_equality(result_json, expected) -def test_cannot_serialize_more_than_9_dimensions(): - # arrange - dimensions = {} - dimension_pointers = [] - allowed_dimensions = 9 - dimensions_to_add = 15 - - for i in range(0, dimensions_to_add): - print(i) - expected_key = f"{i}" - expected_value = fake.word() - dimensions[expected_key] = expected_value - dimension_pointers.append(expected_key) - - expected_dimensions_pointers = dimension_pointers[0:allowed_dimensions] - - expected = {**get_empty_payload(), **dimensions} - expected["_aws"]["CloudWatchMetrics"][0]["Dimensions"].append( - expected_dimensions_pointers - ) - - context = get_context() - context.put_dimensions(dimensions) - - # act - result_json = serializer.serialize(context)[0] - - # assert - assert_json_equality(result_json, expected) - - def test_serialize_properties(): # arrange expected_key = fake.word() From 04869d2b6ef8f63af993e764852ac436df6c303b Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Thu, 23 Jun 2022 12:43:49 -0700 Subject: [PATCH 2/6] Add more tests --- aws_embedded_metrics/constants.py | 2 +- aws_embedded_metrics/exceptions.py | 2 +- .../logger/metrics_context.py | 14 +++++++------ .../serializers/log_serializer.py | 10 ++++----- tests/logger/test_metrics_context.py | 17 ++++++++------- tests/serializer/test_log_serializer.py | 21 +++++++++++++++++++ tox.ini | 2 +- 7 files changed, 47 insertions(+), 21 deletions(-) diff --git a/aws_embedded_metrics/constants.py b/aws_embedded_metrics/constants.py index 6cab4a3..0b975b7 100644 --- a/aws_embedded_metrics/constants.py +++ b/aws_embedded_metrics/constants.py @@ -12,6 +12,6 @@ # limitations under the License. DEFAULT_NAMESPACE = "aws-embedded-metrics" -MAX_DIMENSIONS = 30 +MAX_DIMENSION_SET_SIZE = 30 MAX_METRICS_PER_EVENT = 100 MAX_DATAPOINTS_PER_METRIC = 100 diff --git a/aws_embedded_metrics/exceptions.py b/aws_embedded_metrics/exceptions.py index 60839d4..2747924 100644 --- a/aws_embedded_metrics/exceptions.py +++ b/aws_embedded_metrics/exceptions.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -class DimensionsExceededError(Exception): +class DimensionSetExceededError(Exception): def __init__(self, message: str) -> None: # Call the base class constructor with the parameters it needs super().__init__(message) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index a8b3fd2..4ea97ee 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -14,8 +14,8 @@ from aws_embedded_metrics import constants, utils from aws_embedded_metrics.config import get_config -from aws_embedded_metrics.constants import MAX_DIMENSIONS -from aws_embedded_metrics.exceptions import DimensionsExceededError +from aws_embedded_metrics.constants import MAX_DIMENSION_SET_SIZE +from aws_embedded_metrics.exceptions import DimensionSetExceededError from aws_embedded_metrics.logger.metric import Metric from typing import List, Dict, Any @@ -68,8 +68,9 @@ def put_dimensions(self, dimensions: Dict[str, str]) -> None: # TODO add ability to define failure strategy return - if len(dimensions) > MAX_DIMENSIONS: - raise DimensionsExceededError(f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}") + if len(dimensions) > MAX_DIMENSION_SET_SIZE: + raise DimensionSetExceededError( + f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}") self.dimensions.append(dimensions) @@ -85,8 +86,9 @@ def set_dimensions(self, dimensionSets: List[Dict[str, str]]) -> None: self.should_use_default_dimensions = False for dimensionSet in dimensionSets: - if len(dimensionSet) > MAX_DIMENSIONS: - raise DimensionsExceededError(f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}") + if len(dimensionSet) > MAX_DIMENSION_SET_SIZE: + raise DimensionSetExceededError( + f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}") self.dimensions = dimensionSets diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index 01750a8..156e868 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -15,9 +15,9 @@ from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers import Serializer from aws_embedded_metrics.constants import ( - MAX_DIMENSIONS, MAX_METRICS_PER_EVENT, MAX_DATAPOINTS_PER_METRIC + MAX_DIMENSION_SET_SIZE, MAX_METRICS_PER_EVENT, MAX_DATAPOINTS_PER_METRIC ) -from aws_embedded_metrics.exceptions import DimensionsExceededError +from aws_embedded_metrics.exceptions import DimensionSetExceededError import json from typing import Any, Dict, List @@ -32,10 +32,10 @@ def serialize(context: MetricsContext) -> List[str]: for dimension_set in context.get_dimensions(): keys = list(dimension_set.keys()) - if len(keys) > MAX_DIMENSIONS: - err_msg = (f"Maximum number of dimensions allowed are {MAX_DIMENSIONS}. " + if len(keys) > MAX_DIMENSION_SET_SIZE: + err_msg = (f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}. " f"Account for default dimensions if not using set_dimensions.") - raise DimensionsExceededError(err_msg) + raise DimensionSetExceededError(err_msg) dimension_keys.append(keys) dimensions_properties = {**dimensions_properties, **dimension_set} diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 4fd8b79..d1f8b0c 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,7 +1,7 @@ from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.constants import DEFAULT_NAMESPACE -from aws_embedded_metrics.exceptions import DimensionsExceededError +from aws_embedded_metrics.exceptions import DimensionSetExceededError from importlib import reload from faker import Faker import pytest @@ -44,14 +44,14 @@ def test_put_dimension_adds_to_dimensions(): # arrange context = MetricsContext() - dimension_key = fake.word() - dimension_value = fake.word() + dimensions_to_add = 30 + dimension_set = generate_dimension_set(dimensions_to_add) # act - context.put_dimensions({dimension_key: dimension_value}) + context.put_dimensions(dimension_set) # assert - assert context.dimensions == [{dimension_key: dimension_value}] + assert context.dimensions == [dimension_set] def test_get_dimensions_returns_only_custom_dimensions_if_no_default_dimensions_not_set(): @@ -273,7 +273,7 @@ def test_cannot_set_more_than_30_dimensions(): dimensions_to_add = 32 dimension_set = generate_dimension_set(dimensions_to_add) - with pytest.raises(DimensionsExceededError): + with pytest.raises(DimensionSetExceededError): context.set_dimensions([dimension_set]) @@ -282,10 +282,13 @@ def test_cannot_put_more_than_30_dimensions(): dimensions_to_add = 32 dimension_set = generate_dimension_set(dimensions_to_add) - with pytest.raises(DimensionsExceededError): + with pytest.raises(DimensionSetExceededError): context.put_dimensions(dimension_set) +# Test utility method + + def generate_dimension_set(dimensions_to_add): dimension_set = {} for i in range(0, dimensions_to_add): diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 91f2474..e892922 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -1,9 +1,11 @@ from aws_embedded_metrics.config import get_config +from aws_embedded_metrics.exceptions import DimensionSetExceededError from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers.log_serializer import LogSerializer from collections import Counter from faker import Faker import json +import pytest fake = Faker() @@ -48,6 +50,25 @@ def test_serialize_properties(): assert_json_equality(result_json, expected) +def test_default_and_custom_dimensions_combined_limit_exceeded(): + # While serializing default dimensions are added to the custom dimension set, + # and the combined size of the dimension set should not be more than 30 + dimensions = {} + default_dimension_key = fake.word() + default_dimension_value = fake.word() + custom_dimensions_to_add = 30 + + for i in range(0, custom_dimensions_to_add): + dimensions[f"{i}"] = fake.word() + + context = get_context() + context.set_default_dimensions({default_dimension_key: default_dimension_value}) + context.put_dimensions(dimensions) + + with pytest.raises(DimensionSetExceededError): + serializer.serialize(context) + + def test_serialize_metrics(): # arrange expected_key = fake.word() diff --git a/tox.ini b/tox.ini index fe51c57..1b62783 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,mypy,py37 +envlist = flake8,mypy,python3.7 [testenv] deps = From 30af327e93e8c70ff21f1cd3c5044b04c3c5135b Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Thu, 23 Jun 2022 17:39:19 -0700 Subject: [PATCH 3/6] Undo tox.ini changes --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1b62783..fe51c57 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,mypy,python3.7 +envlist = flake8,mypy,py37 [testenv] deps = From 3083bc842acfbad1328be8bb4763b7c39ee02236 Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Mon, 27 Jun 2022 11:56:11 -0700 Subject: [PATCH 4/6] validate dimension set in a separate method --- aws_embedded_metrics/logger/metrics_context.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 4ea97ee..cd4f63b 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -57,6 +57,15 @@ def put_metric(self, key: str, value: float, unit: str = None) -> None: else: self.metrics[key] = Metric(value, unit) + @staticmethod + def validate_dimension_set(dimensions): + """ + Validates dimension set length is not more than MAX_DIMENSION_SET_SIZE + """ + if len(dimensions) > MAX_DIMENSION_SET_SIZE: + raise DimensionSetExceededError( + f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}") + def put_dimensions(self, dimensions: Dict[str, str]) -> None: """ Adds dimensions to the context. @@ -68,9 +77,7 @@ def put_dimensions(self, dimensions: Dict[str, str]) -> None: # TODO add ability to define failure strategy return - if len(dimensions) > MAX_DIMENSION_SET_SIZE: - raise DimensionSetExceededError( - f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}") + self.validate_dimension_set(dimensions) self.dimensions.append(dimensions) @@ -86,9 +93,7 @@ def set_dimensions(self, dimensionSets: List[Dict[str, str]]) -> None: self.should_use_default_dimensions = False for dimensionSet in dimensionSets: - if len(dimensionSet) > MAX_DIMENSION_SET_SIZE: - raise DimensionSetExceededError( - f"Maximum number of dimensions per dimension set allowed are {MAX_DIMENSION_SET_SIZE}") + self.validate_dimension_set(dimensionSet) self.dimensions = dimensionSets From 7bd1993448e8171e6972053c565519a6c0eb08e4 Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Mon, 27 Jun 2022 12:08:52 -0700 Subject: [PATCH 5/6] add type annotation --- aws_embedded_metrics/logger/metrics_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index cd4f63b..47ed1dd 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -58,7 +58,7 @@ def put_metric(self, key: str, value: float, unit: str = None) -> None: self.metrics[key] = Metric(value, unit) @staticmethod - def validate_dimension_set(dimensions): + def validate_dimension_set(dimensions: Dict[str, str]) -> None: """ Validates dimension set length is not more than MAX_DIMENSION_SET_SIZE """ From 7f6cb1ca7aa14cc583d70a4658a3679dc8fec3cc Mon Sep 17 00:00:00 2001 From: Himtanaya Bhadada Date: Wed, 29 Jun 2022 10:24:03 -0700 Subject: [PATCH 6/6] Bump version to 2.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab81d6a..5ae82a3 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="aws-embedded-metrics", - version="1.0.8", + version="2.0.0", author="Amazon Web Services", author_email="jarnance@amazon.com", description="AWS Embedded Metrics Package",