From ab7fb69d8922cc53c450698159225a47d0bf1637 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 13 Dec 2022 08:07:25 -0300 Subject: [PATCH 1/4] docs(idempotency): fix batch sample for idempotent_function section --- docs/utilities/idempotency.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 2e3f5f1e88d..f89515a1160 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -162,11 +162,20 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo return {"data": data} - @batch_processor(record_handler=record_handler, processor=processor) def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section + config.register_lambda_context(context) # see Lambda timeouts section + # `data` parameter must be called as a keyword argument to work dummy("hello", "universe", data="test") + + # with Lambda context registered for Idempotency + # we can now kick in the Bach processing logic + batch = event["Records"] + with processor(records=batch, handler=record_handler): + # in case you want to access each record processed by your record_handler + # otherwise ignore the result variable assignment + processed_messages = processor.process() + return processor.response() ``` From e597d7349b9c277affe2d573d760ecf918d2164c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 13 Dec 2022 08:21:04 -0300 Subject: [PATCH 2/4] docs(idempotency): split and improve batch example for visibility --- docs/utilities/idempotency.md | 155 +++++++++++++++++----------------- 1 file changed, 77 insertions(+), 78 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index f89515a1160..149005f6294 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -132,84 +132,6 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo DynamoDB Persistency layer uses a Resource client [which is not thread-safe](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html?highlight=multithreading#multithreading-or-multiprocessing-with-resources){target="_blank"}. -=== "batch_sample.py" - - This example also demonstrates how you can integrate with [Batch utility](batch.md), so you can process each record in an idempotent manner. - - ```python hl_lines="4-5 16 21 30" - from aws_lambda_powertools.utilities.batch import (BatchProcessor, EventType, - batch_processor) - from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) - - - processor = BatchProcessor(event_type=EventType.SQS) - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="messageId", # see Choosing a payload subset section - use_local_cache=True, - ) - - - @idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb) - def record_handler(record: SQSRecord): - return {"message": record["body"]} - - - @idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb) - def dummy(arg_one, arg_two, data: dict, **kwargs): - return {"data": data} - - - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section - - # `data` parameter must be called as a keyword argument to work - dummy("hello", "universe", data="test") - - # with Lambda context registered for Idempotency - # we can now kick in the Bach processing logic - batch = event["Records"] - with processor(records=batch, handler=record_handler): - # in case you want to access each record processed by your record_handler - # otherwise ignore the result variable assignment - processed_messages = processor.process() - - return processor.response() - ``` - -=== "Batch event" - - ```json hl_lines="4" - { - "Records": [ - { - "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", - "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", - "body": "Test message.", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1545082649183", - "SenderId": "AIDAIENQZJOLO23YVJ4VO", - "ApproximateFirstReceiveTimestamp": "1545082649185" - }, - "messageAttributes": { - "testAttr": { - "stringValue": "100", - "binaryValue": "base64Str", - "dataType": "Number" - } - }, - "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", - "awsRegion": "us-east-2" - } - ] - } - ``` - === "dataclass_sample.py" ```python hl_lines="3-4 23 33" @@ -285,6 +207,83 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo process_order(order=order) ``` +#### Batch integration + +You can can easily integrate with [Batch utility](batch.md) via context manager. This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. + +???+ "Choosing an unique batch record attribute" + In this example, we choose `messageId` as our idempotency token since we know it'll be unique. + + Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. + +=== "batch_sample.py" + + ```python hl_lines="4-5 11 16 22 26-27 30 32" + from aws_lambda_powertools.utilities.batch import (BatchProcessor, EventType, + batch_processor) + from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) + + + processor = BatchProcessor(event_type=EventType.SQS) + dynamodb = DynamoDBPersistenceLayer(table_name="idem") + config = IdempotencyConfig( + event_key_jmespath="messageId", # see Choosing a payload subset section + use_local_cache=True, + ) + + + @idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb) + def record_handler(record: SQSRecord): + return {"message": record.body} + + + def lambda_handler(event, context): + config.register_lambda_context(context) # see Lambda timeouts section + + # with Lambda context registered for Idempotency + # we can now kick in the Bach processing logic + batch = event["Records"] + with processor(records=batch, handler=record_handler): + # in case you want to access each record processed by your record_handler + # otherwise ignore the result variable assignment + processed_messages = processor.process() + + return processor.response() + ``` + +=== "batch_event.json" + + ```json hl_lines="4" + { + "Records": [ + { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "body": "Test message.", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1545082649183", + "SenderId": "AIDAIENQZJOLO23YVJ4VO", + "ApproximateFirstReceiveTimestamp": "1545082649185" + }, + "messageAttributes": { + "testAttr": { + "stringValue": "100", + "binaryValue": "base64Str", + "dataType": "Number" + } + }, + "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "awsRegion": "us-east-2" + } + ] + } + ``` + ### Choosing a payload subset for idempotency ???+ tip "Tip: Dealing with always changing payloads" From 5c749577533c45a97a7cc4ce692a6ea7d4c659b3 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 13 Dec 2022 08:22:56 -0300 Subject: [PATCH 3/4] docs(idempotency): lint --- docs/utilities/idempotency.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 149005f6294..37f237d6f8e 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -218,9 +218,8 @@ You can can easily integrate with [Batch utility](batch.md) via context manager. === "batch_sample.py" - ```python hl_lines="4-5 11 16 22 26-27 30 32" - from aws_lambda_powertools.utilities.batch import (BatchProcessor, EventType, - batch_processor) + ```python hl_lines="3-4 10 15 21 25-26 29 31" + from aws_lambda_powertools.utilities.batch import BatchProcessor, EventType from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord from aws_lambda_powertools.utilities.idempotency import ( DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) From 867a7a11ddf1e460c1213d8a6065c72ab0573f2a Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 13 Dec 2022 08:29:35 -0300 Subject: [PATCH 4/4] docs(idempotency): update compatibility index --- docs/utilities/idempotency.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 37f237d6f8e..89cd3003b77 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -989,6 +989,10 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer): ## Compatibility with other utilities +### Batch + +See [Batch integration](#batch-integration) above. + ### Validation utility The idempotency utility can be used with the `validator` decorator. Ensure that idempotency is the innermost decorator.