From 07840bb49adf06893746ad8c856dc58d5499d8bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20D=C3=A9fosse?= Date: Fri, 28 Jul 2023 17:53:42 +0200 Subject: [PATCH 1/4] docs(batchprocessor): mention success/failure handlers record arg discrepancy --- docs/utilities/batch.md | 16 +++++++++++----- ...ailure.py => extending_processor_handlers.py} | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) rename examples/batch_processing/src/{extending_failure.py => extending_processor_handlers.py} (79%) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 57e0a6020b1..fd8aed7eac0 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -522,14 +522,20 @@ You might want to bring custom logic to the existing `BatchProcessor` to slightl For these scenarios, you can subclass `BatchProcessor` and quickly override `success_handler` and `failure_handler` methods: -* **`success_handler()`** – Keeps track of successful batch records -* **`failure_handler()`** – Keeps track of failed batch records +* **`success_handler()`** – is called for each successfully processed record +* **`failure_handler()`** – is called for each failed record + +???+ warning + These functions have a common `record` argument. For backward compatibility reasons, their type is not the same: + + - `success_handler`: `record` type is `dict[str, Any]`, the raw record data + - `failure_handler`: `record` type is a Pydantic model, either the parsed record or the `EventSourceDataClass` record in case of validation error. Refer to [Accessing processed messages](#accessing-processed-messages) for more details ???+ example - Let's suppose you'd like to add a metric named `BatchRecordFailures` for each batch record that failed processing + Let's suppose you'd like to add metrics tracking processing successes and failures of your batch records. -```python hl_lines="8 9 16-19 22 38" title="Extending failure handling mechanism in BatchProcessor" ---8<-- "examples/batch_processing/src/extending_failure.py" +```python hl_lines="8 9 18-25 28 44" title="Extending failure handling mechanism in BatchProcessor" +--8<-- "examples/batch_processing/src/extending_processor_handlers.py" ``` ### Create your own partial processor diff --git a/examples/batch_processing/src/extending_failure.py b/examples/batch_processing/src/extending_processor_handlers.py similarity index 79% rename from examples/batch_processing/src/extending_failure.py rename to examples/batch_processing/src/extending_processor_handlers.py index 424c9a5189b..73af5710981 100644 --- a/examples/batch_processing/src/extending_failure.py +++ b/examples/batch_processing/src/extending_processor_handlers.py @@ -1,4 +1,5 @@ import json +from typing import Any from aws_lambda_powertools import Logger, Metrics, Tracer from aws_lambda_powertools.metrics import MetricUnit @@ -9,11 +10,16 @@ FailureResponse, process_partial_response, ) +from aws_lambda_powertools.utilities.batch.base import SuccessResponse from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord from aws_lambda_powertools.utilities.typing import LambdaContext class MyProcessor(BatchProcessor): + def success_handler(self, record: dict[str, Any], result: Any) -> SuccessResponse: + metrics.add_metric(name="BatchRecordSuccesses", unit=MetricUnit.Count, value=1) + return super().success_handler(record, result) + def failure_handler(self, record: SQSRecord, exception: ExceptionInfo) -> FailureResponse: metrics.add_metric(name="BatchRecordFailures", unit=MetricUnit.Count, value=1) return super().failure_handler(record, exception) From 408b35f15eccff1dc7d92d4234138aa25c448472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20D=C3=A9fosse?= Date: Fri, 28 Jul 2023 18:52:39 +0200 Subject: [PATCH 2/4] docs(batchprocessor): address minor change requests --- docs/utilities/batch.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index debee67b345..2617e613a25 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -522,19 +522,19 @@ You might want to bring custom logic to the existing `BatchProcessor` to slightl For these scenarios, you can subclass `BatchProcessor` and quickly override `success_handler` and `failure_handler` methods: -* **`success_handler()`** – is called for each successfully processed record -* **`failure_handler()`** – is called for each failed record +* **`success_handler()`** is called for each successfully processed record +* **`failure_handler()`** is called for each failed record ???+ warning These functions have a common `record` argument. For backward compatibility reasons, their type is not the same: - - `success_handler`: `record` type is `dict[str, Any]`, the raw record data - - `failure_handler`: `record` type is a Pydantic model, either the parsed record or the `EventSourceDataClass` record in case of validation error. Refer to [Accessing processed messages](#accessing-processed-messages) for more details + - `success_handler`: `record` type is `dict[str, Any]`, the raw record data. + - `failure_handler`: `record` type can be an Event Source Data Class or your [Pydantic model](#pydantic-integration). During Pydantic validation errors, we fall back and serialize `record` to Event Source Data Class to not break the processing pipeline. ???+ example - Let's suppose you'd like to add metrics tracking processing successes and failures of your batch records. + Let's suppose you'd like to add metrics to track successes and failures of your batch records. -```python hl_lines="8 9 18-25 28 44" title="Extending failure handling mechanism in BatchProcessor" +```python hl_lines="8-10 18-25 28 44" title="Extending failure handling mechanism in BatchProcessor" --8<-- "examples/batch_processing/src/extending_processor_handlers.py" ``` From 543fe9f78766cd6cd488d532e229d33e0b5a814e Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Fri, 28 Jul 2023 19:12:06 +0200 Subject: [PATCH 3/4] docs: use note callout instead of warning Signed-off-by: Heitor Lessa --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 2617e613a25..ca6cc5f9256 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -525,7 +525,7 @@ For these scenarios, you can subclass `BatchProcessor` and quickly override `suc * **`success_handler()`** is called for each successfully processed record * **`failure_handler()`** is called for each failed record -???+ warning +???+ note These functions have a common `record` argument. For backward compatibility reasons, their type is not the same: - `success_handler`: `record` type is `dict[str, Any]`, the raw record data. From 0580763a88ec83ca1ee41738098b72e89814d724 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 28 Jul 2023 19:14:37 +0200 Subject: [PATCH 4/4] docs(batch): make callout a normal sentence --- docs/utilities/batch.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index ca6cc5f9256..ada05766ab4 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -531,8 +531,7 @@ For these scenarios, you can subclass `BatchProcessor` and quickly override `suc - `success_handler`: `record` type is `dict[str, Any]`, the raw record data. - `failure_handler`: `record` type can be an Event Source Data Class or your [Pydantic model](#pydantic-integration). During Pydantic validation errors, we fall back and serialize `record` to Event Source Data Class to not break the processing pipeline. -???+ example - Let's suppose you'd like to add metrics to track successes and failures of your batch records. +Let's suppose you'd like to add metrics to track successes and failures of your batch records. ```python hl_lines="8-10 18-25 28 44" title="Extending failure handling mechanism in BatchProcessor" --8<-- "examples/batch_processing/src/extending_processor_handlers.py"