Disable flushing metrics in thread

datadog_lambda/metric.py

@@ -41,8 +41,13 @@ def flush(self, value):
 if should_use_extension:
     lambda_stats = StatsDWrapper()
 else:
+    # Periodical flushing in a background thread is NOT guaranteed to succeed
+    # and leads to data loss. When disabled, metrics are only flushed at the
+    # end of invocation. To make metrics submitted from a long-running Lambda
+    # function available sooner, consider using the Datadog Lambda extension.
+    flush_in_thread = os.environ.get("DD_FLUSH_IN_THREAD", "").lower() == "true"
     lambda_stats = ThreadStats(compress_payload=True)
-    lambda_stats.start()
+    lambda_stats.start(flush_in_thread=flush_in_thread)
 
 
 def lambda_metric(metric_name, value, timestamp=None, tags=None, force_async=False):

datadog_lambda/patch.py

@@ -7,6 +7,7 @@
 import os
 import sys
 import logging
+import zlib
 
 from wrapt import wrap_function_wrapper as wrap
 from wrapt.importer import when_imported
@@ -144,6 +145,9 @@ def _print_request_string(request):
 
     # Sort the datapoints POSTed by their name so that snapshots always align
     data = request.body or "{}"
+    if request.headers.get("Content-Encoding") == "deflate":
+        # See metric.py: lambda_stats = ThreadStats(compress_payload=True)
+        data = zlib.decompress(data)
     data_dict = json.loads(data)
     data_dict.get("series", []).sort(key=lambda series: series.get("metric"))
     sorted_data = json.dumps(data_dict)

scripts/run_integration_tests.sh

@@ -16,6 +16,9 @@ CONFIGS=("with-plugin" "without-plugin")
 
 LOGS_WAIT_SECONDS=20
 
+# Force cold start to avoid flaky tests
+export COLD_START_ENFORCER=$((1 + $RANDOM % 100000))
+
 script_path=${BASH_SOURCE[0]}
 scripts_dir=$(dirname $script_path)
 repo_dir=$(dirname $scripts_dir)
@@ -42,6 +45,10 @@ else
 fi
 
 cd $integration_tests_dir
+
+# Install the specified plugin version
+yarn install
+
 input_event_files=$(ls ./input_events)
 # Sort event files by name so that snapshots stay consistent
 input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort))
@@ -127,6 +134,8 @@ for _sls_type in "${CONFIGS[@]}"; do
                 echo "$raw_logs" |
                     # Filter serverless cli errors
                     sed '/Serverless: Recoverable error occurred/d' |
+                    # Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
+                    sed '/RequestsDependencyWarning/d' |
                     # Remove blank lines
                     sed '/^$/d' |
                     # Normalize Lambda runtime report logs

tests/integration/package.json

@@ -1,5 +1,5 @@
 {
   "devDependencies": {
-    "serverless-plugin-datadog": "^2.2.1"
+    "serverless-plugin-datadog": "^2.18.0"
   }
 }

tests/integration/serverless-plugin.yml

@@ -6,6 +6,7 @@ provider:
     DD_INTEGRATION_TEST: true
     DD_API_KEY: ${env:DD_API_KEY}
     WITH_PLUGIN: true
+    COLD_START_ENFORCER: ${env:COLD_START_ENFORCER}
   lambdaHashingVersion: 20201221
 
 layers:
@@ -29,32 +30,24 @@ functions:
     runtime: python2.7
     layers:
       - { Ref: Python27LambdaLayer }
-    environment:
-      DD_FLUSH_TO_LOG: true
 
   async-metrics_python36_with_plugin:
     handler: handle.handle
     runtime: python3.6
     layers:
       - { Ref: Python36LambdaLayer }
-    environment:
-      DD_FLUSH_TO_LOG: true
 
   async-metrics_python37_with_plugin:
     handler: handle.handle
     runtime: python3.7
     layers:
       - { Ref: Python37LambdaLayer }
-    environment:
-      DD_FLUSH_TO_LOG: true
 
   async-metrics_python38_with_plugin:
     handler: handle.handle
     runtime: python3.8
     layers:
       - { Ref: Python38LambdaLayer }
-    environment:
-      DD_FLUSH_TO_LOG: true
 
   # sync-metrics
   sync-metrics_python27_with_plugin:
@@ -85,65 +78,49 @@ functions:
   http-requests_python27_with_plugin:
     handler: http_requests.handle
     runtime: python2.7
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python27LambdaLayer }
 
   http-requests_python36_with_plugin:
     handler: http_requests.handle
     runtime: python3.6
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python36LambdaLayer }
 
   http-requests_python37_with_plugin:
     handler: http_requests.handle
     runtime: python3.7
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python37LambdaLayer }
 
   http-requests_python38_with_plugin:
     handler: http_requests.handle
     runtime: python3.8
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python38LambdaLayer }
 
 # http-error
   http-error_python27_with_plugin:
     handler: http_error.handle
     runtime: python2.7
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python27LambdaLayer }
 
   http-error_python36_with_plugin:
     handler: http_error.handle
     runtime: python3.6
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python36LambdaLayer }
 
   http-error_python37_with_plugin:
     handler: http_error.handle
     runtime: python3.7
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python37LambdaLayer }
 
   http-error_python38_with_plugin:
     handler: http_error.handle
     runtime: python3.8
-    environment:
-      DD_TRACE_ENABLED: true
     layers:
       - { Ref: Python38LambdaLayer }
 

tests/integration/serverless.yml

@@ -8,6 +8,7 @@ provider:
   environment:
     DD_INTEGRATION_TEST: true
     DD_API_KEY: ${env:DD_API_KEY}
+    COLD_START_ENFORCER: ${env:COLD_START_ENFORCER}
   lambdaHashingVersion: 20201221
 
 layers:

tests/integration/snapshots/logs/async-metrics_python27.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-dev-async-metrics_python27", "resource:integration-dev-async-metrics_python27", "cold_start:true", "memorysize:1024", "runtime:python2.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}
 {"e": XXXX, "m": "hello.dog", "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}

tests/integration/snapshots/logs/async-metrics_python27_with_plugin.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python27_with_plugin", "resource:integration-plugin-dev-async-metrics_python27_with_plugin", "cold_start:true", "memorysize:1024", "runtime:python2.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}
 {"e": XXXX, "m": "hello.dog", "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}
@@ -20,7 +18,7 @@ START RequestId: XXXX Version: $LATEST
 {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python27_with_plugin", "resource:integration-plugin-dev-async-metrics_python27_with_plugin", "cold_start:false", "memorysize:1024", "runtime:python2.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}
 {"e": XXXX, "m": "hello.dog", "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1}
 {"e": XXXX, "m": "tests.integration.count", "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 21}
-{"traces": [[{"resource": "integration-plugin-dev-async-metrics_python27_with_plugin", "name": "aws.lambda", "service": "aws.lambda", "start": XXXX, "trace_id": "XXXX", "metrics": {"_sampling_priority_v1": 2, "system.pid": XXXX}, "parent_id": "XXXX", "meta": {"runtime-id": "XXXX", "request_id": "XXXX", "function_trigger.event_source": "sqs", "cold_start": "false", "datadog_lambda": "X.X.X", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python27_with_plugin", "dd_trace": "X.X.X", "_dd.origin": "lambda", "_dd.parent_source": "xray", "resource_names": "integration-plugin-dev-async-metrics_python27_with_plugin", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", "function_version": "$LATEST"}, "error": 0, "duration": XXXX, "type": "serverless", "span_id": "XXXX"}]]}
+{"traces": [[{"resource": "integration-plugin-dev-async-metrics_python27_with_plugin", "name": "aws.lambda", "service": "aws.lambda", "start": XXXX, "trace_id": "XXXX", "metrics": {"_sampling_priority_v1": 1, "system.pid": XXXX}, "parent_id": "XXXX", "meta": {"runtime-id": "XXXX", "request_id": "XXXX", "function_trigger.event_source": "sqs", "cold_start": "false", "datadog_lambda": "X.X.X", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python27_with_plugin", "dd_trace": "X.X.X", "_dd.origin": "lambda", "resource_names": "integration-plugin-dev-async-metrics_python27_with_plugin", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue", "function_version": "$LATEST"}, "error": 0, "duration": XXXX, "type": "serverless", "span_id": "XXXX"}]]}
 END RequestId: XXXX
 REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	
 XRAY TraceId: XXXX	SegmentId: XXXX	Sampled: true	

tests/integration/snapshots/logs/async-metrics_python36.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-dev-async-metrics_python36", "resource:integration-dev-async-metrics_python36", "cold_start:true", "memorysize:1024", "runtime:python3.6", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python36_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"]}

tests/integration/snapshots/logs/async-metrics_python36_with_plugin.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python36_with_plugin", "resource:integration-plugin-dev-async-metrics_python36_with_plugin", "cold_start:true", "memorysize:1024", "runtime:python3.6", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python36_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"]}
@@ -20,7 +18,7 @@ START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python36_with_plugin", "resource:integration-plugin-dev-async-metrics_python36_with_plugin", "cold_start:false", "memorysize:1024", "runtime:python3.6", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python36_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"]}
 {"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"]}
-{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python36_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python36_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python36_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "_dd.parent_source": "xray", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 2}, "type": "serverless"}]]}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python36_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python36_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python36_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]}
 END RequestId: XXXX
 REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	
 XRAY TraceId: XXXX	SegmentId: XXXX	Sampled: true	

tests/integration/snapshots/logs/async-metrics_python37.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-dev-async-metrics_python37", "resource:integration-dev-async-metrics_python37", "cold_start:true", "memorysize:1024", "runtime:python3.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python37_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"]}

tests/integration/snapshots/logs/async-metrics_python37_with_plugin.log

@@ -1,5 +1,3 @@
-/var/runtime/botocore/vendored/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.2) or chardet (4.0.0) doesn't match a supported version!
-RequestsDependencyWarning)
 START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python37_with_plugin", "resource:integration-plugin-dev-async-metrics_python37_with_plugin", "cold_start:true", "memorysize:1024", "runtime:python3.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python37_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"]}
@@ -20,7 +18,7 @@ START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python37_with_plugin", "resource:integration-plugin-dev-async-metrics_python37_with_plugin", "cold_start:false", "memorysize:1024", "runtime:python3.7", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python37_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"]}
 {"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"]}
-{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python37_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python37_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python37_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "_dd.parent_source": "xray", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 2}, "type": "serverless"}]]}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python37_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python37_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python37_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]}
 END RequestId: XXXX
 REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	
 XRAY TraceId: XXXX	SegmentId: XXXX	Sampled: true	

tests/integration/snapshots/logs/async-metrics_python38_with_plugin.log

@@ -18,7 +18,7 @@ START RequestId: XXXX Version: $LATEST
 {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-plugin-dev-async-metrics_python38_with_plugin", "resource:integration-plugin-dev-async-metrics_python38_with_plugin", "cold_start:false", "memorysize:1024", "runtime:python3.8", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python38_2.XX.0"]}
 {"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"]}
 {"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"]}
-{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python38_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python38_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python38_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "_dd.parent_source": "xray", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 2}, "type": "serverless"}]]}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-plugin-dev-async-metrics_python38_with_plugin", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "runtime-id": "XXXX", "cold_start": "false", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-plugin-dev-async-metrics_python38_with_plugin", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-plugin-dev-async-metrics_python38_with_plugin", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]}
 END RequestId: XXXX
 REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	
 XRAY TraceId: XXXX	SegmentId: XXXX	Sampled: true