Run Integration Tests in GitHub

.github/workflows/build.yml

@@ -32,7 +32,7 @@ jobs:
           source venv/bin/activate
           flake8 datadog_lambda/
 
-  test:
+  unit-test:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
@@ -59,3 +59,38 @@ jobs:
         run: |
           source venv/bin/activate
           nose2 -v
+
+  integration-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Node 14
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      - name: Cache Node modules
+        id: cache-node-modules
+        uses: actions/cache@v2
+        with:
+          path: "**/node_modules"
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install Serverless Framework
+        run: sudo yarn global add serverless --prefix /usr/local
+
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        working-directory: tests/integration
+        run: yarn install
+
+      - name: Run tests
+        env:
+          BUILD_LAYERS: true
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: ./scripts/run_integration_tests.sh

datadog_lambda/patch.py

@@ -145,8 +145,8 @@ def _print_request_string(request):
 
     # Sort the datapoints POSTed by their name so that snapshots always align
     data = request.body or "{}"
+    # If payload is compressed, decompress it so we can parse it
     if request.headers.get("Content-Encoding") == "deflate":
-        # See metric.py: lambda_stats = ThreadStats(compress_payload=True)
         data = zlib.decompress(data)
     data_dict = json.loads(data)
     data_dict.get("series", []).sort(key=lambda series: series.get("metric"))

scripts/run_integration_tests.sh

@@ -12,19 +12,15 @@ set -e
 # defined for every handler_runtime combination
 LAMBDA_HANDLERS=("async-metrics" "sync-metrics" "http-requests" "http-error")
 RUNTIMES=("python27" "python36" "python37" "python38")
-CONFIGS=("with-plugin" "without-plugin")
 
 LOGS_WAIT_SECONDS=20
 
-# Force cold start to avoid flaky tests
-export COLD_START_ENFORCER=$((1 + $RANDOM % 100000))
-
 script_path=${BASH_SOURCE[0]}
 scripts_dir=$(dirname $script_path)
 repo_dir=$(dirname $scripts_dir)
 integration_tests_dir="$repo_dir/tests/integration"
 
-script_start_time=$(date --iso-8601=seconds)
+script_utc_start_time=$(date -u +"%Y%m%dT%H%M%S")
 
 mismatch_found=false
 
@@ -46,62 +42,49 @@ fi
 
 cd $integration_tests_dir
 
-# Install the specified plugin version
-yarn install
-
 input_event_files=$(ls ./input_events)
 # Sort event files by name so that snapshots stay consistent
 input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort))
 
-echo "Deploying functions with plugin"
-serverless deploy -c "./serverless-plugin.yml"
-echo "Deploying functions without plugin"
-serverless deploy
+# Generate a random 8-character ID to avoid collisions with other runs
+run_id=$(xxd -l 4 -c 4 -p < /dev/random)
+
+echo "Deploying functions"
+serverless deploy --stage $run_id
 
 echo "Invoking functions"
 set +e # Don't exit this script if an invocation fails or there's a diff
-for _sls_type in "${CONFIGS[@]}"; do
-    for handler_name in "${LAMBDA_HANDLERS[@]}"; do
-        for runtime in "${RUNTIMES[@]}"; do
-            if [ "$_sls_type" = "with-plugin" ]; then
-                function_name="${handler_name}_${runtime}_with_plugin"
+for handler_name in "${LAMBDA_HANDLERS[@]}"; do
+    for runtime in "${RUNTIMES[@]}"; do
+        function_name="${handler_name}_${runtime}"
+
+        # Invoke function once for each input event
+        for input_event_file in "${input_event_files[@]}"; do
+            # Get event name without trailing ".json" so we can build the snapshot file name
+            input_event_name=$(echo "$input_event_file" | sed "s/.json//")
+            snapshot_path="./snapshots/return_values/${function_name}_${input_event_name}.json"
+
+            return_value=$(serverless invoke -f $function_name --stage $run_id --path "./input_events/$input_event_file")
+
+            if [ ! -f $snapshot_path ]; then
+                # If the snapshot file doesn't exist yet, we create it
+                echo "Writing return value to $snapshot_path because no snapshot exists yet"
+                echo "$return_value" >$snapshot_path
+            elif [ -n "$UPDATE_SNAPSHOTS" ]; then
+                # If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
+                echo "Overwriting return value snapshot for $snapshot_path"
+                echo "$return_value" >$snapshot_path
             else
-                function_name="${handler_name}_${runtime}"
-            fi
-
-            # Invoke function once for each input event
-            for input_event_file in "${input_event_files[@]}"; do
-                # Get event name without trailing ".json" so we can build the snapshot file name
-                input_event_name=$(echo "$input_event_file" | sed "s/.json//")
-                # Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event}
-                snapshot_path="./snapshots/return_values/${handler_name}_${runtime}_${input_event_name}.json"
-
-                if [ "$_sls_type" = "with-plugin" ]; then
-                    return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file" -c "serverless-plugin.yml")
-                else
-                    return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file")
-                fi
-
-                if [ ! -f $snapshot_path ]; then
-                    # If the snapshot file doesn't exist yet, we create it
-                    echo "Writing return value to $snapshot_path because no snapshot exists yet"
-                    echo "$return_value" >$snapshot_path
-                elif [ -n "$UPDATE_SNAPSHOTS" ]; then
-                    # If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
-                    echo "Overwriting return value snapshot for $snapshot_path"
-                    echo "$return_value" >$snapshot_path
+                # Compare new return value to snapshot
+                diff_output=$(echo "$return_value" | diff - $snapshot_path)
+                if [ $? -eq 1 ]; then
+                    echo "Failed: Return value for $function_name does not match snapshot:"
+                    echo "$diff_output"
+                    mismatch_found=true
                 else
-                    # Compare new return value to snapshot
-                    diff_output=$(echo "$return_value" | diff - $snapshot_path)
-                    if [ $? -eq 1 ]; then
-                        echo "Failed: Return value for $function_name does not match snapshot:"
-                        echo "$diff_output"
-                        mismatch_found=true
-                    else
-                        echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
-                    fi
+                    echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
                 fi
-            done
+            fi
         done
     done
 done
@@ -110,82 +93,100 @@ set -e
 echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
 sleep $LOGS_WAIT_SECONDS
 
+set +e # Don't exit this script if there is a diff or the logs endpoint fails
 echo "Fetching logs for invocations and comparing to snapshots"
-for _sls_type in "${CONFIGS[@]}"; do
-    for handler_name in "${LAMBDA_HANDLERS[@]}"; do
-        for runtime in "${RUNTIMES[@]}"; do
-            if [ "$_sls_type" = "with-plugin" ]; then
-                function_name="${handler_name}_${runtime}_with_plugin"
-            else
-                function_name="${handler_name}_${runtime}"
-            fi
-
-            function_snapshot_path="./snapshots/logs/$function_name.log"
-
-            # Fetch logs with serverless cli
-            if [ "$_sls_type" = "with-plugin" ]; then
-                raw_logs=$(serverless logs -f $function_name --startTime $script_start_time -c "serverless-plugin.yml")
-            else
-                raw_logs=$(serverless logs -f $function_name --startTime $script_start_time)
+for handler_name in "${LAMBDA_HANDLERS[@]}"; do
+    for runtime in "${RUNTIMES[@]}"; do
+        function_name="${handler_name}_${runtime}"
+        function_snapshot_path="./snapshots/logs/$function_name.log"
+
+        # Fetch logs with serverless cli, retrying to avoid AWS account-wide rate limit error
+        retry_counter=0
+        while [ $retry_counter -lt 10 ]; do
+            raw_logs=$(serverless logs -f $function_name --stage $run_id --startTime $script_utc_start_time)
+            fetch_logs_exit_code=$?
+            if [ $fetch_logs_exit_code -eq 1 ]; then
+                echo "Retrying fetch logs for $function_name..."
+                retry_counter=$(($retry_counter + 1))
+                sleep 10
+                continue
             fi
+            break
+        done
 
-            # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
-            logs=$(
-                echo "$raw_logs" |
-                    # Filter serverless cli errors
-                    sed '/Serverless: Recoverable error occurred/d' |
-                    # Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
-                    sed '/RequestsDependencyWarning/d' |
-                    # Remove blank lines
-                    sed '/^$/d' |
-                    # Normalize Lambda runtime report logs
-                    sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
-                    # Normalize DD APM headers and AWS account ID
-                    sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:)[0-9]+/\1XXXX/g" |
-                    # Normalize timestamps in datapoints POSTed to DD
-                    sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
-                    # Strip API key from logged requests
-                    sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
-                    # Normalize minor package version so that these snapshots aren't broken on version bumps
-                    sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" |
-                    sed -E "s/(datadog_lambda:v)([0-9]+\.[0-9]+\.[0-9])/\1XX/g" |
-                    # Strip out trace/span/parent/timestamps
-                    sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
-                    sed -E "s/(\"datadog_lambda\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
-                    sed -E "s/(\"dd_trace\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g"
-            )
-
-            if [ ! -f $function_snapshot_path ]; then
-                # If no snapshot file exists yet, we create one
-                echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
-                echo "$logs" >$function_snapshot_path
-            elif [ -n "$UPDATE_SNAPSHOTS" ]; then
-                # If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
-                echo "Overwriting log snapshot for $function_snapshot_path"
-                echo "$logs" >$function_snapshot_path
+        if [ $retry_counter -eq 9 ]; then
+            echo "FAILURE: Could not retrieve logs for $function_name"
+            echo "Error from final attempt to retrieve logs:"
+            echo $raw_logs
+
+            echo "Removing functions"
+            serverless remove --stage $run_id
+
+            exit 1
+        fi
+
+        # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
+        logs=$(
+            echo "$raw_logs" |
+                # Filter serverless cli errors
+                sed '/Serverless: Recoverable error occurred/d' |
+                # Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
+                sed '/RequestsDependencyWarning/d' |
+                # Remove blank lines
+                sed '/^$/d' |
+                # Normalize Lambda runtime REPORT logs
+                sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
+                # Normalize HTTP headers
+                sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|Content-Length:)[0-9]+/\1XXXX/g" |
+                # Remove Account ID
+                sed -E "s/(account_id:)[0-9]+/\1XXXX/g" |
+                # Normalize timestamps in datapoints POSTed to DD
+                sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
+                # Strip API key from logged requests
+                sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
+                # Normalize minor package version so that these snapshots aren't broken on version bumps
+                sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" |
+                sed -E "s/(datadog_lambda:v)([0-9]+\.[0-9]+\.[0-9])/\1XX/g" |
+                # Strip out run ID (from function name, resource, etc.)
+                sed -E "s/$run_id/XXXX/g" |
+                # Strip out trace/span/parent/timestamps
+                sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"datadog_lambda\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
+                sed -E "s/(\"dd_trace\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g"
+        )
+
+        if [ ! -f $function_snapshot_path ]; then
+            # If no snapshot file exists yet, we create one
+            echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
+            echo "$logs" >$function_snapshot_path
+        elif [ -n "$UPDATE_SNAPSHOTS" ]; then
+            # If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
+            echo "Overwriting log snapshot for $function_snapshot_path"
+            echo "$logs" >$function_snapshot_path
+        else
+            # Compare new logs to snapshots
+            diff_output=$(echo "$logs" | diff - $function_snapshot_path)
+            if [ $? -eq 1 ]; then
+                echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
+                echo "$diff_output"
+                mismatch_found=true
             else
-                # Compare new logs to snapshots
-                set +e # Don't exit this script if there is a diff
-                diff_output=$(echo "$logs" | diff - $function_snapshot_path)
-                if [ $? -eq 1 ]; then
-                    echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
-                    echo "$diff_output"
-                    mismatch_found=true
-                else
-                    echo "Ok: New logs for $function_name match snapshot"
-                fi
-                set -e
+                echo "Ok: New logs for $function_name match snapshot"
             fi
-        done
+        fi
     done
 done
+set -e
+
+echo "Removing functions"
+serverless remove --stage $run_id
 
 if [ "$mismatch_found" = true ]; then
     echo "FAILURE: A mismatch between new data and a snapshot was found and printed above."

tests/integration/handle.py

@@ -1,13 +1,8 @@
-import os
-
-from decorator import conditional_decorator
 from datadog_lambda.metric import lambda_metric
 from datadog_lambda.wrapper import datadog_lambda_wrapper
 
-with_plugin = os.getenv("WITH_PLUGIN", False)
-
 
-@conditional_decorator(datadog_lambda_wrapper, with_plugin)
+@datadog_lambda_wrapper
 def handle(event, context):
     # Parse request ID and record ids out of the event to include in the response
     request_id = event.get("requestContext", {}).get("requestId")

tests/integration/http_error.py

@@ -1,17 +1,10 @@
-import os
 import requests
 
-from decorator import conditional_decorator
 from datadog_lambda.metric import lambda_metric
 from datadog_lambda.wrapper import datadog_lambda_wrapper
-from ddtrace import tracer
-from ddtrace.internal.writer import LogWriter
 
-tracer.writer = LogWriter()
-with_plugin = os.getenv("WITH_PLUGIN", False)
 
-
-@conditional_decorator(datadog_lambda_wrapper, with_plugin)
+@datadog_lambda_wrapper
 def handle(event, context):
     lambda_metric("hello.dog", 1, tags=["team:serverless", "role:hello"])
     lambda_metric(