From a7e617d60f51df8df00f5f4d9a7db105e4b97ac0 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 18 Mar 2020 14:08:47 -0400 Subject: [PATCH 01/15] Implement dd-trace logic --- datadog_lambda/constants.py | 19 ++++-- datadog_lambda/trace_wrapper.py | 66 +++++++++++++++++++ datadog_lambda/tracing.py | 112 +++++++++++++++++++------------- datadog_lambda/wrapper.py | 35 +++++++++- 4 files changed, 178 insertions(+), 54 deletions(-) create mode 100644 datadog_lambda/trace_wrapper.py diff --git a/datadog_lambda/constants.py b/datadog_lambda/constants.py index 4e4e690d..e31fd026 100644 --- a/datadog_lambda/constants.py +++ b/datadog_lambda/constants.py @@ -14,13 +14,20 @@ class SamplingPriority(object): # Datadog trace headers class TraceHeader(object): - TRACE_ID = 'x-datadog-trace-id' - PARENT_ID = 'x-datadog-parent-id' - SAMPLING_PRIORITY = 'x-datadog-sampling-priority' + TRACE_ID = "x-datadog-trace-id" + PARENT_ID = "x-datadog-parent-id" + SAMPLING_PRIORITY = "x-datadog-sampling-priority" # X-Ray subsegment to save Datadog trace metadata class XraySubsegment(object): - NAME = 'datadog-metadata' - KEY = 'trace' - NAMESPACE = 'datadog' + NAME = "datadog-metadata" + KEY = "trace" + NAMESPACE = "datadog" + + +# Source of datadog context +class Source(object): + XRAY = "xray" + EVENT = "event" + DDTRACE = "ddtrace" diff --git a/datadog_lambda/trace_wrapper.py b/datadog_lambda/trace_wrapper.py new file mode 100644 index 00000000..884d36c6 --- /dev/null +++ b/datadog_lambda/trace_wrapper.py @@ -0,0 +1,66 @@ +import sys +from datadog_lambda.constants import SamplingPriority, TraceHeader, Source + + +class TraceWrapper: + """ + TraceWrapper wraps dd-trace, to make this library usable when + dd-trace hasn't been installed/initialised + """ + + def __init__(self): + self._tracer = None + + def extract(self, event): + tracer = self._load_tracer() + if not tracer: + return None + return _propagator.extract(event) + + def start_span(self, name, **kwargs): + tracer = self._load_tracer() + if not tracer: + return None + + return self._tracer.start_span(name, **kwargs) + + def _load_tracer(self): + if not TraceWrapper.tracer_enabled(): + return None + try: + if not self._tracer: + from ddtrace import tracer + from ddtrace.propagation.http import HTTPPropagator + + self._tracer = tracer + self._propagator = HTTPPropagator() + except: + pass + return self._tracer + + @property + def trace_context(self): + tracer = self._load_tracer() + if not tracer: + return None + span = tracer.current_span() + if not span: + return None + + parent_id = span.context.span_id + trace_id = span.context.trace_id + return { + "parent_id": parent_id, + "trace_id": trace_id, + "sampling_priority": SamplingPriority.AUTO_KEEP, + "source": Source.DDTRACE, + } + + @staticmethod + def tracer_enabled(): + mods = sys.modules.keys() + # Check whether user has imported ddtrace + return "ddtrace" in mods + + +trace_wrapper = TraceWrapper() diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index af967f64..f57f7aa2 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -13,7 +13,9 @@ SamplingPriority, TraceHeader, XraySubsegment, + Source, ) +from datadog_lambda.trace_wrapper import trace_wrapper logger = logging.getLogger(__name__) @@ -38,8 +40,32 @@ def _convert_xray_sampling(xray_sampled): """ Convert X-Ray sampled (True/False) to its Datadog counterpart. """ - return str(SamplingPriority.USER_KEEP) if xray_sampled \ + return ( + str(SamplingPriority.USER_KEEP) + if xray_sampled else str(SamplingPriority.USER_REJECT) + ) + + +def _get_xray_trace_context(): + if not is_lambda_context(): + return None + + xray_trace_entity = xray_recorder.get_trace_entity() # xray (sub)segment + return { + "trace-id": _convert_xray_trace_id(xray_trace_entity.trace_id), + "parent-id": _convert_xray_entity_id(xray_trace_entity.id), + "sampling-priority": _convert_xray_sampling(xray_trace_entity.sampled), + "source": Source.XRAY, + } + + +def _context_obj_to_headers(obj): + return { + TraceHeader.TRACE_ID: obj.get("trace_id"), + TraceHeader.PARENT_ID: obj.get("parent_id"), + TraceHeader.SAMPLING_PRIORITY: obj.get("sampling_priority"), + } def extract_dd_trace_context(event): @@ -54,33 +80,32 @@ def extract_dd_trace_context(event): the correct context. """ global dd_trace_context - headers = event.get('headers', {}) + headers = event.get("headers", {}) lowercase_headers = {k.lower(): v for k, v in headers.items()} trace_id = lowercase_headers.get(TraceHeader.TRACE_ID) parent_id = lowercase_headers.get(TraceHeader.PARENT_ID) sampling_priority = lowercase_headers.get(TraceHeader.SAMPLING_PRIORITY) if trace_id and parent_id and sampling_priority: - logger.debug('Extracted Datadog trace context from headers') + logger.debug("Extracted Datadog trace context from headers") dd_trace_context = { - 'trace-id': trace_id, - 'parent-id': parent_id, - 'sampling-priority': sampling_priority, + "trace-id": trace_id, + "parent-id": parent_id, + "sampling-priority": sampling_priority, + "source": Source.EVENT, } xray_recorder.begin_subsegment(XraySubsegment.NAME) subsegment = xray_recorder.current_subsegment() subsegment.put_metadata( - XraySubsegment.KEY, - dd_trace_context, - XraySubsegment.NAMESPACE + XraySubsegment.KEY, dd_trace_context, XraySubsegment.NAMESPACE ) xray_recorder.end_subsegment() else: # AWS Lambda runtime caches global variables between invocations, # reset to avoid using the context from the last invocation. - dd_trace_context = {} - - logger.debug('extracted dd trace context %s', dd_trace_context) + dd_trace_context = _get_xray_trace_context() + logger.debug("extracted dd trace context %s", dd_trace_context) + return dd_trace_context def get_dd_trace_context(): @@ -95,30 +120,25 @@ def get_dd_trace_context(): automatically, but this function can be used to manually inject the trace context to an outgoing request. """ - if not is_lambda_context(): - logger.debug('get_dd_trace_context is only supported in LambdaContext') - return {} - global dd_trace_context - xray_trace_entity = xray_recorder.get_trace_entity() # xray (sub)segment - if dd_trace_context: - return { - TraceHeader.TRACE_ID: - dd_trace_context['trace-id'], - TraceHeader.PARENT_ID: _convert_xray_entity_id( - xray_trace_entity.id), - TraceHeader.SAMPLING_PRIORITY: - dd_trace_context['sampling-priority'], - } - else: - return { - TraceHeader.TRACE_ID: _convert_xray_trace_id( - xray_trace_entity.trace_id), - TraceHeader.PARENT_ID: _convert_xray_entity_id( - xray_trace_entity.id), - TraceHeader.SAMPLING_PRIORITY: _convert_xray_sampling( - xray_trace_entity.sampled), - } + + if not dd_trace_context: + return None + trace_context = _context_obj_to_headers(dd_trace_context) + datadog_context = trace_wrapper.trace_context + if datadog_context: + logger.debug("get_dd_trace_context using dd-trace context") + return datadog_context + try: + xray_context = _get_xray_trace_context() # xray (sub)segment + if xray_context: + trace_context[TraceHeader.PARENT_ID] = xray_context["parent_id"] + except Exception as e: + logger.debug( + "get_dd_trace_context couldn't read from segment from x-ray, with error %s" + % e + ) + return trace_context def set_correlation_ids(): @@ -130,16 +150,16 @@ def set_correlation_ids(): TODO: Remove me when Datadog tracer is natively supported in Lambda. """ if not is_lambda_context(): - logger.debug('set_correlation_ids is only supported in LambdaContext') + logger.debug("set_correlation_ids is only supported in LambdaContext") return context = get_dd_trace_context() - span = tracer.trace('dummy.span') + span = tracer.trace("dummy.span") span.trace_id = context[TraceHeader.TRACE_ID] span.span_id = context[TraceHeader.PARENT_ID] - logger.debug('correlation ids set') + logger.debug("correlation ids set") def inject_correlation_ids(): @@ -153,17 +173,19 @@ def inject_correlation_ids(): # Override the log format of the AWS provided LambdaLoggerHandler root_logger = logging.getLogger() for handler in root_logger.handlers: - if handler.__class__.__name__ == 'LambdaLoggerHandler': - handler.setFormatter(logging.Formatter( - '[%(levelname)s]\t%(asctime)s.%(msecs)dZ\t%(aws_request_id)s\t' - '[dd.trace_id=%(dd.trace_id)s dd.span_id=%(dd.span_id)s]\t%(message)s\n', - '%Y-%m-%dT%H:%M:%S' - )) + if handler.__class__.__name__ == "LambdaLoggerHandler": + handler.setFormatter( + logging.Formatter( + "[%(levelname)s]\t%(asctime)s.%(msecs)dZ\t%(aws_request_id)s\t" + "[dd.trace_id=%(dd.trace_id)s dd.span_id=%(dd.span_id)s]\t%(message)s\n", + "%Y-%m-%dT%H:%M:%S", + ) + ) # Patch `logging.Logger.makeRecord` to actually inject correlation ids patch(logging=True) - logger.debug('logs injection configured') + logger.debug("logs injection configured") def is_lambda_context(): diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index 35323117..c29f4854 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -7,7 +7,7 @@ import logging import traceback -from datadog_lambda.cold_start import set_cold_start +from datadog_lambda.cold_start import set_cold_start, is_cold_start from datadog_lambda.metric import ( lambda_stats, submit_invocations_metric, @@ -18,7 +18,10 @@ extract_dd_trace_context, set_correlation_ids, inject_correlation_ids, + get_dd_trace_context, ) +from datadog_lambda.trace_wrapper import trace_wrapper +from datadog_lambda.constants import Source logger = logging.getLogger(__name__) @@ -39,7 +42,6 @@ def my_lambda_handle(event, context): class _NoopDecorator(object): - def __init__(self, func): self.func = func @@ -82,6 +84,8 @@ def __init__(self, func): self.logs_injection = ( os.environ.get("DD_LOGS_INJECTION", "true").lower() == "true" ) + self.handler_name = os.environ.get("_HANDLER", "handler") + self.function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "function") # Inject trace correlation ids to logs if self.logs_injection: @@ -106,10 +110,33 @@ def __call__(self, event, context, **kwargs): def _before(self, event, context): try: + set_cold_start() submit_invocations_metric(context) # Extract Datadog trace context from incoming requests - extract_dd_trace_context(event) + dd_context = extract_dd_trace_context(event) + span_context = None + if dd_context["source"] == Source.EVENT: + span_context = trace_wrapper.extract(dd_context) + + tags = {} + if context: + tags = { + "cold_start": is_cold_start(), + "function_arn": context.invoked_function_arn, + "request_id": context.aws_request_id, + "resource_names": context.function_name, + } + args = { + "service": self.function_name, + "resource": self.handler_name, + "span_type": "serverless", + "child_of": span_context, + } + + self.span = trace_wrapper.start_span("aws.lambda", **args) + if self.span: + self.span.set_tags(tags) # Set log correlation ids using extracted trace context set_correlation_ids() @@ -119,6 +146,8 @@ def _before(self, event, context): def _after(self, event, context): try: + if self.span: + self.span.finish() if not self.flush_to_log: lambda_stats.flush(float("inf")) logger.debug("datadog_lambda_wrapper _after() done") From 666658d50ecfd31823f3bb36245f829290bae243 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 19 Mar 2020 15:22:37 -0400 Subject: [PATCH 02/15] Fix issues with reading trace context from ddtrace --- datadog_lambda/trace_wrapper.py | 6 ++--- datadog_lambda/tracing.py | 29 +++++++++++---------- setup.py | 46 ++++++++++++++------------------- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/datadog_lambda/trace_wrapper.py b/datadog_lambda/trace_wrapper.py index 884d36c6..27bc3cbd 100644 --- a/datadog_lambda/trace_wrapper.py +++ b/datadog_lambda/trace_wrapper.py @@ -22,7 +22,7 @@ def start_span(self, name, **kwargs): if not tracer: return None - return self._tracer.start_span(name, **kwargs) + return tracer.start_span(name, **kwargs) def _load_tracer(self): if not TraceWrapper.tracer_enabled(): @@ -50,8 +50,8 @@ def trace_context(self): parent_id = span.context.span_id trace_id = span.context.trace_id return { - "parent_id": parent_id, - "trace_id": trace_id, + "parent_id": str(parent_id), + "trace_id": str(trace_id), "sampling_priority": SamplingPriority.AUTO_KEEP, "source": Source.DDTRACE, } diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index f57f7aa2..1fac7e36 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -62,9 +62,9 @@ def _get_xray_trace_context(): def _context_obj_to_headers(obj): return { - TraceHeader.TRACE_ID: obj.get("trace_id"), - TraceHeader.PARENT_ID: obj.get("parent_id"), - TraceHeader.SAMPLING_PRIORITY: obj.get("sampling_priority"), + TraceHeader.TRACE_ID: str(obj.get("trace_id")), + TraceHeader.PARENT_ID: str(obj.get("parent_id")), + TraceHeader.SAMPLING_PRIORITY: str(obj.get("sampling_priority")), } @@ -122,23 +122,26 @@ def get_dd_trace_context(): """ global dd_trace_context - if not dd_trace_context: - return None - trace_context = _context_obj_to_headers(dd_trace_context) - datadog_context = trace_wrapper.trace_context - if datadog_context: - logger.debug("get_dd_trace_context using dd-trace context") - return datadog_context + native_trace_context = trace_wrapper.trace_context + if native_trace_context: + logger.info("get_dd_trace_context using dd-trace context") + return _context_obj_to_headers(native_trace_context) + try: + trace_headers = _context_obj_to_headers(dd_trace_context) xray_context = _get_xray_trace_context() # xray (sub)segment - if xray_context: - trace_context[TraceHeader.PARENT_ID] = xray_context["parent_id"] + if xray_context and not trace_headers: + return _context_obj_to_headers(xray_context) + if xray_context and trace_headers: + trace_headers[TraceHeader.PARENT_ID] = xray_context["parent-id"] + return trace_headers except Exception as e: logger.debug( "get_dd_trace_context couldn't read from segment from x-ray, with error %s" % e ) - return trace_context + + return {} def set_correlation_ids(): diff --git a/setup.py b/setup.py index 8b252553..186e5b1b 100644 --- a/setup.py +++ b/setup.py @@ -6,40 +6,34 @@ here = path.abspath(path.dirname(__file__)) -with open(path.join(here, 'README.md'), encoding='utf-8') as f: +with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() setup( - name='datadog_lambda', + name="datadog_lambda", version=__version__, - description='The Datadog AWS Lambda Layer', + description="The Datadog AWS Lambda Layer", long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/DataDog/datadog-lambda-layer-python', - author='Datadog, Inc.', - author_email='dev@datadoghq.com', + long_description_content_type="text/markdown", + url="https://github.com/DataDog/datadog-lambda-layer-python", + author="Datadog, Inc.", + author_email="dev@datadoghq.com", classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], - keywords='datadog aws lambda layer', - packages=['datadog_lambda'], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4', + keywords="datadog aws lambda layer", + packages=["datadog_lambda"], + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4", install_requires=[ - 'aws-xray-sdk==2.4.3', - 'datadog==0.32.0', - 'ddtrace==0.31.0', - 'wrapt==1.11.2', - 'setuptools==42.0.2', + "aws-xray-sdk==2.4.3", + "datadog==0.32.0", + "wrapt==1.11.2", + "setuptools==42.0.2", ], extras_require={ - 'dev': [ - 'nose2==0.9.1', - 'flake8==3.7.9', - 'requests==2.22.0', - 'boto3==1.10.33', - ] - } + "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"] + }, ) From c32bdec09c8c129160922696426bd72c2e3d820e Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 19 Mar 2020 15:23:08 -0400 Subject: [PATCH 03/15] Add publish_staging script --- scripts/publish_staging.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 scripts/publish_staging.sh diff --git a/scripts/publish_staging.sh b/scripts/publish_staging.sh new file mode 100755 index 00000000..4d4787ed --- /dev/null +++ b/scripts/publish_staging.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +./scripts/build_layers.sh +./scripts/publish_layers.sh us-east-1 \ No newline at end of file From 353dde6a32e02507826dacbf8c80b0e7aa5a9556 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Tue, 24 Mar 2020 13:23:37 -0400 Subject: [PATCH 04/15] Add DD_TRACE_ENABLED and DD_MERGE_XRAY_TRACES options --- Dockerfile | 1 + datadog_lambda/trace_wrapper.py | 66 --------------------------------- datadog_lambda/tracing.py | 40 +++++++++++++++----- datadog_lambda/wrapper.py | 27 +++++++++----- 4 files changed, 50 insertions(+), 84 deletions(-) delete mode 100644 datadog_lambda/trace_wrapper.py diff --git a/Dockerfile b/Dockerfile index fc838627..b753a31d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ WORKDIR /build # Install datadog_lambda and dependencies from local COPY . . RUN pip install . -t ./python/lib/$runtime/site-packages +RUN pip install --find-links=https://s3.amazonaws.com/pypi.datadoghq.com/trace-dev/index.html ddtrace==0.35.1.dev5+g1e11b2dd -t ./python/lib/$runtime/site-packages # Remove *.pyc files RUN find ./python/lib/$runtime/site-packages -name \*.pyc -delete diff --git a/datadog_lambda/trace_wrapper.py b/datadog_lambda/trace_wrapper.py deleted file mode 100644 index 27bc3cbd..00000000 --- a/datadog_lambda/trace_wrapper.py +++ /dev/null @@ -1,66 +0,0 @@ -import sys -from datadog_lambda.constants import SamplingPriority, TraceHeader, Source - - -class TraceWrapper: - """ - TraceWrapper wraps dd-trace, to make this library usable when - dd-trace hasn't been installed/initialised - """ - - def __init__(self): - self._tracer = None - - def extract(self, event): - tracer = self._load_tracer() - if not tracer: - return None - return _propagator.extract(event) - - def start_span(self, name, **kwargs): - tracer = self._load_tracer() - if not tracer: - return None - - return tracer.start_span(name, **kwargs) - - def _load_tracer(self): - if not TraceWrapper.tracer_enabled(): - return None - try: - if not self._tracer: - from ddtrace import tracer - from ddtrace.propagation.http import HTTPPropagator - - self._tracer = tracer - self._propagator = HTTPPropagator() - except: - pass - return self._tracer - - @property - def trace_context(self): - tracer = self._load_tracer() - if not tracer: - return None - span = tracer.current_span() - if not span: - return None - - parent_id = span.context.span_id - trace_id = span.context.trace_id - return { - "parent_id": str(parent_id), - "trace_id": str(trace_id), - "sampling_priority": SamplingPriority.AUTO_KEEP, - "source": Source.DDTRACE, - } - - @staticmethod - def tracer_enabled(): - mods = sys.modules.keys() - # Check whether user has imported ddtrace - return "ddtrace" in mods - - -trace_wrapper = TraceWrapper() diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index 1fac7e36..2ed3cd05 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -4,22 +4,25 @@ # Copyright 2019 Datadog, Inc. import logging +import os from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core.lambda_launcher import LambdaContext -from ddtrace import patch, tracer from datadog_lambda.constants import ( SamplingPriority, TraceHeader, XraySubsegment, Source, ) -from datadog_lambda.trace_wrapper import trace_wrapper +from ddtrace import tracer, patch logger = logging.getLogger(__name__) dd_trace_context = {} +dd_native_tracing_enabled = ( + os.environ.get("DD_TRACE_ENABLED", "false").lower() == "true" +) def _convert_xray_trace_id(xray_trace_id): @@ -60,11 +63,26 @@ def _get_xray_trace_context(): } +def _get_dd_trace_native_context(): + span = tracer.current_span() + if not span: + return None + + parent_id = span.context.span_id + trace_id = span.context.trace_id + return { + "parent_id": str(parent_id), + "trace_id": str(trace_id), + "sampling_priority": SamplingPriority.AUTO_KEEP, + "source": Source.DDTRACE, + } + + def _context_obj_to_headers(obj): return { - TraceHeader.TRACE_ID: str(obj.get("trace_id")), - TraceHeader.PARENT_ID: str(obj.get("parent_id")), - TraceHeader.SAMPLING_PRIORITY: str(obj.get("sampling_priority")), + TraceHeader.TRACE_ID: str(obj.get("trace-id")), + TraceHeader.PARENT_ID: str(obj.get("parent-id")), + TraceHeader.SAMPLING_PRIORITY: str(obj.get("sampling-priority")), } @@ -122,10 +140,11 @@ def get_dd_trace_context(): """ global dd_trace_context - native_trace_context = trace_wrapper.trace_context - if native_trace_context: - logger.info("get_dd_trace_context using dd-trace context") - return _context_obj_to_headers(native_trace_context) + if dd_native_tracing_enabled: + native_trace_context = _get_dd_trace_native_context() + if native_trace_context is not None: + logger.info("get_dd_trace_context using dd-trace context") + return _context_obj_to_headers(native_trace_context) try: trace_headers = _context_obj_to_headers(dd_trace_context) @@ -155,6 +174,9 @@ def set_correlation_ids(): if not is_lambda_context(): logger.debug("set_correlation_ids is only supported in LambdaContext") return + if dd_native_tracing_enabled: + logger.debug("using ddtrace implementation for spans") + return context = get_dd_trace_context() diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index c29f4854..6bcb9d48 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -16,12 +16,14 @@ from datadog_lambda.patch import patch_all from datadog_lambda.tracing import ( extract_dd_trace_context, - set_correlation_ids, inject_correlation_ids, get_dd_trace_context, + dd_native_tracing_enabled, + set_correlation_ids, ) -from datadog_lambda.trace_wrapper import trace_wrapper from datadog_lambda.constants import Source +from ddtrace import tracer, Span +from ddtrace.propagation.http import HTTPPropagator logger = logging.getLogger(__name__) @@ -84,8 +86,12 @@ def __init__(self, func): self.logs_injection = ( os.environ.get("DD_LOGS_INJECTION", "true").lower() == "true" ) + self.merge_xray_traces = ( + os.environ.get("DD_MERGE_XRAY_TRACES", "false").lower() == "true" + ) self.handler_name = os.environ.get("_HANDLER", "handler") self.function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "function") + self.propagator = HTTPPropagator() # Inject trace correlation ids to logs if self.logs_injection: @@ -116,8 +122,9 @@ def _before(self, event, context): # Extract Datadog trace context from incoming requests dd_context = extract_dd_trace_context(event) span_context = None - if dd_context["source"] == Source.EVENT: - span_context = trace_wrapper.extract(dd_context) + if dd_context["source"] == Source.EVENT or self.merge_xray_traces: + headers = get_dd_trace_context() + span_context = self.propagator.extract(headers) tags = {} if context: @@ -134,12 +141,14 @@ def _before(self, event, context): "child_of": span_context, } - self.span = trace_wrapper.start_span("aws.lambda", **args) - if self.span: - self.span.set_tags(tags) + self.span = None + if dd_native_tracing_enabled: + self.span = tracer.start_span("aws.lambda", **args) + if self.span: + self.span.set_tags(tags) + else: + set_correlation_ids() - # Set log correlation ids using extracted trace context - set_correlation_ids() logger.debug("datadog_lambda_wrapper _before() done") except Exception: traceback.print_exc() From 082683c5bbfbdc4e8986d2bc3208c57b0cd8f28f Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Tue, 24 Mar 2020 13:55:09 -0400 Subject: [PATCH 05/15] Fix tests --- datadog_lambda/tracing.py | 16 ++++++++-------- datadog_lambda/wrapper.py | 2 +- setup.py | 1 + tests/Dockerfile | 3 ++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index 2ed3cd05..ec4e370a 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -71,9 +71,9 @@ def _get_dd_trace_native_context(): parent_id = span.context.span_id trace_id = span.context.trace_id return { - "parent_id": str(parent_id), - "trace_id": str(trace_id), - "sampling_priority": SamplingPriority.AUTO_KEEP, + "parent-id": str(parent_id), + "trace-id": str(trace_id), + "sampling-priority": SamplingPriority.AUTO_KEEP, "source": Source.DDTRACE, } @@ -106,17 +106,17 @@ def extract_dd_trace_context(event): sampling_priority = lowercase_headers.get(TraceHeader.SAMPLING_PRIORITY) if trace_id and parent_id and sampling_priority: logger.debug("Extracted Datadog trace context from headers") - dd_trace_context = { + metadata = { "trace-id": trace_id, "parent-id": parent_id, "sampling-priority": sampling_priority, - "source": Source.EVENT, } xray_recorder.begin_subsegment(XraySubsegment.NAME) subsegment = xray_recorder.current_subsegment() - subsegment.put_metadata( - XraySubsegment.KEY, dd_trace_context, XraySubsegment.NAMESPACE - ) + + subsegment.put_metadata(XraySubsegment.KEY, metadata, XraySubsegment.NAMESPACE) + dd_trace_context = metadata.copy() + dd_trace_context["source"] = Source.EVENT xray_recorder.end_subsegment() else: # AWS Lambda runtime caches global variables between invocations, diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index 6bcb9d48..5750d2da 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -22,7 +22,7 @@ set_correlation_ids, ) from datadog_lambda.constants import Source -from ddtrace import tracer, Span +from ddtrace import tracer from ddtrace.propagation.http import HTTPPropagator diff --git a/setup.py b/setup.py index 186e5b1b..707e1c04 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ install_requires=[ "aws-xray-sdk==2.4.3", "datadog==0.32.0", + "ddtrace==0.31.0", "wrapt==1.11.2", "setuptools==42.0.2", ], diff --git a/tests/Dockerfile b/tests/Dockerfile index 218fe3b2..1d4d6767 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -8,4 +8,5 @@ WORKDIR /test # Install datadog-lambda with dev dependencies from local COPY . . -RUN pip install .[dev] \ No newline at end of file +RUN pip install .[dev] +RUN pip install --find-links=https://s3.amazonaws.com/pypi.datadoghq.com/trace-dev/index.html ddtrace==0.35.1.dev5+g1e11b2dd -t .[dev] From fc8b1fd4562588dd473bc146f8745fe9cb7243f0 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Tue, 24 Mar 2020 14:57:57 -0400 Subject: [PATCH 06/15] Extend tracing tests --- tests/test_tracing.py | 177 +++++++++++++++++++++++------------------- 1 file changed, 98 insertions(+), 79 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index c7d2e14d..1763db79 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -1,15 +1,14 @@ import unittest + try: from unittest.mock import MagicMock, patch except ImportError: from mock import MagicMock, patch from ddtrace.helpers import get_correlation_ids -from datadog_lambda.constants import ( - SamplingPriority, - TraceHeader, - XraySubsegment, -) +from ddtrace import tracer + +from datadog_lambda.constants import SamplingPriority, TraceHeader, XraySubsegment from datadog_lambda.tracing import ( extract_dd_trace_context, get_dd_trace_context, @@ -17,160 +16,179 @@ _convert_xray_trace_id, _convert_xray_entity_id, _convert_xray_sampling, + dd_native_tracing_enabled, ) class TestExtractAndGetDDTraceContext(unittest.TestCase): - def setUp(self): - patcher = patch('datadog_lambda.tracing.xray_recorder') + global dd_native_tracing_enabled + dd_native_tracing_enabled = False + patcher = patch("datadog_lambda.tracing.xray_recorder") self.mock_xray_recorder = patcher.start() self.mock_xray_recorder.get_trace_entity.return_value = MagicMock( - id='ffff', - trace_id='1111', - sampled=True, + id="ffff", trace_id="1111", sampled=True ) self.mock_current_subsegment = MagicMock() - self.mock_xray_recorder.current_subsegment.return_value = \ + self.mock_xray_recorder.current_subsegment.return_value = ( self.mock_current_subsegment + ) self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.tracing.is_lambda_context') + patcher = patch("datadog_lambda.tracing.is_lambda_context") self.mock_is_lambda_context = patcher.start() self.mock_is_lambda_context.return_value = True self.addCleanup(patcher.stop) + def tearDown(self): + global dd_native_tracing_enabled + dd_native_tracing_enabled = False + def test_without_datadog_trace_headers(self): - extract_dd_trace_context({}) + ctx = extract_dd_trace_context({}) + self.assertDictEqual( + ctx, + { + "trace-id": "4369", + "parent-id": "65535", + "sampling-priority": "2", + "source": "xray", + }, + ) self.assertDictEqual( get_dd_trace_context(), { - TraceHeader.TRACE_ID: '4369', - TraceHeader.PARENT_ID: '65535', - TraceHeader.SAMPLING_PRIORITY: '2', - } + TraceHeader.TRACE_ID: "4369", + TraceHeader.PARENT_ID: "65535", + TraceHeader.SAMPLING_PRIORITY: "2", + }, ) def test_with_incomplete_datadog_trace_headers(self): - extract_dd_trace_context({ - 'headers': { - TraceHeader.TRACE_ID: '123', - TraceHeader.PARENT_ID: '321', - } - }) + ctx = extract_dd_trace_context( + {"headers": {TraceHeader.TRACE_ID: "123", TraceHeader.PARENT_ID: "321"}} + ) + self.assertDictEqual( + ctx, + { + "trace-id": "4369", + "parent-id": "65535", + "sampling-priority": "2", + "source": "xray", + }, + ) self.assertDictEqual( get_dd_trace_context(), { - TraceHeader.TRACE_ID: '4369', - TraceHeader.PARENT_ID: '65535', - TraceHeader.SAMPLING_PRIORITY: '2', - } + TraceHeader.TRACE_ID: "4369", + TraceHeader.PARENT_ID: "65535", + TraceHeader.SAMPLING_PRIORITY: "2", + }, ) def test_with_complete_datadog_trace_headers(self): - extract_dd_trace_context({ - 'headers': { - TraceHeader.TRACE_ID: '123', - TraceHeader.PARENT_ID: '321', - TraceHeader.SAMPLING_PRIORITY: '1', + ctx = extract_dd_trace_context( + { + "headers": { + TraceHeader.TRACE_ID: "123", + TraceHeader.PARENT_ID: "321", + TraceHeader.SAMPLING_PRIORITY: "1", + } } - }) + ) + self.assertDictEqual( + ctx, + { + "trace-id": "123", + "parent-id": "321", + "sampling-priority": "1", + "source": "event", + }, + ) self.assertDictEqual( get_dd_trace_context(), { - TraceHeader.TRACE_ID: '123', - TraceHeader.PARENT_ID: '65535', - TraceHeader.SAMPLING_PRIORITY: '1', - } + TraceHeader.TRACE_ID: "123", + TraceHeader.PARENT_ID: "65535", + TraceHeader.SAMPLING_PRIORITY: "1", + }, ) self.mock_xray_recorder.begin_subsegment.assert_called() self.mock_xray_recorder.end_subsegment.assert_called() self.mock_current_subsegment.put_metadata.assert_called_with( XraySubsegment.KEY, - { - 'trace-id': '123', - 'parent-id': '321', - 'sampling-priority': '1', - }, - XraySubsegment.NAMESPACE + {"trace-id": "123", "parent-id": "321", "sampling-priority": "1"}, + XraySubsegment.NAMESPACE, ) def test_with_complete_datadog_trace_headers_with_mixed_casing(self): - extract_dd_trace_context({ - 'headers': { - 'X-Datadog-Trace-Id': '123', - 'X-Datadog-Parent-Id': '321', - 'X-Datadog-Sampling-Priority': '1', + extract_dd_trace_context( + { + "headers": { + "X-Datadog-Trace-Id": "123", + "X-Datadog-Parent-Id": "321", + "X-Datadog-Sampling-Priority": "1", + } } - }) + ) self.assertDictEqual( get_dd_trace_context(), { - TraceHeader.TRACE_ID: '123', - TraceHeader.PARENT_ID: '65535', - TraceHeader.SAMPLING_PRIORITY: '1', - } + TraceHeader.TRACE_ID: "123", + TraceHeader.PARENT_ID: "65535", + TraceHeader.SAMPLING_PRIORITY: "1", + }, ) class TestXRayContextConversion(unittest.TestCase): - def test_convert_xray_trace_id(self): self.assertEqual( - _convert_xray_trace_id('00000000e1be46a994272793'), - '7043144561403045779' + _convert_xray_trace_id("00000000e1be46a994272793"), "7043144561403045779" ) self.assertEqual( - _convert_xray_trace_id('bd862e3fe1be46a994272793'), - '7043144561403045779' + _convert_xray_trace_id("bd862e3fe1be46a994272793"), "7043144561403045779" ) self.assertEqual( - _convert_xray_trace_id('ffffffffffffffffffffffff'), - '9223372036854775807' # 0x7FFFFFFFFFFFFFFF + _convert_xray_trace_id("ffffffffffffffffffffffff"), + "9223372036854775807", # 0x7FFFFFFFFFFFFFFF ) def test_convert_xray_entity_id(self): self.assertEqual( - _convert_xray_entity_id('53995c3f42cd8ad8'), - '6023947403358210776' + _convert_xray_entity_id("53995c3f42cd8ad8"), "6023947403358210776" ) self.assertEqual( - _convert_xray_entity_id('1000000000000000'), - '1152921504606846976' + _convert_xray_entity_id("1000000000000000"), "1152921504606846976" ) self.assertEqual( - _convert_xray_entity_id('ffffffffffffffff'), - '18446744073709551615' + _convert_xray_entity_id("ffffffffffffffff"), "18446744073709551615" ) def test_convert_xray_sampling(self): - self.assertEqual( - _convert_xray_sampling(True), - str(SamplingPriority.USER_KEEP) - ) + self.assertEqual(_convert_xray_sampling(True), str(SamplingPriority.USER_KEEP)) self.assertEqual( - _convert_xray_sampling(False), - str(SamplingPriority.USER_REJECT) + _convert_xray_sampling(False), str(SamplingPriority.USER_REJECT) ) class TestLogsInjection(unittest.TestCase): - def setUp(self): - patcher = patch('datadog_lambda.tracing.get_dd_trace_context') + + patcher = patch("datadog_lambda.tracing.get_dd_trace_context") self.mock_get_dd_trace_context = patcher.start() self.mock_get_dd_trace_context.return_value = { - TraceHeader.TRACE_ID: '123', - TraceHeader.PARENT_ID: '456', + TraceHeader.TRACE_ID: "123", + TraceHeader.PARENT_ID: "456", } self.addCleanup(patcher.stop) - patcher = patch('datadog_lambda.tracing.is_lambda_context') + patcher = patch("datadog_lambda.tracing.is_lambda_context") self.mock_is_lambda_context = patcher.start() self.mock_is_lambda_context.return_value = True self.addCleanup(patcher.stop) @@ -178,5 +196,6 @@ def setUp(self): def test_set_correlation_ids(self): set_correlation_ids() trace_id, span_id = get_correlation_ids() - self.assertEqual(trace_id, '123') - self.assertEqual(span_id, '456') + self.assertEqual(trace_id, "123") + self.assertEqual(span_id, "456") + From 7e8d8b950d095af5224f7f544f718ceb7cd247c1 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Tue, 24 Mar 2020 15:20:52 -0400 Subject: [PATCH 07/15] Add traces to integration tests --- tests/integration/http_requests.py | 4 ++++ tests/integration/serverless.yml | 8 ++++++++ .../integration/snapshots/logs/http-requests_python27.log | 3 +++ .../integration/snapshots/logs/http-requests_python36.log | 3 +++ .../integration/snapshots/logs/http-requests_python37.log | 3 +++ .../integration/snapshots/logs/http-requests_python38.log | 3 +++ 6 files changed, 24 insertions(+) diff --git a/tests/integration/http_requests.py b/tests/integration/http_requests.py index 4a48e0a3..a79eaecc 100644 --- a/tests/integration/http_requests.py +++ b/tests/integration/http_requests.py @@ -2,6 +2,10 @@ from datadog_lambda.metric import lambda_metric from datadog_lambda.wrapper import datadog_lambda_wrapper +from ddtrace import tracer +from ddtrace.internal.writer import LogWriter + +tracer.writer = LogWriter() @datadog_lambda_wrapper diff --git a/tests/integration/serverless.yml b/tests/integration/serverless.yml index b4422c50..6ec3b956 100644 --- a/tests/integration/serverless.yml +++ b/tests/integration/serverless.yml @@ -86,23 +86,31 @@ functions: http-requests_python27: handler: http_requests.handle runtime: python2.7 + environment: + DD_TRACE_ENABLED: true layers: - { Ref: Python27LambdaLayer } http-requests_python36: handler: http_requests.handle runtime: python3.6 + environment: + DD_TRACE_ENABLED: true layers: - { Ref: Python36LambdaLayer } http-requests_python37: handler: http_requests.handle runtime: python3.7 + environment: + DD_TRACE_ENABLED: true layers: - { Ref: Python37LambdaLayer } http-requests_python38: handler: http_requests.handle runtime: python3.8 + environment: + DD_TRACE_ENABLED: true layers: - { Ref: Python38LambdaLayer } diff --git a/tests/integration/snapshots/logs/http-requests_python27.log b/tests/integration/snapshots/logs/http-requests_python27.log index 26003f91..96c50dc7 100644 --- a/tests/integration/snapshots/logs/http-requests_python27.log +++ b/tests/integration/snapshots/logs/http-requests_python27.log @@ -2,6 +2,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:true", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077483318449000, "trace_id": "06C0653FC881D935", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "da026ab0-1d6c-4980-9720-1b4bdd843314", "cold_start": "True", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 389814000, "type": "serverless", "span_id": "3DA8AA128268BFA1"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -10,6 +11,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:false", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077485080991000, "trace_id": "6BDE1FAE35B84C6C", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "f1b1655b-ce14-4540-bf54-8e128af7b8ab", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 80971000, "type": "serverless", "span_id": "393ECB596ED13F61"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -18,6 +20,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:false", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077486596975000, "trace_id": "0A7F7B686FDCC310", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "6f90344e-1fa5-48cb-8933-d7f523868746", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 100314000, "type": "serverless", "span_id": "58E9AB1D40B41342"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python36.log b/tests/integration/snapshots/logs/http-requests_python36.log index c1c839d8..17f4d049 100644 --- a/tests/integration/snapshots/logs/http-requests_python36.log +++ b/tests/integration/snapshots/logs/http-requests_python36.log @@ -2,6 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:true", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "4053D36F25626FD9", "parent_id": "0000000000000000", "span_id": "DF176893913ECE4B", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077489263429000, "duration": 350020000, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "302f8d2f-1dde-4900-842a-843569fcd8ed", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -10,6 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:false", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "DE7F76F612DE39A5", "parent_id": "0000000000000000", "span_id": "99D9DCF60D385A31", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077491033326000, "duration": 50311000, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "6300019b-1fde-426f-be33-1beeaa544558", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -18,6 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:false", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "92B803175CBD933F", "parent_id": "0000000000000000", "span_id": "595ECB5ECF268A15", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077492445331000, "duration": 48984000, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "a425b59a-0ac7-4829-be54-a196046eb77f", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python37.log b/tests/integration/snapshots/logs/http-requests_python37.log index 2c037c43..58106b3a 100644 --- a/tests/integration/snapshots/logs/http-requests_python37.log +++ b/tests/integration/snapshots/logs/http-requests_python37.log @@ -2,6 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:true", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "340D486CEA851ED7", "parent_id": "0000000000000000", "span_id": "188FAAF208CDB655", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077495107217297, "duration": 126437375, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "58168111-8cc5-405a-bcc1-6220f384b624", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -10,6 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:false", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "539467D47212D0E0", "parent_id": "0000000000000000", "span_id": "0DAA998808ADBDFD", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077496608270347, "duration": 61312667, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "d32c332c-bfa5-468c-a8ca-c41636e6af80", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -18,6 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:false", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "32FEE973B78DF478", "parent_id": "0000000000000000", "span_id": "BE4A443C7DBFDAA8", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077498057088472, "duration": 67836145, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "621124bf-df82-4899-b649-988ce04e3b04", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python38.log b/tests/integration/snapshots/logs/http-requests_python38.log index a67728d3..6fb2764a 100644 --- a/tests/integration/snapshots/logs/http-requests_python38.log +++ b/tests/integration/snapshots/logs/http-requests_python38.log @@ -2,6 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:true", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "8CA21AC0190B2697", "parent_id": "0000000000000000", "span_id": "5AF191B8A447D3B4", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077500714066586, "duration": 194363876, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "b8bce9a7-b658-45a8-acac-b545400dc262", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -10,6 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:false", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "639333669E050530", "parent_id": "0000000000000000", "span_id": "95443944385B1F8B", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077502348814608, "duration": 47081832, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "3d11aa90-13b7-4daf-afbd-7a9f2034f9c9", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -18,6 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:false", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} +{"traces": [[{"trace_id": "E454223AE41C9DD3", "parent_id": "0000000000000000", "span_id": "84092FB0842D94E1", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077503759824478, "duration": 44339090, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "73e245ff-801f-4c5d-8896-3be6bf09b899", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB From 47a622b7ab5ecf0e912f4dca562287e79bcc968d Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Tue, 24 Mar 2020 17:06:03 -0400 Subject: [PATCH 08/15] Add README data --- README.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a2d58489..1ef9e2c8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Datadog Lambda Layer for Python (2.7, 3.6, 3.7 and 3.8) enables custom metric su ## IMPORTANT NOTE -AWS Lambda is expected to recieve a [breaking change](https://aws.amazon.com/blogs/compute/upcoming-changes-to-the-python-sdk-in-aws-lambda/) on **January 30, 2021**. If you are using Datadog Python Lambda layer version 7 or below, please upgrade to version 11. +AWS Lambda is expected to recieve a [breaking change](https://aws.amazon.com/blogs/compute/upcoming-changes-to-the-python-sdk-in-aws-lambda/) on **January 30, 2021**. If you are using Datadog Python Lambda layer version 7 or below, please upgrade to version 11. ## Installation @@ -21,6 +21,7 @@ arn:aws:lambda::464622532012:layer:Datadog-:` with the AWS region where your Lambda function is published to. Replace `` with one of the following that matches your Lambda's Python runtime: + - `Datadog-Python27` - `Datadog-Python36` - `Datadog-Python37` @@ -81,7 +82,7 @@ If `DD_FLUSH_TO_LOG` is set to false (not recommended), the Datadog API Key must - DD_KMS_API_KEY - the KMS-encrypted API Key, requires the `kms:Decrypt` permission - DD_API_KEY_SECRET_ARN - the Secret ARN to fetch API Key from the Secrets Manager, requires the `secretsmanager:GetSecretValue` permission (and `kms:Decrypt` if using a customer managed CMK) -You can also supply or override the API key at runtime (not recommended): +You can also supply or override the API key at runtime (not recommended): ```python # Override DD API Key after importing datadog_lambda packages @@ -243,6 +244,7 @@ If your Lambda function is triggered by API Gateway via [the non-proxy integrati If your Lambda function is deployed by the Serverless Framework, such a mapping template gets created by default. ## Log and Trace Correlations + By default, the Datadog trace id gets automatically injected into the logs for correlation, if using the standard python `logging` library. If you use a custom logger handler to log in json, you can inject the ids using the helper function `get_correlation_ids` [manually](https://docs.datadoghq.com/tracing/connect_logs_and_traces/?tab=python#manual-trace-id-injection). @@ -265,6 +267,35 @@ def lambda_handler(event, context): }) ``` +## Datadog Tracer (**Experimental**) + +You can now trace Lambda functions using Datadog APM's tracing libraries ([dd-trace-py](https://github.com/DataDog/dd-trace-py)). + +1. If you are using the Lambda layer, upgrade it to at least version X. +1. If you are using the pip package `datadog-lambda-python`, upgrade it to at least version `v0.X.0`. +1. Install (or update to) the latest version of [Datadog forwarder Lambda function](https://docs.datadoghq.com/integrations/amazon_web_services/?tab=allpermissions#set-up-the-datadog-lambda-function). Ensure the trace forwarding layer is attached to the forwarder, e.g., ARN for Python 2.7 `arn:aws:lambda::464622532012:layer:Datadog-Trace-Forwarder-Python27:4`. +1. Set the environment variable `DD_TRACE_ENABLED` to true on your function. +1. Instrument your function using `dd-trace`. + +```py +from datadog_lambda.metric import lambda_metric +from datadog_lambda.wrapper import datadog_lambda_wrapper + +from ddtrace import tracer +@datadog_lambda_wrapper +def hello(event, context): + return { + "statusCode": 200, + "body": get_message() + } + +@tracer.wrap() +def get_message(): + return "hello world" +``` + +You can also use `dd-trace` and the X-Ray tracer together and merge the traces into one, using the environment variable `DD_MERGE_XRAY_TRACES` to true on your function. + ## Opening Issues If you encounter a bug with this package, we want to hear about it. Before opening a new issue, search the existing issues to avoid duplicates. From 281e791585dff4ce1bd60dddb098f1f99515b8f7 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Wed, 25 Mar 2020 13:51:21 -0400 Subject: [PATCH 09/15] Implement PR feedback --- datadog_lambda/constants.py | 6 +- datadog_lambda/tracing.py | 48 +++++++------- datadog_lambda/wrapper.py | 63 +++++++++++-------- scripts/run_integration_tests.sh | 10 ++- .../snapshots/logs/http-requests_python27.log | 6 +- .../snapshots/logs/http-requests_python36.log | 6 +- .../snapshots/logs/http-requests_python37.log | 6 +- .../snapshots/logs/http-requests_python38.log | 6 +- tests/test_tracing.py | 10 ++- 9 files changed, 90 insertions(+), 71 deletions(-) diff --git a/datadog_lambda/constants.py b/datadog_lambda/constants.py index e31fd026..5e0b1d9b 100644 --- a/datadog_lambda/constants.py +++ b/datadog_lambda/constants.py @@ -26,8 +26,10 @@ class XraySubsegment(object): NAMESPACE = "datadog" -# Source of datadog context -class Source(object): +# TraceContextSource of datadog context. The DD_MERGE_XRAY_TRACES +# feature uses this to determine when to use X-Ray as the parent +# trace. +class TraceContextSource(object): XRAY = "xray" EVENT = "event" DDTRACE = "ddtrace" diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index ec4e370a..6d0f4ed4 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -13,16 +13,14 @@ SamplingPriority, TraceHeader, XraySubsegment, - Source, + TraceContextSource, ) from ddtrace import tracer, patch logger = logging.getLogger(__name__) dd_trace_context = {} -dd_native_tracing_enabled = ( - os.environ.get("DD_TRACE_ENABLED", "false").lower() == "true" -) +dd_tracing_enabled = os.environ.get("DD_TRACE_ENABLED", "false").lower() == "true" def _convert_xray_trace_id(xray_trace_id): @@ -59,22 +57,23 @@ def _get_xray_trace_context(): "trace-id": _convert_xray_trace_id(xray_trace_entity.trace_id), "parent-id": _convert_xray_entity_id(xray_trace_entity.id), "sampling-priority": _convert_xray_sampling(xray_trace_entity.sampled), - "source": Source.XRAY, + "source": TraceContextSource.XRAY, } -def _get_dd_trace_native_context(): +def _get_dd_trace_py_context(): span = tracer.current_span() if not span: return None parent_id = span.context.span_id trace_id = span.context.trace_id + sampling_priority = span.context.sampling_priority return { "parent-id": str(parent_id), "trace-id": str(trace_id), - "sampling-priority": SamplingPriority.AUTO_KEEP, - "source": Source.DDTRACE, + "sampling-priority": str(sampling_priority), + "source": TraceContextSource.DDTRACE, } @@ -116,7 +115,7 @@ def extract_dd_trace_context(event): subsegment.put_metadata(XraySubsegment.KEY, metadata, XraySubsegment.NAMESPACE) dd_trace_context = metadata.copy() - dd_trace_context["source"] = Source.EVENT + dd_trace_context["source"] = TraceContextSource.EVENT xray_recorder.end_subsegment() else: # AWS Lambda runtime caches global variables between invocations, @@ -132,7 +131,7 @@ def get_dd_trace_context(): If the Lambda function is invoked by a Datadog-traced service, a Datadog trace context may already exist, and it should be used. Otherwise, use the - current X-Ray trace entity. + current X-Ray trace entity, or the dd-trace-py context if DD_TRACE_ENABLED is true. Most of widely-used HTTP clients are patched to inject the context automatically, but this function can be used to manually inject the trace @@ -140,27 +139,30 @@ def get_dd_trace_context(): """ global dd_trace_context - if dd_native_tracing_enabled: - native_trace_context = _get_dd_trace_native_context() - if native_trace_context is not None: - logger.info("get_dd_trace_context using dd-trace context") - return _context_obj_to_headers(native_trace_context) + context = None + xray_context = None try: - trace_headers = _context_obj_to_headers(dd_trace_context) xray_context = _get_xray_trace_context() # xray (sub)segment - if xray_context and not trace_headers: - return _context_obj_to_headers(xray_context) - if xray_context and trace_headers: - trace_headers[TraceHeader.PARENT_ID] = xray_context["parent-id"] - return trace_headers except Exception as e: logger.debug( "get_dd_trace_context couldn't read from segment from x-ray, with error %s" % e ) - return {} + if xray_context and not dd_trace_context: + context = xray_context + elif xray_context and dd_trace_context: + context = dd_trace_context.copy() + context["parent-id"] = xray_context["parent-id"] + + if dd_tracing_enabled: + dd_trace_py_context = _get_dd_trace_py_context() + if dd_trace_py_context is not None: + logger.debug("get_dd_trace_context using dd-trace context") + context = dd_trace_py_context + + return _context_obj_to_headers(context) if context is not None else {} def set_correlation_ids(): @@ -174,7 +176,7 @@ def set_correlation_ids(): if not is_lambda_context(): logger.debug("set_correlation_ids is only supported in LambdaContext") return - if dd_native_tracing_enabled: + if dd_tracing_enabled: logger.debug("using ddtrace implementation for spans") return diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index 5750d2da..ad629744 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -18,10 +18,10 @@ extract_dd_trace_context, inject_correlation_ids, get_dd_trace_context, - dd_native_tracing_enabled, + dd_tracing_enabled, set_correlation_ids, ) -from datadog_lambda.constants import Source +from datadog_lambda.constants import TraceContextSource from ddtrace import tracer from ddtrace.propagation.http import HTTPPropagator @@ -97,8 +97,10 @@ def __init__(self, func): if self.logs_injection: inject_correlation_ids() - # Patch HTTP clients to propagate Datadog trace context - patch_all() + if not dd_tracing_enabled: + # When using dd_trace_py it will patch all the http clients for us, + # Patch HTTP clients to propagate Datadog trace context + patch_all() logger.debug("datadog_lambda_wrapper initialized") except Exception: traceback.print_exc() @@ -121,31 +123,10 @@ def _before(self, event, context): submit_invocations_metric(context) # Extract Datadog trace context from incoming requests dd_context = extract_dd_trace_context(event) - span_context = None - if dd_context["source"] == Source.EVENT or self.merge_xray_traces: - headers = get_dd_trace_context() - span_context = self.propagator.extract(headers) - - tags = {} - if context: - tags = { - "cold_start": is_cold_start(), - "function_arn": context.invoked_function_arn, - "request_id": context.aws_request_id, - "resource_names": context.function_name, - } - args = { - "service": self.function_name, - "resource": self.handler_name, - "span_type": "serverless", - "child_of": span_context, - } self.span = None - if dd_native_tracing_enabled: - self.span = tracer.start_span("aws.lambda", **args) - if self.span: - self.span.set_tags(tags) + if dd_tracing_enabled: + self.span = self._create_dd_trace_py_span(context, dd_context) else: set_correlation_ids() @@ -163,5 +144,33 @@ def _after(self, event, context): except Exception: traceback.print_exc() + def _create_dd_trace_py_span(self, context, trace_context): + span_context = None + if ( + trace_context["source"] == TraceContextSource.EVENT + or self.merge_xray_traces + ): + headers = get_dd_trace_context() + span_context = self.propagator.extract(headers) + + tags = {} + if context: + tags = { + "cold_start": is_cold_start(), + "function_arn": context.invoked_function_arn, + "request_id": context.aws_request_id, + "resource_names": context.function_name, + } + args = { + "service": self.function_name, + "resource": self.handler_name, + "span_type": "serverless", + "child_of": span_context, + } + span = tracer.start_span("aws.lambda", **args) + if span: + span.set_tags(tags) + return span + datadog_lambda_wrapper = _LambdaDecorator diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index c9f7df55..6619210d 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -116,7 +116,15 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do # Strip API key from logged requests sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" | # Normalize minor package version so that these snapshots aren't broken on version bumps - sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" + sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" | + # Strip out trace/span/parent/timestamps + sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" | + sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" ) if [ ! -f $function_snapshot_path ]; then diff --git a/tests/integration/snapshots/logs/http-requests_python27.log b/tests/integration/snapshots/logs/http-requests_python27.log index 96c50dc7..14b9dc0c 100644 --- a/tests/integration/snapshots/logs/http-requests_python27.log +++ b/tests/integration/snapshots/logs/http-requests_python27.log @@ -2,7 +2,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:true", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077483318449000, "trace_id": "06C0653FC881D935", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "da026ab0-1d6c-4980-9720-1b4bdd843314", "cold_start": "True", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 389814000, "type": "serverless", "span_id": "3DA8AA128268BFA1"}]]} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": XXXX, "trace_id": "XXXX", "metrics": {"_sampling_priority_v1": 1, "system.pid": XXXX, "_dd.agent_psr": 1.0}, "parent_id": "XXXX", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "XXXX", "cold_start": "True", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": XXXX, "type": "serverless", "span_id": "XXXX"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -11,7 +11,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:false", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077485080991000, "trace_id": "6BDE1FAE35B84C6C", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "f1b1655b-ce14-4540-bf54-8e128af7b8ab", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 80971000, "type": "serverless", "span_id": "393ECB596ED13F61"}]]} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": XXXX, "trace_id": "XXXX", "metrics": {"_sampling_priority_v1": 1, "system.pid": XXXX, "_dd.agent_psr": 1.0}, "parent_id": "XXXX", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "XXXX", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": XXXX, "type": "serverless", "span_id": "XXXX"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -20,7 +20,7 @@ START RequestId: XXXX Version: $LATEST {"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python27", "cold_start:false", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.XX.0"], "v": 1} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": 1585077486596975000, "trace_id": "0A7F7B686FDCC310", "metrics": {"_sampling_priority_v1": 1, "system.pid": 1, "_dd.agent_psr": 1.0}, "parent_id": "0000000000000000", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "6f90344e-1fa5-48cb-8933-d7f523868746", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": 100314000, "type": "serverless", "span_id": "58E9AB1D40B41342"}]]} +{"traces": [[{"resource": "http_requests.handle", "name": "aws.lambda", "service": "integration-tester-dev-http-requests_python27", "start": XXXX, "trace_id": "XXXX", "metrics": {"_sampling_priority_v1": 1, "system.pid": XXXX, "_dd.agent_psr": 1.0}, "parent_id": "XXXX", "meta": {"function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python27", "request_id": "XXXX", "cold_start": "False", "resource_names": "integration-tester-dev-http-requests_python27"}, "error": 0, "duration": XXXX, "type": "serverless", "span_id": "XXXX"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "hello.dog", "interval": 10, "host": null, "points": [[XXXX, [1.0]]], "device": null, "type": "distribution"}, {"tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.XX.0"], "metric": "tests.integration.count", "interval": 10, "host": null, "points": [[XXXX, [21.0]]], "device": null, "type": "distribution"}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python36.log b/tests/integration/snapshots/logs/http-requests_python36.log index 17f4d049..67c00196 100644 --- a/tests/integration/snapshots/logs/http-requests_python36.log +++ b/tests/integration/snapshots/logs/http-requests_python36.log @@ -2,7 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:true", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "4053D36F25626FD9", "parent_id": "0000000000000000", "span_id": "DF176893913ECE4B", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077489263429000, "duration": 350020000, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "302f8d2f-1dde-4900-842a-843569fcd8ed", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -11,7 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:false", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "DE7F76F612DE39A5", "parent_id": "0000000000000000", "span_id": "99D9DCF60D385A31", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077491033326000, "duration": 50311000, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "6300019b-1fde-426f-be33-1beeaa544558", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -20,7 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python36", "cold_start:false", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "92B803175CBD933F", "parent_id": "0000000000000000", "span_id": "595ECB5ECF268A15", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077492445331000, "duration": 48984000, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "a425b59a-0ac7-4829-be54-a196046eb77f", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 1, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python36", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python36", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python36"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python37.log b/tests/integration/snapshots/logs/http-requests_python37.log index 58106b3a..0f93de6d 100644 --- a/tests/integration/snapshots/logs/http-requests_python37.log +++ b/tests/integration/snapshots/logs/http-requests_python37.log @@ -2,7 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:true", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "340D486CEA851ED7", "parent_id": "0000000000000000", "span_id": "188FAAF208CDB655", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077495107217297, "duration": 126437375, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "58168111-8cc5-405a-bcc1-6220f384b624", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -11,7 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:false", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "539467D47212D0E0", "parent_id": "0000000000000000", "span_id": "0DAA998808ADBDFD", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077496608270347, "duration": 61312667, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "d32c332c-bfa5-468c-a8ca-c41636e6af80", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -20,7 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python37", "cold_start:false", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "32FEE973B78DF478", "parent_id": "0000000000000000", "span_id": "BE4A443C7DBFDAA8", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077498057088472, "duration": 67836145, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "621124bf-df82-4899-b649-988ce04e3b04", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 7, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python37", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python37", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python37"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/integration/snapshots/logs/http-requests_python38.log b/tests/integration/snapshots/logs/http-requests_python38.log index 6fb2764a..30e214d5 100644 --- a/tests/integration/snapshots/logs/http-requests_python38.log +++ b/tests/integration/snapshots/logs/http-requests_python38.log @@ -2,7 +2,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:true", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "8CA21AC0190B2697", "parent_id": "0000000000000000", "span_id": "5AF191B8A447D3B4", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077500714066586, "duration": 194363876, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "b8bce9a7-b658-45a8-acac-b545400dc262", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "True", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms @@ -11,7 +11,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:false", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "639333669E050530", "parent_id": "0000000000000000", "span_id": "95443944385B1F8B", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077502348814608, "duration": 47081832, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "3d11aa90-13b7-4daf-afbd-7a9f2034f9c9", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB @@ -20,7 +20,7 @@ START RequestId: XXXX Version: $LATEST {"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:XXXX", "functionname:integration-tester-dev-http-requests_python38", "cold_start:false", "memorysize:1024", "runtime:python3.8", "dd_lambda_layer:datadog-python38_2.XX.0"]} HTTP GET https://ip-ranges.datadoghq.com/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} HTTP GET https://ip-ranges.datadoghq.eu/ Headers: ["x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {} -{"traces": [[{"trace_id": "E454223AE41C9DD3", "parent_id": "0000000000000000", "span_id": "84092FB0842D94E1", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": 1585077503759824478, "duration": 44339090, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "73e245ff-801f-4c5d-8896-3be6bf09b899", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": 8, "_sampling_priority_v1": 1}, "type": "serverless"}]]} +{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "integration-tester-dev-http-requests_python38", "resource": "http_requests.handle", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"cold_start": "False", "function_arn": "arn:aws:lambda:us-east-1:601427279990:function:integration-tester-dev-http-requests_python38", "request_id": "XXXX", "resource_names": "integration-tester-dev-http-requests_python38"}, "metrics": {"_dd.agent_psr": 1.0, "system.pid": XXXX, "_sampling_priority_v1": 1}, "type": "serverless"}]]} HTTP POST https://api.datadoghq.com/api/v1/distribution_points Headers: ["Content-Type:application/json", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:2", "x-datadog-trace-id:XXXX"] Data: {"series": [{"metric": "hello.dog", "points": [[XXXX, [1.0]]], "type": "distribution", "host": null, "device": null, "tags": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}, {"metric": "tests.integration.count", "points": [[XXXX, [21.0]]], "type": "distribution", "host": null, "device": null, "tags": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python38_2.XX.0"], "interval": 10}]} END RequestId: XXXX REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 732a0be7..4f4a6ab1 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -6,7 +6,6 @@ from mock import MagicMock, patch from ddtrace.helpers import get_correlation_ids -from ddtrace import tracer from datadog_lambda.constants import SamplingPriority, TraceHeader, XraySubsegment from datadog_lambda.tracing import ( @@ -16,14 +15,13 @@ _convert_xray_trace_id, _convert_xray_entity_id, _convert_xray_sampling, - dd_native_tracing_enabled, ) class TestExtractAndGetDDTraceContext(unittest.TestCase): def setUp(self): - global dd_native_tracing_enabled - dd_native_tracing_enabled = False + global dd_tracing_enabled + dd_tracing_enabled = False patcher = patch("datadog_lambda.tracing.xray_recorder") self.mock_xray_recorder = patcher.start() self.mock_xray_recorder.get_trace_entity.return_value = MagicMock( @@ -41,8 +39,8 @@ def setUp(self): self.addCleanup(patcher.stop) def tearDown(self): - global dd_native_tracing_enabled - dd_native_tracing_enabled = False + global dd_tracing_enabled + dd_tracing_enabled = False def test_without_datadog_trace_headers(self): ctx = extract_dd_trace_context({}) From 903d7c5fc313adc1dc336349b859786537037a4a Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 2 Apr 2020 11:19:36 -0400 Subject: [PATCH 10/15] Use latest release to dd-trace-py --- Dockerfile | 1 - README.md | 2 +- datadog_lambda/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index b753a31d..fc838627 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ WORKDIR /build # Install datadog_lambda and dependencies from local COPY . . RUN pip install . -t ./python/lib/$runtime/site-packages -RUN pip install --find-links=https://s3.amazonaws.com/pypi.datadoghq.com/trace-dev/index.html ddtrace==0.35.1.dev5+g1e11b2dd -t ./python/lib/$runtime/site-packages # Remove *.pyc files RUN find ./python/lib/$runtime/site-packages -name \*.pyc -delete diff --git a/README.md b/README.md index 1ef9e2c8..714409b7 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ def lambda_handler(event, context): You can now trace Lambda functions using Datadog APM's tracing libraries ([dd-trace-py](https://github.com/DataDog/dd-trace-py)). -1. If you are using the Lambda layer, upgrade it to at least version X. +1. If you are using the Lambda layer, upgrade it to at least version 15. 1. If you are using the pip package `datadog-lambda-python`, upgrade it to at least version `v0.X.0`. 1. Install (or update to) the latest version of [Datadog forwarder Lambda function](https://docs.datadoghq.com/integrations/amazon_web_services/?tab=allpermissions#set-up-the-datadog-lambda-function). Ensure the trace forwarding layer is attached to the forwarder, e.g., ARN for Python 2.7 `arn:aws:lambda::464622532012:layer:Datadog-Trace-Forwarder-Python27:4`. 1. Set the environment variable `DD_TRACE_ENABLED` to true on your function. diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 49d69f5e..a6aff66f 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "2.14.0" +__version__ = "2.15.0" import os diff --git a/setup.py b/setup.py index 707e1c04..745558ec 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ "aws-xray-sdk==2.4.3", "datadog==0.32.0", - "ddtrace==0.31.0", + "ddtrace==0.36.0", "wrapt==1.11.2", "setuptools==42.0.2", ], From b40dcb6aac98235849e29433b40e0be51f9f605d Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 2 Apr 2020 11:36:44 -0400 Subject: [PATCH 11/15] Add empty line to readme example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 714409b7..6bc64dcd 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,7 @@ def lambda_handler(event, context): You can now trace Lambda functions using Datadog APM's tracing libraries ([dd-trace-py](https://github.com/DataDog/dd-trace-py)). 1. If you are using the Lambda layer, upgrade it to at least version 15. -1. If you are using the pip package `datadog-lambda-python`, upgrade it to at least version `v0.X.0`. +1. If you are using the pip package `datadog-lambda-python`, upgrade it to at least version `v2.15.0`. 1. Install (or update to) the latest version of [Datadog forwarder Lambda function](https://docs.datadoghq.com/integrations/amazon_web_services/?tab=allpermissions#set-up-the-datadog-lambda-function). Ensure the trace forwarding layer is attached to the forwarder, e.g., ARN for Python 2.7 `arn:aws:lambda::464622532012:layer:Datadog-Trace-Forwarder-Python27:4`. 1. Set the environment variable `DD_TRACE_ENABLED` to true on your function. 1. Instrument your function using `dd-trace`. @@ -282,6 +282,7 @@ from datadog_lambda.metric import lambda_metric from datadog_lambda.wrapper import datadog_lambda_wrapper from ddtrace import tracer + @datadog_lambda_wrapper def hello(event, context): return { From 8bd253df97792c8a93812a52237a5e000292fc92 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 2 Apr 2020 13:57:44 -0400 Subject: [PATCH 12/15] Implement PR feedback --- datadog_lambda/tracing.py | 36 +++++++++++++++++++++++++++++++ datadog_lambda/wrapper.py | 45 +++++++++------------------------------ tests/Dockerfile | 3 +-- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index 6d0f4ed4..ed769fd1 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -16,12 +16,15 @@ TraceContextSource, ) from ddtrace import tracer, patch +from ddtrace.propagation.http import HTTPPropagator logger = logging.getLogger(__name__) dd_trace_context = {} dd_tracing_enabled = os.environ.get("DD_TRACE_ENABLED", "false").lower() == "true" +propagator = HTTPPropagator() + def _convert_xray_trace_id(xray_trace_id): """ @@ -221,3 +224,36 @@ def is_lambda_context(): regular `Context` (e.g., when testing lambda functions locally). """ return type(xray_recorder.context) == LambdaContext + + +def create_function_execution_span( + context, + function_name, + handler_name, + is_cold_start, + trace_context, + merge_xray_traces, +): + span_context = None + if trace_context["source"] == TraceContextSource.EVENT or merge_xray_traces: + headers = get_dd_trace_context() + span_context = propagator.extract(headers) + + tags = {} + if context: + tags = { + "cold_start": is_cold_start, + "function_arn": context.invoked_function_arn, + "request_id": context.aws_request_id, + "resource_names": context.function_name, + } + args = { + "service": function_name, + "resource": handler_name, + "span_type": "serverless", + "child_of": span_context, + } + span = tracer.start_span("aws.lambda", **args) + if span: + span.set_tags(tags) + return span diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index ad629744..e057c3c6 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -17,14 +17,10 @@ from datadog_lambda.tracing import ( extract_dd_trace_context, inject_correlation_ids, - get_dd_trace_context, dd_tracing_enabled, set_correlation_ids, + create_function_execution_span, ) -from datadog_lambda.constants import TraceContextSource -from ddtrace import tracer -from ddtrace.propagation.http import HTTPPropagator - logger = logging.getLogger(__name__) @@ -91,7 +87,6 @@ def __init__(self, func): ) self.handler_name = os.environ.get("_HANDLER", "handler") self.function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "function") - self.propagator = HTTPPropagator() # Inject trace correlation ids to logs if self.logs_injection: @@ -126,7 +121,15 @@ def _before(self, event, context): self.span = None if dd_tracing_enabled: - self.span = self._create_dd_trace_py_span(context, dd_context) + + self.span = create_function_execution_span( + context, + self.function_name, + self.handler_name, + is_cold_start(), + dd_context, + self.merge_xray_traces, + ) else: set_correlation_ids() @@ -144,33 +147,5 @@ def _after(self, event, context): except Exception: traceback.print_exc() - def _create_dd_trace_py_span(self, context, trace_context): - span_context = None - if ( - trace_context["source"] == TraceContextSource.EVENT - or self.merge_xray_traces - ): - headers = get_dd_trace_context() - span_context = self.propagator.extract(headers) - - tags = {} - if context: - tags = { - "cold_start": is_cold_start(), - "function_arn": context.invoked_function_arn, - "request_id": context.aws_request_id, - "resource_names": context.function_name, - } - args = { - "service": self.function_name, - "resource": self.handler_name, - "span_type": "serverless", - "child_of": span_context, - } - span = tracer.start_span("aws.lambda", **args) - if span: - span.set_tags(tags) - return span - datadog_lambda_wrapper = _LambdaDecorator diff --git a/tests/Dockerfile b/tests/Dockerfile index 1d4d6767..218fe3b2 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -8,5 +8,4 @@ WORKDIR /test # Install datadog-lambda with dev dependencies from local COPY . . -RUN pip install .[dev] -RUN pip install --find-links=https://s3.amazonaws.com/pypi.datadoghq.com/trace-dev/index.html ddtrace==0.35.1.dev5+g1e11b2dd -t .[dev] +RUN pip install .[dev] \ No newline at end of file From e9d26ecdd236af7e189b293702344624d0f5faec Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 2 Apr 2020 17:52:40 -0400 Subject: [PATCH 13/15] Update with PR feedback --- datadog_lambda/tracing.py | 23 +++++++++-------------- datadog_lambda/wrapper.py | 12 ++++-------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index ed769fd1..1747b794 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -226,34 +226,29 @@ def is_lambda_context(): return type(xray_recorder.context) == LambdaContext -def create_function_execution_span( - context, - function_name, - handler_name, - is_cold_start, - trace_context, - merge_xray_traces, -): - span_context = None +def set_dd_trace_py_root(trace_context, merge_xray_traces): if trace_context["source"] == TraceContextSource.EVENT or merge_xray_traces: headers = get_dd_trace_context() span_context = propagator.extract(headers) + tracer.context_provider.activate(span_context) + +def create_function_execution_span(context, function_name, is_cold_start): tags = {} if context: tags = { - "cold_start": is_cold_start, + "cold_start": str(is_cold_start).lower(), "function_arn": context.invoked_function_arn, "request_id": context.aws_request_id, "resource_names": context.function_name, } args = { - "service": function_name, - "resource": handler_name, + "service": "aws.lambda", + "resource": function_name, "span_type": "serverless", - "child_of": span_context, } - span = tracer.start_span("aws.lambda", **args) + tracer.set_tags({"_dd.origin": "lambda"}) + span = tracer.trace("aws.lambda", **args) if span: span.set_tags(tags) return span diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index e057c3c6..c38347d1 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -19,6 +19,7 @@ inject_correlation_ids, dd_tracing_enabled, set_correlation_ids, + set_dd_trace_py_root, create_function_execution_span, ) @@ -85,7 +86,6 @@ def __init__(self, func): self.merge_xray_traces = ( os.environ.get("DD_MERGE_XRAY_TRACES", "false").lower() == "true" ) - self.handler_name = os.environ.get("_HANDLER", "handler") self.function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "function") # Inject trace correlation ids to logs @@ -120,15 +120,11 @@ def _before(self, event, context): dd_context = extract_dd_trace_context(event) self.span = None - if dd_tracing_enabled: + if dd_tracing_enabled: + set_dd_trace_py_root(dd_context, self.merge_xray_traces) self.span = create_function_execution_span( - context, - self.function_name, - self.handler_name, - is_cold_start(), - dd_context, - self.merge_xray_traces, + context, self.function_name, is_cold_start() ) else: set_correlation_ids() From 8d6fbfd44d5e361fe61e9444abe422dc62be5788 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Fri, 3 Apr 2020 11:49:42 -0400 Subject: [PATCH 14/15] Apply patch all when ddtrace is enabled --- datadog_lambda/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index c38347d1..2e38a4a0 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -22,6 +22,7 @@ set_dd_trace_py_root, create_function_execution_span, ) +from ddtrace import patch_all as patch_all_dd logger = logging.getLogger(__name__) @@ -96,6 +97,8 @@ def __init__(self, func): # When using dd_trace_py it will patch all the http clients for us, # Patch HTTP clients to propagate Datadog trace context patch_all() + else: + patch_all_dd() logger.debug("datadog_lambda_wrapper initialized") except Exception: traceback.print_exc() From 3b47b3963edaeff801006fb3d60a037d461f9839 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Fri, 3 Apr 2020 15:11:05 -0400 Subject: [PATCH 15/15] Add metadata to track X-Ray segments --- datadog_lambda/tracing.py | 8 +++++++- datadog_lambda/wrapper.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/datadog_lambda/tracing.py b/datadog_lambda/tracing.py index 1747b794..132b3884 100644 --- a/datadog_lambda/tracing.py +++ b/datadog_lambda/tracing.py @@ -233,7 +233,9 @@ def set_dd_trace_py_root(trace_context, merge_xray_traces): tracer.context_provider.activate(span_context) -def create_function_execution_span(context, function_name, is_cold_start): +def create_function_execution_span( + context, function_name, is_cold_start, trace_context +): tags = {} if context: tags = { @@ -242,6 +244,10 @@ def create_function_execution_span(context, function_name, is_cold_start): "request_id": context.aws_request_id, "resource_names": context.function_name, } + source = trace_context["source"] + if source != TraceContextSource.DDTRACE: + tags["_dd.parent_source"] = source + args = { "service": "aws.lambda", "resource": function_name, diff --git a/datadog_lambda/wrapper.py b/datadog_lambda/wrapper.py index 2e38a4a0..db30e0c3 100644 --- a/datadog_lambda/wrapper.py +++ b/datadog_lambda/wrapper.py @@ -123,11 +123,10 @@ def _before(self, event, context): dd_context = extract_dd_trace_context(event) self.span = None - if dd_tracing_enabled: set_dd_trace_py_root(dd_context, self.merge_xray_traces) self.span = create_function_execution_span( - context, self.function_name, is_cold_start() + context, self.function_name, is_cold_start(), dd_context ) else: set_correlation_ids()