diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 69f50e0774..35d8e52382 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -483,6 +483,7 @@ tests/: Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # updates the operation name of the span not the resource name Test_Parametric_Otel_Baggage: incomplete_test_app (otel baggage endpoints are not implemented) Test_Parametric_Otel_Current_Span: incomplete_test_app (otel current span endpoint are not implemented) + test_process_discovery.py: missing_feature test_span_events.py: missing_feature test_span_links.py: missing_feature test_telemetry.py: diff --git a/manifests/golang.yml b/manifests/golang.yml index a754e1cbfd..abe4f891ca 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -611,6 +611,7 @@ tests/: Test_Parametric_OtelSpan_Start: bug (APMAPI-778) # String attributes are incorrectly stored/serialized in a list Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported) Test_Parametric_Otel_Current_Span: missing_feature (otel current span endpoint is not defined) + test_process_discovery.py: missing_feature test_span_events.py: missing_feature test_span_links.py: missing_feature test_telemetry.py: diff --git a/manifests/java.yml b/manifests/java.yml index 3b2f624a24..3785ca9d83 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1699,6 +1699,7 @@ tests/: Test_Parametric_DDTrace_Current_Span: bug (APMAPI-778) # Fails to retreive the current span after a span has finished Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported) Test_Parametric_Otel_Current_Span: bug (APMAPI-778) # Current span endpoint does not return DataDog spans created by the otel api + test_process_discovery.py: missing_feature test_span_events.py: missing_feature test_span_links.py: missing_feature test_telemetry.py: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 376f77fd06..fb9029f706 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -873,6 +873,7 @@ tests/: Test_Parametric_Otel_Current_Span: incomplete_test_app (otel current_span endpoint is not supported) test_partial_flushing.py: Test_Partial_Flushing: bug (APMLP-270) + test_process_discovery.py: missing_feature test_span_events.py: missing_feature test_span_links.py: Test_Span_Links: *ref_5_3_0 diff --git a/manifests/php.yml b/manifests/php.yml index 3cd7ad567c..b3df327b08 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -461,6 +461,7 @@ tests/: Test_Parametric_Otel_Current_Span: bug (APMAPI-778) # otel current span endpoint should return a span and trace id of zero if no span is "active" test_partial_flushing.py: Test_Partial_Flushing: missing_feature + test_process_discovery.py: missing_feature test_sampling_delegation.py: Test_Decisionless_Extraction: >- missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index 28322c175f..b0f0d6b02d 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -858,6 +858,7 @@ tests/: Test_Parametric_Otel_Baggage: v2.16.0 test_partial_flushing.py: Test_Partial_Flushing: flaky (APMAPI-734) + test_process_discovery.py: missing_feature test_sampling_delegation.py: Test_Decisionless_Extraction: v2.8.0 test_sampling_span_tags.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 285eb9d7f4..ffb9ac7000 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -479,6 +479,7 @@ tests/: Test_Parametric_Otel_Current_Span: incomplete_test_app (otel current span endpoint is not supported) test_partial_flushing.py: # Not configurable in a standard way Test_Partial_Flushing: missing_feature + test_process_discovery.py: missing_feature test_sampling_delegation.py: Test_Decisionless_Extraction: v2.4.0 test_span_events.py: missing_feature diff --git a/tests/parametric/test_process_discovery.py b/tests/parametric/test_process_discovery.py new file mode 100644 index 0000000000..b3b0e2474b --- /dev/null +++ b/tests/parametric/test_process_discovery.py @@ -0,0 +1,69 @@ +"""Test the instrumented process discovery mechanism feature.""" + +import pytest +import json +import msgpack +from jsonschema import validate as validation_jsonschema +from utils import features, scenarios + + +def find_dd_memfds(test_library, pid: int) -> list[str]: + rc, out = test_library.container_exec_run(f"find /proc/{pid}/fd -lname '/memfd:datadog-tracer-info*'") + if not rc: + return [] + + return out.split() + + +def validate_schema(payload: str) -> bool: + schema = None + with open("utils/interfaces/schemas/library/process-discovery.json", "r") as f: + schema = json.load(f) + + try: + validation_jsonschema(payload, schema) + return True + except Exception: + return False + + +def read_memfd(test_library, memfd_path: str): + rc, output = test_library.container_exec_run_raw(f"cat {memfd_path}") + if not rc: + return rc, output + + return rc, msgpack.unpackb(output) + + +@scenarios.parametric +@features.process_discovery +class Test_ProcessDiscovery: + @pytest.mark.parametrize( + "library_env", + [ + {"DD_SERVICE": "a", "DD_ENV": "test", "DD_VERSION": "0.1.0"}, + {"DD_SERVICE": "b", "DD_ENV": "second-test", "DD_VERSION": "0.2.0"}, + ], + ) + def test_metadata_content(self, test_library, library_env): + """Verify the content of the memfd file matches the expected metadata format and structure""" + with test_library: + # NOTE(@dmehala): the server is started on container is always pid 1. + # That's a strong assumption :hehe: + # Maybe we should use `pidof pidof parametric-http-server` instead. + memfds = find_dd_memfds(test_library, 1) + assert len(memfds) == 1 + + rc, tracer_metadata = read_memfd(test_library, memfds[0]) + assert rc + assert validate_schema(tracer_metadata) + + assert tracer_metadata["schema_version"] == 1 + assert tracer_metadata["runtime_id"] + # assert tracer_metadata["hostname"] + # TODO(@dmehala): how to get the version? + # assert tracer_metadata["tracer_version"] == + assert tracer_metadata["tracer_language"] == test_library.lang + assert tracer_metadata["service_name"] == library_env["DD_SERVICE"] + assert tracer_metadata["service_version"] == library_env["DD_VERSION"] + assert tracer_metadata["service_env"] == library_env["DD_ENV"] diff --git a/utils/_features.py b/utils/_features.py index 17f42f55a7..b64bd0e22e 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -1724,7 +1724,9 @@ def semantic_core_validations(test_object): return test_object @staticmethod - def aws_sqs_span_creationcontext_propagation_via_xray_header_with_dd_trace(test_object): + def aws_sqs_span_creationcontext_propagation_via_xray_header_with_dd_trace( + test_object, + ): """[AWS-SQS][Span Creation][Context Propagation][AWS X-Ray] with dd-trace https://feature-parity.us1.prod.dog/#/?feature=263 @@ -1733,7 +1735,9 @@ def aws_sqs_span_creationcontext_propagation_via_xray_header_with_dd_trace(test_ return test_object @staticmethod - def aws_sqs_span_creationcontext_propagation_via_message_attributes_with_dd_trace(test_object): + def aws_sqs_span_creationcontext_propagation_via_message_attributes_with_dd_trace( + test_object, + ): """[AWS-SQS][Span Creation][Context Propagation][AWS Message Attributes] with dd-trace https://feature-parity.us1.prod.dog/#/?feature=264 @@ -1796,7 +1800,9 @@ def rabbitmq_span_creationcontext_propagation_with_dd_trace(test_object): return test_object @staticmethod - def aws_sns_span_creationcontext_propagation_via_message_attributes_with_dd_trace(test_object): + def aws_sns_span_creationcontext_propagation_via_message_attributes_with_dd_trace( + test_object, + ): """[AWS-SNS][Span Creation][Context Propagation] with dd-trace https://feature-parity.us1.prod.dog/#/?feature=271 @@ -1850,7 +1856,9 @@ def host_block_list(test_object): return test_object @staticmethod - def aws_kinesis_span_creationcontext_propagation_via_message_attributes_with_dd_trace(test_object): + def aws_kinesis_span_creationcontext_propagation_via_message_attributes_with_dd_trace( + test_object, + ): """[AWS-Kinesis][Span Creation][Context Propagation] with dd-trace https://feature-parity.us1.prod.dog/#/?feature=280 @@ -2417,5 +2425,14 @@ def single_span_ingestion_control(test_object): pytest.mark.features(feature_id=366)(test_object) return test_object + @staticmethod + def process_discovery(test_object): + """Process Disocvery + + https://feature-parity.us1.prod.dog/#/?feature=367 + """ + pytest.mark.features(feature_id=362)(test_object) + return test_object + features = _Features() diff --git a/utils/interfaces/schemas/library/process-discovery.json b/utils/interfaces/schemas/library/process-discovery.json new file mode 100644 index 0000000000..e8e9ad2466 --- /dev/null +++ b/utils/interfaces/schemas/library/process-discovery.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "/library/process-discovery.json", + "title": "Tracer Metadata", + "description": "Metadata format used for process discovery", + "type": "object", + "properties": { + "schema_version": { + "type": "integer", + "description": "Version of the schema" + }, + "runtime_id": { + "type": "string", + "description": "Runtime UUID" + }, + "tracer_version": { + "type": "string", + "description": "Version of the Datadog tracer library" + }, + "tracer_language": { + "type": "string", + "description": "Programming language of the tracer library" + }, + "hostname": { + "type": "string", + "description": "An identifier for the machine running the process" + }, + "service_name": { + "type": "string", + "description": "Name of the service being instrumented" + }, + "service_env": { + "type": "string", + "description": "Environment of the service being instrumented" + }, + "service_version": { + "type": "string", + "description": "Version of the service being instrumented" + } + }, + "required": [ + "schema_version", + "tracer_version", + "tracer_language", + "hostname" + ] +} + + diff --git a/utils/parametric/_library_client.py b/utils/parametric/_library_client.py index 3df25ccb0c..9264ed01aa 100644 --- a/utils/parametric/_library_client.py +++ b/utils/parametric/_library_client.py @@ -99,6 +99,26 @@ def crash(self) -> None: except: logger.info("Expected exception when calling /trace/crash") + def container_exec_run_raw(self, command: str) -> tuple[bool, str]: + try: + code, (stdout, _) = self.container.exec_run(command, demux=True) + if code is None: + success = False + message = "Exit code from command in the parametric app container is None" + elif stdout is None: + success = False + message = "Stdout from command in the parametric app container is None" + else: + success = True + message = stdout + except BaseException: + return ( + False, + "Encountered an issue running command in the parametric app container", + ) + + return success, message + def container_exec_run(self, command: str) -> tuple[bool, str]: try: code, (stdout, stderr) = self.container.exec_run(command, demux=True) @@ -112,7 +132,10 @@ def container_exec_run(self, command: str) -> tuple[bool, str]: success = True message = stdout.decode() if stdout is not None else "" except BaseException: - return False, "Encountered an issue running command in the parametric app container" + return ( + False, + "Encountered an issue running command in the parametric app container", + ) return success, message @@ -160,39 +183,68 @@ def finish_span(self, span_id: int) -> None: self._session.post(self._url("/trace/span/finish"), json={"span_id": span_id}) def span_set_resource(self, span_id: int, resource: str) -> None: - self._session.post(self._url("/trace/span/set_resource"), json={"span_id": span_id, "resource": resource}) + self._session.post( + self._url("/trace/span/set_resource"), + json={"span_id": span_id, "resource": resource}, + ) def span_set_meta(self, span_id: int, key: str, value) -> None: - self._session.post(self._url("/trace/span/set_meta"), json={"span_id": span_id, "key": key, "value": value}) + self._session.post( + self._url("/trace/span/set_meta"), + json={"span_id": span_id, "key": key, "value": value}, + ) def span_set_baggage(self, span_id: int, key: str, value: str) -> None: - self._session.post(self._url("/trace/span/set_baggage"), json={"span_id": span_id, "key": key, "value": value}) + self._session.post( + self._url("/trace/span/set_baggage"), + json={"span_id": span_id, "key": key, "value": value}, + ) def span_remove_baggage(self, span_id: int, key: str) -> None: - self._session.post(self._url("/trace/span/remove_baggage"), json={"span_id": span_id, "key": key}) + self._session.post( + self._url("/trace/span/remove_baggage"), + json={"span_id": span_id, "key": key}, + ) def span_remove_all_baggage(self, span_id: int) -> None: self._session.post(self._url("/trace/span/remove_all_baggage"), json={"span_id": span_id}) def span_set_metric(self, span_id: int, key: str, value: float) -> None: - self._session.post(self._url("/trace/span/set_metric"), json={"span_id": span_id, "key": key, "value": value}) + self._session.post( + self._url("/trace/span/set_metric"), + json={"span_id": span_id, "key": key, "value": value}, + ) def span_set_error(self, span_id: int, typestr: str, message: str, stack: str) -> None: self._session.post( self._url("/trace/span/error"), - json={"span_id": span_id, "type": typestr, "message": message, "stack": stack}, + json={ + "span_id": span_id, + "type": typestr, + "message": message, + "stack": stack, + }, ) def span_add_link(self, span_id: int, parent_id: int, attributes: dict | None = None): self._session.post( self._url("/trace/span/add_link"), - json={"span_id": span_id, "parent_id": parent_id, "attributes": attributes or {}}, + json={ + "span_id": span_id, + "parent_id": parent_id, + "attributes": attributes or {}, + }, ) def span_add_event(self, span_id: int, name: str, timestamp: int, attributes: dict | None = None): self._session.post( self._url("/trace/span/add_event"), - json={"span_id": span_id, "name": name, "timestamp": timestamp, "attributes": attributes or {}}, + json={ + "span_id": span_id, + "name": name, + "timestamp": timestamp, + "attributes": attributes or {}, + }, ) def span_get_baggage(self, span_id: int, key: str) -> str: @@ -212,7 +264,10 @@ def trace_inject_headers(self, span_id): return resp.json()["http_headers"] def trace_extract_headers(self, http_headers: list[tuple[str, str]]): - resp = self._session.post(self._url("/trace/span/extract_headers"), json={"http_headers": http_headers}) + resp = self._session.post( + self._url("/trace/span/extract_headers"), + json={"http_headers": http_headers}, + ) return resp.json()["span_id"] def trace_flush(self) -> bool: @@ -248,10 +303,16 @@ def otel_trace_start_span( return StartSpanResponse(span_id=resp["span_id"], trace_id=resp["trace_id"]) def otel_end_span(self, span_id: int, timestamp: int | None) -> None: - self._session.post(self._url("/trace/otel/end_span"), json={"id": span_id, "timestamp": timestamp}) + self._session.post( + self._url("/trace/otel/end_span"), + json={"id": span_id, "timestamp": timestamp}, + ) def otel_set_attributes(self, span_id: int, attributes) -> None: - self._session.post(self._url("/trace/otel/set_attributes"), json={"span_id": span_id, "attributes": attributes}) + self._session.post( + self._url("/trace/otel/set_attributes"), + json={"span_id": span_id, "attributes": attributes}, + ) def otel_set_name(self, span_id: int, name: str) -> None: self._session.post(self._url("/trace/otel/set_name"), json={"span_id": span_id, "name": name}) @@ -265,7 +326,12 @@ def otel_set_status(self, span_id: int, code: StatusCode, description: str) -> N def otel_add_event(self, span_id: int, name: str, timestamp: int | None, attributes) -> None: self._session.post( self._url("/trace/otel/add_event"), - json={"span_id": span_id, "name": name, "timestamp": timestamp, "attributes": attributes}, + json={ + "span_id": span_id, + "name": name, + "timestamp": timestamp, + "attributes": attributes, + }, ) def otel_record_exception(self, span_id: int, message: str, attributes) -> None: @@ -294,7 +360,8 @@ def otel_flush(self, timeout: int) -> bool: def otel_set_baggage(self, span_id: int, key: str, value: str) -> None: resp = self._session.post( - self._url("/trace/otel/otel_set_baggage"), json={"span_id": span_id, "key": key, "value": value} + self._url("/trace/otel/otel_set_baggage"), + json={"span_id": span_id, "key": key, "value": value}, ) data = resp.json() return data["value"] @@ -431,6 +498,9 @@ def crash(self) -> None: def container_exec_run(self, command: str) -> tuple[bool, str]: return self._client.container_exec_run(command) + def container_exec_run_raw(self, command: str) -> tuple[bool, str]: + return self._client.container_exec_run_raw(command) + def container_restart(self): self._client.container_restart() @@ -445,7 +515,12 @@ def dd_start_span( tags: list[tuple[str, str]] | None = None, ) -> Generator[_TestSpan, None, None]: resp = self._client.trace_start_span( - name=name, service=service, resource=resource, parent_id=parent_id, typestr=typestr, tags=tags + name=name, + service=service, + resource=resource, + parent_id=parent_id, + typestr=typestr, + tags=tags, ) span = _TestSpan(self._client, resp["span_id"], resp["trace_id"]) yield span