From 22f3e49e8191c5ba19118634ad37994a9a04a76a Mon Sep 17 00:00:00 2001 From: hynnot Date: Thu, 23 Jan 2025 15:43:20 +0100 Subject: [PATCH] Add support for parsing echcheck related metadata in pipeline --- .../src/oonidata/models/nettests/__init__.py | 2 + .../src/oonidata/models/nettests/echcheck.py | 33 ++++++++++++++++ .../transforms/nettests/echcheck.py | 38 +++++++++++++++++++ .../oonipipeline/transforms/observations.py | 2 + oonipipeline/tests/_fixtures.py | 2 + oonipipeline/tests/test_transforms.py | 33 ++++++++++++++++ 6 files changed, 110 insertions(+) create mode 100644 oonidata/src/oonidata/models/nettests/echcheck.py create mode 100644 oonipipeline/src/oonipipeline/transforms/nettests/echcheck.py diff --git a/oonidata/src/oonidata/models/nettests/__init__.py b/oonidata/src/oonidata/models/nettests/__init__.py index 3a9a015e..247fd39d 100644 --- a/oonidata/src/oonidata/models/nettests/__init__.py +++ b/oonidata/src/oonidata/models/nettests/__init__.py @@ -13,6 +13,7 @@ from .whatsapp import Whatsapp from .http_invalid_request_line import HTTPInvalidRequestLine from .http_header_field_manipulation import HTTPHeaderFieldManipulation +from .echcheck import ECHCheck SUPPORTED_CLASSES = [ HTTPHeaderFieldManipulation, @@ -28,6 +29,7 @@ FacebookMessenger, Whatsapp, BaseMeasurement, + ECHCheck, ] SupportedDataformats = Union[ HTTPHeaderFieldManipulation, diff --git a/oonidata/src/oonidata/models/nettests/echcheck.py b/oonidata/src/oonidata/models/nettests/echcheck.py new file mode 100644 index 00000000..ef987f94 --- /dev/null +++ b/oonidata/src/oonidata/models/nettests/echcheck.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +from typing import List, Optional + +from oonidata.models.nettests.web_connectivity import WebConnectivityControl + +from ...compat import add_slots +from ..dataformats import ( + BaseTestKeys, + DNSQuery, + NetworkEvent, + TCPConnect, + TLSHandshake, +) +from oonidata.models.nettests.base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class ECHCheckTestKeys(BaseTestKeys): + tls_handshakes: Optional[List[TLSHandshake]] = None + control: Optional[TLSHandshake] = None + target: Optional[TLSHandshake] = None + network_events: Optional[List[NetworkEvent]] = None + queries: Optional[List[DNSQuery]] = None + tcp_connect: Optional[List[TCPConnect]] = None + + +@add_slots +@dataclass +class ECHCheck(BaseMeasurement): + __test_name__ = "echcheck" + + test_keys: ECHCheckTestKeys diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/echcheck.py b/oonipipeline/src/oonipipeline/transforms/nettests/echcheck.py new file mode 100644 index 00000000..580619c0 --- /dev/null +++ b/oonipipeline/src/oonipipeline/transforms/nettests/echcheck.py @@ -0,0 +1,38 @@ +from typing import List, Tuple + +from oonidata.models.nettests import ECHCheck +from oonidata.models.observations import WebObservation + +from ..measurement_transformer import MeasurementTransformer + + +class ECHCheckTransformer(MeasurementTransformer): + def make_observations(self, msmt: ECHCheck) -> Tuple[List[WebObservation]]: + dns_observations = [] + tcp_observations = [] + tls_observations = [] + + if msmt.test_keys.queries: + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + if msmt.test_keys.tcp_connect: + tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) + + if msmt.test_keys.tls_handshakes: + tls_observations = self.make_tls_observations( + tls_handshakes=msmt.test_keys.tls_handshakes, + network_events=msmt.test_keys.network_events, + ) + elif msmt.test_keys.control and msmt.test_keys.target: + target = msmt.test_keys.target + target.echconfig = "GREASE" + tls_observations = self.make_tls_observations( + tls_handshakes=[msmt.test_keys.control, target], + network_events=msmt.test_keys.network_events, + ) + + return (self.consume_web_observations( + dns_observations=dns_observations, + tcp_observations=tcp_observations, + tls_observations=tls_observations, + http_observations=[] + ), ) diff --git a/oonipipeline/src/oonipipeline/transforms/observations.py b/oonipipeline/src/oonipipeline/transforms/observations.py index 5cd728e7..4645bdd6 100644 --- a/oonipipeline/src/oonipipeline/transforms/observations.py +++ b/oonipipeline/src/oonipipeline/transforms/observations.py @@ -37,6 +37,7 @@ from .nettests.http_invalid_request_line import ( HTTPInvalidRequestLineTransformer, ) +from .nettests.echcheck import ECHCheckTransformer from ..netinfo import NetinfoDB @@ -53,6 +54,7 @@ "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, "http_invalid_request_line": HTTPInvalidRequestLineTransformer, "web_connectivity": WebConnectivityTransformer, + "echcheck": ECHCheckTransformer, } TypeWebConnectivityObservations = Tuple[ diff --git a/oonipipeline/tests/_fixtures.py b/oonipipeline/tests/_fixtures.py index 96f150c8..f8a81020 100644 --- a/oonipipeline/tests/_fixtures.py +++ b/oonipipeline/tests/_fixtures.py @@ -39,6 +39,8 @@ "20240302000305.316064_EG_webconnectivity_397bca9091b07444", # nxdomain blocked, unknown_failure and from the future "20240309112858.009725_SE_webconnectivity_dce757ef4ec9b6c8", # blockpage for Iran in Sweden "20241101171509.547086_CN_webconnectivity_f0ec3f0e369cec9b", # web_connectivity 0.5 which was failing + "20250120145930.582606_US_echcheck_899a304b7beef05c", # echcheck tls_handshakes + "20240714111032.898994_GB_echcheck_f10079cac5cdf770" # echcheck control and target ] SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"] diff --git a/oonipipeline/tests/test_transforms.py b/oonipipeline/tests/test_transforms.py index c8931ad6..34d589ec 100644 --- a/oonipipeline/tests/test_transforms.py +++ b/oonipipeline/tests/test_transforms.py @@ -2,6 +2,7 @@ from oonidata.dataclient import load_measurement from oonidata.models.nettests.dnscheck import DNSCheck +from oonidata.models.nettests.echcheck import ECHCheck from oonidata.models.nettests.telegram import Telegram from oonidata.models.nettests.signal import Signal from oonidata.models.nettests.facebook_messenger import FacebookMessenger @@ -408,3 +409,35 @@ def test_facebook_messenger_obs(netinfodb, measurements): hostname_set.add(wo.hostname) assert hostname_set == spec_hostname_set assert len(web_obs) == 14 + + +def test_echcheck_obs_tls_handshakes(netinfodb, measurements): + msmt = load_measurement( + msmt_path=measurements["20250120145930.582606_US_echcheck_899a304b7beef05c"] + ) + assert isinstance(msmt, ECHCheck) + assert msmt.test_version == '0.2.0' + + obs_tup = measurement_to_observations( + msmt=msmt, netinfodb=netinfodb, bucket_date="2022-10-13" + ) + assert len(obs_tup) == 1 + web_obs = obs_tup[0] + assert len(web_obs) == 3 + assert any(wo.tls_echconfig == "GREASE" for wo in web_obs) + + +def test_echcheck_obs_control_and_target(netinfodb, measurements): + msmt = load_measurement( + msmt_path=measurements["20240714111032.898994_GB_echcheck_f10079cac5cdf770"] + ) + assert isinstance(msmt, ECHCheck) + assert msmt.test_version == '0.1.2' + + obs_tup = measurement_to_observations( + msmt=msmt, netinfodb=netinfodb, bucket_date="2022-10-13" + ) + assert len(obs_tup) == 1 + web_obs = obs_tup[0] + assert len(web_obs) == 2 + assert any(wo.tls_echconfig == "GREASE" for wo in web_obs)