Start implementing some tests

api/fastapi/dataapi/tests/gen-db-fixtures.py

@@ -0,0 +1,86 @@
+import csv
+
+TABLE_CREATE_QUERIES = """
+CREATE TABLE fastpath (
+  measurement_uid TEXT,
+  report_id TEXT,
+  input TEXT,
+  probe_cc TEXT,
+  probe_asn INTEGER,
+  test_name TEXT,
+  test_start_time TEXT,
+  measurement_start_time TEXT,
+  filename TEXT,
+  scores TEXT,
+  platform TEXT,
+  anomaly TEXT,
+  confirmed TEXT,
+  msm_failure TEXT,
+  domain TEXT,
+  software_name TEXT,
+  software_version TEXT,
+  control_failure TEXT,
+  blocking_general REAL,
+  is_ssl_expected INTEGER,
+  page_len INTEGER,
+  page_len_ratio REAL,
+  server_cc TEXT,
+  server_asn INTEGER,
+  server_as_name TEXT,
+  update_time TEXT,
+  test_version TEXT,
+  architecture TEXT,
+  engine_name TEXT,
+  engine_version TEXT,
+  test_runtime REAL,
+  blocking_type TEXT,
+  test_helper_address TEXT,
+  test_helper_type TEXT,
+  ooni_run_link_id TEXT
+);
+
+CREATE TABLE citizenlab (
+    domain TEXT, 
+    url TEXT,
+    cc TEXT,
+    category_code TEXT
+);
+"""
+
+# Dumps for fastpath.csv and citizenlab.csv can be generated running the following commands on the backend-fsn host:
+"""
+clickhouse-client -q "SELECT * FROM fastpath WHERE measurement_start_time > '2024-01-01' AND measurement_start_time < '2024-01-10' ORDER BY measurement_start_time LIMIT 1000 INTO OUTFILE '20240110-fastpath.csv' FORMAT CSVWithNames"
+clickhouse-client -q "SELECT * FROM citizenlab INTO OUTFILE '20240110-citizenlab.csv' FORMAT CSVWithNames"
+"""
+FASTPATH_CSV = "fastpath.csv"
+CITIZENLAB_CSV = "citizenlab.csv"
+
+OUTPUT_SQL = "db-fixtures.sql"
+
+with open(OUTPUT_SQL, "w") as f:
+    f.write(TABLE_CREATE_QUERIES)
+
+    input_set = set()
+    # Process fastpath.csv
+    with open(FASTPATH_CSV) as fastpath_csv:
+        reader = csv.DictReader(fastpath_csv)
+        for row in reader:
+            input_set.add(row["input"])
+            columns = ",".join(row.keys())
+            values = ",".join([f"'{v.strip()}'" for v in row.values()])
+            query = f"INSERT INTO fastpath ({columns}) VALUES ({values});\n"
+            f.write(query)
+
+    # Process citizenlab.csv
+    with open(CITIZENLAB_CSV) as citizenlab_csv:
+        reader = csv.DictReader(citizenlab_csv)
+        for row in reader:
+            if row["url"] not in input_set:
+                continue
+            row["cc"] = row["cc"][:2]
+            columns = ",".join(row.keys())
+            values = ",".join([f"'{v.strip()}'" for v in row.values()])
+            query = f"INSERT INTO citizenlab ({columns}) VALUES ({values});\n"
+            f.write(query)
+
+print("SQL file generated successfully!")

api/fastapi/dataapi/tests/test_measurements.py

@@ -0,0 +1,78 @@
+import os
+import re
+import sqlite3
+import tempfile
+import pytest
+
+from fastapi.testclient import TestClient
+
+from ..main import app
+from ..dependencies import get_clickhouse_client
+
+THIS_DIR = os.path.dirname(__file__)
+
+
+@pytest.fixture(name="clickhouse")
+def clickhouse_fixture():
+    fd, path = tempfile.mkstemp()
+    print(f"created sqlite file {path}")
+
+    conn = sqlite3.connect(path, check_same_thread=False)
+
+    with open(os.path.join(THIS_DIR, "db-fixtures.sql")) as in_file:
+        sql = in_file.read()
+    statements = sql.split(";")
+    cur = conn.cursor()
+    for statement in statements:
+        if statement.strip():
+            cur.execute(statement)
+    conn.commit()
+
+    def replace_template_params(sql):
+        sql = re.sub(r"%\((.+?)\)s", r":\1", sql)
+        return sql
+
+    class MockClick:
+        def __init__(self, conn):
+            self.conn = conn
+
+        def execute(self, sql, query_params=(), *arg, **kwargs):
+            cursor = conn.cursor()
+            sql = replace_template_params(sql)
+            cursor.execute(sql, query_params)
+            rows = cursor.fetchall()
+            colnames = [description[0] for description in cursor.description]
+            return rows, [(cn, None) for cn in colnames]
+
+    yield MockClick(conn)
+    conn.close()
+    os.close(fd)
+    # os.remove(path)
+
+
+@pytest.fixture(name="client")
+def client_fixture(clickhouse):
+    def get_clickhouse_override():
+        return clickhouse
+
+    app.dependency_overrides[get_clickhouse_client] = get_clickhouse_override
+
+    client = TestClient(app)
+    yield client
+    app.dependency_overrides.clear()
+
+
+def test_list_measurements(client, clickhouse):
+    clickhouse.execute("SELECT * FROM fastpath")
+    response = client.get("/api/v1/measurements")
+    assert response.status_code == 200
+    j = response.json()
+    assert len(j["results"]) == 100
+
+    response = client.get("/api/v1/measurements?probe_cc=IT")
+    assert response.status_code == 200
+    j = response.json()
+    for res in j["results"]:
+        assert res["probe_cc"] == "IT"
+
+    app.dependency_overrides.clear()

api/fastapi/pyproject.toml

@@ -17,8 +17,12 @@ pydantic-settings = "^2.1.0"
 statsd = "^4.0.1"
 uvicorn = "^0.25.0"
 psycopg2 = "^2.9.9"
+httpx = "^0.26.0"
 
 
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4.4"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"