From 0cb0d29124d89f734c083b3e32296151295cb5a0 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 16 May 2024 11:38:12 +0530 Subject: [PATCH 1/5] refactor: measurements -> oonimeasurements --- .../test_ooniapi_oonimeasurements.yml | 25 + ooniapi/common/src/common/auth.py | 70 +++ ooniapi/common/src/common/clickhouse_utils.py | 2 +- ooniapi/common/src/common/dependencies.py | 5 +- ooniapi/common/src/common/utils.py | 62 +-- ooniapi/services/measurements/pyproject.toml | 54 --- .../src/measurements/dependencies.py | 7 - .../measurements/src/measurements/main.py | 53 --- .../measurements/tests/run_smoketest.py | 53 --- .../services/oonimeasurements/.dockerignore | 10 + ooniapi/services/oonimeasurements/.gitignore | 3 + ooniapi/services/oonimeasurements/Dockerfile | 34 ++ ooniapi/services/oonimeasurements/LICENSE.txt | 26 ++ ooniapi/services/oonimeasurements/Makefile | 70 +++ .../__init__.py | 0 .../services/oonimeasurements/buildspec.yml | 29 ++ .../services/oonimeasurements/pyproject.toml | 83 ++++ .../scripts/docker-smoketest.sh | 34 ++ .../src/oonimeasurements}/__init__.py | 0 .../src/oonimeasurements}/common | 0 .../src/oonimeasurements/dependencies.py | 15 + .../src/oonimeasurements/main.py | 112 +++++ .../src/oonimeasurements}/routers/__init__.py | 0 .../oonimeasurements}/routers/aggregation.py | 25 +- .../oonimeasurements}/routers/measurements.py | 436 +++++++++--------- .../tests/__init__.py | 0 .../tests/conftest.py | 0 .../tests/fixtures/1_schema.sql | 0 .../tests/fixtures/2_fastpath_fixtures.sql | 0 .../tests/fixtures/clickhouse_1_schema.sql | 0 .../tests/fixtures/clickhouse_2_fixtures.sql | 0 .../tests/fixtures/oonipg-legacy.sql | 0 .../tests/fixtures/oonipg.sql | 0 .../tests/gen-db-fixtures.py | 0 .../tests/integ/__init__.py | 0 .../tests/integ/conftest.py | 0 .../tests/integ/test_aggregation.py | 0 .../tests/integ/test_measurements.py | 0 .../tests/test_auth.py | 0 .../tests/test_measurements.py | 0 40 files changed, 763 insertions(+), 445 deletions(-) create mode 100644 .github/workflows/test_ooniapi_oonimeasurements.yml create mode 100644 ooniapi/common/src/common/auth.py delete mode 100644 ooniapi/services/measurements/pyproject.toml delete mode 100644 ooniapi/services/measurements/src/measurements/dependencies.py delete mode 100644 ooniapi/services/measurements/src/measurements/main.py delete mode 100644 ooniapi/services/measurements/tests/run_smoketest.py create mode 100644 ooniapi/services/oonimeasurements/.dockerignore create mode 100644 ooniapi/services/oonimeasurements/.gitignore create mode 100644 ooniapi/services/oonimeasurements/Dockerfile create mode 100644 ooniapi/services/oonimeasurements/LICENSE.txt create mode 100644 ooniapi/services/oonimeasurements/Makefile rename ooniapi/services/{measurements => oonimeasurements}/__init__.py (100%) create mode 100644 ooniapi/services/oonimeasurements/buildspec.yml create mode 100644 ooniapi/services/oonimeasurements/pyproject.toml create mode 100644 ooniapi/services/oonimeasurements/scripts/docker-smoketest.sh rename ooniapi/services/{measurements/src/measurements => oonimeasurements/src/oonimeasurements}/__init__.py (100%) rename ooniapi/services/{measurements/src/measurements => oonimeasurements/src/oonimeasurements}/common (100%) create mode 100644 ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py create mode 100644 ooniapi/services/oonimeasurements/src/oonimeasurements/main.py rename ooniapi/services/{measurements/src/measurements => oonimeasurements/src/oonimeasurements}/routers/__init__.py (100%) rename ooniapi/services/{measurements/src/measurements => oonimeasurements/src/oonimeasurements}/routers/aggregation.py (96%) rename ooniapi/services/{measurements/src/measurements => oonimeasurements/src/oonimeasurements}/routers/measurements.py (81%) rename ooniapi/services/{measurements => oonimeasurements}/tests/__init__.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/conftest.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/1_schema.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/2_fastpath_fixtures.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/clickhouse_1_schema.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/clickhouse_2_fixtures.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/oonipg-legacy.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/fixtures/oonipg.sql (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/gen-db-fixtures.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/integ/__init__.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/integ/conftest.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/integ/test_aggregation.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/integ/test_measurements.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/test_auth.py (100%) rename ooniapi/services/{measurements => oonimeasurements}/tests/test_measurements.py (100%) diff --git a/.github/workflows/test_ooniapi_oonimeasurements.yml b/.github/workflows/test_ooniapi_oonimeasurements.yml new file mode 100644 index 00000000..bc6b0515 --- /dev/null +++ b/.github/workflows/test_ooniapi_oonimeasurements.yml @@ -0,0 +1,25 @@ +name: test ooniapi/oonimeasurements +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Run all tests + run: make test-cov + working-directory: ./ooniapi/services/oonimeasurements/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: oonimeasurements + working-directory: ./ooniapi/services/oonimeasurements/ diff --git a/ooniapi/common/src/common/auth.py b/ooniapi/common/src/common/auth.py new file mode 100644 index 00000000..e6740bf7 --- /dev/null +++ b/ooniapi/common/src/common/auth.py @@ -0,0 +1,70 @@ +import hashlib +from typing import Optional, Dict, Any +import jwt + + +def hash_email_address(email_address: str, key: str) -> str: + em = email_address.encode() + return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest() + + +def check_email_address( + authorization: str, + jwt_encryption_key: str, + email_address: str, + key: str +) -> bool: + account_id = get_account_id_or_raise(authorization, jwt_encryption_key=jwt_encryption_key) + hashed = hash_email_address(email_address, key=key) + if account_id == hashed: + return True + return False + + +def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]: + tok = jwt.decode(token, key, algorithms=["HS256"], **kw) + return tok + + +def create_jwt(payload: dict, key: str) -> str: + token = jwt.encode(payload, key, algorithm="HS256") + if isinstance(token, bytes): + return token.decode() + else: + return token + + +def get_client_token(authorization: str, jwt_encryption_key: str): + try: + assert authorization.startswith("Bearer ") + token = authorization[7:] + return decode_jwt(token, audience="user_auth", key=jwt_encryption_key) + except: + return None + + +def get_client_role(authorization: str, jwt_encryption_key: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + try: + assert tok + return tok["role"] + except: + return None + +def get_account_id_or_none( + authorization: str, jwt_encryption_key: str +) -> Optional[str]: + """Returns None for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + if tok: + return tok["account_id"] + return None + + +def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str: + """Raise exception for unlogged users""" + tok = get_client_token(authorization, jwt_encryption_key) + if tok: + return tok["account_id"] + raise Exception diff --git a/ooniapi/common/src/common/clickhouse_utils.py b/ooniapi/common/src/common/clickhouse_utils.py index 221041b3..71407aa4 100644 --- a/ooniapi/common/src/common/clickhouse_utils.py +++ b/ooniapi/common/src/common/clickhouse_utils.py @@ -61,7 +61,7 @@ def optimize_table(db: clickhouse_driver.Client, tblname: str) -> None: def raw_query( db: clickhouse_driver.Client, query: Query, query_params: dict, query_prio=1 -): +) -> int: settings = {"priority": query_prio, "max_execution_time": 300} q = db.execute(query, query_params, with_column_types=True, settings=settings) return q diff --git a/ooniapi/common/src/common/dependencies.py b/ooniapi/common/src/common/dependencies.py index 476a6910..22beafa4 100644 --- a/ooniapi/common/src/common/dependencies.py +++ b/ooniapi/common/src/common/dependencies.py @@ -3,7 +3,7 @@ from fastapi import Depends from fastapi import HTTPException, Header -from .utils import get_client_token +from .auth import get_client_token from .config import Settings @@ -29,6 +29,9 @@ async def verify_jwt( tok = get_client_token(authorization, settings.jwt_encryption_key) except: raise HTTPException(detail="Authentication required", status_code=401) + + if not tok: + raise HTTPException(detail="Authentication required", status_code=401) if tok["role"] not in roles: raise HTTPException(detail="Role not authorized", status_code=401) diff --git a/ooniapi/common/src/common/utils.py b/ooniapi/common/src/common/utils.py index 63bca052..c1a62fd3 100644 --- a/ooniapi/common/src/common/utils.py +++ b/ooniapi/common/src/common/utils.py @@ -1,11 +1,12 @@ from csv import DictWriter from io import StringIO +from sys import byteorder +from os import urandom import logging -from typing import Any, Dict, List, Optional, Union +from typing import List +from fastapi import Response from fastapi.responses import JSONResponse -import jwt - log = logging.getLogger(__name__) @@ -31,6 +32,15 @@ def jerror(msg, code=400, **kw) -> JSONResponse: return JSONResponse(content=dict(msg=msg, **kw), status_code=code, headers=headers) +def setcacheresponse(interval: str, response: Response): + max_age = int(interval[:-1]) * INTERVAL_UNITS[interval[-1]] + response.headers["Cache-Control"] = f"max-age={max_age}" + + +def setnocacheresponse(response: Response): + response.headers["Cache-Control"] = "no-cache" + + def commasplit(p: str) -> List[str]: assert p is not None out = set(p.split(",")) @@ -60,44 +70,10 @@ def convert_to_csv(r) -> str: return result -def decode_jwt(token: str, key: str, **kw) -> Dict[str, Any]: - tok = jwt.decode(token, key, algorithms=["HS256"], **kw) - return tok - - -def create_jwt(payload: dict, key: str) -> str: - token = jwt.encode(payload, key, algorithm="HS256") - if isinstance(token, bytes): - return token.decode() - else: - return token - - -def get_client_token(authorization: str, jwt_encryption_key: str) -> Dict[str, Any]: - assert authorization.startswith("Bearer ") - token = authorization[7:] - return decode_jwt(token, audience="user_auth", key=jwt_encryption_key) - - -def get_client_role(authorization: str, jwt_encryption_key: str) -> str: - """Raise exception for unlogged users""" - tok = get_client_token(authorization, jwt_encryption_key) - assert tok - return tok["role"] - - -def get_account_id_or_none( - authorization: str, jwt_encryption_key: str -) -> Optional[str]: - """Returns None for unlogged users""" +def generate_random_intuid(collector_id: str) -> int: try: - tok = get_client_token(authorization, jwt_encryption_key) - return tok["account_id"] - except: - return None - - -def get_account_id_or_raise(authorization: str, jwt_encryption_key: str) -> str: - """Raise exception for unlogged users""" - tok = get_client_token(authorization, jwt_encryption_key) - return tok["account_id"] + collector_id = int(collector_id) + except ValueError: + collector_id = 0 + randint = int.from_bytes(urandom(4), byteorder) + return randint * 100 + collector_id diff --git a/ooniapi/services/measurements/pyproject.toml b/ooniapi/services/measurements/pyproject.toml deleted file mode 100644 index b2763d1b..00000000 --- a/ooniapi/services/measurements/pyproject.toml +++ /dev/null @@ -1,54 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "measurements" -version = "0.5.0.dev1" -description = '' - -dependencies = [ - "fastapi ~= 0.108.0", - "clickhouse-driver ~= 0.2.6", - "sqlalchemy ~= 2.0.27", - "ujson ~= 5.9.0", - "urllib3 ~= 2.1.0", - "python-dateutil ~= 2.8.2", - "pydantic-settings ~= 2.1.0", - "statsd ~= 4.0.1", - "uvicorn ~= 0.25.0", - "httpx ~= 0.26.0", - "pyjwt ~= 2.8.0", - "alembic ~= 1.13.1", -] -readme = "README.md" -requires-python = ">=3.8" -license = "MIT" -keywords = [] -authors = [{ name = "OONI", email = "contact@ooni.org" }] -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] - -[project.urls] -Documentation = "https://github.com/ooni/backend" -Issues = "https://github.com/ooni/backend/issues" -Source = "https://github.com/ooni/backend" - -[tool.hatch.envs.test] -dependencies = [ - "coverage[toml]", - "pytest", - "pytest-cov", - "click", - "black", - "docker", -] diff --git a/ooniapi/services/measurements/src/measurements/dependencies.py b/ooniapi/services/measurements/src/measurements/dependencies.py deleted file mode 100644 index b788825d..00000000 --- a/ooniapi/services/measurements/src/measurements/dependencies.py +++ /dev/null @@ -1,7 +0,0 @@ -from clickhouse_driver import Client as ClickhouseClient - -from ooniapi.common.src.common.config import settings - - -def get_clickhouse_client() -> ClickhouseClient: - return ClickhouseClient.from_url(settings.clickhouse_url) diff --git a/ooniapi/services/measurements/src/measurements/main.py b/ooniapi/services/measurements/src/measurements/main.py deleted file mode 100644 index 99c6fdbc..00000000 --- a/ooniapi/services/measurements/src/measurements/main.py +++ /dev/null @@ -1,53 +0,0 @@ -from fastapi import FastAPI - -from .routers import measurements -from .routers import aggregation - -from ooniapi.common.config import settings -from fastapi.middleware.cors import CORSMiddleware - -import logging - -logging.basicConfig(level=getattr(logging, settings.log_level.upper())) - -app = FastAPI() -# TODO: temporarily enable all -origins = ["*"] -app.add_middleware( - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(measurements.router, prefix="/api") -app.include_router(aggregation.router, prefix="/api") - - -from importlib.metadata import version as importlib_version -from importlib.resources import files as importlib_files - -pkg_name = "oonidataapi" - -try: - pkg_version = importlib_version(pkg_name) -except: - # This happens when we are not installed, for example in development - pkg_version = None - -try: - with importlib_files(pkg_name).joinpath("BUILD_LABEL").open("r") as in_file: - build_label = in_file.read().strip() -except: - build_label = None - - -@app.get("/version") -async def version(): - return {"version": pkg_version, "build_label": build_label} - - -@app.get("/") -async def root(): - return {"message": "Hello OONItarian!"} diff --git a/ooniapi/services/measurements/tests/run_smoketest.py b/ooniapi/services/measurements/tests/run_smoketest.py deleted file mode 100644 index 37fd465d..00000000 --- a/ooniapi/services/measurements/tests/run_smoketest.py +++ /dev/null @@ -1,53 +0,0 @@ -import httpx -import time -import click -import random - - -def test_oonirun(client): - r = client.get("/api/v2/oonirun-links") - j = r.json() - assert r.status_code == 200, j - desc = j["links"] - assert isinstance(desc, list) - if len(desc) > 0: - for _ in range(5): - d = random.choice(desc) - client.get(f'/api/v2/oonirun/{d["oonirun_link_id"]}').raise_for_status() - - -def wait_for_backend(backend_base_url, timeout=10): - start_time = time.time() - - while True: - try: - with httpx.Client(base_url=backend_base_url) as client: - r = client.get("/version") - if r.status_code == 200: - print("Service ready") - break - except Exception as e: - print(f"Connection failed: {e}") - - if time.time() - start_time > timeout: - raise TimeoutError("Service did not become available in time") - - time.sleep(1) - - -@click.command() -@click.option( - "--backend-base-url", - default="http://localhost:8000", - help="Base URL of the backend", -) -def smoketest(backend_base_url): - """Run a smoke test against a running backend""" - wait_for_backend(backend_base_url) - - with httpx.Client(base_url=backend_base_url) as client: - test_oonirun(client) - - -if __name__ == "__main__": - smoketest() diff --git a/ooniapi/services/oonimeasurements/.dockerignore b/ooniapi/services/oonimeasurements/.dockerignore new file mode 100644 index 00000000..4f7a82b5 --- /dev/null +++ b/ooniapi/services/oonimeasurements/.dockerignore @@ -0,0 +1,10 @@ +.DS_Store +*.log +*.pyc +*.swp +*.env +.coverage +coverage.xml +dist/ +.venv/ +__pycache__/ diff --git a/ooniapi/services/oonimeasurements/.gitignore b/ooniapi/services/oonimeasurements/.gitignore new file mode 100644 index 00000000..9a1b4f54 --- /dev/null +++ b/ooniapi/services/oonimeasurements/.gitignore @@ -0,0 +1,3 @@ +/dist +/coverage_html +*.coverage* diff --git a/ooniapi/services/oonimeasurements/Dockerfile b/ooniapi/services/oonimeasurements/Dockerfile new file mode 100644 index 00000000..52a73615 --- /dev/null +++ b/ooniapi/services/oonimeasurements/Dockerfile @@ -0,0 +1,34 @@ +# Python builder +FROM python:3.11-bookworm as builder +ARG BUILD_LABEL=docker + +WORKDIR /build + +RUN python -m pip install hatch + +COPY . /build + +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + +RUN echo "$BUILD_LABEL" > /build/src/oonimeasurements/BUILD_LABEL + +RUN make clean +RUN hatch build + +### Actual image running on the host +FROM python:3.11-bookworm as runner + +WORKDIR /app + +COPY --from=builder /build/README.md /app/ +COPY --from=builder /build/dist/*.whl /app/ +RUN pip install /app/*whl && rm /app/*whl + +COPY --from=builder /build/src/oonimeasurements/common/alembic/ /app/alembic/ +COPY --from=builder /build/src/oonimeasurements/common/alembic.ini /app/ +RUN rm -rf /app/alembic/__pycache__ + +CMD ["uvicorn", "oonimeasurements.main:app", "--host", "0.0.0.0", "--port", "80"] +EXPOSE 80 diff --git a/ooniapi/services/oonimeasurements/LICENSE.txt b/ooniapi/services/oonimeasurements/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/ooniapi/services/oonimeasurements/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ooniapi/services/oonimeasurements/Makefile b/ooniapi/services/oonimeasurements/Makefile new file mode 100644 index 00000000..7876c8f1 --- /dev/null +++ b/ooniapi/services/oonimeasurements/Makefile @@ -0,0 +1,70 @@ +SERVICE_NAME ?= oonimeasurements + +ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME) +IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) +DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") +GIT_FULL_SHA ?= $(shell git rev-parse HEAD) +SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) +PKG_VERSION := $(shell hatch version) + +BUILD_LABEL := $(DATE)-$(SHORT_SHA) +VERSION_LABEL = v$(PKG_VERSION) +ENV_LABEL ?= latest + +print-labels: + echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}" + echo "PKG_VERSION=${PKG_VERSION}" + echo "BUILD_LABEL=${BUILD_LABEL}" + echo "VERSION_LABEL=${VERSION_LABEL}" + echo "ENV_LABEL=${ENV_LABEL}" + +init: + hatch env create + +docker-build: + # We need to use tar -czh to resolve the common dir symlink + tar -czh . | docker build \ + --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${VERSION_LABEL} \ + -t ${IMAGE_NAME}:${ENV_LABEL} \ + - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" + +docker-push: + # We need to use tar -czh to resolve the common dir symlink + docker push ${IMAGE_NAME}:${BUILD_LABEL} + docker push ${IMAGE_NAME}:${VERSION_LABEL} + docker push ${IMAGE_NAME}:${ENV_LABEL} + +docker-smoketest: + ./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL} + +imagedefinitions.json: + echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json + +test: + hatch run test + +test-cov: + hatch run test-cov + +build: + hatch build + +clean: + hatch clean + rm -f imagedefinitions.json + rm -rf build dist *eggs *.egg-info + rm -rf .venv + +run: + hatch run uvicorn $(SERVICE_NAME).main:app + +db-migration: docker-build + docker run \ + -e OONI_PG_URL=$(shell aws secretsmanager get-secret-value --secret-id oonidevops/ooni-tier0-postgres/postgresql_url | jq .SecretString) \ + ${IMAGE_NAME}:${BUILD_LABEL} \ + alembic upgrade head + +.PHONY: init test build clean docker print-labels diff --git a/ooniapi/services/measurements/__init__.py b/ooniapi/services/oonimeasurements/__init__.py similarity index 100% rename from ooniapi/services/measurements/__init__.py rename to ooniapi/services/oonimeasurements/__init__.py diff --git a/ooniapi/services/oonimeasurements/buildspec.yml b/ooniapi/services/oonimeasurements/buildspec.yml new file mode 100644 index 00000000..0b967f30 --- /dev/null +++ b/ooniapi/services/oonimeasurements/buildspec.yml @@ -0,0 +1,29 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: ooniapi/services/oonimeasurements + DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token + +phases: + install: + runtime-versions: + python: 3.11 + + pre_build: + commands: + - echo "Logging in to dockerhub" + - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text) + - echo $DOCKER_SECRET | docker login --username ooni --password-stdin + + build: + commands: + - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION} + - cd $OONI_CODE_PATH + - make docker-build + - make docker-smoketest + - make docker-push + - make imagedefinitions.json + - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/ooniapi/services/oonimeasurements/pyproject.toml b/ooniapi/services/oonimeasurements/pyproject.toml new file mode 100644 index 00000000..9e0a1d05 --- /dev/null +++ b/ooniapi/services/oonimeasurements/pyproject.toml @@ -0,0 +1,83 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "oonimeasurements" +version = "0.5.0.dev1" +description = '' + +dependencies = [ + "fastapi ~= 0.108.0", + "psycopg2 ~= 2.9.5", + "clickhouse-driver ~= 0.2.6", + "sqlalchemy ~= 2.0.27", + "ujson ~= 5.9.0", + "urllib3 ~= 2.1.0", + "python-dateutil ~= 2.8.2", + "pydantic-settings ~= 2.1.0", + "statsd ~= 4.0.1", + "uvicorn ~= 0.25.0", + "httpx ~= 0.26.0", + "pyjwt ~= 2.8.0", + "alembic ~= 1.13.1", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", +] +readme = "README.md" +requires-python = ">=3.11" +license = "BSD-3-Clause" +keywords = [] +authors = [{ name = "OONI", email = "contact@ooni.org" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://github.com/ooni/backend" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/oonimeasurements/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/oonimeasurements"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.envs.default] +dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio"] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[tool.pytest.ini_options] +addopts = ["--import-mode=importlib"] + +[tool.coverage.run] +branch = true +parallel = true +source_pkgs = ["oonimeasurements", "tests"] +omit = ["src/oonimeasurements/common/*", "src/oonimeasurements/__about__.py"] + +[tool.coverage.paths] +oonimeasurements = ["src/oonimeasurements"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] diff --git a/ooniapi/services/oonimeasurements/scripts/docker-smoketest.sh b/ooniapi/services/oonimeasurements/scripts/docker-smoketest.sh new file mode 100644 index 00000000..fdc964cc --- /dev/null +++ b/ooniapi/services/oonimeasurements/scripts/docker-smoketest.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi diff --git a/ooniapi/services/measurements/src/measurements/__init__.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/__init__.py similarity index 100% rename from ooniapi/services/measurements/src/measurements/__init__.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/__init__.py diff --git a/ooniapi/services/measurements/src/measurements/common b/ooniapi/services/oonimeasurements/src/oonimeasurements/common similarity index 100% rename from ooniapi/services/measurements/src/measurements/common rename to ooniapi/services/oonimeasurements/src/oonimeasurements/common diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py new file mode 100644 index 00000000..f3d86738 --- /dev/null +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/dependencies.py @@ -0,0 +1,15 @@ +from typing import Annotated + +from fastapi import Depends + +from clickhouse_driver import Client as Clickhouse + +from .common.config import Settings +from .common.dependencies import get_settings + +def get_clickhouse_session(settings: Annotated[Settings, Depends(get_settings)]): + db = Clickhouse.from_url(settings.clickhouse_url) + try: + yield db + finally: + db.disconnect() diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py new file mode 100644 index 00000000..e529a483 --- /dev/null +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -0,0 +1,112 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware + +from pydantic import BaseModel + +from prometheus_fastapi_instrumentator import Instrumentator + +from .routers import aggregation, measurements + +from .dependencies import get_clickhouse_session +from .common.dependencies import get_settings +from .common.version import get_build_label, get_pkg_version +from .common.clickhouse_utils import query_click +from .common.metrics import mount_metrics + + +pkg_name = "oonimeasurements" + +pkg_version = get_pkg_version(pkg_name) +build_label = get_build_label(pkg_name) + +log = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) + yield + + +app = FastAPI(lifespan=lifespan) + +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="oonimeasurements" +) + +# TODO: temporarily enable all +origins = ["*"] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(measurements.router, prefix="/api") +app.include_router(aggregation.router, prefix="/api") + + +@app.get("/version") +async def version(): + return {"version": pkg_version, "build_label": build_label} + + +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + +# TODO(decfox): Add minimal health check functionality +@app.get("/health") +async def health( + settings=Depends(get_settings), + db=Depends(get_clickhouse_session), +): + errors = [] + + # try: + # query = f"""SELECT id, update_time, start_time, end_time, reported_by, + # title, event_type, published, CCs, ASNs, domains, tags, test_names, + # links, short_description, email_address, create_time, creator_account_id + # FROM incidents FINAL + # """ + # query_click(db=db, query=query, query_params={}) + # except Exception as exc: + # log.error(exc) + # errors.append("db_error") + + if settings.jwt_encryption_key == "CHANGEME": + err = "bad_jwt_secret" + log.error(err) + errors.append(err) + + if settings.prometheus_metrics_password == "CHANGEME": + err = "bad_prometheus_password" + log.error(err) + errors.append(err) + + if len(errors) > 0: + raise HTTPException(status_code=400, detail="health check failed") + + status = "ok" + + return { + "status": status, + "errors": errors, + "version": pkg_version, + "build_label": build_label, + } + + +@app.get("/") +async def root(): + return {"message": "Hello OONItarian"} \ No newline at end of file diff --git a/ooniapi/services/measurements/src/measurements/routers/__init__.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/__init__.py similarity index 100% rename from ooniapi/services/measurements/src/measurements/routers/__init__.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/__init__.py diff --git a/ooniapi/services/measurements/src/measurements/routers/aggregation.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py similarity index 96% rename from ooniapi/services/measurements/src/measurements/routers/aggregation.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py index 23ee1562..b299ad8c 100644 --- a/ooniapi/services/measurements/src/measurements/routers/aggregation.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py @@ -7,24 +7,18 @@ from typing import List, Any, Dict, Optional, Union import logging -from fastapi import APIRouter, Depends, Query, Request +from fastapi import APIRouter, Depends, Query, HTTPException from fastapi.responses import Response from pydantic import BaseModel from typing_extensions import Annotated # debdeps: python3-sqlalchemy -from sqlalchemy.sql.expression import and_, select, column, table +from sqlalchemy.sql.expression import and_, select, column from sqlalchemy.sql.expression import table as sql_table from sqlalchemy.sql.expression import text as sql_text -from ooniapi.common.config import settings, metrics -from ..utils import ( - jerror, - convert_to_csv, - commasplit, - query_click, - query_click_one_row, -) +from oonimeasurements.common.clickhouse_utils import query_click, query_click_one_row +from oonimeasurements.common.utils import jerror, commasplit, convert_to_csv from ..dependencies import ClickhouseClient, get_clickhouse_client router = APIRouter() @@ -137,12 +131,13 @@ class MeasurementAggregation(BaseModel): result: Union[List[AggregationResult], AggregationResult] -@router.get("/v1/aggregation", response_model_exclude_none=True) -@metrics.timer("get_aggregated") +@router.get( + "/v1/aggregation", + response_model_exclude_none=True +) async def get_measurements( db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], response: Response, - request: Request, input: Annotated[ Optional[str], Query( @@ -250,7 +245,7 @@ async def get_measurements( int(i[2:]) if i.startswith("AS") else i for i in commasplit(probe_asn) ] except ValueError: - raise ValueError(f"Invalid ASN value in parameter probe_asn") + raise HTTPException(status_code=400, detail="Invalid ASN value in parameter probe_asn") probe_cc_s = [] if probe_cc: @@ -416,4 +411,4 @@ async def get_measurements( ).model_dump(exclude_none=True) except Exception as e: - return jerror(str(e), v=0) + raise HTTPException(status_code=400, detail=str(e)) diff --git a/ooniapi/services/measurements/src/measurements/routers/measurements.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py similarity index 81% rename from ooniapi/services/measurements/src/measurements/routers/measurements.py rename to ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py index 778c4c78..261d0cc6 100644 --- a/ooniapi/services/measurements/src/measurements/routers/measurements.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py @@ -3,7 +3,7 @@ The routes are mounted under /api """ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import List, Optional, Any, Dict, Union import gzip @@ -12,33 +12,40 @@ import math import time -import ujson # debdeps: python3-ujson -import urllib3 # debdeps: python3-urllib3 - -from fastapi import APIRouter, Depends, Query, HTTPException, Header, Request -from fastapi.responses import Response, JSONResponse -from pydantic import BaseModel, validator +import ujson +import urllib3 + +from fastapi import ( + APIRouter, + Depends, + Query, + HTTPException, + Header, + Response, + Request +) from typing_extensions import Annotated -# debdeps: python3-sqlalchemy from sqlalchemy.sql.expression import and_, text, select, column from sqlalchemy.sql.expression import text as sql_text from sqlalchemy.sql.expression import table as sql_table from sqlalchemy.exc import OperationalError -from psycopg2.extensions import QueryCanceledError # debdeps: python3-psycopg2 +from psycopg2.extensions import QueryCanceledError + +from clickhouse_driver import Client as ClickhouseClient from urllib.request import urlopen from urllib.parse import urljoin, urlencode -from ..common.config import settings, metrics -from ..common.utils import ( - jerror, - cachedjson, - commasplit, - query_click, - query_click_one_row, -) -from ..dependencies import ClickhouseClient, get_clickhouse_client +from ..common.config import Settings +from ..common.dependencies import get_settings +from ..common.models import BaseModel +from ..common.utils import setcacheresponse, commasplit, setnocacheresponse +from ..common.clickhouse_utils import query_click, query_click_one_row +from ..dependencies import get_clickhouse_session + + +log = logging.getLogger(__name__) router = APIRouter() @@ -46,31 +53,10 @@ FASTPATH_SERVER = "fastpath.ooni.nu" FASTPATH_PORT = 8000 -log = logging.getLogger(__name__) urllib_pool = urllib3.PoolManager() -# type hints -ostr = Optional[str] - - -class MsmtNotFound(Exception): - pass - - -""" -TODO(art): do we care to have this redirect in place? -@api_msm_blueprint.route("/") -def show_apidocs(): - Route to https://api.ooni.io/api/ to /apidocs/ - return redirect("/apidocs") -""" - - -@router.get("/v1/files", tags=["files"]) -def list_files() -> JSONResponse: - """List files - unsupported""" - return cachedjson("1d", msg="not implemented") +MsmtNotFound = HTTPException(status_code=500, detail="Measurement not found") def measurement_uid_to_s3path_linenum(db: ClickhouseClient, measurement_uid: str): @@ -83,64 +69,81 @@ def measurement_uid_to_s3path_linenum(db: ClickhouseClient, measurement_uid: str query_params = dict(uid=measurement_uid) lookup = query_click_one_row(db, sql_text(query), query_params, query_prio=3) if lookup is None: - raise HTTPException(status_code=500, detail="Measurement not found") + raise MsmtNotFound s3path = lookup["s3path"] linenum = lookup["linenum"] return s3path, linenum -@metrics.timer("get_measurement") -@router.get("/v1/measurement/{measurement_uid}") +def _fetch_jsonl_measurement_body_from_s3( + s3path: str, + linenum: int, +) -> bytes: + baseurl = f"https://{settings.s3_bucket_name}.s3.amazonaws.com/" + url = urljoin(baseurl, s3path) + log.info(f"Fetching {url}") + r = urlopen(url) + f = gzip.GzipFile(fileobj=r, mode="r") + for n, line in enumerate(f): + if n == linenum: + return line + + raise MsmtNotFound + + +class NotImplemented(BaseModel): + msg: str + + +@router.get( + "/v1/files", + tags=["files"], + response_model=NotImplemented, +) +def list_files( + response: Response, +): + """List files - unsupported""" + setcacheresponse("1d", response) + return NotImplemented(msg="not implemented") + + +@router.get( + "/v1/measurement/{measurement_uid}", + tags=["oonimeasurements"], +) def get_measurement( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], measurement_uid: str, - download: bool = False, -) -> Response: - """Get one measurement by measurement_id, + download: bool, + response: Response, + db=Depends(get_clickhouse_session) +): + """ + Get one measurement by measurement_id, Returns only the measurement without extra data from the database """ assert measurement_uid try: s3path, linenum = measurement_uid_to_s3path_linenum(db, measurement_uid) - except MsmtNotFound: - return jerror("Incorrect or inexistent measurement_uid") + except: + raise log.debug(f"Fetching file {s3path} from S3") - try: - body = _fetch_jsonl_measurement_body_from_s3(s3path, linenum) - except Exception: # pragma: no cover - log.error(f"Failed to fetch file {s3path} from S3") - return jerror("Incorrect or inexistent measurement_uid") + body = _fetch_jsonl_measurement_body_from_s3(s3path, linenum) - headers = {"Cache-Control": "max-age=3600"} if download: - headers["Content-Disposition"] = ( + response.headers["Content-Disposition"] = ( f"attachment; filename=ooni_measurement-{measurement_uid}.json" ) - return Response(content=body, media_type="application/json", headers=headers) - + setcacheresponse("1h", response) + response.content = body + response.media_type = "application/json" + return response -# # Fetching measurement bodies - - -@metrics.timer("_fetch_jsonl_measurement_body_from_s3") -def _fetch_jsonl_measurement_body_from_s3( - s3path: str, - linenum: int, -) -> bytes: - baseurl = f"https://{settings.s3_bucket_name}.s3.amazonaws.com/" - url = urljoin(baseurl, s3path) - log.info(f"Fetching {url}") - r = urlopen(url) - f = gzip.GzipFile(fileobj=r, mode="r") - for n, line in enumerate(f): - if n == linenum: - return line - - raise MsmtNotFound +### Fetching measurement bodies def report_id_input_to_s3path_linenum(db: ClickhouseClient, report_id: str, input: str): query = """SELECT s3path, linenum FROM jsonl @@ -152,15 +155,13 @@ def report_id_input_to_s3path_linenum(db: ClickhouseClient, report_id: str, inpu if lookup is None: m = f"Missing row in jsonl table: {report_id} {input}" log.error(m) - metrics.incr("msmt_not_found_in_jsonl") - raise MsmtNotFound + raise HTTPException s3path = lookup["s3path"] linenum = lookup["linenum"] return s3path, linenum -@metrics.timer("_fetch_jsonl_measurement_body_clickhouse") def _fetch_jsonl_measurement_body_clickhouse( db: ClickhouseClient, report_id: str, @@ -202,9 +203,9 @@ def _unwrap_post(post: dict) -> dict: raise Exception("Unexpected format") -@metrics.timer("_fetch_measurement_body_on_disk_by_msmt_uid") def _fetch_measurement_body_on_disk_by_msmt_uid(msmt_uid: str) -> Optional[bytes]: - """Fetch raw POST from disk, extract msmt + """ + Fetch raw POST from disk, extract msmt This is used only for msmts that have been processed by the fastpath but are not uploaded to S3 yet. YAML msmts not supported: requires implementing normalization here @@ -225,22 +226,9 @@ def _fetch_measurement_body_on_disk_by_msmt_uid(msmt_uid: str) -> Optional[bytes return ujson.dumps(body).encode() -def _fetch_measurement_body_by_uid(db: ClickhouseClient, msmt_uid: str) -> bytes: - """Fetch measurement body from either disk or jsonl on S3""" - log.debug(f"Fetching body for UID {msmt_uid}") - body = _fetch_measurement_body_on_disk_by_msmt_uid(msmt_uid) - if body is not None: - # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands - return body # type: ignore - - log.debug(f"Fetching body for UID {msmt_uid} from jsonl on S3") - s3path, linenum = measurement_uid_to_s3path_linenum(db, msmt_uid) - return _fetch_jsonl_measurement_body_from_s3(s3path, linenum) # type: ignore - - -@metrics.timer("_fetch_measurement_body_from_hosts") -def _fetch_measurement_body_from_hosts(msmt_uid: str) -> Optional[bytes]: - """Fetch raw POST from another API host, extract msmt +def _fetch_measurement_body_from_hosts(other_collectors: List[str], msmt_uid: str) -> Optional[bytes]: + """ + Fetch raw POST from another API host, extract msmt This is used only for msmts that have been processed by the fastpath but are not uploaded to S3 yet. """ @@ -254,7 +242,7 @@ def _fetch_measurement_body_from_hosts(msmt_uid: str) -> Optional[bytes]: log.info("Error", exc_info=True) return None - for hostname in settings.other_collectors: + for hostname in other_collectors: url = urljoin(f"https://{hostname}/measurement_spool/", path) log.debug(f"Attempt to load {url}") try: @@ -276,14 +264,18 @@ def _fetch_measurement_body_from_hosts(msmt_uid: str) -> Optional[bytes]: return None -@metrics.timer("fetch_measurement_body") def _fetch_measurement_body( - db: ClickhouseClient, report_id: str, input: Optional[str], measurement_uid: str + db: ClickhouseClient, + settings: Settings, + report_id: str, + input: Optional[str], + measurement_uid: str ) -> bytes: - """Fetch measurement body from either: - - local measurement spool dir (.post files) - - JSONL files on S3 - - remote measurement spool dir (another API/collector host) + """ + Fetch measurement body from either: + - local measurement spool dir (.post files) + - JSONL files on S3 + - remote measurement spool dir (another API/collector host) """ # TODO: uid_cleanup log.debug(f"Fetching body for {report_id} {input}") @@ -294,14 +286,15 @@ def _fetch_measurement_body( fresh = False if new_format: - ts = (datetime.utcnow() - timedelta(hours=1)).strftime("%Y%m%d%H%M") + ts = (datetime.now(timezone.utc) - timedelta(hours=1)).strftime("%Y%m%d%H%M") fresh = measurement_uid > ts + other_collectors = settings.other_collectors # Do the fetching in different orders based on the likelyhood of success if new_format and fresh: body = ( _fetch_measurement_body_on_disk_by_msmt_uid(measurement_uid) - or _fetch_measurement_body_from_hosts(measurement_uid) + or _fetch_measurement_body_from_hosts(other_collectors, measurement_uid) or _fetch_jsonl_measurement_body_clickhouse( db, report_id, input, measurement_uid ) @@ -313,7 +306,7 @@ def _fetch_measurement_body( db, report_id, input, measurement_uid ) or _fetch_measurement_body_on_disk_by_msmt_uid(measurement_uid) - or _fetch_measurement_body_from_hosts(measurement_uid) + or _fetch_measurement_body_from_hosts(other_collectors, measurement_uid) ) else: @@ -322,67 +315,11 @@ def _fetch_measurement_body( ) if body: - metrics.incr("msmt_body_found") - # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands return body # type: ignore - metrics.incr("msmt_body_not_found") raise MsmtNotFound -def genurl(path: str, **kw) -> str: - """Generate absolute URL for the API""" - base = settings.base_url - return urljoin(base, path) + "?" + urlencode(kw) - - -@router.get("/v1/raw_measurement") -@metrics.timer("get_raw_measurement") -async def get_raw_measurement( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], - report_id: Annotated[ - Optional[str], - Query(description="The report_id to search measurements for", min_length=3), - ] = None, - input: Annotated[ - Optional[str], - Query( - description="The input (for example a URL or IP address) to search measurements for", - min_length=3, - ), - ] = None, - measurement_uid: Annotated[ - Optional[str], - Query( - description="The measurement_uid to search measurements for", min_length=3 - ), - ] = None, -) -> Response: - """Get raw measurement body by report_id + input - responses: - '200': - description: raw measurement body, served as JSON file to be dowloaded - """ - # This is used by Explorer to let users download msmts - if measurement_uid: - # TODO: uid_cleanup - msmt_meta = _get_measurement_meta_by_uid(db, measurement_uid) - elif report_id: - # _fetch_measurement_body needs the UID - msmt_meta = _get_measurement_meta_clickhouse(db, report_id, input) - else: - raise Exception("Either report_id or measurement_uid must be provided") - - body = "{}" - if msmt_meta: - # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands - body = _fetch_measurement_body( - db, msmt_meta["report_id"], msmt_meta["input"], msmt_meta["measurement_uid"] # type: ignore - ) - - headers = {"Cache-Control": f"max-age={24*3600}"} - return Response(content=body, media_type="application/json", headers=headers) - def format_msmt_meta(msmt_meta: dict) -> dict: keys = ( @@ -404,7 +341,6 @@ def format_msmt_meta(msmt_meta: dict) -> dict: return out -@metrics.timer("get_measurement_meta_clickhouse") def _get_measurement_meta_clickhouse( db: ClickhouseClient, report_id: str, input_: Optional[str] ) -> dict: @@ -433,7 +369,6 @@ def _get_measurement_meta_clickhouse( return format_msmt_meta(msmt_meta) -@metrics.timer("get_measurement_meta_by_uid") def _get_measurement_meta_by_uid(db: ClickhouseClient, measurement_uid: str) -> dict: query = """SELECT * FROM fastpath LEFT OUTER JOIN citizenlab ON citizenlab.url = fastpath.input @@ -452,6 +387,55 @@ def _get_measurement_meta_by_uid(db: ClickhouseClient, measurement_uid: str) -> return format_msmt_meta(msmt_meta) +@router.get("/v1/raw_measurement") +async def get_raw_measurement( + report_id: Annotated[ + Optional[str], + Query(description="The report_id to search measurements for", min_length=3), + ], + input: Annotated[ + Optional[str], + Query( + description="The input (for example a URL or IP address) to search measurements for", + min_length=3, + ), + ], + measurement_uid: Annotated[ + Optional[str], + Query( + description="The measurement_uid to search measurements for", min_length=3 + ), + ], + response: Response, + db=Depends(get_clickhouse_session), + settings=Depends(get_settings), +) -> Response: + """ + Get raw measurement body by report_id + input + """ + # This is used by Explorer to let users download msmts + if measurement_uid: + # TODO: uid_cleanup + msmt_meta = _get_measurement_meta_by_uid(db, measurement_uid) + elif report_id: + # _fetch_measurement_body needs the UID + msmt_meta = _get_measurement_meta_clickhouse(db, report_id, input) + else: + HTTPException(status_code=400, detail="Either report_id or measurement_uid must be provided") + + body = "{}" + if msmt_meta: + # TODO(arturo): remove ignore once https://github.com/jsocol/pystatsd/pull/184 lands + body = _fetch_measurement_body( + db, settings, msmt_meta["report_id"], msmt_meta["input"], msmt_meta["measurement_uid"] # type: ignore + ) + + setcacheresponse("1d", response) + response.content = body + response.media_type = "application/json" + return response + + class MeasurementMeta(BaseModel): anomaly: bool confirmed: bool @@ -468,9 +452,7 @@ class MeasurementMeta(BaseModel): @router.get("/v1/measurement_meta") -@metrics.timer("get_measurement_meta") async def get_measurement_meta( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], response: Response, measurement_uid: Annotated[ Optional[str], @@ -496,9 +478,15 @@ async def get_measurement_meta( min_length=3, ), ] = None, - full: Annotated[bool, Query(description="Include JSON measurement data")] = False, + full: Annotated[ + bool, + Query(description="Include JSON measurement data") + ] = False, + db=Depends(get_clickhouse_session), ) -> MeasurementMeta: - """Get metadata on one measurement by measurement_uid or report_id + input""" + """ + Get metadata on one measurement by measurement_uid or report_id + input + """ # TODO: input can be '' or NULL in the fastpath table - fix it # TODO: see integ tests for TODO items @@ -509,15 +497,15 @@ async def get_measurement_meta( log.info(f"get_measurement_meta {report_id} {input}") msmt_meta = _get_measurement_meta_clickhouse(db, report_id, input) else: - raise Exception("Either report_id or measurement_uid must be provided") + raise HTTPException(status_code=400, detail="Either report_id or measurement_uid must be provided") assert isinstance(msmt_meta, dict) + + setcacheresponse("1m", response) if not full: - response.headers["Cache-Control"] = f"max-age=60" return MeasurementMeta(**msmt_meta) if msmt_meta == {}: # measurement not found - response.headers["Cache-Control"] = f"max-age=60" return MeasurementMeta( raw_measurement="", **msmt_meta, @@ -534,15 +522,13 @@ async def get_measurement_meta( log.error(e, exc_info=True) body = "" - response.headers["Cache-Control"] = f"max-age=60" return MeasurementMeta( raw_measurement=body, **msmt_meta, ) -# # Listing measurements - +### Listing measurements # TODO(art): Isn't this the same as the above MeasurementMeta? Check it class MeasurementMeta2(BaseModel): @@ -575,50 +561,57 @@ class MeasurementList(BaseModel): results: List[MeasurementMeta2] +def genurl(base_url: str, path: str, **kw) -> str: + """Generate absolute URL for the API""" + return urljoin(base_url, path) + "?" + urlencode(kw) + + @router.get("/v1/measurements") -@metrics.timer("list_measurements") async def list_measurements( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], - response: Response, request: Request, + response: Response, report_id: Annotated[ Optional[str], Query(description="Report_id to search measurements for", min_length=3), - ] = None, + ], input: Annotated[ Optional[str], Query( description="Input (for example a URL or IP address) to search measurements for", min_length=3, ), - ] = None, + ], domain: Annotated[ Optional[str], Query(description="Domain to search measurements for", min_length=3), - ] = None, + ], probe_cc: Annotated[ Optional[str], Query(description="Two letter country code") - ] = None, + ], probe_asn: Annotated[ Union[str, int, None], Query(description='Autonomous system number in the format "ASXXX"'), - ] = None, - test_name: Annotated[Optional[str], Query(description="Name of the test")] = None, + ], + test_name: Annotated[ + Optional[str], + Query(description="Name of the test"), + ], category_code: Annotated[ - Optional[str], Query(description="Category code from the citizenlab list") - ] = None, + Optional[str], + Query(description="Category code from the citizenlab list"), + ], since: Annotated[ Optional[str], Query( description='Start date of when measurements were run (ex. "2016-10-20T10:30:00")' ), - ] = None, + ], until: Annotated[ Optional[str], Query( description='End date of when measurement were run (ex. "2016-10-20T10:30:00")' ), - ] = None, + ], confirmed: Annotated[ Optional[bool], Query( @@ -627,7 +620,7 @@ async def list_measurements( "Default: no filtering (show both true and false)" ) ), - ] = None, + ], anomaly: Annotated[ Optional[bool], Query( @@ -636,7 +629,7 @@ async def list_measurements( "Default: no filtering (show both true and false)" ) ), - ] = None, + ], failure: Annotated[ Optional[bool], Query( @@ -645,22 +638,22 @@ async def list_measurements( "Default: no filtering (show both true and false)" ) ), - ] = None, + ], software_version: Annotated[ Optional[str], Query(description="Filter measurements by software version. Comma-separated."), - ] = None, + ], test_version: Annotated[ Optional[str], Query(description="Filter measurements by test version. Comma-separated."), - ] = None, + ], engine_version: Annotated[ Optional[str], Query(description="Filter measurements by engine version. Comma-separated."), - ] = None, + ], ooni_run_link_id: Annotated[ Optional[str], Query(description="Filter measurements by OONIRun ID.") - ] = None, + ], order_by: Annotated[ Optional[str], Query( @@ -674,7 +667,7 @@ async def list_measurements( "test_name", ], ), - ] = None, + ], order: Annotated[ str, Query( @@ -689,8 +682,12 @@ async def list_measurements( int, Query(description="Number of records to return (default: 100)") ] = 100, user_agent: Annotated[str | None, Header()] = None, + db=Depends(get_clickhouse_session), + settings=Depends(get_settings), ) -> MeasurementList: - """Search for measurements using only the database. Provide pagination.""" + """ + Search for measurements using only the database. Provide pagination. + """ # x-code-samples: # - lang: 'curl' # source: | @@ -727,7 +724,7 @@ async def list_measurements( results=[MeasurementMeta2(measurement_url="")], ) - # # Prepare query parameters + ### Prepare query parameters until_dt = None if until is not None: @@ -737,10 +734,10 @@ async def list_measurements( try: if until is None: if report_id is None: - t = datetime.utcnow() + timedelta(days=1) + t = datetime.now(timezone.utc) + timedelta(days=1) until_dt = datetime(t.year, t.month, t.day) except ValueError: - raise HTTPException(status_code=400, detail="Invalid until") + raise HTTPException(status_code=400, detail="Invalid until parameter") since_dt = None if since is not None: @@ -751,12 +748,12 @@ async def list_measurements( if report_id is None and until_dt is not None: since_dt = until_dt - timedelta(days=30) except ValueError: - raise HTTPException(status_code=400, detail="Invalid since") + raise HTTPException(status_code=400, detail="Invalid since parameter") if order.lower() not in ("asc", "desc"): - raise HTTPException(status_code=400, detail="Invalid order") + raise HTTPException(status_code=400, detail="Invalid order parameter") - # # Perform query + ### Perform query INULL = "" # Special value for input = NULL to merge rows with FULL OUTER JOIN @@ -892,7 +889,7 @@ async def list_measurements( results = [] for row in rows: msmt_uid = row["measurement_uid"] - url = genurl("/api/v1/raw_measurement", measurement_uid=msmt_uid) + url = genurl(settings.base_url, "/api/v1/raw_measurement", measurement_uid=msmt_uid) results.append( MeasurementMeta2( measurement_uid=msmt_uid, @@ -944,7 +941,7 @@ async def list_measurements( next_args = dict(request.query_params) next_args["offset"] = str(offset + limit) next_args["limit"] = str(limit) - next_url = genurl("/api/v1/measurements", **next_args) + next_url = genurl(settings.base_url, "/api/v1/measurements", **next_args) query_time = time.time() - iter_start_time metadata = ResultsMetadata( @@ -956,15 +953,17 @@ async def list_measurements( next_url=next_url, query_time=query_time, ) - response.headers["Cache-Control"] = "max-age=60" + setcacheresponse("1m", response) return MeasurementList(metadata=metadata, results=results[:limit]) +class ErrorResponse(BaseModel): + v: int + msg: str + + @router.get("/v1/torsf_stats") -@metrics.timer("get_torsf_stats") async def get_torsf_stats( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], - response: Response, probe_cc: Annotated[Optional[str], Query(description="Two letter country code")], since: Annotated[ Optional[datetime], @@ -978,14 +977,14 @@ async def get_torsf_stats( description='End date of when measurement were run (ex. "2016-10-20T10:30:00")' ), ], + response: Response, + db=Depends(get_clickhouse_session), ) -> Response: - """Tor Pluggable Transports statistics - Average / percentiles / total_count grouped by day - Either group-by or filter by probe_cc - Returns a format similar to get_aggregated - responses: - '200': - description: Returns aggregated counters + """ + Tor Pluggable Transports statistics + Average / percentiles / total_count grouped by day + Either group-by or filter by probe_cc + Returns a format similar to get_aggregated """ cacheable = False @@ -1031,8 +1030,9 @@ async def get_torsf_stats( row["anomaly_rate"] = row["anomaly_count"] / row["measurement_count"] result.append(row) if cacheable: - response.headers["Cache-Control"] = f"max_age={3600 * 24}" + setcacheresponse("1d", response) return Response({"v": 0, "result": result}) except Exception as e: - return jerror(str(e), v=0) + setnocacheresponse(response) + return ErrorResponse(msg=str(e), v=0) diff --git a/ooniapi/services/measurements/tests/__init__.py b/ooniapi/services/oonimeasurements/tests/__init__.py similarity index 100% rename from ooniapi/services/measurements/tests/__init__.py rename to ooniapi/services/oonimeasurements/tests/__init__.py diff --git a/ooniapi/services/measurements/tests/conftest.py b/ooniapi/services/oonimeasurements/tests/conftest.py similarity index 100% rename from ooniapi/services/measurements/tests/conftest.py rename to ooniapi/services/oonimeasurements/tests/conftest.py diff --git a/ooniapi/services/measurements/tests/fixtures/1_schema.sql b/ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/1_schema.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql diff --git a/ooniapi/services/measurements/tests/fixtures/2_fastpath_fixtures.sql b/ooniapi/services/oonimeasurements/tests/fixtures/2_fastpath_fixtures.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/2_fastpath_fixtures.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/2_fastpath_fixtures.sql diff --git a/ooniapi/services/measurements/tests/fixtures/clickhouse_1_schema.sql b/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/clickhouse_1_schema.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql diff --git a/ooniapi/services/measurements/tests/fixtures/clickhouse_2_fixtures.sql b/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/clickhouse_2_fixtures.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql diff --git a/ooniapi/services/measurements/tests/fixtures/oonipg-legacy.sql b/ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/oonipg-legacy.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql diff --git a/ooniapi/services/measurements/tests/fixtures/oonipg.sql b/ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql similarity index 100% rename from ooniapi/services/measurements/tests/fixtures/oonipg.sql rename to ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql diff --git a/ooniapi/services/measurements/tests/gen-db-fixtures.py b/ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py similarity index 100% rename from ooniapi/services/measurements/tests/gen-db-fixtures.py rename to ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py diff --git a/ooniapi/services/measurements/tests/integ/__init__.py b/ooniapi/services/oonimeasurements/tests/integ/__init__.py similarity index 100% rename from ooniapi/services/measurements/tests/integ/__init__.py rename to ooniapi/services/oonimeasurements/tests/integ/__init__.py diff --git a/ooniapi/services/measurements/tests/integ/conftest.py b/ooniapi/services/oonimeasurements/tests/integ/conftest.py similarity index 100% rename from ooniapi/services/measurements/tests/integ/conftest.py rename to ooniapi/services/oonimeasurements/tests/integ/conftest.py diff --git a/ooniapi/services/measurements/tests/integ/test_aggregation.py b/ooniapi/services/oonimeasurements/tests/integ/test_aggregation.py similarity index 100% rename from ooniapi/services/measurements/tests/integ/test_aggregation.py rename to ooniapi/services/oonimeasurements/tests/integ/test_aggregation.py diff --git a/ooniapi/services/measurements/tests/integ/test_measurements.py b/ooniapi/services/oonimeasurements/tests/integ/test_measurements.py similarity index 100% rename from ooniapi/services/measurements/tests/integ/test_measurements.py rename to ooniapi/services/oonimeasurements/tests/integ/test_measurements.py diff --git a/ooniapi/services/measurements/tests/test_auth.py b/ooniapi/services/oonimeasurements/tests/test_auth.py similarity index 100% rename from ooniapi/services/measurements/tests/test_auth.py rename to ooniapi/services/oonimeasurements/tests/test_auth.py diff --git a/ooniapi/services/measurements/tests/test_measurements.py b/ooniapi/services/oonimeasurements/tests/test_measurements.py similarity index 100% rename from ooniapi/services/measurements/tests/test_measurements.py rename to ooniapi/services/oonimeasurements/tests/test_measurements.py From ae3b10d7a4e11312d53f2f362d9f2e762f35e828 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 16 May 2024 18:28:46 +0530 Subject: [PATCH 2/5] refactor: improve health check and bug fixes --- .../src/oonimeasurements/__about__.py | 1 + .../src/oonimeasurements/main.py | 18 ++++++++---------- .../oonimeasurements/routers/aggregation.py | 8 +++++--- .../oonimeasurements/routers/measurements.py | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 ooniapi/services/oonimeasurements/src/oonimeasurements/__about__.py diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/__about__.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/__about__.py new file mode 100644 index 00000000..342f1d78 --- /dev/null +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/__about__.py @@ -0,0 +1 @@ +VERSION = "0.5.0dev1" diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index e529a483..449f22b0 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -73,16 +73,14 @@ async def health( ): errors = [] - # try: - # query = f"""SELECT id, update_time, start_time, end_time, reported_by, - # title, event_type, published, CCs, ASNs, domains, tags, test_names, - # links, short_description, email_address, create_time, creator_account_id - # FROM incidents FINAL - # """ - # query_click(db=db, query=query, query_params={}) - # except Exception as exc: - # log.error(exc) - # errors.append("db_error") + try: + query = f"""SELECT * + FROM fastpath FINAL + """ + query_click(db=db, query=query, query_params={}) + except Exception as exc: + log.error(exc) + errors.append("db_error") if settings.jwt_encryption_key == "CHANGEME": err = "bad_jwt_secret" diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py index b299ad8c..36ab79f1 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/aggregation.py @@ -12,14 +12,16 @@ from pydantic import BaseModel from typing_extensions import Annotated -# debdeps: python3-sqlalchemy +from clickhouse_driver import Client as ClickhouseClient + from sqlalchemy.sql.expression import and_, select, column from sqlalchemy.sql.expression import table as sql_table from sqlalchemy.sql.expression import text as sql_text from oonimeasurements.common.clickhouse_utils import query_click, query_click_one_row from oonimeasurements.common.utils import jerror, commasplit, convert_to_csv -from ..dependencies import ClickhouseClient, get_clickhouse_client +from ..dependencies import get_clickhouse_session + router = APIRouter() @@ -136,7 +138,6 @@ class MeasurementAggregation(BaseModel): response_model_exclude_none=True ) async def get_measurements( - db: Annotated[ClickhouseClient, Depends(get_clickhouse_client)], response: Response, input: Annotated[ Optional[str], @@ -226,6 +227,7 @@ async def get_measurements( download: Annotated[ Optional[bool], Query(description="If we should be triggering a file download") ] = False, + db=Depends(get_clickhouse_session), ): # TODO(art): figure out how to define either CSV or JSON data format in the response """Aggregate counters data""" # TODO: diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py index 261d0cc6..5672ef1c 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/routers/measurements.py @@ -39,7 +39,7 @@ from ..common.config import Settings from ..common.dependencies import get_settings -from ..common.models import BaseModel +from ..common.routers import BaseModel from ..common.utils import setcacheresponse, commasplit, setnocacheresponse from ..common.clickhouse_utils import query_click, query_click_one_row from ..dependencies import get_clickhouse_session @@ -979,7 +979,7 @@ async def get_torsf_stats( ], response: Response, db=Depends(get_clickhouse_session), -) -> Response: +): """ Tor Pluggable Transports statistics Average / percentiles / total_count grouped by day From e4b14f062179c3d9db899122a6e7c889a8415e4b Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 16 May 2024 18:29:35 +0530 Subject: [PATCH 3/5] refactor: tests for oonimeasurements service --- .../{__init__.py => README.md} | 0 .../services/oonimeasurements/pyproject.toml | 49 +- .../oonimeasurements/tests/conftest.py | 116 +- .../oonimeasurements/tests/docker-compose.yml | 6 + .../tests/fixtures/1_schema.sql | 44 - .../tests/fixtures/clickhouse_1_schema.sql | 261 ---- .../tests/fixtures/clickhouse_2_fixtures.sql | 23 - .../tests/fixtures/oonipg-legacy.sql | 182 --- .../tests/fixtures/oonipg.sql | 396 ------ .../oonimeasurements/tests/gen-db-fixtures.py | 86 -- .../oonimeasurements/tests/integ/__init__.py | 0 .../oonimeasurements/tests/integ/conftest.py | 91 -- .../tests/integ/test_measurements.py | 1087 ----------------- .../migrations/clickhouse_init_tables.sql | 66 + .../clickhouse_populate_citizenlab.sql | 641 ++++++++++ .../clickhouse_populate_fastpath.sql} | 633 +--------- .../migrations/clickhouse_populate_jsonl.sql | 1 + .../tests/{integ => }/test_aggregation.py | 27 +- .../oonimeasurements/tests/test_auth.py | 22 - .../tests/test_integration.py | 40 + .../oonimeasurements/tests/test_main.py | 35 + .../tests/test_measurements.py | 662 +++++++++- 22 files changed, 1502 insertions(+), 2966 deletions(-) rename ooniapi/services/oonimeasurements/{__init__.py => README.md} (100%) create mode 100644 ooniapi/services/oonimeasurements/tests/docker-compose.yml delete mode 100644 ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql delete mode 100644 ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql delete mode 100644 ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql delete mode 100644 ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql delete mode 100644 ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql delete mode 100644 ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py delete mode 100644 ooniapi/services/oonimeasurements/tests/integ/__init__.py delete mode 100644 ooniapi/services/oonimeasurements/tests/integ/conftest.py delete mode 100644 ooniapi/services/oonimeasurements/tests/integ/test_measurements.py create mode 100644 ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql create mode 100644 ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql rename ooniapi/services/oonimeasurements/tests/{fixtures/2_fastpath_fixtures.sql => migrations/clickhouse_populate_fastpath.sql} (92%) create mode 100644 ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql rename ooniapi/services/oonimeasurements/tests/{integ => }/test_aggregation.py (94%) delete mode 100644 ooniapi/services/oonimeasurements/tests/test_auth.py create mode 100644 ooniapi/services/oonimeasurements/tests/test_integration.py create mode 100644 ooniapi/services/oonimeasurements/tests/test_main.py diff --git a/ooniapi/services/oonimeasurements/__init__.py b/ooniapi/services/oonimeasurements/README.md similarity index 100% rename from ooniapi/services/oonimeasurements/__init__.py rename to ooniapi/services/oonimeasurements/README.md diff --git a/ooniapi/services/oonimeasurements/pyproject.toml b/ooniapi/services/oonimeasurements/pyproject.toml index 9e0a1d05..fcb19189 100644 --- a/ooniapi/services/oonimeasurements/pyproject.toml +++ b/ooniapi/services/oonimeasurements/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "oonimeasurements" -version = "0.5.0.dev1" +dynamic = ["version"] description = '' dependencies = [ @@ -24,11 +24,14 @@ dependencies = [ "prometheus-fastapi-instrumentator ~= 6.1.0", "prometheus-client", ] + readme = "README.md" requires-python = ">=3.11" license = "BSD-3-Clause" keywords = [] -authors = [{ name = "OONI", email = "contact@ooni.org" }] +authors = [ + { name = "OONI", email = "contact@ooni.org" }, +] classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", @@ -42,7 +45,7 @@ classifiers = [ ] [project.urls] -Documentation = "https://github.com/ooni/backend" +Documentation = "https://docs.ooni.org" Issues = "https://github.com/ooni/backend/issues" Source = "https://github.com/ooni/backend" @@ -56,8 +59,18 @@ include = ["BUILD_LABEL"] packages = ["src/oonimeasurements"] artifacts = ["BUILD_LABEL"] +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.envs.default] -dependencies = ["pytest", "pytest-cov", "click", "black", "pytest-postgresql", "pytest-asyncio"] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-asyncio", + "pytest-docker", +] path = ".venv/" [tool.hatch.envs.default.scripts] @@ -66,18 +79,32 @@ test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --s cov-report = ["coverage report"] cov = ["test-cov", "cov-report"] -[tool.pytest.ini_options] -addopts = ["--import-mode=importlib"] +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/oonimeasurements tests}" [tool.coverage.run] +source_pkgs = ["oonimeasurements", "tests"] branch = true parallel = true -source_pkgs = ["oonimeasurements", "tests"] -omit = ["src/oonimeasurements/common/*", "src/oonimeasurements/__about__.py"] +omit = [ + "src/oonimeasurements/common/*", + "src/oonimeasurements/__about__.py" +] [tool.coverage.paths] -oonimeasurements = ["src/oonimeasurements"] -tests = ["tests"] +oonimeasurements = ["src/oonimeasurements", "*/oonimeasurements/src/oonimeasurements"] +tests = ["tests", "*/oonimeasurements/tests"] [tool.coverage.report] -exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] \ No newline at end of file diff --git a/ooniapi/services/oonimeasurements/tests/conftest.py b/ooniapi/services/oonimeasurements/tests/conftest.py index cfb5f698..b943a131 100644 --- a/ooniapi/services/oonimeasurements/tests/conftest.py +++ b/ooniapi/services/oonimeasurements/tests/conftest.py @@ -1,84 +1,110 @@ +from pathlib import Path import pytest import time import jwt -from pathlib import Path from fastapi.testclient import TestClient +from clickhouse_driver import Client as ClickhouseClient -from ..config import settings -from ..main import app -from ..dependencies import get_postgresql_session -from .. import models +from oonimeasurements.common.config import Settings +from oonimeasurements.dependencies import get_settings +from oonimeasurements.main import app -import sqlalchemy as sa -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker +THIS_DIR = Path(__file__).parent.resolve() -def setup_db_with_alembic(db_url): - from alembic import command - from alembic.config import Config +def is_clickhouse_running(url): + try: + with ClickhouseClient.from_url(url) as client: + client.execute("SELECT 1") + return True + except Exception: + return False - migrations_path = (Path(__file__).parent.parent / "alembic").resolve() - alembic_cfg = Config() - alembic_cfg.set_main_option("script_location", str(migrations_path)) - alembic_cfg.set_main_option("sqlalchemy.url", db_url) +@pytest.fixture(scope="session") +def clickhouse_server(docker_ip, docker_services): + port = docker_services.port_for("clickhouse", 9000) + url = "clickhouse://{}:{}".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) + ) + yield url - ret = command.upgrade(alembic_cfg, "head") - print(ret) +def run_migration(path: Path, click: ClickhouseClient): + sql_no_comment = "\n".join( + filter(lambda x: not x.startswith("--"), path.read_text().split("\n")) + ) + queries = sql_no_comment.split(";") + for q in queries: + q = q.strip() + if not q: + continue + click.execute(q) -def override_pg(db_url): - def f(): - engine = create_engine(db_url) - SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - db = SessionLocal() - try: - yield db - finally: - db.close() - return f +def create_db_for_fixture(conn_url): + try: + with ClickhouseClient.from_url(conn_url) as client: + migrations_dir = THIS_DIR / "migrations" + for fn in sorted(migrations_dir.iterdir()): + migration_path = fn.resolve() + run_migration(migration_path, click=client) + return conn_url + except Exception as e: + pytest.skip("database migration failed") -@pytest.fixture -def postgresql_app_override(postgresql): - db_url = f"postgresql://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}" +@pytest.fixture(scope="session") +def db(clickhouse_server): + yield create_db_for_fixture(conn_url=clickhouse_server) - setup_db_with_alembic(db_url) - app.dependency_overrides[get_postgresql_session] = override_pg(db_url) - yield +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings @pytest.fixture -def client(postgresql_app_override): +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + clickhouse_url = "clickhouse://badhost:9000" + ) + client = TestClient(app) - return client + yield client + + +@pytest.fixture +def client(db): + app.dependency_overrides[get_settings] = make_override_get_settings( + clickhouse_url=db, + jwt_encryption_key="super_secure", + prometheus_metrics_password="super_secure", + account_id_hashing_key="super_secure" + ) + + client = TestClient(app) + yield client def create_jwt(payload: dict) -> str: - key = settings.jwt_encryption_key - token = jwt.encode(payload, key, algorithm="HS256") - if isinstance(token, bytes): - return token.decode() - else: - return token + return jwt.encode(payload, "super_secure", algorithm="HS256") -def create_session_token(account_id: str, role: str, login_time=None) -> str: +def create_session_token(account_id: str, role: str) -> str: now = int(time.time()) - if login_time is None: - login_time = now payload = { "nbf": now, "iat": now, "exp": now + 10 * 86400, "aud": "user_auth", "account_id": account_id, - "login_time": login_time, + "login_time": None, "role": role, } return create_jwt(payload) diff --git a/ooniapi/services/oonimeasurements/tests/docker-compose.yml b/ooniapi/services/oonimeasurements/tests/docker-compose.yml new file mode 100644 index 00000000..7546ca5b --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2' +services: + clickhouse: + image: "clickhouse/clickhouse-server" + ports: + - "9000:9000" diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql b/ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql deleted file mode 100644 index 85496df7..00000000 --- a/ooniapi/services/oonimeasurements/tests/fixtures/1_schema.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE TABLE fastpath ( - measurement_uid TEXT, - report_id TEXT, - input TEXT, - probe_cc TEXT, - probe_asn INTEGER, - test_name TEXT, - test_start_time TEXT, - measurement_start_time TEXT, - filename TEXT, - scores TEXT, - platform TEXT, - anomaly TEXT, - confirmed TEXT, - msm_failure TEXT, - domain TEXT, - software_name TEXT, - software_version TEXT, - control_failure TEXT, - blocking_general REAL, - is_ssl_expected INTEGER, - page_len INTEGER, - page_len_ratio REAL, - server_cc TEXT, - server_asn INTEGER, - server_as_name TEXT, - update_time TEXT, - test_version TEXT, - architecture TEXT, - engine_name TEXT, - engine_version TEXT, - test_runtime REAL, - blocking_type TEXT, - test_helper_address TEXT, - test_helper_type TEXT, - ooni_run_link_id TEXT -); - -CREATE TABLE citizenlab ( - domain TEXT, - url TEXT, - cc TEXT, - category_code TEXT -); \ No newline at end of file diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql b/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql deleted file mode 100644 index 271e494b..00000000 --- a/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_1_schema.sql +++ /dev/null @@ -1,261 +0,0 @@ --- Create tables for Clickhouse integ tests - --- Main tables - -CREATE TABLE default.fastpath -( - `measurement_uid` String, - `report_id` String, - `input` String, - `probe_cc` String, - `probe_asn` UInt32, - `test_name` String, - `test_start_time` DateTime, - `measurement_start_time` DateTime, - `filename` String, - `scores` String, - `platform` String, - `anomaly` String, - `confirmed` String, - `msm_failure` String, - `domain` String, - `software_name` String, - `software_version` String, - `control_failure` String, - `blocking_general` Float32, - `is_ssl_expected` Int8, - `page_len` Int32, - `page_len_ratio` Float32, - `server_cc` String, - `server_asn` Int8, - `server_as_name` String, - `update_time` DateTime64(3) MATERIALIZED now64(), - `test_version` String, - `test_runtime` Float32, - `architecture` String, - `engine_name` String, - `engine_version` String, - `blocking_type` String, - `test_helper_address` LowCardinality(String), - `test_helper_type` LowCardinality(String), - `ooni_run_link_id` Nullable(UInt64) -) -ENGINE = ReplacingMergeTree -ORDER BY (measurement_start_time, report_id, input) -SETTINGS index_granularity = 8192; - -CREATE TABLE default.jsonl -( - `report_id` String, - `input` String, - `s3path` String, - `linenum` Int32, - `measurement_uid` String -) -ENGINE = MergeTree -ORDER BY (report_id, input) -SETTINGS index_granularity = 8192; - -CREATE TABLE default.url_priorities ( - `sign` Int8, - `category_code` String, - `cc` String, - `domain` String, - `url` String, - `priority` Int32 -) -ENGINE = CollapsingMergeTree(sign) -ORDER BY (category_code, cc, domain, url, priority) -SETTINGS index_granularity = 1024; - -CREATE TABLE default.citizenlab -( - `domain` String, - `url` String, - `cc` FixedString(32), - `category_code` String -) -ENGINE = ReplacingMergeTree -ORDER BY (domain, url, cc, category_code) -SETTINGS index_granularity = 4; - -CREATE TABLE default.citizenlab_flip AS default.citizenlab; - -CREATE TABLE test_groups ( - `test_name` String, - `test_group` String -) -ENGINE = Join(ANY, LEFT, test_name); - - --- Auth - -CREATE TABLE accounts -( - `account_id` FixedString(32), - `role` String -) -ENGINE = EmbeddedRocksDB -PRIMARY KEY account_id; - -CREATE TABLE session_expunge -( - `account_id` FixedString(32), - `threshold` DateTime DEFAULT now() -) -ENGINE = EmbeddedRocksDB -PRIMARY KEY account_id; - --- Materialized views - -CREATE MATERIALIZED VIEW default.counters_test_list -( - `day` DateTime, - `probe_cc` String, - `input` String, - `msmt_cnt` UInt64 -) -ENGINE = SummingMergeTree -PARTITION BY day -ORDER BY (probe_cc, input) -SETTINGS index_granularity = 8192 AS -SELECT - toDate(measurement_start_time) AS day, - probe_cc, - input, - count() AS msmt_cnt -FROM default.fastpath -INNER JOIN default.citizenlab ON fastpath.input = citizenlab.url -WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') -GROUP BY - day, - probe_cc, - input; - -CREATE MATERIALIZED VIEW default.counters_asn_test_list -( - `week` DateTime, - `probe_cc` String, - `probe_asn` UInt32, - `input` String, - `msmt_cnt` UInt64 -) -ENGINE = SummingMergeTree -ORDER BY (probe_cc, probe_asn, input) -SETTINGS index_granularity = 8192 AS -SELECT - toStartOfWeek(measurement_start_time) AS week, - probe_cc, - probe_asn, - input, - count() AS msmt_cnt -FROM default.fastpath -INNER JOIN default.citizenlab ON fastpath.input = citizenlab.url -WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') -GROUP BY - week, - probe_cc, - probe_asn, - input; - -CREATE TABLE msmt_feedback -( - `measurement_uid` String, - `account_id` String, - `status` String, - `update_time` DateTime64(3) MATERIALIZED now64() -) -ENGINE = ReplacingMergeTree -ORDER BY (measurement_uid, account_id) -SETTINGS index_granularity = 4; - -CREATE TABLE default.fingerprints_dns -( - `name` String, - `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6), - `other_names` String, - `location_found` String, - `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), - `pattern` String, - `confidence_no_fp` UInt8, - `expected_countries` String, - `source` String, - `exp_url` String, - `notes` String -) -ENGINE = EmbeddedRocksDB -PRIMARY KEY name; - -CREATE TABLE default.fingerprints_http -( - `name` String, - `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8), - `other_names` String, - `location_found` String, - `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), - `pattern` String, - `confidence_no_fp` UInt8, - `expected_countries` String, - `source` String, - `exp_url` String, - `notes` String -) -ENGINE = EmbeddedRocksDB -PRIMARY KEY name; - -CREATE TABLE asnmeta -( - `asn` UInt32, - `org_name` String, - `cc` String, - `changed` Date, - `aut_name` String, - `source` String -) -ENGINE = MergeTree -ORDER BY (asn, changed); - --- Disabled due to CREATE query errors --- CREATE TABLE IF NOT EXISTS default.incidents --- ( --- `update_time` DateTime DEFAULT now(), --- `create_time` DateTime DEFAULT now(), --- `start_time` DateTime DEFAULT now(), --- `end_time` Nullable(DateTime), --- `creator_account_id` FixedString(32), --- `reported_by` String, --- `email_address` String, --- `id` String, --- `title` String, --- `text` String, --- `event_type` LowCardinality(String), --- `published` UInt8, --- `deleted` UInt8 DEFAULT 0, --- `CCs` Array(FixedString(2)), --- `ASNs` Array(UInt32), --- `domains` Array(String), --- `tags` Array(String), --- `links` Array(String), --- `test_names` Array(String), --- `short_description` String, --- ) --- ENGINE = ReplacingMergeTree(update_time) --- ORDER BY (id) --- SETTINGS index_granularity = 1; --- --- CREATE TABLE IF NOT EXISTS default.oonirun --- ( --- `ooni_run_link_id` UInt64, --- `descriptor_creation_time` DateTime64(3), --- `translation_creation_time` DateTime64(3), --- `creator_account_id` FixedString(32), --- `archived` UInt8 DEFAULT 0, --- `descriptor` String, --- `author` String, --- `name` String, --- `short_description` String, --- `icon` String --- ) --- ENGINE = ReplacingMergeTree(translation_creation_time) --- ORDER BY (ooni_run_link_id, descriptor_creation_time) --- SETTINGS index_granularity = 1; \ No newline at end of file diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql b/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql deleted file mode 100644 index 6943e6b1..00000000 --- a/ooniapi/services/oonimeasurements/tests/fixtures/clickhouse_2_fixtures.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Populate lookup tables - -INSERT INTO test_groups (test_name, test_group) VALUES ('bridge_reachability', 'circumvention'), ('meek_fronted_requests_test', 'circumvention'), ('psiphon', 'circumvention'), ('riseupvpn', 'circumvention'), ('tcp_connect', 'circumvention'), ('tor', 'circumvention'), ('torsf', 'circumvention'), ('vanilla_tor', 'circumvention'), ('dnscheck', 'experimental'), ('urlgetter', 'experimental'), ('facebook_messenger', 'im'), ('signal', 'im'), ('telegram', 'im'), ('whatsapp', 'im'), ('dns_consistency', 'legacy'), ('http_host', 'legacy'), ('http_requests', 'legacy'), ('multi_protocol_traceroute', 'legacy'), ('http_header_field_manipulation', 'middlebox'), ('http_invalid_request_line', 'middlebox'), ('dash', 'performance'), ('ndt', 'performance')('web_connectivity', 'websites') ; - --- Create integ test data for Clickhouse - -INSERT INTO citizenlab VALUES ('www.ushmm.org', 'https://www.ushmm.org/', 'ZZ', 'CULTR'); -INSERT INTO citizenlab VALUES ('www.cabofrio.rj.gov.br', 'http://www.cabofrio.rj.gov.br/', 'BR', 'CULTR'); -INSERT INTO citizenlab VALUES ('ncac.org', 'http://ncac.org/', 'ZZ', 'NEWS'); -INSERT INTO citizenlab VALUES ('ncac.org', 'https://ncac.org/', 'ZZ', 'NEWS'); -INSERT INTO citizenlab VALUES ('www.facebook.com','http://www.facebook.com/saakashvilimikheil','ge','NEWS'); -INSERT INTO citizenlab VALUES ('www.facebook.com','http://www.facebook.com/somsakjeam/videos/1283095981743678/','th','POLR'); -INSERT INTO citizenlab VALUES ('www.facebook.com','https://www.facebook.com/','ZZ','GRP'); -INSERT INTO citizenlab VALUES ('facebook.com','http://facebook.com/','ua','GRP'); -INSERT INTO citizenlab VALUES ('facebook.com','https://facebook.com/watch','jo','GRP'); -INSERT INTO citizenlab VALUES ('twitter.com','http://twitter.com/ghonim','kw','POLR'); -INSERT INTO citizenlab VALUES ('twitter.com','http://twitter.com/ghonim','so','POLR'); -INSERT INTO citizenlab VALUES ('twitter.com','https://twitter.com/','ZZ','GRP'); - --- get_measurement_meta integ tests -INSERT INTO jsonl (report_id, input, s3path, linenum) VALUES ('20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3', 'https://www.backtrack-linux.org/', 'raw/20210709/00/MY/webconnectivity/2021070900_MY_webconnectivity.n0.2.jsonl.gz', 35) - - diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql b/ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql deleted file mode 100644 index 2cc6e895..00000000 --- a/ooniapi/services/oonimeasurements/tests/fixtures/oonipg-legacy.sql +++ /dev/null @@ -1,182 +0,0 @@ --- --- Name: alembic_version; Type: TABLE; Schema: public; Owner: oonipg --- - -CREATE TABLE public.alembic_version ( - version_num character varying(32) NOT NULL -); - - -ALTER TABLE public.alembic_version OWNER TO oonipg; - --- --- Name: oonirun; Type: TABLE; Schema: public; Owner: oonipg --- - -CREATE TABLE public.oonirun ( - oonirun_link_id text NOT NULL, - revision integer NOT NULL, - date_created timestamp without time zone NOT NULL, - date_updated timestamp without time zone NOT NULL, - creator_account_id character varying NOT NULL, - name character varying, - name_intl json, - short_description character varying, - short_description_intl json, - description character varying, - description_intl json, - author character varying, - icon character varying, - nettests json NOT NULL, - expiration_date timestamp without time zone NOT NULL, - color character varying -); - - -ALTER TABLE public.oonirun OWNER TO oonipg; - --- --- Data for Name: alembic_version; Type: TABLE DATA; Schema: public; Owner: oonipg --- - -COPY public.alembic_version (version_num) FROM stdin; -7d5841cb9549 -\. - - --- --- Data for Name: oonirun; Type: TABLE DATA; Schema: public; Owner: oonipg --- - -COPY public.oonirun (oonirun_link_id, revision, date_created, date_updated, creator_account_id, name, name_intl, short_description, short_description_intl, description, description_intl, author, icon, nettests, expiration_date, color) FROM stdin; -9408643002 1 2023-07-18 15:38:21 2023-07-18 15:38:21 19c854be61b934ad1902a5940eb9c204 OONIMkAll Integration Testing \N Integration testing descriptor for ooni/probe-cli/v3/pkg/oonimkall. \N We use this OONI Run descriptor for writing integration tests for ooni/probe-cli/v3/pkg/oonimkall. \N simone@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://www.example.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-01-14 15:38:21 \N -12964651802 1 2023-12-06 14:34:24.15 2023-12-06 14:34:24.15 2dbebf8200d9672894c6930ebf315964 testy test \N a short description \N a longer description a longer description a longer discription \N jessie@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://twitter.com ", "https://facebook.com"], "options": {"": ""}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-06-03 14:34:24.15 #000000 -14808260102 1 2023-10-16 20:05:19.365 2023-10-16 20:09:17.864 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (test2) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-04-13 20:05:19.365 \N -14808260102 2 2023-10-16 20:05:19.365 2023-10-16 20:10:17.441 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (test2) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"test_name": "signal", "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false, "inputs": []}] 2024-04-13 20:10:17.441 \N -14808260102 3 2023-10-16 20:05:19.365 2023-10-16 20:11:00.152 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (test2) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "signal"}, {"test_name": "telegram", "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false, "inputs": []}] 2024-04-13 20:11:00.152 \N -23600827702 1 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 Kazakhstan: Extra URLs to test \N Automatically imported from https://ooni.org/get-involved/run/. \N Kazakhstan: Extra URLs to test \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["http://www.adilsoz.kz/", "https://avaaz.org/", "https://azh.kz/", "https://bureau.kz/", "https://erkindikqanaty.com/", "https://femagora.org/", "https://fergana.agency/", "https://fergana.news/", "https://informburo.kz/", "https://kaztag.kz/", "https://masa.media/", "https://medianet.kz/", "https://orda.kz/", "https://otyrar.kz/", "https://oyan.digital/", "https://qazaqtimes.com/", "https://rus.azattyq.org/", "https://sarpa.media/", "https://the-village-kz.com/", "https://time.kz/", "https://tobolinfo.kz/", "https://vlast.kz/", "https://www.azattyq.org/", "https://www.kaztag.kz/", "https://www.lada.kz/", "https://www.uralskweek.kz/", "https://yvision.kz/", "https://www.currenttime.tv/", "https://ulysmedia.kz/", "https://twitter.com/", "https://web.whatsapp.com/", "https://www.whatsapp.com/", "https://telegram.org/", "https://web.telegram.org/", "https://www.facebook.com/", "https://www.instagram.com/", "https://www.youtube.com/", "https://signal.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:02 \N -24971849002 1 2023-08-02 15:25:06.81 2023-08-02 15:25:06.81 33fa58e2ae78590aa6750a5bf4f11d45 00 With Icon \N Sample short description \N Sample description \N aanorbel Fa500Px [{"test_name": "dash", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-01-29 15:25:06.81 \N -24971849002 2 2023-08-02 15:25:06.81 2024-02-15 07:41:42.641 33fa58e2ae78590aa6750a5bf4f11d45 00 With Icon \N Sample short description \N Sample description \N aanorbel FaSuitcaseMedical [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "dash"}] 2024-08-13 07:41:42.641 #194d33 -31395275002 1 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 Thematic Testing: Environment \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: Environment \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.greenpeace.org/", "https://www.unenvironment.org/", "https://www.fridaysforfuture.org/", "https://rebellion.earth/", "http://www.earthsystemgovernance.org/", "https://gggi.org/", "https://www.ipcc.ch/", "https://www.eea.europa.eu/", "http://www.pemsea.org/", "https://350.org/", "https://www.rainforesttrust.org/", "https://amazonwatch.org/", "https://www.climaterealityproject.org/", "https://www.letsdoitworld.org/", "https://earthwatch.org/", "http://www.climatenetwork.org/", "https://oceanswatch.org/", "https://www.chasingcoral.com/", "https://oceanconservancy.org/", "https://seashepherd.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:00 \N -45674709602 1 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 Azerbaijan: Extra URLs to test \N Automatically imported from https://ooni.org/get-involved/run/. \N Azerbaijan: Extra URLs to test \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["http://azerireport.com/", "http://azeritoday.com/", "http://imgur.com/", "https://kavkazcenter.com/", "http://www.jw.org/", "http://www.onetwotrip.com/", "https://www.azadliq.info/", "https://www.azadliq.org/", "https://www.meydan.tv/", "http://www.abzas.net/", "http://www.cumhuriyyet.net/", "https://www.gununsesi.org/", "https://www.dosye.org/", "http://www.24saat.org/", "https://www.irfs.org/", "http://azerbaycansaati.tv/", "http://sonolay.org/", "http://aztoday.az/", "https://www.24saat.org/", "https://www.occrp.org/en", "http://masallilar.az/", "http://criminal.az/", "http://kulis.az/", "https://axar.az/news/kult/", "http://bastainfo.com/", "http://www.turan.az/", "http://www.femida.az/", "http://www.gndem.org/", "http://gozetci.az/", "https://www.humanrightsclub.net/", "https://novator.az/", "https://www.xural.com/", "http://smdtaz.org/", "https://az.wikipedia.org/", "https://tr.wikipedia.org/", "https://az.wikisource.org/", "https://az.wikiquote.org/", "https://ru.wiktionary.org/", "https://az.wikibooks.org/", "https://www.rferl.org/", "https://ru.wikisource.org/", "https://ru.m.wikisource.org/", "https://www.azerbaycanrealligi.com/", "http://azpolitika.info/", "http://www.kanal13.tv/", "https://cenubxeberleri.com/", "http://arqument.az/", "http://strateq.az/", "http://istiqlal.az/", "http://mia.az/", "https://qaynarinfo.az/", "http://gundeminfo.az/", "https://qafqazinfo.az/", "http://teref.az/", "http://7gun.az/", "http://hefteaz.info/", "http://www.gununsesi.info/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:02 \N -163832393602 8 2023-08-02 07:48:58.537 2023-08-02 15:31:51.947 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 15:31:23.367 \N -53880296502 1 2023-07-14 08:03:47 2023-07-14 08:03:47 33fa58e2ae78590aa6750a5bf4f11d45 Webconnectivity XS \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"test_name": "web_connectivity", "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2024-01-10 08:03:47 \N -53880296502 2 2023-07-14 08:03:47 2023-07-14 10:09:01 33fa58e2ae78590aa6750a5bf4f11d45 Webconnectivity XS \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}, {"test_name": "torsf", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "vanilla_tor", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "stunreachability", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "dnscheck", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}] 2024-01-10 10:09:01 \N -53880296502 3 2023-07-14 08:03:47 2023-07-14 11:00:31 33fa58e2ae78590aa6750a5bf4f11d45 Webconnectivity XS \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}, {"test_name": "signal", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}] 2024-01-10 11:00:31 \N -53880296502 4 2023-07-14 08:03:47 2023-08-14 09:33:15.618 33fa58e2ae78590aa6750a5bf4f11d45 Webconnectivity XS \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "signal"}, {"test_name": "dash", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}] 2024-02-10 09:33:15.618 \N -53880296502 5 2023-07-14 08:03:47 2024-02-15 07:39:45.843 33fa58e2ae78590aa6750a5bf4f11d45 Webconnectivity XS \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel FaComputer [{"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "signal"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dash"}] 2024-08-13 07:39:45.843 #0062b1 -59329370102 1 2023-12-05 10:37:44.369 2023-12-05 10:37:44.369 b1abc32ef7c2fbfa48551d4bff42e77f Test run link \N this has been autopopulated \N this has been autopopulated \N unknown-author [{"test_name": "web_connectivity", "inputs": ["https://rtvslo.si", "https://twitter.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-06-02 10:37:44.369 #000000 -61747276002 1 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 Thematic Testing: COVID-19 \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: COVID-19 \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.who.int/emergencies/diseases/novel-coronavirus-2019/", "https://coronavirus.jhu.edu/map.html", "https://gisanddata.maps.arcgis.com/apps/opsdashboard/index.html#/bda7594740fd40299423467b48e9ecf6", "https://experience.arcgis.com/experience/685d0ace521648f8a5beeeee1b9125cd", "https://www.worldometers.info/coronavirus/", "https://www.healthmap.org/covid-19/", "https://hgis.uw.edu/virus/", "https://nssac.bii.virginia.edu/covid-19/dashboard/", "https://app.developer.here.com/coronavirus/", "https://apps.crowdtangle.com/public-hub/covid19", "https://google.org/crisisresponse/covid19-map", "https://www.bing.com/covid", "https://covidtracking.com/", "https://www.kff.org/global-health-policy/fact-sheet/coronavirus-tracker/", "https://storymaps.arcgis.com/stories/4fdc0d03d3a34aa485de1fb0d2650ee0", "https://www.projectbaseline.com/study/covid-19/", "https://en.wikipedia.org/wiki/2019%E2%80%9320_coronavirus_pandemic", "https://www.cdc.gov/coronavirus/2019-ncov/cases-updates/world-map.html", "http://www.euro.who.int/en/health-topics/health-emergencies/coronavirus-covid-19", "https://www.ecdc.europa.eu/en/geographical-distribution-2019-ncov-cases", "https://www.caixinglobal.com/2020/coronavirus-live-update/", "https://www.nytimes.com/interactive/2020/world/coronavirus-maps.html", "https://multimedia.scmp.com/infographics/news/china/article/3047038/wuhan-virus/index.html?src=article-launcher", "https://www.nbcnewyork.com/news/national-international/map-watch-the-coronavirus-cases-spread-across-the-world/2303276/", "https://nextstrain.org/ncov"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:01 \N -90334195102 1 2023-12-06 15:52:37.245 2023-12-06 15:52:37.245 2dbebf8200d9672894c6930ebf315964 test 2 \N this has been autopopulated \N this has been autopopulated \N jessie@openobservatory.org FaCircleNodes [{"test_name": "web_connectivity", "inputs": ["https://cbc.com", "https://bbc.co.uk", "https://cnn.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-06-03 15:52:37.245 #000000 -93187783002 1 2023-06-05 08:41:32 2023-06-05 08:41:32 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked \N ooni Md10K [{"test_name": "whatsapp", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2023-12-02 08:41:32 \N -93187783002 2 2023-06-05 08:41:32 2023-08-09 18:55:28.013 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps. \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-02-05 18:50:58.044 \N -163832393602 9 2023-08-02 07:48:58.537 2023-08-02 15:45:34.222 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 15:45:34.222 \N -95746934202 1 2023-07-14 06:20:11 2023-07-14 06:20:11 33fa58e2ae78590aa6750a5bf4f11d45 Test XL \N Test various tools \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"test_name": "dash", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}, {"test_name": "facebook_messenger", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "http_header_field_manipulation", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "http_invalid_request_line", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "ndt", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "psiphon", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "riseupvpn", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "signal", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "telegram", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "tor", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "whatsapp", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}, {"test_name": "web_connectivity", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"]}] 2024-01-10 06:20:11 \N -95746934202 2 2023-07-14 06:20:11 2023-08-04 12:26:05.66 33fa58e2ae78590aa6750a5bf4f11d45 Test XL \N Test various tools \N Test various tools available in the OONI toolbelt \N aanorbel FaBuffer [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dash"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "facebook_messenger"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "http_header_field_manipulation"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "http_invalid_request_line"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "ndt"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "psiphon"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "riseupvpn"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "signal"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "telegram"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-31 12:26:05.66 \N -95746934202 3 2023-07-14 06:20:11 2024-02-15 07:41:09.614 33fa58e2ae78590aa6750a5bf4f11d45 Test XL \N Test various tools \N Test various tools available in the OONI toolbelt \N aanorbel FaMap [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dash"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "facebook_messenger"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "http_header_field_manipulation"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "http_invalid_request_line"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "ndt"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "psiphon"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "riseupvpn"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "signal"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "telegram"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://ooni.org/nettest/web-connectivity", "https://twitter.com", "https://facebook.com", "https://instagram.com", "https://signal.org"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-08-13 07:41:09.614 #ab149e -101760961902 1 2023-08-02 12:27:36.911 2023-08-02 15:23:38.836 33fa58e2ae78590aa6750a5bf4f11d45 Sample With Wrong Icon name \N Sample short description \N Sample description \N aanorbel shapes [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "dash"}] 2024-01-29 12:27:36.911 \N -101760961902 2 2023-08-02 12:27:36.911 2023-08-02 16:03:54.401 33fa58e2ae78590aa6750a5bf4f11d45 Sample With Wrong Icon name \N Sample short description \N Sample description \N aanorbel shapes12 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "dash"}] 2024-01-29 16:03:54.401 \N -102496846102 1 2023-07-20 13:10:21 2023-07-20 13:10:21 08c79c0eb5e325f01353e107bd6c9458 This is a test \N This is a test \N This is a description \N arturo [{"test_name": "web_connectivity", "inputs": ["https://apple.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-01-16 13:10:21 \N -118518729102 1 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 Thematic Testing: DNS Over HTTPS \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: DNS Over HTTPS \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://dns.adguard.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns-family.adguard.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.google/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.google.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://mozilla.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://ooni.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://family.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns9.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns11.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.opendns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.familyshield.opendns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.cleanbrowsing.org/doh/family-filter/?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh-jp.blahdns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://1.1.1.1/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://1.0.0.1/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://1.1.1.3/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://1.0.0.3/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://8.8.8.8/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://8.8.4.4/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://9.9.9.9/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://9.9.9.11/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://149.112.112.9/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://149.112.112.12/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://ibuki.cgnat.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.rubyfish.cn/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://resolver-eu.lelux.fi/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://odvr.nic.cz/doh?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.nextdns.io/c7fee2?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.dns.sb/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://private.canadianshield.cira.ca/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.centraleu.pi-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://doh.li/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://rdns.faelix.net/?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://jp.tiar.app/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.switch.ch/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", "https://dns.flatuslifir.is/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:02 \N -118697266602 1 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 Thematic Testing: Human Rights \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: Human Rights \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.amnesty.org/", "https://www.amnestyusa.org/", "https://www.hrw.org/", "https://www.unhcr.org/", "https://freedomhouse.org/", "https://rsf.org/", "https://www.article19.org/", "https://www.accessnow.org/", "https://www.antislavery.org/", "https://www.care-international.org/", "https://crd.org/", "https://www.equalitynow.org/", "https://www.freedomfromtorture.org/", "https://www.frontlinedefenders.org/", "https://www.savethechildren.net/", "https://minorityrights.org/", "https://www.refugeesinternational.org/", "https://www.omct.org/", "https://www.transparency.org/", "https://unpo.org/", "https://www.awid.org/", "https://www.aclu.org/", "https://www.childrensdefense.org/", "https://www.humanrightsactioncenter.com/", "https://www.liberties.eu/", "https://www.womenonwaves.org/", "https://www.womenonweb.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:00 \N -139288675902 1 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 Thematic Testing: Social Media \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: Social Media \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.facebook.com/", "https://www.youtube.com/", "https://www.instagram.com/", "https://whatsapp.com/", "https://web.whatsapp.com/", "https://www.skype.com/", "https://www.linkedin.com/", "https://twitter.com/", "https://upload.twitter.com/", "https://pbs.twimg.com/profile_images/941658150889054208/lFeD3554_200x200.jpg", "https://video.twimg.com/tweet_video/EC6DSXDXsAEhHAa.mp4", "https://video-0.twimg.com/tweet_video/EC6DSXDXsAEhHAa.mp4", "https://api.twitter.com/1.1/search/tweets.json?q=from%3Atwitterdev&result_type=mixed&count=2", "https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png", "https://abs-0.twimg.com/sticky/default_profile_images/default_profile_normal.png", "https://t.co/Usz3cIuO6v?amp=1", "https://www.reddit.com/", "https://web.telegram.org/", "https://telegram.org/", "https://plus.google.com/", "https://hangouts.google.com/", "https://meet.google.com/", "https://meet.jit.si/", "https://www.snapchat.com/", "https://www.truecaller.com/", "https://www.viber.com/", "https://www.tumblr.com/", "https://www.pinterest.com/", "https://4chan.org/", "https://discordapp.com/", "https://line.me/", "https://www.wechat.com/", "https://www.imqq.com/", "https://qzone.qq.com/", "https://www.douban.com/", "https://www.tiktok.com/", "https://www.weibo.com/", "https://tieba.baidu.com/", "https://signal.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:08:59 \N -141159615602 1 2023-07-25 14:22:06.102 2023-07-25 14:22:06.102 de75e682df2acba70dc5e863870fe114 Federico test \N Short desc \N Desc \N Federico test [{"test_name": "web_connectivity", "inputs": ["https://meow.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": false}] 2024-01-21 14:22:06.102 \N -158234241102 1 2024-02-05 13:09:19.6 2024-02-05 13:09:19.6 2dbebf8200d9672894c6930ebf315964 Test name \N this has been autopopulated \N this has been autopopulated \N jessie@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://cbc.com", "https://facebook.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-08-03 13:09:19.6 #000000 -158234241102 2 2024-02-05 13:09:19.6 2024-02-05 13:11:02.441 2dbebf8200d9672894c6930ebf315964 Test name \N this has been autopopulated \N this has been autopopulated \N jessie@openobservatory.org FaGauge [{"backend_options": {}, "inputs": ["https://cbc.com", "https://facebook.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-08-03 13:11:02.441 #000000 -159102134902 1 2023-08-11 10:36:06.151 2023-08-11 10:36:06.151 33fa58e2ae78590aa6750a5bf4f11d45 Experimental XS v2 \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "torsf"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "stunreachability"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-02-07 10:36:06.151 \N -159102134902 2 2023-08-11 10:36:06.151 2023-08-11 10:43:10.109 33fa58e2ae78590aa6750a5bf4f11d45 Experimental XS v2 \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "stunreachability"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-02-07 10:43:10.109 \N -159102134902 3 2023-08-11 10:36:06.151 2023-08-11 10:49:22.933 33fa58e2ae78590aa6750a5bf4f11d45 Experimental XS v2 \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-02-07 10:49:22.933 \N -159102134902 4 2023-08-11 10:36:06.151 2024-02-15 07:40:07.008 33fa58e2ae78590aa6750a5bf4f11d45 Experimental XS v2 \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel FaSocks [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-08-13 07:40:07.008 #7b64ff -159682723702 1 2023-11-01 19:22:46.315 2023-11-01 19:22:46.315 b1abc32ef7c2fbfa48551d4bff42e77f test link \N this has been autopopulated \N this has been autopopulated \N maja.komel@gmail.com [{"test_name": "web_connectivity", "inputs": ["https://twitter.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-04-29 19:22:46.315 \N -159682723702 2 2023-11-01 19:22:46.315 2023-11-30 18:34:34.973 b1abc32ef7c2fbfa48551d4bff42e77f test link \N this has been autopopulated \N this has been autopopulated \N unknown-author [{"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-05-28 18:34:34.973 #000000 -161073283602 1 2024-02-13 09:22:43.476 2024-02-13 09:22:43.476 08c79c0eb5e325f01353e107bd6c9458 Sports News Media \N This will test international sports websites \N This will test international sports websites \N arturo@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://gazzetta.it/", "https://esports.net/"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-08-11 09:22:43.476 #000000 -161073283602 2 2024-02-13 09:22:43.476 2024-02-13 09:23:37.674 08c79c0eb5e325f01353e107bd6c9458 Sports News Media \N This will test international sports websites \N This will test international sports websites \N arturo@openobservatory.org FaMicroscope [{"backend_options": {}, "inputs": ["https://gazzetta.it/", "https://esports.net/"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-08-11 09:23:37.674 #68ccca -163832393602 1 2023-08-02 07:48:58.537 2023-08-02 07:48:58.537 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-01-29 07:48:58.537 \N -163832393602 2 2023-08-02 07:48:58.537 2023-08-02 07:52:03.388 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 07:49:38.155 \N -163832393602 3 2023-08-02 07:48:58.537 2023-08-02 07:53:05.095 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 07:53:05.095 \N -163832393602 4 2023-08-02 07:48:58.537 2023-08-02 07:54:36.62 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 07:54:36.62 \N -163832393602 5 2023-08-02 07:48:58.537 2023-08-02 10:46:39.059 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 10:46:39.059 \N -163832393602 6 2023-08-02 07:48:58.537 2023-08-02 10:56:46.078 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 10:56:46.078 \N -163832393602 7 2023-08-02 07:48:58.537 2023-08-02 15:26:00.614 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 15:26:00.614 \N -163832393602 10 2023-08-02 07:48:58.537 2023-08-02 21:39:53.49 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja FaArrowDown19 [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 21:39:53.49 \N -163832393602 11 2023-08-02 07:48:58.537 2023-08-03 09:39:48.562 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja FaArrowUpFromWaterPump [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-01-30 09:21:40.767 \N -163832393602 12 2023-08-02 07:48:58.537 2023-08-08 20:59:34.487 b1abc32ef7c2fbfa48551d4bff42e77f test \N With short description \N And description \N maja FaArrowUpFromWaterPump [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-02-03 11:58:22.773 \N -163832393602 13 2023-08-02 07:48:58.537 2023-08-08 20:59:52.724 b1abc32ef7c2fbfa48551d4bff42e77f test \N With short description \N And description \N maja FaArrowUpFromWaterPump [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com", "https://example.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-02-04 20:59:52.724 \N -163832393602 14 2023-08-02 07:48:58.537 2023-11-30 18:33:08.126 b1abc32ef7c2fbfa48551d4bff42e77f test title \N With short description \N And description \N unknown-author FaArrowUpFromWaterPump [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-05-28 18:33:08.126 #7b64ff -163832393602 15 2023-08-02 07:48:58.537 2023-11-30 18:34:05.664 b1abc32ef7c2fbfa48551d4bff42e77f test title \N With short description \N And description \N maja.komel@gmail.com FaArrowUpFromWaterPump [{"backend_options": {}, "inputs": ["https://twitter.com", "https://www.facebook.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-05-28 18:34:05.664 #7b64ff -163870603602 1 2023-07-12 12:11:17 2023-07-12 12:11:17 33fa58e2ae78590aa6750a5bf4f11d45 Sample AN \N Sample short description \N Sample description \N aanorbel settings [{"test_name": "whatsapp", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}, {"test_name": "web_connectivity", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": ["https://google.com", "https://twitter.com", "https://ubuea.cm", "https://stackoverflow.com"]}] 2024-01-08 12:11:17 \N -163870603602 2 2023-07-12 12:11:17 2023-08-02 12:26:23.727 33fa58e2ae78590aa6750a5bf4f11d45 Sample AN \N Sample short description \N Sample description \N aanorbel shapes [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://google.com", "https://twitter.com", "https://ubuea.cm", "https://stackoverflow.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-29 12:23:13.545 \N -163870603602 3 2023-07-12 12:11:17 2024-02-15 07:40:30.592 33fa58e2ae78590aa6750a5bf4f11d45 Sample AN \N Sample short description \N Sample description \N aanorbel FaPoo [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://google.com", "https://twitter.com", "https://ubuea.cm", "https://stackoverflow.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-08-13 07:40:30.592 #68bc00 -184673504702 1 2023-10-16 17:46:28.761 2023-10-16 19:38:33.387 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (another second edit) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-04-13 17:46:28.761 \N -184673504702 2 2023-10-16 17:46:28.761 2023-11-28 17:48:06.715 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (another second edit) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-05-26 17:48:06.715 #000000 -191221092002 1 2023-08-23 08:48:08.939 2023-08-23 08:48:08.939 b50a2bec3a18db897bc97c0833a94cb3 Maria K Archiving Test Aug 23 \N This is a test (short description) \N Description test \N Maria K FaArrowUpRightDots [{"test_name": "Test test test Archived", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2024-02-19 08:48:08.939 \N -204148402902 1 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 Thematic Testing: Wikipedia \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: Wikipedia \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.wikipedia.org/", "https://en.wikipedia.org/", "https://en.m.wikipedia.org/", "https://ar.wikipedia.org/", "https://ar.m.wikipedia.org/", "https://es.wikipedia.org/", "https://es.m.wikipedia.org/", "https://de.wikipedia.org/", "https://de.m.wikipedia.org/", "https://ru.wikipedia.org/", "https://ru.m.wikipedia.org/", "https://fr.wikipedia.org/", "https://fr.m.wikipedia.org/", "https://ca.wikipedia.org/", "https://ca.m.wikipedia.org/", "https://zh.wikipedia.org/", "https://zh.m.wikipedia.org/", "https://ja.wikipedia.org/", "https://ja.m.wikipedia.org/", "https://it.wikipedia.org/", "https://it.m.wikipedia.org/", "https://commons.wikimedia.org/", "https://wikidata.org/", "https://upload.wikimedia.org/", "https://fa.wikipedia.org/", "https://fa.m.wikipedia.org/", "https://fa.wikinews.org/", "https://en.wikinews.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:00 \N -240422656402 1 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 Uganda: Extra URLs to test \N Automatically imported from https://ooni.org/get-involved/run/. \N Uganda: Extra URLs to test \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://twitter.com/", "http://ugandaforums.com/", "http://www.africanwomenblogs.com/", "http://www.africanzone.com/", "https://www.facebook.com/", "https://www.youtube.com/", "http://acfode.org/", "http://dioceseofnorthernuganda.blogspot.com/", "http://humanistuganda.wordpress.com/", "https://hurinet.or.ug/", "http://huripec.mak.ac.ug/", "http://invisiblechildren.com/", "http://killthebilluganda.blogspot.com/", "http://maureenagena.wordpress.com/", "http://ngoforum.or.ug/", "http://pathwaystopeaceuganda.com/", "http://thereport.amnesty.org/eng/Regions/Africa/Uganda", "http://tortureinuganda.wordpress.com/", "http://www.acholitoday.worldbreak.com/", "http://www.actionaid.org/uganda/", "http://www.actvuganda.org/", "http://www.aglionline.org/", "http://www.child-soldier.org/children-in-conflict-child-soldiers-in-uganda", "http://www.crisisgroup.org/home/index.cfm?id=2346&l=1", "http://www.doctorswithoutborders.org/news/uganda.cfm", "http://www.enoughproject.org/", "http://www.faweuganda.org/", "http://www.fhri.or.ug/", "http://www.fidauganda.org/", "http://www.hrcug.org/", "https://www.hrw.org/africa/uganda", "http://www.hrw.org/reports/2004/uganda0404/", "http://www.humanrightsuganda.org/", "http://www.hurinet.or.ug/", "http://www.irinnews.org/country/ug/uganda", "http://www.kulanu.org/", "http://www.refugeelawproject.org/", "http://www.theguardian.com/world/uganda", "http://www.uls.or.ug/", "http://www.uwonet.org/", "http://www.vday.org/", "http://www.warchild.org.uk/issues/the-lords-resistance-army", "http://www.wougnet.org/", "http://allafrica.com/stories/201509211054.html", "http://chapterfouruganda.com/", "https://defendersprotection.org/", "https://www.albertinewatchdog.org/", "https://unwantedwitness.or.ug/", "http://www.spellingbee.ug/", "http://stories4humanrights.com/", "http://www.kas.de/uganda/en/", "https://leoafricainstitute.org/", "http://mcjl.ug/", "http://bros4bros.com/", "http://gayuganda.blogspot.com/", "http://groups.yahoo.com/group/candies/", "http://whosoever.org/", "http://www.advocate.com/news/world-news/2013/04/02/photos-antigay-rally-held-ugandan-activist-david-katos-grave?page=full", "http://www.allmale.com/", "http://www.geocities.com/gayuganda/", "http://www.gmad.org/", "http://www.hemeetshim.com/", "http://www.mask.org.za/", "http://www.realjock.com/gay-personals/black", "http://www.smug.4t.com/", "https://www.kuchutimes.com/2016/04/smug-launches-publication-highlighting-human-rights-violations-of-lgbti-ugandans/", "https://sexualminoritiesuganda.com/", "https://oblogdee.blog/", "http://www.africanhrc.org/", "http://blogs.monitor.co.ug/", "http://cbsfmbuganda.co.ug/index.php/en/", "http://dembefm.ug/", "http://jssozi.wordpress.com/", "http://kalinaki.blogspot.com/", "http://kfm.co.ug/", "http://mak.ac.ug/", "http://mondaytimes.co.ug/", "http://mugumya.com/", "http://ntv.co.ug/", "http://ntvuganda.co.ug/", "http://owerahbits.blogspot.com/", "http://www.bukedde.co.ug/", "http://www.independent.co.ug/", "http://www.kizito.uklinux.net/", "http://www.maureenagena.com/", "http://www.monitor.co.ug/", "http://chimpreports.com/", "http://kawowo.com/", "http://koonadance.ug/", "http://pctechmag.com/", "http://proggie.ug/", "http://ugandaradionetwork.com/a/", "http://urbanlegendkampala.com/", "http://www.techpost.ug/", "http://xclusive.co.ug/", "http://cpj.org/attacks06/africa06/uga06.html", "http://dennismatanda.blogspot.com/", "http://dying-communist.blogspot.com/", "http://katwe.blogspot.com/", "http://observer.ug/", "http://uganda.blogspirit.com/", "http://ugandawatch.blogspot.com/", "http://www.acme-ug.org/", "http://www.africafoicentre.org/", "http://www.cipesa.org/", "http://www.freeuganda.com/", "http://www.ict4democracy.org/", "http://www.mail-archive.com/ugandanet@kym.net/", "http://www.nabj.org/", "http://www.sudantribune.com/", "http://www.theeastafrican.co.ke/", "http://www.ugandatvnetwork.com/", "http://nilepost.co.ug/", "http://watchdoguganda.com/", "http://matookerepublic.com/", "https://theinvestigatornews.com/", "https://www.softpower.ug/", "https://www.thegrapevine.co.ug/", "http://trumpetnews.co.ug/", "http://newz.ug/", "https://theugandan.com.ug/", "https://www.ceo.co.ug/", "https://campusbee.ug/", "http://businessfocus.co.ug/", "https://kampalapost.com/", "http://witnessradio.org/", "http://www.theinsider.ug/", "http://eagle.co.ug/", "http://www.pmldaily.com/", "http://dailypost.ug/", "http://edge.ug/", "https://spyreports.net/", "https://www.westnileweb.com/", "http://www.olpa-rdc.org/", "http://westnilepress.org/", "http://ujk-ug.org/", "http://activists4change.blogspot.com/", "http://andrewmwendasblog.blogspot.com/", "http://dpuganda.blogspot.com/", "http://fdcuganda.org/", "http://fedsnet.blogspot.com/", "http://groups.yahoo.com/group/ugandagreen", "http://northernugandapost.blogspot.com/", "http://upcparty.net/", "https://www.amamambabazi.com/", "http://www.exposed2013.com/", "http://www.fdc-uganda.org/", "http://www.federo.com/", "http://www.newvision.co.ug/", "http://www.norbertmao.org/", "http://www.splmtoday.com/", "http://www.struggle.ws/africa.html", "http://www.tiuganda.org/", "https://www.transparency.org/", "https://www.transparency.org/news/pressrelease/uganda_tops_east_africa_in_corruption", "https://www.transparency.org/research/cpi/overview", "http://www.uganda.sfgd.org/", "http://www.ugandaonlinelawlibrary.com/", "http://www.upcparty.org/", "http://www.uyonet.or.ug/", "http://www.statehouse.go.ug/", "https://rosebellkagumire.com/", "https://www.theelephant.info/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:02 \N -249356450302 1 2023-10-23 13:02:39.734 2023-10-23 13:02:39.734 33fa58e2ae78590aa6750a5bf4f11d45 [Test] Weconnectivity \N [Test] Weconnectivity during integration \N [Test] Weconnectivity during integration .... \N aanorbel FaEdge [{"test_name": "web_connectivity", "inputs": ["https://ooni.torproject.org/", "https://expired.badssl.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2024-04-20 13:02:39.734 \N -249356450302 2 2023-10-23 13:02:39.734 2024-02-15 07:39:12.977 33fa58e2ae78590aa6750a5bf4f11d45 [Test] Weconnectivity \N [Test] Weconnectivity during integration \N [Test] Weconnectivity during integration .... \N aanorbel FaDroplet [{"backend_options": {}, "inputs": ["https://ooni.torproject.org/", "https://expired.badssl.com/"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-08-13 07:39:12.977 #68bc00 -260165369202 1 2023-10-16 19:51:28.67 2023-10-16 19:51:28.67 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (another second edit) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"test_name": "signal", "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false, "inputs": []}] 2024-04-13 19:51:28.67 \N -270962959102 1 2024-02-15 12:36:52.079 2024-02-15 12:36:52.079 0df0a857508387189f83cdb8cac14537 Test list name \N a short descirption \N a long description \N jessiebonisteel@gmail.com [{"test_name": "web_connectivity", "inputs": ["https://twitter.com", "https://cbc.ca"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-08-13 12:36:52.079 #000000 -276248267502 1 2023-08-18 17:25:35.798 2023-08-18 17:25:35.798 9e949d434e85644245031f4626047787 Maria's test \N This is just a test \N this has been autopopulated \N Maria [{"test_name": "Web Connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-02-14 17:25:35.798 \N -276982785902 1 2023-07-14 10:11:30 2023-08-15 12:49:03.495 33fa58e2ae78590aa6750a5bf4f11d45 Experimental XS --version = 2 \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel about_ooni [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "torsf"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "stunreachability"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-10 10:11:30 \N -276982785902 2 2023-07-14 10:11:30 2024-02-15 07:38:36.218 33fa58e2ae78590aa6750a5bf4f11d45 Carrot Icon \N Sample short description \N Test various tools available in the OONI toolbelt \N aanorbel FaPepperHot [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "torsf"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "vanilla_tor"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "stunreachability"}, {"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "dnscheck"}, {"backend_options": {}, "inputs": ["https://google.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-08-13 07:38:36.218 #009ce0 -281191790802 1 2023-12-05 14:20:51.471 2023-12-06 11:31:30.93 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging New Edit \N Test the blocking of instant messaging apps. New Edit \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author Md10K [{"backend_options": {"backend_option_key": "backend_option_value", "backend_option_key_2": "backedn_option_val_2"}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {"another_reg_option": "opt_val_2", "regular_option": "option_value"}, "test_name": "whatsapp"}] 2024-06-02 14:20:51.471 #000000 -281191790802 2 2023-12-05 14:20:51.471 2023-12-06 11:32:44.6 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging New Edit \N Test the blocking of instant messaging apps. New Edit \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-06-03 11:32:44.6 #000000 -281191790802 3 2023-12-05 14:20:51.471 2024-01-08 22:57:39.6 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging New Edit \N Test the blocking of instant messaging apps. New Edit \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"test_name": "web_connectivity", "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false, "inputs": ["https://twitter.com"]}] 2024-07-06 22:57:39.6 #000000 -281191790802 4 2023-12-05 14:20:51.471 2024-01-24 11:03:52.724 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging New Edit \N Test the blocking of instant messaging apps. New Edit \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author FaCarrot [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-07-22 11:03:52.724 #000000 -281191790802 5 2023-12-05 14:20:51.471 2024-01-31 16:07:34.534 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging New Edit \N Test the blocking of instant messaging apps. New Edit \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N unknown-author FaCarrot [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://twitter.com", " http://www.example.com/brass", " http://example.net/act", " http://arch.example.com/ball/behavior.html", " https://example.com/#baseball", " http://www.example.com/baby/blow", " http://www.example.org/anger", " http://www.example.com/brass", " http://example.net/act", " http://arch.example.com/ball/behavior.html", " https://example.com/#baseball", " http://www.example.com/baby/blow", " http://www.example.org/anger"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-07-29 16:07:34.534 #000000 -282024422102 1 2023-08-23 08:51:43.91 2023-08-23 08:51:43.91 b50a2bec3a18db897bc97c0833a94cb3 Maria K Edited Test Aug 23 \N This is a test (short description) \N Description test \N Maria K FaArchway [{"test_name": "Test test test Edited", "inputs": ["https://golos.plus", "https://www.twitter.com"], "options": {"key": "value"}, "backend_options": {"": "value"}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2024-02-19 08:51:43.91 \N -282024422102 2 2023-08-23 08:51:43.91 2023-08-23 08:51:57.2 b50a2bec3a18db897bc97c0833a94cb3 Maria K Edited Test Aug 23 \N This is a test (short description) \N Description test \N Maria K FaArchway [{"backend_options": {"": "value"}, "inputs": ["https://golos.plus", "https://www.twitter.com", "http://www.facebook.com"], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {"key": "value"}, "test_name": "Test test test Edited"}] 2024-02-19 08:51:57.2 \N -26382735700 1 2024-03-06 11:17:54 2024-03-06 11:17:54 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -282024422102 3 2023-08-23 08:51:43.91 2023-08-23 10:37:57.493 b50a2bec3a18db897bc97c0833a94cb3 Maria K Edited Test Aug 23 \N This is a test (short description) \N Description test \N Maria K FaArchway [{"backend_options": {"": "value"}, "inputs": ["https://golos.plus", "https://www.twitter.com", "http://www.facebook.com"], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {"key": "value"}, "test_name": "Test test test Edited"}] 2024-02-19 10:37:57.493 \N -291048935002 1 2024-02-05 15:12:28.261 2024-02-05 15:12:28.261 2dbebf8200d9672894c6930ebf315964 This is a test name \N Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! \N Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh!Here is a much longer and more verbose description of my test! ahhh! \N jessie@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-08-03 15:12:28.261 #000000 -297500125102 1 2023-06-05 12:35:51 2023-06-05 12:35:51 3760e4b3bb1834c51d49fa7cc07f59f2 Updated Instant Messaging \N Test the blocking of instant messaging apps \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked \N ooni Md123 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "whatsapp"}] 2023-12-02 12:35:51 \N -297500125102 2 2023-06-05 12:35:51 2023-06-06 09:19:41 3760e4b3bb1834c51d49fa7cc07f59f2 Updated 2 Instant Messaging \N Test the blocking of instant messaging apps \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked \N ooni Md123 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "whatsapp"}] 2023-12-03 09:19:41 \N -297500125102 3 2023-06-05 12:35:51 2023-07-18 08:43:26 3760e4b3bb1834c51d49fa7cc07f59f2 Updated 2 Instant Messaging \N Test the blocking of instant messaging apps and twitter \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) and Twitter is blocked \N ooni Md123 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}, {"is_background_run_enabled": true, "backend_options": {}, "options": {}, "inputs": ["https://twitter.com"], "test_name": "web_connectivity"}] 2024-01-14 08:43:26 \N -297500125102 4 2023-06-05 12:35:51 2023-07-18 10:04:46 3760e4b3bb1834c51d49fa7cc07f59f2 Updated 2 Instant Messaging \N Test the blocking of instant messaging apps and twitter \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) and Twitter is blocked \N ooni Md123 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": false, "options": {}, "test_name": "web_connectivity", "is_manual_run_enabled": false}] 2024-01-14 10:04:46 \N -297500125102 5 2023-06-05 12:35:51 2023-07-18 12:42:27 3760e4b3bb1834c51d49fa7cc07f59f2 Updated 2 Instant Messaging \N Test the blocking of instant messaging apps and twitter \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) and Twitter is blocked \N ooni Md123 [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": false, "options": {}, "test_name": "whatsapp"}, {"backend_options": {}, "inputs": ["https://twitter.com"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-14 12:42:27 \N -300815650102 1 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 Thematic Testing: LGBTQI Short \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: LGBTQI Short \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.grindr.com/", "https://weareher.com/", "https://www.hrc.org/", "https://outrightinternational.org/", "https://ilga.org/", "https://www.iglyo.com/", "http://ilglaw.org/", "https://kaleidoscopetrust.com/", "https://www.lgbtqnation.com/", "https://www.pinknews.co.uk/", "https://thegailygrind.com/", "https://www.nostraightnews.com/", "https://www.gaystarnews.com/", "https://www.afterellen.com/", "https://www.towleroad.com/", "https://www.irqr.ca/", "http://transactivists.org/", "http://www.grincampaign.com/", "http://oiiinternational.com/", "https://www.stonewall.org.uk/", "http://www.transmarch.org/", "https://www.advocate.com/", "http://queernet.org/", "http://lesbian.org/", "http://transsexual.org/", "https://lalgbtcenter.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:00 \N -304792721502 1 2023-10-16 19:57:01.88 2023-10-16 20:02:37.084 3760e4b3bb1834c51d49fa7cc07f59f2 Instant Messaging \N Test the blocking of instant messaging apps (another second edit) \N Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. \N maja Md10K [{"backend_options": {}, "inputs": [], "is_background_run_enabled": true, "is_manual_run_enabled": true, "options": {}, "test_name": "whatsapp"}] 2024-04-13 19:57:01.88 \N -305791146102 1 2023-08-23 08:41:59.081 2023-08-23 08:41:59.081 b50a2bec3a18db897bc97c0833a94cb3 Maria K Test Aug 23 \N This is a test (short description) \N Description test \N Maria K FaAngellist [{"test_name": "Test test tes", "inputs": ["https://golos.plus"], "options": {"key": "value"}, "backend_options": {"key": "value"}, "is_background_run_enabled": true, "is_manual_run_enabled": true}, {"test_name": "Second test", "options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true, "backend_options": {}, "inputs": []}] 2024-02-19 08:41:59.081 \N -339182569802 1 2023-08-01 20:25:35.685 2023-08-01 20:25:35.685 b1abc32ef7c2fbfa48551d4bff42e77f test \N this has been autopopulated \N this has been autopopulated \N maja [{"test_name": "test", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-01-28 20:25:35.685 \N -347268696402 1 2023-11-30 19:43:33.383 2023-11-30 19:43:33.383 b1abc32ef7c2fbfa48551d4bff42e77f Another run link \N Some short description \N And longer description \N maja.komel@gmail.com FaAnchorCircleXmark [{"test_name": "web_connectivity", "inputs": ["https://rtvslo.si"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-05-28 19:43:33.383 #000000 -347268696402 2 2023-11-30 19:43:33.383 2023-11-30 19:43:57.024 b1abc32ef7c2fbfa48551d4bff42e77f Another run link \N Some short description \N And longer description \N unknown-author FaAnchorCircleXmark [{"backend_options": {}, "inputs": ["https://rtvslo.si"], "is_background_run_enabled": false, "is_manual_run_enabled": false, "options": {}, "test_name": "web_connectivity"}] 2024-05-28 19:43:57.024 #000000 -347761207002 1 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 Thematic Testing: LGBTQI Long \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: LGBTQI Long \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.advocate.com/", "https://www.autostraddle.com/", "https://www.datalounge.com/threads", "https://www.washingtonblade.com/", "https://www.gaycities.com/", "https://gayauthors.org/", "https://www.outsports.com/", "https://chicago.gopride.com/", "https://www.poz.com/", "https://www.metroweekly.com/", "http://www.logotv.com/", "https://www.ebar.com/", "https://atlantisevents.com/", "https://www.gaytravel.com/", "https://pride.amsterdam/", "http://www.downelink.com/", "https://www.outfest.org//", "http://windycitymediagroup.com/", "http://www.outsmartmagazine.com/", "https://www.nlgja.org/", "https://www.iglta.org/", "https://passportmagazine.com/", "https://lgbtqia.ucdavis.edu/", "https://communitymarketinginc.com/", "http://nglcc.org/", "https://www.growinggenerations.com/", "http://www.logcabin.org/", "https://reelingfilmfestival.org/2019/", "https://cbst.org/", "https://www.rainbowrailroad.com/", "http://kareemazmy.blogspot.com/", "http://emraamethlya.blogspot.com/", "https://egale.ca/", "http://equalitymatters.org/", "https://may17.org/", "http://www.noh8campaign.com/", "https://robynochs.com/", "http://www.newnownext.com/", "http://awesomedude.com/", "https://joy.org.au/", "https://www.frameline.org/", "https://www.gayleague.com/", "http://queerculturalcenter.org/", "https://www.leslielohman.org/", "http://outtv.ca/", "https://www.sfgmc.org/", "http://www.gaypartyblog.com/", "http://www.gay.tv/", "http://glbtqarchive.com/", "https://butch-in-progress.tumblr.com/", "https://www.outonfilm.org/", "http://www.gaytheatre.ie/", "http://lgbt.syr.edu/", "http://www.smackjeeves.com/", "https://www.thetrevorproject.org/", "https://advocatesforyouth.org/", "https://itgetsbetter.org/", "https://www.outyouth.org/", "https://larkinstreetyouth.org/", "http://www.alterheros.com/en/home/", "https://face2face.org/", "https://www.ricsenapresents.com/", "https://www.bisexual.com/", "http://bimen.org/", "http://www.biwriters.org/", "http://bimedia.org/", "http://www.nyacyouth.org/", "https://smyal.org/", "https://dojpride.org/", "http://fordglobe.org/", "https://glifaa.org/", "https://lgbtbar.org/annual/", "https://bi.org/en", "http://www.gayadnetwork.com/", "http://pinkbananamedia.com/", "https://www.outnowconsulting.com/", "http://www.ppt.on.ca/", "http://sendtherightmessage.ca/", "http://emptyclosets.com/", "https://www.gays.com/", "https://www.bglad.com/", "http://makena.libsyn.com/", "https://www.outtraveler.com/", "https://www.glsen.org/", "https://www.campuspride.org/", "https://gsanetwork.org/", "https://pridefoundation.org/", "https://sgdo.utoronto.ca/", "https://www.sageusa.org/", "https://rmnetwork.org/", "http://www.saintvictor.org/", "https://affirmation.org/", "http://whosoever.org/", "https://christiangays.com/", "http://www.sarbat.net/", "http://hgsc.sigs.harvard.edu/", "https://www.sec.sa.utoronto.ca/", "https://spectrumcenter.umich.edu/", "https://www.campusprideindex.org/", "http://lgbtout.sa.utoronto.ca/", "https://www.lgbtcampus.org/", "https://www.ualbertaoutreach.ca/", "http://www.schools-out.org.uk/", "https://gaygames.org/", "http://www.glta.net/", "http://igra.com/", "http://www.igla.org/", "https://chicagomsa.leagueapps.com/", "https://www.gaysport.info/", "https://shaarzahav.org/", "https://christchapel.com/", "http://www.ldsfamilyfellowship.org/", "http://twospirits.org/", "https://www.galva108.org/", "https://lgbt.wikia.org/wiki/Main_Page", "http://www.gaylibrary.com/enter.htm", "https://www.prideangel.com/", "http://www.therainbowbabies.com/", "https://www.rainbowweddingnetwork.com/", "https://www.familyequality.org/", "http://www.proudparenting.com/", "https://www.coparents.co.uk/", "http://www.gayparentmag.com/", "http://www.gayspermbank.com/", "https://www.lgbtcenters.org/LGBTCenters", "https://read.dukeupress.edu/glq", "https://iagsdc.org/", "http://desertadventures.org/", "https://www.gbmcc.co.uk/home", "http://gaycampers.com/", "https://gccc.nl/", "http://www.thegaygamer.com/", "https://www.goc.org.uk/", "https://www.gayoutdoors.org/index.cfm?", "https://gaysailing.org.uk/", "https://ggguards.org/", "http://gayhike.org/", "https://www.rainbowdivers.org/", "https://outventures.org/", "https://alp.org/", "http://www.sfpride.org/", "https://www.capitalpride.org/", "http://pride2020.pridetoronto.com/", "http://www.vancouverpride.ca/", "https://2019-worldpride-stonewall50.nycpride.org/", "http://www.colognepride.de/en/", "https://www.atlantapride.org/", "https://instinctmagazine.com/", "https://www.outeverywhere.com/", "http://www.berkeleyfreeclinic.org/", "https://b-gay.com/", "http://gmhc.org/", "https://www.jeffandwill.com/", "https://www.daddydater.com/", "https://www.gayfriendschat.com/", "https://lgbt.foundation/", "http://pinktherapy.com/", "http://www.nalgap.org/", "http://gal-aa.org/", "http://outhistory.org/", "https://gala.co.za/", "https://www.matthewshepard.org/", "https://pride.fi/", "https://www.pridecymru.com/", "https://www.saskatoonpride.ca/", "https://victoryfund.org/", "http://www.gayrepublicans.org/", "http://georgialogcabin.org/", "http://www.gaypatriot.net/", "https://www.stonewalltx.org/", "http://www.rainbowfund.org/", "https://lgbtfunders.org/", "https://truecolorsunited.org/", "https://gillfoundation.org/", "http://lgbthistorymonth.com/", "https://www.hrc.org/", "https://www.glaad.org/", "https://pflag.org/", "https://ilga.org/", "https://www.lgbtcenters.org/", "https://truthwinsout.org/", "https://www.gayrealestate.com/", "https://www.immigrationequality.org/", "https://www.lambdalegal.org/", "https://www.equaldex.com/", "https://www.eqca.org/", "https://www.gaylawnet.com/", "http://www.nclrights.org/", "https://urbanjustice.org/", "https://uklgig.org.uk/", "https://www.marriageequality.org/", "https://www.gaygirlnet.com/", "https://www.lesbiannews.com/", "http://www.curvemag.com/", "http://glumagazine.com/", "http://www.bintelnas.org/", "http://lesbian.org/", "http://dont.law.stanford.edu/", "https://www.usnaout.org/", "https://www.out.com/", "https://www.queerty.com/", "https://www.lgbtqnation.com/", "https://www.towleroad.com/", "https://www.equalityfederation.org/", "https://www.fflag.org.uk/", "http://pflagaustralia.org.au/", "http://plagal.org/", "https://www.gaystarnews.com", "https://pridesource.com/", "https://www.lgbtqnation.com/commentary/", "http://blacklightonline.com/", "https://echomag.com/", "https://exit.co.za/", "https://gay-nerds.com/", "http://www.therainbowtimesmass.com/", "http://gay.americablog.com/", "https://glreview.org/", "http://aidsnews.blogspot.com/", "http://ambiente.us/", "http://baywindows.com/", "http://genderpsychology.org/", "http://trans-academics.org/", "https://www.sfaf.org", "http://antonyandthejohnsons.com/", "https://www.hyenafilms.com/", "https://dragqueenmissdotcom.co.uk/", "http://sftff.org/", "https://www.crossdresserheaven.com/", "http://transgenderzone.com/", "https://www.mychangingroom.com/", "https://www.crossdresserphotoblog.com/", "http://trans-lide.info/", "http://transfamily.org/", "http://ftmguide.org/", "http://thetransitionalmale.com/", "https://transguys.com/", "https://americantransman.com/", "http://ftmi.org/", "http://eunuch.org/", "https://transgenderwarrior.org/", "https://www.lagendercenter.org/", "https://www.drchristinemcginn.com/", "https://www.transgendermap.com/", "https://www.geneticalliance.org.uk/", "https://transgenderlawcenter.org/", "https://www.crossdreamers.com/", "http://www.genderweb.org/", "http://www.transhouston.com/", "http://www.transgendermichigan.org/", "https://www.transgenderpulse.com/", "http://xdressnj.com/", "http://transdude.com/", "https://transgender-chat.com/", "http://gendertree.com/", "http://agender.org.nz/", "https://espritconf.com/", "http://gendermosaic.com/", "http://gendersanity.com/", "https://www.grsmontreal.com", "https://www.phuketpsi.com/gender-change.html", "http://tgcomics.com/", "http://storysite.org/", "http://transsexual.org/", "https://mermaidsuk.org.uk/", "https://borngay.procon.org/", "https://americansfortruth.com", "https://beyondexgay.com/", "http://www.cuav.org/", "https://www.gaylife.co.za/", "https://www.therapeuticchoice.com/", "https://nea-exgay.org/", "http://www.pfox.org/", "https://www.brothersroad.org/", "https://www.afterellen.com", "http://www.equalitypastore.com/", "https://www.gardenstateequality.org/", "http://www.glma.org/", "https://www.humanrightsfirst.org/", "https://allout.org/en", "https://diversitypro.eu/", "http://www.bisocialnetwork.com/", "http://www.freedomtomarry.org/", "http://transactivists.org/", "https://www.ilga-europe.org/", "http://inakost.sk/", "http://www.imlocal.co.nz/", "https://www.gaynz.com/", "http://www.ianz.org.nz/", "http://www.c-h-e.org.uk/", "https://www.stonewall.org.uk/", "http://affirmationscotland.org.uk/", "https://www.galck.org/", "http://www.ishtarmsm.org/", "http://transgender.or.ke/", "https://bisdakpride.wordpress.com/", "https://www.lgbt.ee/", "http://www.gei.kristlased.ee/", "https://gaysinfronteras.weebly.com/", "http://www.igy.org.il/", "https://www.lgbt.org.il/", "https://www.lsvd.de/", "http://www.maneo.de/en/about-maneo/maneo-in-short.html", "https://transgendervictoria.com/", "https://www.arcigay.it/en/", "https://www.aibai.cn/", "http://www.bds.org.np/", "http://www.boysofbangladesh.org/", "https://gspottt.wordpress.com/", "https://colombiadiversa.org/", "http://www.rightsrwanda.com/index.html", "http://equalityjamaica.org/", "http://www.lambdaistanbul.org/s/", "http://lgbti.org/", "https://lgbtinewsturkey.com/", "https://gladt.de/", "http://www.lgbt.dk", "https://lgbtvoicetz.org/", "http://maltagayrights.org/", "http://nazindia.org/", "http://plu.sg/", "https://pinkalliance.hk/", "http://www.pinkarmenia.org/", "https://samtokin78.is/", "https://hotline.org.tw/", "https://unibam.org/", "http://www.zagreb-pride.net/hr/", "http://gejowo.pl/", "https://queer.pl/", "https://fellow.pl/", "http://fille.pl/", "https://kobiety-kobietom.com/", "http://homiki.pl/", "http://alone-persian-man.blogspot.com/", "http://alqaws.org/siteEn/index", "http://aqueeerdiaries.blogspot.com/", "http://arabgaypride.blogspot.com/", "http://arabiclgbt.blogspot.com/", "http://baantomdy.narak.com/", "http://beurgay.free.fr/", "http://boyahoy.com/", "http://bravingkl.blogspot.com/", "http://cabalasisoy.blogspot.com/", "http://cafe.daum.net/pridecare", "http://comingout.narun.net/", "http://forum.mennawfena.net/", "http://gaia.ru/", "http://galz.org/", "http://gay-uae.blogspot.com/", "http://gayarmenia.blogspot.com/", "http://gaybelarus.by/", "http://gayinkuwait.blogspot.com/", "http://gaylifekenya.blogspot.com/", "http://gaymiddleeast.blogspot.com/", "http://gaysong.blogspot.com/", "http://gaytoday.com/", "http://gayuganda.blogspot.com/", "http://gayzona.com/", "http://happygay.blogspot.com/", "http://harfeakhar.persianblog.com/", "http://helem.net/", "http://hitchhikersgui.de/Alice_Nkom", "http://hitchhikersgui.de/LGBT_rights_in_Cameroon", "http://iraqilgbtuk.blogspot.com/", "http://ivancity.com/", "http://kenyantg.blogspot.com/", "http://kscrc.org/", "http://labrys.kg/", "http://lgbt.by/", "http://lgbt.ge/", "http://lgbt.kdlp.org/", "http://lgbtact.org/", "http://lgbtmap.org/", "http://madikazemi.blogspot.com/2009/07/activists-petition-un-over-violation-of.html", "http://manra.blogspot.com/", "http://my.lesbi.name/", "http://news.cameroon-today.com/tag/cameroon-gays-and-homosexuals/", "http://onglinepodcast.com/", "http://pesareghabile.blogspot.com/", "http://pesari2004.persianblog.com/", "http://phnompenhpride.blogspot.com/", "http://pjhouse.blogspot.com/", "http://queerala.org/be-you-a-gathering-on-coming-out-concerns-and-acceptance-from-families/", "http://queeringyerevan.blogspot.com/", "http://queerquotes.blogspot.com/", "http://queerteam.blogspot.com/", "http://ruchlgbt.pl/", "http://sabaah.dk/", "http://seajunction.org/lgbt-rights-siege-indonesia/", "http://shabbin.blogspot.com/", "http://somosgay.org/", "http://sunshine.by/", "http://the-ipf.com/2016/06/09/homosexuality-nigeria-young-lgbt/", "http://thegaykenyan.blogspot.com/", "http://theladiesfinger.com/i-am-a-queer-feminist-killjoy-and-am-grateful-for-metoo-in-india/", "http://theotherfoundation.org/wp-content/uploads/2017/02/Canaries_Zimbabwe.pdf", "http://trans-armenia.blogspot.com/", "http://www.afrakenya.org/", "http://www.africanhrc.org/", "http://www.allmale.com/", "http://www.aswatmag.com/", "http://www.bangkokrainbow.org/", "http://www.bisialimi.com/", "http://www.buddy79.com/", "http://www.center4civilrights.org/", "http://www.cha.org.ar/", "http://www.colorsrainbow.com/", "http://www.deti-404.com/", "http://www.dreadedned.com/", "http://www.fasiroong.org/", "http://www.forplu.com/", "http://www.galangphilippines.org/", "http://www.galasla.org/", "http://www.gayarab.org/", "http://www.gayegypt.com/", "http://www.gayegypt.com/tanzania.html", "http://www.gayguatemala.com/", "http://www.gayhealth.com/", "http://www.gaylaxymag.com/latest-news/transgender-rights-bill-and-surrogacy-bill-lapse-as-rajya-sabha-adjourns-sine-die/", "http://www.gayly.ru/", "http://www.gaymexico.com.mx/", "http://www.gaytelaviv.com/", "http://www.gayvideo.ru/", "http://www.glas.org/", "http://www.glas.org/ahbab/", "http://www.glas.org/lazeeza.html", "http://www.glbtqarchive.com/", "http://www.glil.org/", "http://www.globalequality.org/newsroom/latest-news/1-in-the-news/186-the-facts-on-lgbt-rights-in-russia", "http://www.gmhc.org/", "http://www.goodasyou.net/", "http://www.gthai.net/", "http://www.gvantage.com/", "http://www.helem.net/", "http://www.houseofrainbow.org/", "http://www.ifge.org/", "http://www.kqcf.org/", "http://www.lambdaistanbul.org/", "http://www.lesben-gegen-gewalt.de/", "http://www.lesbi.ru/", "http://www.lesbian.org/", "http://www.lgbtglobalfaith.org/", "http://www.lgbtmuslimretreat.com/", "http://www.lgfoundation.org.uk/", "http://www.loveyourself.ph/", "http://www.marchadelorgullo.org.ar/", "http://www.mask.org.za/", "http://www.moscowpride.ru/", "http://www.newnownext.com/franchise/the-backlot/", "http://www.oneinstitute.org/", "http://www.out.org.za/", "http://www.panambi.org.py/", "http://www.parniplus.ru/", "http://www.pembehayat.org/", "http://www.pglo.net/", "http://www.queernet.org/", "http://www.rainbowfoundation.co.kr/", "http://www.richard-gay.com/", "http://www.runa.org.pe/", "http://www.russianopengames.ru/", "http://www.salganyc.org/", "http://www.samesexmarriage.ca/", "http://www.sapaan.org/", "http://www.siamboy.com/", "http://www.suarakita.org/", "http://www.thaigmc.com/", "http://www.theadvocatesforhumanrights.org/cameroon_upr_lgbti_2017", "http://www.turkgayclub.com/", "http://www.xgay.ru/", "http://xs.gay.ru/", "https://76crimes.com/2014/06/25/zambia-to-west-dont-watch-when-we-jail-lgbt-people/", "https://ahwaa.org/?lang=ar", "https://aseansogiecaucus.org/", "https://bedayaa.webs.com/", "https://boylovealone.blogspot.com/", "https://caracasqueer.blogspot.com/", "https://galz.org/", "https://gayguide.net/", "https://gaymenshealth.org/", "https://gender.do.am/", "https://hornet.com/", "https://humsafar.org/", "https://identidadydiversidad.adc.org.ar/", "https://igualdaddegenero.tedic.org/", "https://ilga.org/upr-cameroon", "https://ilga.org/upr-paraguay", "https://imaanlondon.wordpress.com/", "https://justiceforsisters.wordpress.com/", "https://kaleidoscopetrust.com/", "https://kwirmedia.com/", "https://lagablab.wordpress.com/", "https://lalgbtcenter.org/", "https://lgbtbar.org/", "https://lgbtnewsnow.org/", "https://lgbtvacationplanners.com/", "https://lgbtvoicetz.org/our-mission/", "https://londonqueermuslims.com/", "https://majal.org/ahwaa", "https://mexgay.com/", "https://mutsaween.wordpress.com/", "https://nostringsng.com/", "https://oblogdee.blog/", "https://outincolombiatravel.com/", "https://panafricailga.org/category/cameroon/", "https://paulshalala.blogspot.com/2015/05/investigation-reveals-zambias-600.html", "https://queeralliancenigeria.blogspot.com/", "https://queeralliancenigeria.wordpress.com/", "https://rainbowegypt.org/", "https://sexualminoritiesuganda.com/", "https://solidaritywithegyptlgbt.wordpress.com/", "https://tapcpr.org/", "https://thequeerarabs.com/", "https://transatsite.com/", "https://transequality.org/", "https://transnational-queer-underground.net/", "https://travesti-diva.io.ua/", "https://twitter.com/rulgbtnet", "https://www.aireana.org.py/", "https://www.beirutpride.org/", "https://www.bisialimifoundation.org/", "https://www.buzzarab.com/chat/gay-arabs/", "https://www.equaldex.com/region/mexico", "https://www.gaystarnews.com/", "https://www.globalgayz.com/", "https://www.hidayahlgbt.co.uk/", "https://www.homochat.org/ar/", "https://www.humandignitytrust.org/country-profile/cameroon/", "https://www.igualdadlgbt.org/", "https://www.kuchutimes.com/2016/04/smug-launches-publication-highlighting-human-rights-violations-of-lgbti-ugandans/", "https://www.lavaplace.com/", "https://www.lgbtarabic.com/", "https://www.mesahatfoundation.org/", "https://www.metrodate.com/gh/", "https://www.mombian.com/", "https://www.mykalimag.com/", "https://www.nostraightnews.com/", "https://outrightinternational.org/", "https://www.pinknews.co.uk/", "https://www.planetromeo.com/", "https://www.prideconnectionmexico.com/", "https://www.proudcampaign.ph/", "https://www.queer-refugees.hamburg/", "https://www.queerlapis.com/", "https://www.rainbow-street.org/", "https://www.salaamcanada.info/", "https://www.scruff.com/", "https://www.soyhomosensual.com/", "https://www.unfe.org/", "https://blogcritics.org", "http://bluesystem.info/", "https://caribeafirmativo.lgbt/", "https://empressmaruja.wordpress.com/", "https://equalitymyanmar.org/", "http://gay.kz/", "https://aglany.org/", "http://www.gaykenya.com/", "https://gaymorocco.yoo7.com/", "https://www.planetromeo.com/auth/signup", "https://gaycity.love/", "https://groups.yahoo.com/neo/groups/candies/info", "https://groups.yahoo.com/neo/groups/lesbianmuslims/info", "https://gukira.wordpress.com/", "https://kenyangayman.wordpress.com/", "https://kph.org.pl/", "https://lgbt.ie/", "http://www.lgbtph.org/", "https://love.gay.ru/en/search.phtml", "https://mingle2.com/online-dating/carriacou/ghana/gay-personals", "https://mnw.org.pl/", "https://naijalez.com/", "https://outragemag.com/", "https://www.rbth.com/lgbt-in-russia", "https://rwaho.wordpress.com/", "https://saudislgbt.tumblr.com/", "https://turkeygay.net/", "https://alturi.org/", "https://www.aswatgroup.org/", "https://chingusai.net/xe/main", "https://www.exgay.com/", "http://falgbt.url.ph/", "https://www.fridae.asia/", "https://www.fundacionreflejosdevenezuela.com/", "http://www.gaycambodianews.com/", "https://gaycationmagazine.com/", "https://gmad.org/", "https://www.grindr.com/", "https://www.istscare.org/", "https://www.kaosgl.org/", "https://www.baby-boy.fr/", "https://lgbti-ep.eu/", "https://www.pridemedia.com/", "http://lgbtpride.or.kr/", "https://oia.co.za/", "https://www.refworld.org/docid/53ecb2de4.html", "http://satrang.org/", "https://thegailygrind.com/", "https://www.transparentgender.com/", "https://www.utopia-asia.com/", "https://www.blued.com/en/index.html", "https://www.lgbtarabic.com/ar/", "https://www.mykalimag.com/en/home-page/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:01 \N -349628539702 1 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 Thematic Testing: News Media \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: News Media \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.reddit.com/", "https://edition.cnn.com/", "https://www.nytimes.com/", "https://news.google.com/", "https://www.theguardian.com/", "https://www.indiatimes.com/", "https://www.washingtonpost.com/", "https://news.yahoo.com/", "https://www.forbes.com/", "https://www.foxnews.com/", "https://timesofindia.indiatimes.com/", "https://www.cnbc.com/", "https://economictimes.indiatimes.com/", "https://www.bloomberg.com/", "https://www.wsj.com/", "https://www.bbc.com/", "https://www.dw.com/", "https://www.voanews.com/", "https://www.reuters.com/", "https://www.rt.com/", "http://www.chinadaily.com.cn/", "https://www.nbcnews.com/", "https://abcnews.go.com/", "https://nypost.com/", "https://www.cbsnews.com/", "https://www.aljazeera.net/", "https://time.com/", "https://www.theatlantic.com/", "https://www.alarabiya.net/", "https://www.alhurra.com/", "https://africatimes.com/", "https://allafrica.com/", "https://www.africanews.com/", "https://www.asiatimes.com/", "https://www.sina.com.cn/", "https://www.rambler.ru/", "https://www.lemonde.fr/", "https://www.rfa.org/", "https://www.rferl.org/", "https://elpais.com/", "https://www.elmundo.es/", "https://www.globo.com/", "https://www.irna.ir/", "https://www.euronews.com/", "http://ru.rfi.fr/", "https://ar.qantara.de/", "https://www.radiotelevisionmarti.com/", "https://www.radiosawa.com/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:08:59 \N -361132445202 1 2023-08-08 12:26:15.453 2023-08-08 12:26:15.453 9e949d434e85644245031f4626047787 Maria's experiment \N This is a test \N this has been autopopulated \N Maria [{"test_name": "Web Connectivity", "inputs": ["https://twitter.com/", "https://www.facebook.com/", "https://www.instagram.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": false}] 2024-02-04 12:26:15.453 \N -366386230002 1 2023-09-26 13:33:00.068 2023-09-26 13:33:00.068 08c79c0eb5e325f01353e107bd6c9458 Social Media testing \N Testing of social media websites \N This is for testing of social media websites \N arturo@ooni.org FaWhatsapp [{"test_name": "web_connectivity", "inputs": ["https://www.facebook.com/", "https://twitter.com/"], "options": {}, "backend_options": {}, "is_background_run_enabled": true, "is_manual_run_enabled": true}] 2024-03-24 13:33:00.068 \N -373393593802 1 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 Thematic Testing: VPNs \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: VPNs \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.torproject.org/", "https://psiphon.ca/", "https://getlantern.org/", "https://www.tunnelbear.com/", "https://www.getoutline.org/", "https://www.hotspotshield.com/", "https://nordvpn.com/", "https://www.expressvpn.com/", "https://www.ipvanish.com/", "https://www.privateinternetaccess.com/", "https://www.purevpn.com/", "https://www.cyberghostvpn.com/", "https://mullvad.net/", "https://windscribe.com/", "https://strongvpn.com/", "https://www.vyprvpn.com/", "https://surfshark.com/", "https://goosevpn.com/", "https://buffered.com/", "https://ultrasurf.us/", "http://anonymouse.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:08:59 \N -424988552202 1 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 Thematic Testing: Reproduction Rights \N Automatically imported from https://ooni.org/get-involved/run/. \N Thematic Testing: Reproduction Rights \N simone@openobservatory.org [{"backend_options": {}, "inputs": ["https://www.womenonwaves.org/", "https://www.womenonweb.org/", "http://www.womenonwaves.org/", "http://www.womenonweb.org/"], "is_background_run_enabled": false, "is_manual_run_enabled": true, "options": {}, "test_name": "web_connectivity"}] 2024-01-15 11:09:01 \N -115983820800 1 2024-03-06 10:11:34 2024-03-06 10:11:34 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -156737021500 1 2024-03-06 10:23:33 2024-03-06 10:23:33 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://twitter.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -381898594100 1 2024-03-06 10:26:47 2024-03-06 10:26:47 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -32447275800 1 2024-03-06 10:32:48 2024-03-06 10:32:48 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -95930853000 1 2024-03-06 10:32:59 2024-03-06 10:32:59 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -662549500 1 2024-03-06 10:39:27 2024-03-06 10:39:27 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -166744503000 1 2024-03-06 10:40:07 2024-03-06 10:40:07 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -164213156400 1 2024-03-06 11:13:31 2024-03-06 11:13:31 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -113145430800 1 2024-03-06 10:34:36 2024-03-08 09:17:12 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-03-08 00:00:00 #000000 -201675761900 1 2024-03-07 12:52:26 2024-03-07 12:52:26 556680adb71c49ebc35d6cd4ab269999 Social Media List Edited {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://twitter.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-06-30 00:00:00 #000000 -285684714800 1 2024-03-06 12:14:33 2024-03-08 09:32:38 556680adb71c49ebc35d6cd4ab269999 Social Media List Expired 2 {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-03-08 00:00:00 #000000 -181619554500 1 2024-03-06 10:42:08 2024-03-06 10:42:08 556680adb71c49ebc35d6cd4ab269999 Social Media List {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -181619554500 2 2024-03-07 13:11:52 2024-03-07 13:11:52 556680adb71c49ebc35d6cd4ab269999 Social Media List ------ {} Short desc ---- {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://rumble.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -421206342800 1 2024-03-07 13:00:22 2024-03-07 13:00:22 556680adb71c49ebc35d6cd4ab269999 Social Media List Edited 2 {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://twitter.com", "https://www.facebook.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-06-30 00:00:00 #000000 -51580415900 1 2024-03-07 18:17:18 2024-03-07 18:17:18 556680adb71c49ebc35d6cd4ab269999 Messaging Apps {} short description {} long description {} maja@openobservatory.org [{"test_name": "facebook_messenger", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}, {"test_name": "signal", "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false, "inputs": []}] 2024-03-20 00:00:00 #000000 -194411950900 1 2024-03-06 10:42:26 2024-03-08 09:31:07 556680adb71c49ebc35d6cd4ab269999 Social Media List Expired {} Short desc {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": [], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2024-03-07 00:00:00 #000000 -181619554500 3 2024-03-08 13:41:33 2024-03-08 13:41:33 556680adb71c49ebc35d6cd4ab269999 Social Media List ------ {} Short desc ---- {} Desc {} maja@openobservatory.org [{"test_name": "web_connectivity", "inputs": ["https://rumble.com", "https://twitter.com"], "options": {}, "backend_options": {}, "is_background_run_enabled": false, "is_manual_run_enabled": false}] 2025-10-10 00:00:00 #000000 -\. - - --- --- Name: alembic_version alembic_version_pkc; Type: CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.alembic_version - ADD CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num); - - --- --- Name: oonirun oonirun_pkey; Type: CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.oonirun - ADD CONSTRAINT oonirun_pkey PRIMARY KEY (oonirun_link_id, revision); diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql b/ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql deleted file mode 100644 index 5222558c..00000000 --- a/ooniapi/services/oonimeasurements/tests/fixtures/oonipg.sql +++ /dev/null @@ -1,396 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 16.2 (Debian 16.2-1.pgdg120+2) --- Dumped by pg_dump version 16.2 (Homebrew) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: alembic_version; Type: TABLE; Schema: public; Owner: oonipg --- - -CREATE TABLE public.alembic_version ( - version_num character varying(32) NOT NULL -); - - -ALTER TABLE public.alembic_version OWNER TO oonipg; - --- --- Name: oonirun; Type: TABLE; Schema: public; Owner: oonipg --- - -CREATE TABLE public.oonirun ( - oonirun_link_id character varying NOT NULL, - date_created timestamp without time zone NOT NULL, - date_updated timestamp without time zone NOT NULL, - creator_account_id character varying NOT NULL, - expiration_date timestamp without time zone NOT NULL, - name character varying NOT NULL, - short_description character varying NOT NULL, - description character varying NOT NULL, - author character varying, - icon character varying, - color character varying, - name_intl json, - short_description_intl json, - description_intl json -); - - -ALTER TABLE public.oonirun OWNER TO oonipg; - --- --- Name: oonirun_nettest; Type: TABLE; Schema: public; Owner: oonipg --- - -CREATE TABLE public.oonirun_nettest ( - oonirun_link_id character varying NOT NULL, - revision integer DEFAULT 1 NOT NULL, - nettest_index integer DEFAULT 0 NOT NULL, - date_created timestamp without time zone, - test_name character varying, - test_inputs json, - test_options json, - backend_config json, - is_background_run_enabled_default boolean DEFAULT false, - is_manual_run_enabled_default boolean DEFAULT false -); - - -ALTER TABLE public.oonirun_nettest OWNER TO oonipg; - --- --- Data for Name: alembic_version; Type: TABLE DATA; Schema: public; Owner: oonipg --- - -COPY public.alembic_version (version_num) FROM stdin; -9973a7e1f96c -\. - - --- --- Data for Name: oonirun; Type: TABLE DATA; Schema: public; Owner: oonipg --- - -COPY public.oonirun (oonirun_link_id, date_created, date_updated, creator_account_id, expiration_date, name, short_description, description, author, icon, color, name_intl, short_description_intl, description_intl) FROM stdin; -101760961902 2023-08-02 12:27:36.911 2023-08-02 16:03:54.401 33fa58e2ae78590aa6750a5bf4f11d45 2024-01-29 16:03:54.401 Sample With Wrong Icon name Sample short description Sample description aanorbel shapes12 \N \N \N \N -102496846102 2023-07-20 13:10:21 2023-07-20 13:10:21 08c79c0eb5e325f01353e107bd6c9458 2024-01-16 13:10:21 This is a test This is a test This is a description arturo \N \N \N \N -113145430800 2024-03-06 10:34:36 2024-03-08 09:17:12 556680adb71c49ebc35d6cd4ab269999 2024-03-08 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -115983820800 2024-03-06 10:11:34 2024-03-06 10:11:34 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -118518729102 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:02 Thematic Testing: DNS Over HTTPS Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: DNS Over HTTPS simone@openobservatory.org \N \N \N \N -118697266602 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:00 Thematic Testing: Human Rights Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: Human Rights simone@openobservatory.org \N \N \N \N -12964651802 2023-12-06 14:34:24.15 2023-12-06 14:34:24.15 2dbebf8200d9672894c6930ebf315964 2024-06-03 14:34:24.15 testy test a short description a longer description a longer description a longer discription jessie@openobservatory.org #000000 \N \N \N -139288675902 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:08:59 Thematic Testing: Social Media Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: Social Media simone@openobservatory.org \N \N \N \N -141159615602 2023-07-25 14:22:06.102 2023-07-25 14:22:06.102 de75e682df2acba70dc5e863870fe114 2024-01-21 14:22:06.102 Federico test Short desc Desc Federico test \N \N \N \N -14808260102 2023-10-16 20:05:19.365 2023-10-16 20:11:00.152 3760e4b3bb1834c51d49fa7cc07f59f2 2024-04-13 20:11:00.152 Instant Messaging Test the blocking of instant messaging apps (test2) Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. maja Md10K \N \N \N \N -156737021500 2024-03-06 10:23:33 2024-03-06 10:23:33 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -158234241102 2024-02-05 13:09:19.6 2024-02-05 13:11:02.441 2dbebf8200d9672894c6930ebf315964 2024-08-03 13:11:02.441 Test name this has been autopopulated this has been autopopulated jessie@openobservatory.org FaGauge #000000 \N \N \N -159102134902 2023-08-11 10:36:06.151 2024-02-15 07:40:07.008 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:40:07.008 Experimental XS v2 Sample short description Test various tools available in the OONI toolbelt aanorbel FaSocks #7b64ff \N \N \N -159682723702 2023-11-01 19:22:46.315 2023-11-30 18:34:34.973 b1abc32ef7c2fbfa48551d4bff42e77f 2024-05-28 18:34:34.973 test link this has been autopopulated this has been autopopulated unknown-author #000000 \N \N \N -161073283602 2024-02-13 09:22:43.476 2024-02-13 09:23:37.674 08c79c0eb5e325f01353e107bd6c9458 2024-08-11 09:23:37.674 Sports News Media This will test international sports websites This will test international sports websites arturo@openobservatory.org FaMicroscope #68ccca \N \N \N -163832393602 2023-08-02 07:48:58.537 2023-11-30 18:34:05.664 b1abc32ef7c2fbfa48551d4bff42e77f 2024-05-28 18:34:05.664 test title With short description And description maja.komel@gmail.com FaArrowUpFromWaterPump #7b64ff \N \N \N -163870603602 2023-07-12 12:11:17 2024-02-15 07:40:30.592 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:40:30.592 Sample AN Sample short description Sample description aanorbel FaPoo #68bc00 \N \N \N -164213156400 2024-03-06 11:13:31 2024-03-06 11:13:31 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -166744503000 2024-03-06 10:40:07 2024-03-06 10:40:07 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -181619554500 2024-03-08 13:41:33 2024-03-08 13:41:33 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List ------ Short desc ---- Desc maja@openobservatory.org #000000 {} {} {} -184673504702 2023-10-16 17:46:28.761 2023-11-28 17:48:06.715 3760e4b3bb1834c51d49fa7cc07f59f2 2024-05-26 17:48:06.715 Instant Messaging Test the blocking of instant messaging apps (another second edit) Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. unknown-author Md10K #000000 \N \N \N -191221092002 2023-08-23 08:48:08.939 2023-08-23 08:48:08.939 b50a2bec3a18db897bc97c0833a94cb3 2024-02-19 08:48:08.939 Maria K Archiving Test Aug 23 This is a test (short description) Description test Maria K FaArrowUpRightDots \N \N \N \N -194411950900 2024-03-06 10:42:26 2024-03-08 09:31:07 556680adb71c49ebc35d6cd4ab269999 2024-03-07 00:00:00 Social Media List Expired Short desc Desc maja@openobservatory.org #000000 {} {} {} -201675761900 2024-03-07 12:52:26 2024-03-07 12:52:26 556680adb71c49ebc35d6cd4ab269999 2024-06-30 00:00:00 Social Media List Edited Short desc Desc maja@openobservatory.org #000000 {} {} {} -204148402902 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:00 Thematic Testing: Wikipedia Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: Wikipedia simone@openobservatory.org \N \N \N \N -23600827702 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:02 Kazakhstan: Extra URLs to test Automatically imported from https://ooni.org/get-involved/run/. Kazakhstan: Extra URLs to test simone@openobservatory.org \N \N \N \N -240422656402 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:02 Uganda: Extra URLs to test Automatically imported from https://ooni.org/get-involved/run/. Uganda: Extra URLs to test simone@openobservatory.org \N \N \N \N -249356450302 2023-10-23 13:02:39.734 2024-02-15 07:39:12.977 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:39:12.977 [Test] Weconnectivity [Test] Weconnectivity during integration [Test] Weconnectivity during integration .... aanorbel FaDroplet #68bc00 \N \N \N -24971849002 2023-08-02 15:25:06.81 2024-02-15 07:41:42.641 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:41:42.641 00 With Icon Sample short description Sample description aanorbel FaSuitcaseMedical #194d33 \N \N \N -260165369202 2023-10-16 19:51:28.67 2023-10-16 19:51:28.67 3760e4b3bb1834c51d49fa7cc07f59f2 2024-04-13 19:51:28.67 Instant Messaging Test the blocking of instant messaging apps (another second edit) Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. maja Md10K \N \N \N \N -26382735700 2024-03-06 11:17:54 2024-03-06 11:17:54 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -270962959102 2024-02-15 12:36:52.079 2024-02-15 12:36:52.079 0df0a857508387189f83cdb8cac14537 2024-08-13 12:36:52.079 Test list name a short descirption a long description jessiebonisteel@gmail.com #000000 \N \N \N -276248267502 2023-08-18 17:25:35.798 2023-08-18 17:25:35.798 9e949d434e85644245031f4626047787 2024-02-14 17:25:35.798 Maria's test This is just a test this has been autopopulated Maria \N \N \N \N -276982785902 2023-07-14 10:11:30 2024-02-15 07:38:36.218 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:38:36.218 Carrot Icon Sample short description Test various tools available in the OONI toolbelt aanorbel FaPepperHot #009ce0 \N \N \N -281191790802 2023-12-05 14:20:51.471 2024-01-31 16:07:34.534 3760e4b3bb1834c51d49fa7cc07f59f2 2024-07-29 16:07:34.534 Instant Messaging New Edit Test the blocking of instant messaging apps. New Edit Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. unknown-author FaCarrot #000000 \N \N \N -282024422102 2023-08-23 08:51:43.91 2023-08-23 10:37:57.493 b50a2bec3a18db897bc97c0833a94cb3 2024-02-19 10:37:57.493 Maria K Edited Test Aug 23 This is a test (short description) Description test Maria K FaArchway \N \N \N \N -285684714800 2024-03-06 12:14:33 2024-03-08 09:32:38 556680adb71c49ebc35d6cd4ab269999 2024-03-08 00:00:00 Social Media List Expired 2 Short desc Desc maja@openobservatory.org #000000 {} {} {} -291048935002 2024-02-05 15:12:28.261 2024-02-05 15:12:28.261 2dbebf8200d9672894c6930ebf315964 2024-08-03 15:12:28.261 This is a test name Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! Here is a short description of my test! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh! Here is a much longer and more verbose description of my test! ahhh!Here is a much longer and more verbose description of my test! ahhh! jessie@openobservatory.org #000000 \N \N \N -297500125102 2023-06-05 12:35:51 2023-07-18 12:42:27 3760e4b3bb1834c51d49fa7cc07f59f2 2024-01-14 12:42:27 Updated 2 Instant Messaging Test the blocking of instant messaging apps and twitter Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) and Twitter is blocked ooni Md123 \N \N \N \N -300815650102 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:00 Thematic Testing: LGBTQI Short Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: LGBTQI Short simone@openobservatory.org \N \N \N \N -304792721502 2023-10-16 19:57:01.88 2023-10-16 20:02:37.084 3760e4b3bb1834c51d49fa7cc07f59f2 2024-04-13 19:57:01.88 Instant Messaging Test the blocking of instant messaging apps (another second edit) Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. maja Md10K \N \N \N \N -305791146102 2023-08-23 08:41:59.081 2023-08-23 08:41:59.081 b50a2bec3a18db897bc97c0833a94cb3 2024-02-19 08:41:59.081 Maria K Test Aug 23 This is a test (short description) Description test Maria K FaAngellist \N \N \N \N -31395275002 2023-07-19 11:09:00 2023-07-19 11:09:00 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:00 Thematic Testing: Environment Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: Environment simone@openobservatory.org \N \N \N \N -32447275800 2024-03-06 10:32:48 2024-03-06 10:32:48 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -339182569802 2023-08-01 20:25:35.685 2023-08-01 20:25:35.685 b1abc32ef7c2fbfa48551d4bff42e77f 2024-01-28 20:25:35.685 test this has been autopopulated this has been autopopulated maja \N \N \N \N -347268696402 2023-11-30 19:43:33.383 2023-11-30 19:43:57.024 b1abc32ef7c2fbfa48551d4bff42e77f 2024-05-28 19:43:57.024 Another run link Some short description And longer description unknown-author FaAnchorCircleXmark #000000 \N \N \N -347761207002 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:01 Thematic Testing: LGBTQI Long Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: LGBTQI Long simone@openobservatory.org \N \N \N \N -349628539702 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:08:59 Thematic Testing: News Media Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: News Media simone@openobservatory.org \N \N \N \N -361132445202 2023-08-08 12:26:15.453 2023-08-08 12:26:15.453 9e949d434e85644245031f4626047787 2024-02-04 12:26:15.453 Maria's experiment This is a test this has been autopopulated Maria \N \N \N \N -366386230002 2023-09-26 13:33:00.068 2023-09-26 13:33:00.068 08c79c0eb5e325f01353e107bd6c9458 2024-03-24 13:33:00.068 Social Media testing Testing of social media websites This is for testing of social media websites arturo@ooni.org FaWhatsapp \N \N \N \N -373393593802 2023-07-19 11:08:59 2023-07-19 11:08:59 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:08:59 Thematic Testing: VPNs Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: VPNs simone@openobservatory.org \N \N \N \N -381898594100 2024-03-06 10:26:47 2024-03-06 10:26:47 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -421206342800 2024-03-07 13:00:22 2024-03-07 13:00:22 556680adb71c49ebc35d6cd4ab269999 2024-06-30 00:00:00 Social Media List Edited 2 Short desc Desc maja@openobservatory.org #000000 {} {} {} -424988552202 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:01 Thematic Testing: Reproduction Rights Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: Reproduction Rights simone@openobservatory.org \N \N \N \N -45674709602 2023-07-19 11:09:02 2023-07-19 11:09:02 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:02 Azerbaijan: Extra URLs to test Automatically imported from https://ooni.org/get-involved/run/. Azerbaijan: Extra URLs to test simone@openobservatory.org \N \N \N \N -51580415900 2024-03-07 18:17:18 2024-03-07 18:17:18 556680adb71c49ebc35d6cd4ab269999 2024-03-20 00:00:00 Messaging Apps short description long description maja@openobservatory.org #000000 {} {} {} -53880296502 2023-07-14 08:03:47 2024-02-15 07:39:45.843 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:39:45.843 Webconnectivity XS Sample short description Test various tools available in the OONI toolbelt aanorbel FaComputer #0062b1 \N \N \N -59329370102 2023-12-05 10:37:44.369 2023-12-05 10:37:44.369 b1abc32ef7c2fbfa48551d4bff42e77f 2024-06-02 10:37:44.369 Test run link this has been autopopulated this has been autopopulated unknown-author #000000 \N \N \N -61747276002 2023-07-19 11:09:01 2023-07-19 11:09:01 19c854be61b934ad1902a5940eb9c204 2024-01-15 11:09:01 Thematic Testing: COVID-19 Automatically imported from https://ooni.org/get-involved/run/. Thematic Testing: COVID-19 simone@openobservatory.org \N \N \N \N -662549500 2024-03-06 10:39:27 2024-03-06 10:39:27 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -90334195102 2023-12-06 15:52:37.245 2023-12-06 15:52:37.245 2dbebf8200d9672894c6930ebf315964 2024-06-03 15:52:37.245 test 2 this has been autopopulated this has been autopopulated jessie@openobservatory.org FaCircleNodes #000000 \N \N \N -93187783002 2023-06-05 08:41:32 2023-08-09 18:55:28.013 3760e4b3bb1834c51d49fa7cc07f59f2 2024-02-05 18:50:58.044 Instant Messaging Test the blocking of instant messaging apps. Check whether [WhatsApp](https://ooni.org/nettest/whatsapp/) is blocked. maja Md10K \N \N \N \N -9408643002 2023-07-18 15:38:21 2023-07-18 15:38:21 19c854be61b934ad1902a5940eb9c204 2024-01-14 15:38:21 OONIMkAll Integration Testing Integration testing descriptor for ooni/probe-cli/v3/pkg/oonimkall. We use this OONI Run descriptor for writing integration tests for ooni/probe-cli/v3/pkg/oonimkall. simone@openobservatory.org \N \N \N \N -95746934202 2023-07-14 06:20:11 2024-02-15 07:41:09.614 33fa58e2ae78590aa6750a5bf4f11d45 2024-08-13 07:41:09.614 Test XL Test various tools Test various tools available in the OONI toolbelt aanorbel FaMap #ab149e \N \N \N -95930853000 2024-03-06 10:32:59 2024-03-06 10:32:59 556680adb71c49ebc35d6cd4ab269999 2025-10-10 00:00:00 Social Media List Short desc Desc maja@openobservatory.org #000000 {} {} {} -\. - - --- --- Data for Name: oonirun_nettest; Type: TABLE DATA; Schema: public; Owner: oonipg --- - -COPY public.oonirun_nettest (oonirun_link_id, revision, nettest_index, date_created, test_name, test_inputs, test_options, backend_config, is_background_run_enabled_default, is_manual_run_enabled_default) FROM stdin; -9408643002 1 0 2023-07-18 15:38:21 web_connectivity [{"url": "https://www.example.com/"}] {} {} f f -12964651802 1 0 2023-12-06 14:34:24.15 web_connectivity [{"url": "https://twitter.com "}, {"url": "https://facebook.com"}] {"": ""} {} f f -14808260102 1 0 2023-10-16 20:05:19.365 whatsapp [] {} {} t t -14808260102 2 0 2023-10-16 20:05:19.365 whatsapp [] {} {} t t -14808260102 2 1 2023-10-16 20:05:19.365 signal [] {} {} f f -14808260102 3 0 2023-10-16 20:05:19.365 whatsapp [] {} {} t t -14808260102 3 1 2023-10-16 20:05:19.365 signal [] {} {} f f -14808260102 3 2 2023-10-16 20:05:19.365 telegram [] {} {} f f -23600827702 1 0 2023-07-19 11:09:02 web_connectivity [{"url": "http://www.adilsoz.kz/"}, {"url": "https://avaaz.org/"}, {"url": "https://azh.kz/"}, {"url": "https://bureau.kz/"}, {"url": "https://erkindikqanaty.com/"}, {"url": "https://femagora.org/"}, {"url": "https://fergana.agency/"}, {"url": "https://fergana.news/"}, {"url": "https://informburo.kz/"}, {"url": "https://kaztag.kz/"}, {"url": "https://masa.media/"}, {"url": "https://medianet.kz/"}, {"url": "https://orda.kz/"}, {"url": "https://otyrar.kz/"}, {"url": "https://oyan.digital/"}, {"url": "https://qazaqtimes.com/"}, {"url": "https://rus.azattyq.org/"}, {"url": "https://sarpa.media/"}, {"url": "https://the-village-kz.com/"}, {"url": "https://time.kz/"}, {"url": "https://tobolinfo.kz/"}, {"url": "https://vlast.kz/"}, {"url": "https://www.azattyq.org/"}, {"url": "https://www.kaztag.kz/"}, {"url": "https://www.lada.kz/"}, {"url": "https://www.uralskweek.kz/"}, {"url": "https://yvision.kz/"}, {"url": "https://www.currenttime.tv/"}, {"url": "https://ulysmedia.kz/"}, {"url": "https://twitter.com/"}, {"url": "https://web.whatsapp.com/"}, {"url": "https://www.whatsapp.com/"}, {"url": "https://telegram.org/"}, {"url": "https://web.telegram.org/"}, {"url": "https://www.facebook.com/"}, {"url": "https://www.instagram.com/"}, {"url": "https://www.youtube.com/"}, {"url": "https://signal.org/"}] {} {} f t -24971849002 1 0 2023-08-02 15:25:06.81 dash [] {} {} f f -24971849002 2 0 2023-08-02 15:25:06.81 dash [] {} {} f f -31395275002 1 0 2023-07-19 11:09:00 web_connectivity [{"url": "https://www.greenpeace.org/"}, {"url": "https://www.unenvironment.org/"}, {"url": "https://www.fridaysforfuture.org/"}, {"url": "https://rebellion.earth/"}, {"url": "http://www.earthsystemgovernance.org/"}, {"url": "https://gggi.org/"}, {"url": "https://www.ipcc.ch/"}, {"url": "https://www.eea.europa.eu/"}, {"url": "http://www.pemsea.org/"}, {"url": "https://350.org/"}, {"url": "https://www.rainforesttrust.org/"}, {"url": "https://amazonwatch.org/"}, {"url": "https://www.climaterealityproject.org/"}, {"url": "https://www.letsdoitworld.org/"}, {"url": "https://earthwatch.org/"}, {"url": "http://www.climatenetwork.org/"}, {"url": "https://oceanswatch.org/"}, {"url": "https://www.chasingcoral.com/"}, {"url": "https://oceanconservancy.org/"}, {"url": "https://seashepherd.org/"}] {} {} f t -45674709602 1 0 2023-07-19 11:09:02 web_connectivity [{"url": "http://azerireport.com/"}, {"url": "http://azeritoday.com/"}, {"url": "http://imgur.com/"}, {"url": "https://kavkazcenter.com/"}, {"url": "http://www.jw.org/"}, {"url": "http://www.onetwotrip.com/"}, {"url": "https://www.azadliq.info/"}, {"url": "https://www.azadliq.org/"}, {"url": "https://www.meydan.tv/"}, {"url": "http://www.abzas.net/"}, {"url": "http://www.cumhuriyyet.net/"}, {"url": "https://www.gununsesi.org/"}, {"url": "https://www.dosye.org/"}, {"url": "http://www.24saat.org/"}, {"url": "https://www.irfs.org/"}, {"url": "http://azerbaycansaati.tv/"}, {"url": "http://sonolay.org/"}, {"url": "http://aztoday.az/"}, {"url": "https://www.24saat.org/"}, {"url": "https://www.occrp.org/en"}, {"url": "http://masallilar.az/"}, {"url": "http://criminal.az/"}, {"url": "http://kulis.az/"}, {"url": "https://axar.az/news/kult/"}, {"url": "http://bastainfo.com/"}, {"url": "http://www.turan.az/"}, {"url": "http://www.femida.az/"}, {"url": "http://www.gndem.org/"}, {"url": "http://gozetci.az/"}, {"url": "https://www.humanrightsclub.net/"}, {"url": "https://novator.az/"}, {"url": "https://www.xural.com/"}, {"url": "http://smdtaz.org/"}, {"url": "https://az.wikipedia.org/"}, {"url": "https://tr.wikipedia.org/"}, {"url": "https://az.wikisource.org/"}, {"url": "https://az.wikiquote.org/"}, {"url": "https://ru.wiktionary.org/"}, {"url": "https://az.wikibooks.org/"}, {"url": "https://www.rferl.org/"}, {"url": "https://ru.wikisource.org/"}, {"url": "https://ru.m.wikisource.org/"}, {"url": "https://www.azerbaycanrealligi.com/"}, {"url": "http://azpolitika.info/"}, {"url": "http://www.kanal13.tv/"}, {"url": "https://cenubxeberleri.com/"}, {"url": "http://arqument.az/"}, {"url": "http://strateq.az/"}, {"url": "http://istiqlal.az/"}, {"url": "http://mia.az/"}, {"url": "https://qaynarinfo.az/"}, {"url": "http://gundeminfo.az/"}, {"url": "https://qafqazinfo.az/"}, {"url": "http://teref.az/"}, {"url": "http://7gun.az/"}, {"url": "http://hefteaz.info/"}, {"url": "http://www.gununsesi.info/"}] {} {} f t -163832393602 8 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}] {} {} f f -53880296502 1 0 2023-07-14 08:03:47 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -53880296502 2 0 2023-07-14 08:03:47 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -53880296502 2 1 2023-07-14 08:03:47 torsf [] {} {} t t -53880296502 2 2 2023-07-14 08:03:47 vanilla_tor [] {} {} t t -53880296502 2 3 2023-07-14 08:03:47 stunreachability [] {} {} t t -53880296502 2 4 2023-07-14 08:03:47 dnscheck [] {} {} t t -53880296502 3 0 2023-07-14 08:03:47 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -53880296502 3 1 2023-07-14 08:03:47 signal [] {} {} t t -53880296502 4 0 2023-07-14 08:03:47 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -53880296502 4 1 2023-07-14 08:03:47 signal [] {} {} t t -53880296502 4 2 2023-07-14 08:03:47 dash [] {} {} t t -53880296502 5 0 2023-07-14 08:03:47 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -53880296502 5 1 2023-07-14 08:03:47 signal [] {} {} t t -53880296502 5 2 2023-07-14 08:03:47 dash [] {} {} t t -59329370102 1 0 2023-12-05 10:37:44.369 web_connectivity [{"url": "https://rtvslo.si"}, {"url": "https://twitter.com"}] {} {} f f -118697266602 1 0 2023-07-19 11:09:00 web_connectivity [{"url": "https://www.amnesty.org/"}, {"url": "https://www.amnestyusa.org/"}, {"url": "https://www.hrw.org/"}, {"url": "https://www.unhcr.org/"}, {"url": "https://freedomhouse.org/"}, {"url": "https://rsf.org/"}, {"url": "https://www.article19.org/"}, {"url": "https://www.accessnow.org/"}, {"url": "https://www.antislavery.org/"}, {"url": "https://www.care-international.org/"}, {"url": "https://crd.org/"}, {"url": "https://www.equalitynow.org/"}, {"url": "https://www.freedomfromtorture.org/"}, {"url": "https://www.frontlinedefenders.org/"}, {"url": "https://www.savethechildren.net/"}, {"url": "https://minorityrights.org/"}, {"url": "https://www.refugeesinternational.org/"}, {"url": "https://www.omct.org/"}, {"url": "https://www.transparency.org/"}, {"url": "https://unpo.org/"}, {"url": "https://www.awid.org/"}, {"url": "https://www.aclu.org/"}, {"url": "https://www.childrensdefense.org/"}, {"url": "https://www.humanrightsactioncenter.com/"}, {"url": "https://www.liberties.eu/"}, {"url": "https://www.womenonwaves.org/"}, {"url": "https://www.womenonweb.org/"}] {} {} f t -61747276002 1 0 2023-07-19 11:09:01 web_connectivity [{"url": "https://www.who.int/emergencies/diseases/novel-coronavirus-2019/"}, {"url": "https://coronavirus.jhu.edu/map.html"}, {"url": "https://gisanddata.maps.arcgis.com/apps/opsdashboard/index.html#/bda7594740fd40299423467b48e9ecf6"}, {"url": "https://experience.arcgis.com/experience/685d0ace521648f8a5beeeee1b9125cd"}, {"url": "https://www.worldometers.info/coronavirus/"}, {"url": "https://www.healthmap.org/covid-19/"}, {"url": "https://hgis.uw.edu/virus/"}, {"url": "https://nssac.bii.virginia.edu/covid-19/dashboard/"}, {"url": "https://app.developer.here.com/coronavirus/"}, {"url": "https://apps.crowdtangle.com/public-hub/covid19"}, {"url": "https://google.org/crisisresponse/covid19-map"}, {"url": "https://www.bing.com/covid"}, {"url": "https://covidtracking.com/"}, {"url": "https://www.kff.org/global-health-policy/fact-sheet/coronavirus-tracker/"}, {"url": "https://storymaps.arcgis.com/stories/4fdc0d03d3a34aa485de1fb0d2650ee0"}, {"url": "https://www.projectbaseline.com/study/covid-19/"}, {"url": "https://en.wikipedia.org/wiki/2019%E2%80%9320_coronavirus_pandemic"}, {"url": "https://www.cdc.gov/coronavirus/2019-ncov/cases-updates/world-map.html"}, {"url": "http://www.euro.who.int/en/health-topics/health-emergencies/coronavirus-covid-19"}, {"url": "https://www.ecdc.europa.eu/en/geographical-distribution-2019-ncov-cases"}, {"url": "https://www.caixinglobal.com/2020/coronavirus-live-update/"}, {"url": "https://www.nytimes.com/interactive/2020/world/coronavirus-maps.html"}, {"url": "https://multimedia.scmp.com/infographics/news/china/article/3047038/wuhan-virus/index.html?src=article-launcher"}, {"url": "https://www.nbcnewyork.com/news/national-international/map-watch-the-coronavirus-cases-spread-across-the-world/2303276/"}, {"url": "https://nextstrain.org/ncov"}] {} {} f t -90334195102 1 0 2023-12-06 15:52:37.245 web_connectivity [{"url": "https://cbc.com"}, {"url": "https://bbc.co.uk"}, {"url": "https://cnn.com"}] {} {} f f -93187783002 1 0 2023-06-05 08:41:32 whatsapp [] {} {} t t -93187783002 2 0 2023-06-05 08:41:32 whatsapp [] {} {} t t -163832393602 9 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} t f -95746934202 1 0 2023-07-14 06:20:11 dash [] {} {} t t -95746934202 1 1 2023-07-14 06:20:11 facebook_messenger [] {} {} t t -95746934202 1 2 2023-07-14 06:20:11 http_header_field_manipulation [] {} {} t t -95746934202 1 3 2023-07-14 06:20:11 http_invalid_request_line [] {} {} t t -95746934202 1 4 2023-07-14 06:20:11 ndt [] {} {} t t -95746934202 1 5 2023-07-14 06:20:11 psiphon [] {} {} t t -95746934202 1 6 2023-07-14 06:20:11 riseupvpn [] {} {} t t -95746934202 1 7 2023-07-14 06:20:11 signal [] {} {} t t -95746934202 1 8 2023-07-14 06:20:11 telegram [] {} {} t t -95746934202 1 9 2023-07-14 06:20:11 tor [] {} {} t t -95746934202 1 10 2023-07-14 06:20:11 whatsapp [] {} {} t t -95746934202 1 11 2023-07-14 06:20:11 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -95746934202 2 0 2023-07-14 06:20:11 dash [] {} {} t t -95746934202 2 1 2023-07-14 06:20:11 facebook_messenger [] {} {} t t -95746934202 2 2 2023-07-14 06:20:11 http_header_field_manipulation [] {} {} t t -95746934202 2 3 2023-07-14 06:20:11 http_invalid_request_line [] {} {} t t -95746934202 2 4 2023-07-14 06:20:11 ndt [] {} {} t t -95746934202 2 5 2023-07-14 06:20:11 psiphon [] {} {} t t -95746934202 2 6 2023-07-14 06:20:11 riseupvpn [] {} {} t t -95746934202 2 7 2023-07-14 06:20:11 signal [] {} {} t t -95746934202 2 8 2023-07-14 06:20:11 telegram [] {} {} t t -95746934202 2 9 2023-07-14 06:20:11 tor [] {} {} t t -95746934202 2 10 2023-07-14 06:20:11 whatsapp [] {} {} t t -95746934202 2 11 2023-07-14 06:20:11 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -95746934202 3 0 2023-07-14 06:20:11 dash [] {} {} t t -95746934202 3 1 2023-07-14 06:20:11 facebook_messenger [] {} {} t t -95746934202 3 2 2023-07-14 06:20:11 http_header_field_manipulation [] {} {} t t -95746934202 3 3 2023-07-14 06:20:11 http_invalid_request_line [] {} {} t t -95746934202 3 4 2023-07-14 06:20:11 ndt [] {} {} t t -95746934202 3 5 2023-07-14 06:20:11 psiphon [] {} {} t t -95746934202 3 6 2023-07-14 06:20:11 riseupvpn [] {} {} t t -95746934202 3 7 2023-07-14 06:20:11 signal [] {} {} t t -95746934202 3 8 2023-07-14 06:20:11 telegram [] {} {} t t -95746934202 3 9 2023-07-14 06:20:11 tor [] {} {} t t -95746934202 3 10 2023-07-14 06:20:11 whatsapp [] {} {} t t -95746934202 3 11 2023-07-14 06:20:11 web_connectivity [{"url": "https://ooni.org/nettest/web-connectivity"}, {"url": "https://twitter.com"}, {"url": "https://facebook.com"}, {"url": "https://instagram.com"}, {"url": "https://signal.org"}] {} {} t t -101760961902 1 0 2023-08-02 12:27:36.911 dash [] {} {} f f -101760961902 2 0 2023-08-02 12:27:36.911 dash [] {} {} f f -102496846102 1 0 2023-07-20 13:10:21 web_connectivity [{"url": "https://apple.com/"}] {} {} f f -118518729102 1 0 2023-07-19 11:09:02 web_connectivity [{"url": "https://dns.adguard.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns-family.adguard.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.google/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.google.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://mozilla.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://ooni.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://family.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns9.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns11.quad9.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.opendns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.familyshield.opendns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.cleanbrowsing.org/doh/family-filter/?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh-jp.blahdns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://1.1.1.1/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://1.0.0.1/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://1.1.1.3/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://1.0.0.3/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://8.8.8.8/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://8.8.4.4/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://9.9.9.9/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://9.9.9.11/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://149.112.112.9/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://149.112.112.12/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://ibuki.cgnat.net/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.rubyfish.cn/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://resolver-eu.lelux.fi/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://odvr.nic.cz/doh?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.nextdns.io/c7fee2?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.dns.sb/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://private.canadianshield.cira.ca/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.centraleu.pi-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://doh.li/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://rdns.faelix.net/?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://jp.tiar.app/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.switch.ch/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}, {"url": "https://dns.flatuslifir.is/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"}] {} {} f t -139288675902 1 0 2023-07-19 11:08:59 web_connectivity [{"url": "https://www.facebook.com/"}, {"url": "https://www.youtube.com/"}, {"url": "https://www.instagram.com/"}, {"url": "https://whatsapp.com/"}, {"url": "https://web.whatsapp.com/"}, {"url": "https://www.skype.com/"}, {"url": "https://www.linkedin.com/"}, {"url": "https://twitter.com/"}, {"url": "https://upload.twitter.com/"}, {"url": "https://pbs.twimg.com/profile_images/941658150889054208/lFeD3554_200x200.jpg"}, {"url": "https://video.twimg.com/tweet_video/EC6DSXDXsAEhHAa.mp4"}, {"url": "https://video-0.twimg.com/tweet_video/EC6DSXDXsAEhHAa.mp4"}, {"url": "https://api.twitter.com/1.1/search/tweets.json?q=from%3Atwitterdev&result_type=mixed&count=2"}, {"url": "https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png"}, {"url": "https://abs-0.twimg.com/sticky/default_profile_images/default_profile_normal.png"}, {"url": "https://t.co/Usz3cIuO6v?amp=1"}, {"url": "https://www.reddit.com/"}, {"url": "https://web.telegram.org/"}, {"url": "https://telegram.org/"}, {"url": "https://plus.google.com/"}, {"url": "https://hangouts.google.com/"}, {"url": "https://meet.google.com/"}, {"url": "https://meet.jit.si/"}, {"url": "https://www.snapchat.com/"}, {"url": "https://www.truecaller.com/"}, {"url": "https://www.viber.com/"}, {"url": "https://www.tumblr.com/"}, {"url": "https://www.pinterest.com/"}, {"url": "https://4chan.org/"}, {"url": "https://discordapp.com/"}, {"url": "https://line.me/"}, {"url": "https://www.wechat.com/"}, {"url": "https://www.imqq.com/"}, {"url": "https://qzone.qq.com/"}, {"url": "https://www.douban.com/"}, {"url": "https://www.tiktok.com/"}, {"url": "https://www.weibo.com/"}, {"url": "https://tieba.baidu.com/"}, {"url": "https://signal.org/"}] {} {} f t -141159615602 1 0 2023-07-25 14:22:06.102 web_connectivity [{"url": "https://meow.com/"}] {} {} t f -158234241102 1 0 2024-02-05 13:09:19.6 web_connectivity [{"url": "https://cbc.com"}, {"url": "https://facebook.com"}] {} {} f f -158234241102 2 0 2024-02-05 13:09:19.6 web_connectivity [{"url": "https://cbc.com"}, {"url": "https://facebook.com"}] {} {} f f -159102134902 1 0 2023-08-11 10:36:06.151 torsf [] {} {} t t -159102134902 1 1 2023-08-11 10:36:06.151 vanilla_tor [] {} {} t t -159102134902 1 2 2023-08-11 10:36:06.151 stunreachability [] {} {} t t -159102134902 1 3 2023-08-11 10:36:06.151 dnscheck [] {} {} t t -159102134902 1 4 2023-08-11 10:36:06.151 web_connectivity [{"url": "https://google.com"}] {} {} t t -159102134902 2 0 2023-08-11 10:36:06.151 tor [] {} {} t t -159102134902 2 1 2023-08-11 10:36:06.151 vanilla_tor [] {} {} t t -159102134902 2 2 2023-08-11 10:36:06.151 stunreachability [] {} {} t t -159102134902 2 3 2023-08-11 10:36:06.151 dnscheck [] {} {} t t -159102134902 2 4 2023-08-11 10:36:06.151 web_connectivity [{"url": "https://google.com"}] {} {} t t -159102134902 3 0 2023-08-11 10:36:06.151 tor [] {} {} t t -159102134902 3 1 2023-08-11 10:36:06.151 vanilla_tor [] {} {} t t -159102134902 3 2 2023-08-11 10:36:06.151 dnscheck [] {} {} t t -159102134902 3 3 2023-08-11 10:36:06.151 web_connectivity [{"url": "https://google.com"}] {} {} t t -159102134902 4 0 2023-08-11 10:36:06.151 tor [] {} {} t t -159102134902 4 1 2023-08-11 10:36:06.151 vanilla_tor [] {} {} t t -159102134902 4 2 2023-08-11 10:36:06.151 dnscheck [] {} {} t t -159102134902 4 3 2023-08-11 10:36:06.151 web_connectivity [{"url": "https://google.com"}] {} {} t t -159682723702 1 0 2023-11-01 19:22:46.315 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -159682723702 2 0 2023-11-01 19:22:46.315 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -161073283602 1 0 2024-02-13 09:22:43.476 web_connectivity [{"url": "https://gazzetta.it/"}, {"url": "https://esports.net/"}] {} {} f f -161073283602 2 0 2024-02-13 09:22:43.476 web_connectivity [{"url": "https://gazzetta.it/"}, {"url": "https://esports.net/"}] {} {} f f -163832393602 1 0 2023-08-02 07:48:58.537 web_connectivity [] {} {} f f -163832393602 2 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -163832393602 3 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}] {} {} t f -163832393602 4 0 2023-08-02 07:48:58.537 web_connectivity [] {} {} t f -163832393602 5 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}] {} {} t f -163832393602 6 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}] {} {} t f -163832393602 7 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} f f -163832393602 10 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} t f -163832393602 11 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} t f -163832393602 12 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} t t -163832393602 13 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}, {"url": "https://example.com"}] {} {} f f -163832393602 14 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}] {} {} f f -163832393602 15 0 2023-08-02 07:48:58.537 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}] {} {} f f -163870603602 1 0 2023-07-12 12:11:17 whatsapp [] {} {} t t -163870603602 1 1 2023-07-12 12:11:17 web_connectivity [{"url": "https://google.com"}, {"url": "https://twitter.com"}, {"url": "https://ubuea.cm"}, {"url": "https://stackoverflow.com"}] {} {} t t -163870603602 2 0 2023-07-12 12:11:17 whatsapp [] {} {} t t -163870603602 2 1 2023-07-12 12:11:17 web_connectivity [{"url": "https://google.com"}, {"url": "https://twitter.com"}, {"url": "https://ubuea.cm"}, {"url": "https://stackoverflow.com"}] {} {} t t -163870603602 3 0 2023-07-12 12:11:17 whatsapp [] {} {} t t -163870603602 3 1 2023-07-12 12:11:17 web_connectivity [{"url": "https://google.com"}, {"url": "https://twitter.com"}, {"url": "https://ubuea.cm"}, {"url": "https://stackoverflow.com"}] {} {} t t -184673504702 1 0 2023-10-16 17:46:28.761 whatsapp [] {} {} t t -184673504702 2 0 2023-10-16 17:46:28.761 whatsapp [] {} {} t t -191221092002 1 0 2023-08-23 08:48:08.939 Test test test Archived [] {} {} t t -204148402902 1 0 2023-07-19 11:09:00 web_connectivity [{"url": "https://www.wikipedia.org/"}, {"url": "https://en.wikipedia.org/"}, {"url": "https://en.m.wikipedia.org/"}, {"url": "https://ar.wikipedia.org/"}, {"url": "https://ar.m.wikipedia.org/"}, {"url": "https://es.wikipedia.org/"}, {"url": "https://es.m.wikipedia.org/"}, {"url": "https://de.wikipedia.org/"}, {"url": "https://de.m.wikipedia.org/"}, {"url": "https://ru.wikipedia.org/"}, {"url": "https://ru.m.wikipedia.org/"}, {"url": "https://fr.wikipedia.org/"}, {"url": "https://fr.m.wikipedia.org/"}, {"url": "https://ca.wikipedia.org/"}, {"url": "https://ca.m.wikipedia.org/"}, {"url": "https://zh.wikipedia.org/"}, {"url": "https://zh.m.wikipedia.org/"}, {"url": "https://ja.wikipedia.org/"}, {"url": "https://ja.m.wikipedia.org/"}, {"url": "https://it.wikipedia.org/"}, {"url": "https://it.m.wikipedia.org/"}, {"url": "https://commons.wikimedia.org/"}, {"url": "https://wikidata.org/"}, {"url": "https://upload.wikimedia.org/"}, {"url": "https://fa.wikipedia.org/"}, {"url": "https://fa.m.wikipedia.org/"}, {"url": "https://fa.wikinews.org/"}, {"url": "https://en.wikinews.org/"}] {} {} f t -240422656402 1 0 2023-07-19 11:09:02 web_connectivity [{"url": "https://twitter.com/"}, {"url": "http://ugandaforums.com/"}, {"url": "http://www.africanwomenblogs.com/"}, {"url": "http://www.africanzone.com/"}, {"url": "https://www.facebook.com/"}, {"url": "https://www.youtube.com/"}, {"url": "http://acfode.org/"}, {"url": "http://dioceseofnorthernuganda.blogspot.com/"}, {"url": "http://humanistuganda.wordpress.com/"}, {"url": "https://hurinet.or.ug/"}, {"url": "http://huripec.mak.ac.ug/"}, {"url": "http://invisiblechildren.com/"}, {"url": "http://killthebilluganda.blogspot.com/"}, {"url": "http://maureenagena.wordpress.com/"}, {"url": "http://ngoforum.or.ug/"}, {"url": "http://pathwaystopeaceuganda.com/"}, {"url": "http://thereport.amnesty.org/eng/Regions/Africa/Uganda"}, {"url": "http://tortureinuganda.wordpress.com/"}, {"url": "http://www.acholitoday.worldbreak.com/"}, {"url": "http://www.actionaid.org/uganda/"}, {"url": "http://www.actvuganda.org/"}, {"url": "http://www.aglionline.org/"}, {"url": "http://www.child-soldier.org/children-in-conflict-child-soldiers-in-uganda"}, {"url": "http://www.crisisgroup.org/home/index.cfm?id=2346&l=1"}, {"url": "http://www.doctorswithoutborders.org/news/uganda.cfm"}, {"url": "http://www.enoughproject.org/"}, {"url": "http://www.faweuganda.org/"}, {"url": "http://www.fhri.or.ug/"}, {"url": "http://www.fidauganda.org/"}, {"url": "http://www.hrcug.org/"}, {"url": "https://www.hrw.org/africa/uganda"}, {"url": "http://www.hrw.org/reports/2004/uganda0404/"}, {"url": "http://www.humanrightsuganda.org/"}, {"url": "http://www.hurinet.or.ug/"}, {"url": "http://www.irinnews.org/country/ug/uganda"}, {"url": "http://www.kulanu.org/"}, {"url": "http://www.refugeelawproject.org/"}, {"url": "http://www.theguardian.com/world/uganda"}, {"url": "http://www.uls.or.ug/"}, {"url": "http://www.uwonet.org/"}, {"url": "http://www.vday.org/"}, {"url": "http://www.warchild.org.uk/issues/the-lords-resistance-army"}, {"url": "http://www.wougnet.org/"}, {"url": "http://allafrica.com/stories/201509211054.html"}, {"url": "http://chapterfouruganda.com/"}, {"url": "https://defendersprotection.org/"}, {"url": "https://www.albertinewatchdog.org/"}, {"url": "https://unwantedwitness.or.ug/"}, {"url": "http://www.spellingbee.ug/"}, {"url": "http://stories4humanrights.com/"}, {"url": "http://www.kas.de/uganda/en/"}, {"url": "https://leoafricainstitute.org/"}, {"url": "http://mcjl.ug/"}, {"url": "http://bros4bros.com/"}, {"url": "http://gayuganda.blogspot.com/"}, {"url": "http://groups.yahoo.com/group/candies/"}, {"url": "http://whosoever.org/"}, {"url": "http://www.advocate.com/news/world-news/2013/04/02/photos-antigay-rally-held-ugandan-activist-david-katos-grave?page=full"}, {"url": "http://www.allmale.com/"}, {"url": "http://www.geocities.com/gayuganda/"}, {"url": "http://www.gmad.org/"}, {"url": "http://www.hemeetshim.com/"}, {"url": "http://www.mask.org.za/"}, {"url": "http://www.realjock.com/gay-personals/black"}, {"url": "http://www.smug.4t.com/"}, {"url": "https://www.kuchutimes.com/2016/04/smug-launches-publication-highlighting-human-rights-violations-of-lgbti-ugandans/"}, {"url": "https://sexualminoritiesuganda.com/"}, {"url": "https://oblogdee.blog/"}, {"url": "http://www.africanhrc.org/"}, {"url": "http://blogs.monitor.co.ug/"}, {"url": "http://cbsfmbuganda.co.ug/index.php/en/"}, {"url": "http://dembefm.ug/"}, {"url": "http://jssozi.wordpress.com/"}, {"url": "http://kalinaki.blogspot.com/"}, {"url": "http://kfm.co.ug/"}, {"url": "http://mak.ac.ug/"}, {"url": "http://mondaytimes.co.ug/"}, {"url": "http://mugumya.com/"}, {"url": "http://ntv.co.ug/"}, {"url": "http://ntvuganda.co.ug/"}, {"url": "http://owerahbits.blogspot.com/"}, {"url": "http://www.bukedde.co.ug/"}, {"url": "http://www.independent.co.ug/"}, {"url": "http://www.kizito.uklinux.net/"}, {"url": "http://www.maureenagena.com/"}, {"url": "http://www.monitor.co.ug/"}, {"url": "http://chimpreports.com/"}, {"url": "http://kawowo.com/"}, {"url": "http://koonadance.ug/"}, {"url": "http://pctechmag.com/"}, {"url": "http://proggie.ug/"}, {"url": "http://ugandaradionetwork.com/a/"}, {"url": "http://urbanlegendkampala.com/"}, {"url": "http://www.techpost.ug/"}, {"url": "http://xclusive.co.ug/"}, {"url": "http://cpj.org/attacks06/africa06/uga06.html"}, {"url": "http://dennismatanda.blogspot.com/"}, {"url": "http://dying-communist.blogspot.com/"}, {"url": "http://katwe.blogspot.com/"}, {"url": "http://observer.ug/"}, {"url": "http://uganda.blogspirit.com/"}, {"url": "http://ugandawatch.blogspot.com/"}, {"url": "http://www.acme-ug.org/"}, {"url": "http://www.africafoicentre.org/"}, {"url": "http://www.cipesa.org/"}, {"url": "http://www.freeuganda.com/"}, {"url": "http://www.ict4democracy.org/"}, {"url": "http://www.mail-archive.com/ugandanet@kym.net/"}, {"url": "http://www.nabj.org/"}, {"url": "http://www.sudantribune.com/"}, {"url": "http://www.theeastafrican.co.ke/"}, {"url": "http://www.ugandatvnetwork.com/"}, {"url": "http://nilepost.co.ug/"}, {"url": "http://watchdoguganda.com/"}, {"url": "http://matookerepublic.com/"}, {"url": "https://theinvestigatornews.com/"}, {"url": "https://www.softpower.ug/"}, {"url": "https://www.thegrapevine.co.ug/"}, {"url": "http://trumpetnews.co.ug/"}, {"url": "http://newz.ug/"}, {"url": "https://theugandan.com.ug/"}, {"url": "https://www.ceo.co.ug/"}, {"url": "https://campusbee.ug/"}, {"url": "http://businessfocus.co.ug/"}, {"url": "https://kampalapost.com/"}, {"url": "http://witnessradio.org/"}, {"url": "http://www.theinsider.ug/"}, {"url": "http://eagle.co.ug/"}, {"url": "http://www.pmldaily.com/"}, {"url": "http://dailypost.ug/"}, {"url": "http://edge.ug/"}, {"url": "https://spyreports.net/"}, {"url": "https://www.westnileweb.com/"}, {"url": "http://www.olpa-rdc.org/"}, {"url": "http://westnilepress.org/"}, {"url": "http://ujk-ug.org/"}, {"url": "http://activists4change.blogspot.com/"}, {"url": "http://andrewmwendasblog.blogspot.com/"}, {"url": "http://dpuganda.blogspot.com/"}, {"url": "http://fdcuganda.org/"}, {"url": "http://fedsnet.blogspot.com/"}, {"url": "http://groups.yahoo.com/group/ugandagreen"}, {"url": "http://northernugandapost.blogspot.com/"}, {"url": "http://upcparty.net/"}, {"url": "https://www.amamambabazi.com/"}, {"url": "http://www.exposed2013.com/"}, {"url": "http://www.fdc-uganda.org/"}, {"url": "http://www.federo.com/"}, {"url": "http://www.newvision.co.ug/"}, {"url": "http://www.norbertmao.org/"}, {"url": "http://www.splmtoday.com/"}, {"url": "http://www.struggle.ws/africa.html"}, {"url": "http://www.tiuganda.org/"}, {"url": "https://www.transparency.org/"}, {"url": "https://www.transparency.org/news/pressrelease/uganda_tops_east_africa_in_corruption"}, {"url": "https://www.transparency.org/research/cpi/overview"}, {"url": "http://www.uganda.sfgd.org/"}, {"url": "http://www.ugandaonlinelawlibrary.com/"}, {"url": "http://www.upcparty.org/"}, {"url": "http://www.uyonet.or.ug/"}, {"url": "http://www.statehouse.go.ug/"}, {"url": "https://rosebellkagumire.com/"}, {"url": "https://www.theelephant.info/"}] {} {} f t -249356450302 1 0 2023-10-23 13:02:39.734 web_connectivity [{"url": "https://ooni.torproject.org/"}, {"url": "https://expired.badssl.com/"}] {} {} t t -249356450302 2 0 2023-10-23 13:02:39.734 web_connectivity [{"url": "https://ooni.torproject.org/"}, {"url": "https://expired.badssl.com/"}] {} {} t t -260165369202 1 0 2023-10-16 19:51:28.67 whatsapp [] {} {} t t -260165369202 1 1 2023-10-16 19:51:28.67 signal [] {} {} f f -270962959102 1 0 2024-02-15 12:36:52.079 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://cbc.ca"}] {} {} f f -276248267502 1 0 2023-08-18 17:25:35.798 Web Connectivity [] {} {} f f -276982785902 1 0 2023-07-14 10:11:30 torsf [] {} {} t t -276982785902 1 1 2023-07-14 10:11:30 vanilla_tor [] {} {} t t -276982785902 1 2 2023-07-14 10:11:30 stunreachability [] {} {} t t -276982785902 1 3 2023-07-14 10:11:30 dnscheck [] {} {} t t -276982785902 1 4 2023-07-14 10:11:30 web_connectivity [{"url": "https://google.com"}] {} {} t t -276982785902 2 0 2023-07-14 10:11:30 torsf [] {} {} t t -276982785902 2 1 2023-07-14 10:11:30 vanilla_tor [] {} {} t t -276982785902 2 2 2023-07-14 10:11:30 stunreachability [] {} {} t t -276982785902 2 3 2023-07-14 10:11:30 dnscheck [] {} {} t t -276982785902 2 4 2023-07-14 10:11:30 web_connectivity [{"url": "https://google.com"}] {} {} t t -281191790802 1 0 2023-12-05 14:20:51.471 whatsapp [] {"another_reg_option": "opt_val_2", "regular_option": "option_value"} {"backend_option_key": "backend_option_value", "backend_option_key_2": "backedn_option_val_2"} t t -281191790802 2 0 2023-12-05 14:20:51.471 whatsapp [] {} {} t t -281191790802 3 0 2023-12-05 14:20:51.471 whatsapp [] {} {} t t -281191790802 3 1 2023-12-05 14:20:51.471 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -281191790802 4 0 2023-12-05 14:20:51.471 whatsapp [] {} {} t t -281191790802 4 1 2023-12-05 14:20:51.471 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -281191790802 5 0 2023-12-05 14:20:51.471 whatsapp [] {} {} t t -281191790802 5 1 2023-12-05 14:20:51.471 web_connectivity [{"url": "https://twitter.com"}, {"url": " http://www.example.com/brass"}, {"url": " http://example.net/act"}, {"url": " http://arch.example.com/ball/behavior.html"}, {"url": " https://example.com/#baseball"}, {"url": " http://www.example.com/baby/blow"}, {"url": " http://www.example.org/anger"}, {"url": " http://www.example.com/brass"}, {"url": " http://example.net/act"}, {"url": " http://arch.example.com/ball/behavior.html"}, {"url": " https://example.com/#baseball"}, {"url": " http://www.example.com/baby/blow"}, {"url": " http://www.example.org/anger"}] {} {} f f -282024422102 1 0 2023-08-23 08:51:43.91 Test test test Edited [{"url": "https://golos.plus"}, {"url": "https://www.twitter.com"}] {"key": "value"} {"": "value"} t t -282024422102 2 0 2023-08-23 08:51:43.91 Test test test Edited [{"url": "https://golos.plus"}, {"url": "https://www.twitter.com"}, {"url": "http://www.facebook.com"}] {"key": "value"} {"": "value"} t t -26382735700 1 0 2024-03-06 11:17:54 web_connectivity [] {} {} f f -282024422102 3 0 2023-08-23 08:51:43.91 Test test test Edited [{"url": "https://golos.plus"}, {"url": "https://www.twitter.com"}, {"url": "http://www.facebook.com"}] {"key": "value"} {"": "value"} t f -291048935002 1 0 2024-02-05 15:12:28.261 web_connectivity [] {} {} f f -297500125102 1 0 2023-06-05 12:35:51 whatsapp [] {} {} f f -297500125102 2 0 2023-06-05 12:35:51 whatsapp [] {} {} t f -297500125102 3 0 2023-06-05 12:35:51 whatsapp [] {} {} f t -297500125102 3 1 2023-06-05 12:35:51 web_connectivity [{"url": "https://twitter.com"}] {} {} t f -297500125102 4 0 2023-06-05 12:35:51 whatsapp [] {} {} f f -297500125102 4 1 2023-06-05 12:35:51 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -297500125102 5 0 2023-06-05 12:35:51 whatsapp [] {} {} t f -297500125102 5 1 2023-06-05 12:35:51 web_connectivity [{"url": "https://twitter.com"}] {} {} f t -300815650102 1 0 2023-07-19 11:09:00 web_connectivity [{"url": "https://www.grindr.com/"}, {"url": "https://weareher.com/"}, {"url": "https://www.hrc.org/"}, {"url": "https://outrightinternational.org/"}, {"url": "https://ilga.org/"}, {"url": "https://www.iglyo.com/"}, {"url": "http://ilglaw.org/"}, {"url": "https://kaleidoscopetrust.com/"}, {"url": "https://www.lgbtqnation.com/"}, {"url": "https://www.pinknews.co.uk/"}, {"url": "https://thegailygrind.com/"}, {"url": "https://www.nostraightnews.com/"}, {"url": "https://www.gaystarnews.com/"}, {"url": "https://www.afterellen.com/"}, {"url": "https://www.towleroad.com/"}, {"url": "https://www.irqr.ca/"}, {"url": "http://transactivists.org/"}, {"url": "http://www.grincampaign.com/"}, {"url": "http://oiiinternational.com/"}, {"url": "https://www.stonewall.org.uk/"}, {"url": "http://www.transmarch.org/"}, {"url": "https://www.advocate.com/"}, {"url": "http://queernet.org/"}, {"url": "http://lesbian.org/"}, {"url": "http://transsexual.org/"}, {"url": "https://lalgbtcenter.org/"}] {} {} f t -304792721502 1 0 2023-10-16 19:57:01.88 whatsapp [] {} {} t t -305791146102 1 0 2023-08-23 08:41:59.081 Test test tes [{"url": "https://golos.plus"}] {"key": "value"} {"key": "value"} t t -305791146102 1 1 2023-08-23 08:41:59.081 Second test [] {} {} t t -339182569802 1 0 2023-08-01 20:25:35.685 test [] {} {} f f -347268696402 1 0 2023-11-30 19:43:33.383 web_connectivity [{"url": "https://rtvslo.si"}] {} {} f f -347268696402 2 0 2023-11-30 19:43:33.383 web_connectivity [{"url": "https://rtvslo.si"}] {} {} f f -347761207002 1 0 2023-07-19 11:09:01 web_connectivity [{"url": "https://www.advocate.com/"}, {"url": "https://www.autostraddle.com/"}, {"url": "https://www.datalounge.com/threads"}, {"url": "https://www.washingtonblade.com/"}, {"url": "https://www.gaycities.com/"}, {"url": "https://gayauthors.org/"}, {"url": "https://www.outsports.com/"}, {"url": "https://chicago.gopride.com/"}, {"url": "https://www.poz.com/"}, {"url": "https://www.metroweekly.com/"}, {"url": "http://www.logotv.com/"}, {"url": "https://www.ebar.com/"}, {"url": "https://atlantisevents.com/"}, {"url": "https://www.gaytravel.com/"}, {"url": "https://pride.amsterdam/"}, {"url": "http://www.downelink.com/"}, {"url": "https://www.outfest.org//"}, {"url": "http://windycitymediagroup.com/"}, {"url": "http://www.outsmartmagazine.com/"}, {"url": "https://www.nlgja.org/"}, {"url": "https://www.iglta.org/"}, {"url": "https://passportmagazine.com/"}, {"url": "https://lgbtqia.ucdavis.edu/"}, {"url": "https://communitymarketinginc.com/"}, {"url": "http://nglcc.org/"}, {"url": "https://www.growinggenerations.com/"}, {"url": "http://www.logcabin.org/"}, {"url": "https://reelingfilmfestival.org/2019/"}, {"url": "https://cbst.org/"}, {"url": "https://www.rainbowrailroad.com/"}, {"url": "http://kareemazmy.blogspot.com/"}, {"url": "http://emraamethlya.blogspot.com/"}, {"url": "https://egale.ca/"}, {"url": "http://equalitymatters.org/"}, {"url": "https://may17.org/"}, {"url": "http://www.noh8campaign.com/"}, {"url": "https://robynochs.com/"}, {"url": "http://www.newnownext.com/"}, {"url": "http://awesomedude.com/"}, {"url": "https://joy.org.au/"}, {"url": "https://www.frameline.org/"}, {"url": "https://www.gayleague.com/"}, {"url": "http://queerculturalcenter.org/"}, {"url": "https://www.leslielohman.org/"}, {"url": "http://outtv.ca/"}, {"url": "https://www.sfgmc.org/"}, {"url": "http://www.gaypartyblog.com/"}, {"url": "http://www.gay.tv/"}, {"url": "http://glbtqarchive.com/"}, {"url": "https://butch-in-progress.tumblr.com/"}, {"url": "https://www.outonfilm.org/"}, {"url": "http://www.gaytheatre.ie/"}, {"url": "http://lgbt.syr.edu/"}, {"url": "http://www.smackjeeves.com/"}, {"url": "https://www.thetrevorproject.org/"}, {"url": "https://advocatesforyouth.org/"}, {"url": "https://itgetsbetter.org/"}, {"url": "https://www.outyouth.org/"}, {"url": "https://larkinstreetyouth.org/"}, {"url": "http://www.alterheros.com/en/home/"}, {"url": "https://face2face.org/"}, {"url": "https://www.ricsenapresents.com/"}, {"url": "https://www.bisexual.com/"}, {"url": "http://bimen.org/"}, {"url": "http://www.biwriters.org/"}, {"url": "http://bimedia.org/"}, {"url": "http://www.nyacyouth.org/"}, {"url": "https://smyal.org/"}, {"url": "https://dojpride.org/"}, {"url": "http://fordglobe.org/"}, {"url": "https://glifaa.org/"}, {"url": "https://lgbtbar.org/annual/"}, {"url": "https://bi.org/en"}, {"url": "http://www.gayadnetwork.com/"}, {"url": "http://pinkbananamedia.com/"}, {"url": "https://www.outnowconsulting.com/"}, {"url": "http://www.ppt.on.ca/"}, {"url": "http://sendtherightmessage.ca/"}, {"url": "http://emptyclosets.com/"}, {"url": "https://www.gays.com/"}, {"url": "https://www.bglad.com/"}, {"url": "http://makena.libsyn.com/"}, {"url": "https://www.outtraveler.com/"}, {"url": "https://www.glsen.org/"}, {"url": "https://www.campuspride.org/"}, {"url": "https://gsanetwork.org/"}, {"url": "https://pridefoundation.org/"}, {"url": "https://sgdo.utoronto.ca/"}, {"url": "https://www.sageusa.org/"}, {"url": "https://rmnetwork.org/"}, {"url": "http://www.saintvictor.org/"}, {"url": "https://affirmation.org/"}, {"url": "http://whosoever.org/"}, {"url": "https://christiangays.com/"}, {"url": "http://www.sarbat.net/"}, {"url": "http://hgsc.sigs.harvard.edu/"}, {"url": "https://www.sec.sa.utoronto.ca/"}, {"url": "https://spectrumcenter.umich.edu/"}, {"url": "https://www.campusprideindex.org/"}, {"url": "http://lgbtout.sa.utoronto.ca/"}, {"url": "https://www.lgbtcampus.org/"}, {"url": "https://www.ualbertaoutreach.ca/"}, {"url": "http://www.schools-out.org.uk/"}, {"url": "https://gaygames.org/"}, {"url": "http://www.glta.net/"}, {"url": "http://igra.com/"}, {"url": "http://www.igla.org/"}, {"url": "https://chicagomsa.leagueapps.com/"}, {"url": "https://www.gaysport.info/"}, {"url": "https://shaarzahav.org/"}, {"url": "https://christchapel.com/"}, {"url": "http://www.ldsfamilyfellowship.org/"}, {"url": "http://twospirits.org/"}, {"url": "https://www.galva108.org/"}, {"url": "https://lgbt.wikia.org/wiki/Main_Page"}, {"url": "http://www.gaylibrary.com/enter.htm"}, {"url": "https://www.prideangel.com/"}, {"url": "http://www.therainbowbabies.com/"}, {"url": "https://www.rainbowweddingnetwork.com/"}, {"url": "https://www.familyequality.org/"}, {"url": "http://www.proudparenting.com/"}, {"url": "https://www.coparents.co.uk/"}, {"url": "http://www.gayparentmag.com/"}, {"url": "http://www.gayspermbank.com/"}, {"url": "https://www.lgbtcenters.org/LGBTCenters"}, {"url": "https://read.dukeupress.edu/glq"}, {"url": "https://iagsdc.org/"}, {"url": "http://desertadventures.org/"}, {"url": "https://www.gbmcc.co.uk/home"}, {"url": "http://gaycampers.com/"}, {"url": "https://gccc.nl/"}, {"url": "http://www.thegaygamer.com/"}, {"url": "https://www.goc.org.uk/"}, {"url": "https://www.gayoutdoors.org/index.cfm?"}, {"url": "https://gaysailing.org.uk/"}, {"url": "https://ggguards.org/"}, {"url": "http://gayhike.org/"}, {"url": "https://www.rainbowdivers.org/"}, {"url": "https://outventures.org/"}, {"url": "https://alp.org/"}, {"url": "http://www.sfpride.org/"}, {"url": "https://www.capitalpride.org/"}, {"url": "http://pride2020.pridetoronto.com/"}, {"url": "http://www.vancouverpride.ca/"}, {"url": "https://2019-worldpride-stonewall50.nycpride.org/"}, {"url": "http://www.colognepride.de/en/"}, {"url": "https://www.atlantapride.org/"}, {"url": "https://instinctmagazine.com/"}, {"url": "https://www.outeverywhere.com/"}, {"url": "http://www.berkeleyfreeclinic.org/"}, {"url": "https://b-gay.com/"}, {"url": "http://gmhc.org/"}, {"url": "https://www.jeffandwill.com/"}, {"url": "https://www.daddydater.com/"}, {"url": "https://www.gayfriendschat.com/"}, {"url": "https://lgbt.foundation/"}, {"url": "http://pinktherapy.com/"}, {"url": "http://www.nalgap.org/"}, {"url": "http://gal-aa.org/"}, {"url": "http://outhistory.org/"}, {"url": "https://gala.co.za/"}, {"url": "https://www.matthewshepard.org/"}, {"url": "https://pride.fi/"}, {"url": "https://www.pridecymru.com/"}, {"url": "https://www.saskatoonpride.ca/"}, {"url": "https://victoryfund.org/"}, {"url": "http://www.gayrepublicans.org/"}, {"url": "http://georgialogcabin.org/"}, {"url": "http://www.gaypatriot.net/"}, {"url": "https://www.stonewalltx.org/"}, {"url": "http://www.rainbowfund.org/"}, {"url": "https://lgbtfunders.org/"}, {"url": "https://truecolorsunited.org/"}, {"url": "https://gillfoundation.org/"}, {"url": "http://lgbthistorymonth.com/"}, {"url": "https://www.hrc.org/"}, {"url": "https://www.glaad.org/"}, {"url": "https://pflag.org/"}, {"url": "https://ilga.org/"}, {"url": "https://www.lgbtcenters.org/"}, {"url": "https://truthwinsout.org/"}, {"url": "https://www.gayrealestate.com/"}, {"url": "https://www.immigrationequality.org/"}, {"url": "https://www.lambdalegal.org/"}, {"url": "https://www.equaldex.com/"}, {"url": "https://www.eqca.org/"}, {"url": "https://www.gaylawnet.com/"}, {"url": "http://www.nclrights.org/"}, {"url": "https://urbanjustice.org/"}, {"url": "https://uklgig.org.uk/"}, {"url": "https://www.marriageequality.org/"}, {"url": "https://www.gaygirlnet.com/"}, {"url": "https://www.lesbiannews.com/"}, {"url": "http://www.curvemag.com/"}, {"url": "http://glumagazine.com/"}, {"url": "http://www.bintelnas.org/"}, {"url": "http://lesbian.org/"}, {"url": "http://dont.law.stanford.edu/"}, {"url": "https://www.usnaout.org/"}, {"url": "https://www.out.com/"}, {"url": "https://www.queerty.com/"}, {"url": "https://www.lgbtqnation.com/"}, {"url": "https://www.towleroad.com/"}, {"url": "https://www.equalityfederation.org/"}, {"url": "https://www.fflag.org.uk/"}, {"url": "http://pflagaustralia.org.au/"}, {"url": "http://plagal.org/"}, {"url": "https://www.gaystarnews.com"}, {"url": "https://pridesource.com/"}, {"url": "https://www.lgbtqnation.com/commentary/"}, {"url": "http://blacklightonline.com/"}, {"url": "https://echomag.com/"}, {"url": "https://exit.co.za/"}, {"url": "https://gay-nerds.com/"}, {"url": "http://www.therainbowtimesmass.com/"}, {"url": "http://gay.americablog.com/"}, {"url": "https://glreview.org/"}, {"url": "http://aidsnews.blogspot.com/"}, {"url": "http://ambiente.us/"}, {"url": "http://baywindows.com/"}, {"url": "http://genderpsychology.org/"}, {"url": "http://trans-academics.org/"}, {"url": "https://www.sfaf.org"}, {"url": "http://antonyandthejohnsons.com/"}, {"url": "https://www.hyenafilms.com/"}, {"url": "https://dragqueenmissdotcom.co.uk/"}, {"url": "http://sftff.org/"}, {"url": "https://www.crossdresserheaven.com/"}, {"url": "http://transgenderzone.com/"}, {"url": "https://www.mychangingroom.com/"}, {"url": "https://www.crossdresserphotoblog.com/"}, {"url": "http://trans-lide.info/"}, {"url": "http://transfamily.org/"}, {"url": "http://ftmguide.org/"}, {"url": "http://thetransitionalmale.com/"}, {"url": "https://transguys.com/"}, {"url": "https://americantransman.com/"}, {"url": "http://ftmi.org/"}, {"url": "http://eunuch.org/"}, {"url": "https://transgenderwarrior.org/"}, {"url": "https://www.lagendercenter.org/"}, {"url": "https://www.drchristinemcginn.com/"}, {"url": "https://www.transgendermap.com/"}, {"url": "https://www.geneticalliance.org.uk/"}, {"url": "https://transgenderlawcenter.org/"}, {"url": "https://www.crossdreamers.com/"}, {"url": "http://www.genderweb.org/"}, {"url": "http://www.transhouston.com/"}, {"url": "http://www.transgendermichigan.org/"}, {"url": "https://www.transgenderpulse.com/"}, {"url": "http://xdressnj.com/"}, {"url": "http://transdude.com/"}, {"url": "https://transgender-chat.com/"}, {"url": "http://gendertree.com/"}, {"url": "http://agender.org.nz/"}, {"url": "https://espritconf.com/"}, {"url": "http://gendermosaic.com/"}, {"url": "http://gendersanity.com/"}, {"url": "https://www.grsmontreal.com"}, {"url": "https://www.phuketpsi.com/gender-change.html"}, {"url": "http://tgcomics.com/"}, {"url": "http://storysite.org/"}, {"url": "http://transsexual.org/"}, {"url": "https://mermaidsuk.org.uk/"}, {"url": "https://borngay.procon.org/"}, {"url": "https://americansfortruth.com"}, {"url": "https://beyondexgay.com/"}, {"url": "http://www.cuav.org/"}, {"url": "https://www.gaylife.co.za/"}, {"url": "https://www.therapeuticchoice.com/"}, {"url": "https://nea-exgay.org/"}, {"url": "http://www.pfox.org/"}, {"url": "https://www.brothersroad.org/"}, {"url": "https://www.afterellen.com"}, {"url": "http://www.equalitypastore.com/"}, {"url": "https://www.gardenstateequality.org/"}, {"url": "http://www.glma.org/"}, {"url": "https://www.humanrightsfirst.org/"}, {"url": "https://allout.org/en"}, {"url": "https://diversitypro.eu/"}, {"url": "http://www.bisocialnetwork.com/"}, {"url": "http://www.freedomtomarry.org/"}, {"url": "http://transactivists.org/"}, {"url": "https://www.ilga-europe.org/"}, {"url": "http://inakost.sk/"}, {"url": "http://www.imlocal.co.nz/"}, {"url": "https://www.gaynz.com/"}, {"url": "http://www.ianz.org.nz/"}, {"url": "http://www.c-h-e.org.uk/"}, {"url": "https://www.stonewall.org.uk/"}, {"url": "http://affirmationscotland.org.uk/"}, {"url": "https://www.galck.org/"}, {"url": "http://www.ishtarmsm.org/"}, {"url": "http://transgender.or.ke/"}, {"url": "https://bisdakpride.wordpress.com/"}, {"url": "https://www.lgbt.ee/"}, {"url": "http://www.gei.kristlased.ee/"}, {"url": "https://gaysinfronteras.weebly.com/"}, {"url": "http://www.igy.org.il/"}, {"url": "https://www.lgbt.org.il/"}, {"url": "https://www.lsvd.de/"}, {"url": "http://www.maneo.de/en/about-maneo/maneo-in-short.html"}, {"url": "https://transgendervictoria.com/"}, {"url": "https://www.arcigay.it/en/"}, {"url": "https://www.aibai.cn/"}, {"url": "http://www.bds.org.np/"}, {"url": "http://www.boysofbangladesh.org/"}, {"url": "https://gspottt.wordpress.com/"}, {"url": "https://colombiadiversa.org/"}, {"url": "http://www.rightsrwanda.com/index.html"}, {"url": "http://equalityjamaica.org/"}, {"url": "http://www.lambdaistanbul.org/s/"}, {"url": "http://lgbti.org/"}, {"url": "https://lgbtinewsturkey.com/"}, {"url": "https://gladt.de/"}, {"url": "http://www.lgbt.dk"}, {"url": "https://lgbtvoicetz.org/"}, {"url": "http://maltagayrights.org/"}, {"url": "http://nazindia.org/"}, {"url": "http://plu.sg/"}, {"url": "https://pinkalliance.hk/"}, {"url": "http://www.pinkarmenia.org/"}, {"url": "https://samtokin78.is/"}, {"url": "https://hotline.org.tw/"}, {"url": "https://unibam.org/"}, {"url": "http://www.zagreb-pride.net/hr/"}, {"url": "http://gejowo.pl/"}, {"url": "https://queer.pl/"}, {"url": "https://fellow.pl/"}, {"url": "http://fille.pl/"}, {"url": "https://kobiety-kobietom.com/"}, {"url": "http://homiki.pl/"}, {"url": "http://alone-persian-man.blogspot.com/"}, {"url": "http://alqaws.org/siteEn/index"}, {"url": "http://aqueeerdiaries.blogspot.com/"}, {"url": "http://arabgaypride.blogspot.com/"}, {"url": "http://arabiclgbt.blogspot.com/"}, {"url": "http://baantomdy.narak.com/"}, {"url": "http://beurgay.free.fr/"}, {"url": "http://boyahoy.com/"}, {"url": "http://bravingkl.blogspot.com/"}, {"url": "http://cabalasisoy.blogspot.com/"}, {"url": "http://cafe.daum.net/pridecare"}, {"url": "http://comingout.narun.net/"}, {"url": "http://forum.mennawfena.net/"}, {"url": "http://gaia.ru/"}, {"url": "http://galz.org/"}, {"url": "http://gay-uae.blogspot.com/"}, {"url": "http://gayarmenia.blogspot.com/"}, {"url": "http://gaybelarus.by/"}, {"url": "http://gayinkuwait.blogspot.com/"}, {"url": "http://gaylifekenya.blogspot.com/"}, {"url": "http://gaymiddleeast.blogspot.com/"}, {"url": "http://gaysong.blogspot.com/"}, {"url": "http://gaytoday.com/"}, {"url": "http://gayuganda.blogspot.com/"}, {"url": "http://gayzona.com/"}, {"url": "http://happygay.blogspot.com/"}, {"url": "http://harfeakhar.persianblog.com/"}, {"url": "http://helem.net/"}, {"url": "http://hitchhikersgui.de/Alice_Nkom"}, {"url": "http://hitchhikersgui.de/LGBT_rights_in_Cameroon"}, {"url": "http://iraqilgbtuk.blogspot.com/"}, {"url": "http://ivancity.com/"}, {"url": "http://kenyantg.blogspot.com/"}, {"url": "http://kscrc.org/"}, {"url": "http://labrys.kg/"}, {"url": "http://lgbt.by/"}, {"url": "http://lgbt.ge/"}, {"url": "http://lgbt.kdlp.org/"}, {"url": "http://lgbtact.org/"}, {"url": "http://lgbtmap.org/"}, {"url": "http://madikazemi.blogspot.com/2009/07/activists-petition-un-over-violation-of.html"}, {"url": "http://manra.blogspot.com/"}, {"url": "http://my.lesbi.name/"}, {"url": "http://news.cameroon-today.com/tag/cameroon-gays-and-homosexuals/"}, {"url": "http://onglinepodcast.com/"}, {"url": "http://pesareghabile.blogspot.com/"}, {"url": "http://pesari2004.persianblog.com/"}, {"url": "http://phnompenhpride.blogspot.com/"}, {"url": "http://pjhouse.blogspot.com/"}, {"url": "http://queerala.org/be-you-a-gathering-on-coming-out-concerns-and-acceptance-from-families/"}, {"url": "http://queeringyerevan.blogspot.com/"}, {"url": "http://queerquotes.blogspot.com/"}, {"url": "http://queerteam.blogspot.com/"}, {"url": "http://ruchlgbt.pl/"}, {"url": "http://sabaah.dk/"}, {"url": "http://seajunction.org/lgbt-rights-siege-indonesia/"}, {"url": "http://shabbin.blogspot.com/"}, {"url": "http://somosgay.org/"}, {"url": "http://sunshine.by/"}, {"url": "http://the-ipf.com/2016/06/09/homosexuality-nigeria-young-lgbt/"}, {"url": "http://thegaykenyan.blogspot.com/"}, {"url": "http://theladiesfinger.com/i-am-a-queer-feminist-killjoy-and-am-grateful-for-metoo-in-india/"}, {"url": "http://theotherfoundation.org/wp-content/uploads/2017/02/Canaries_Zimbabwe.pdf"}, {"url": "http://trans-armenia.blogspot.com/"}, {"url": "http://www.afrakenya.org/"}, {"url": "http://www.africanhrc.org/"}, {"url": "http://www.allmale.com/"}, {"url": "http://www.aswatmag.com/"}, {"url": "http://www.bangkokrainbow.org/"}, {"url": "http://www.bisialimi.com/"}, {"url": "http://www.buddy79.com/"}, {"url": "http://www.center4civilrights.org/"}, {"url": "http://www.cha.org.ar/"}, {"url": "http://www.colorsrainbow.com/"}, {"url": "http://www.deti-404.com/"}, {"url": "http://www.dreadedned.com/"}, {"url": "http://www.fasiroong.org/"}, {"url": "http://www.forplu.com/"}, {"url": "http://www.galangphilippines.org/"}, {"url": "http://www.galasla.org/"}, {"url": "http://www.gayarab.org/"}, {"url": "http://www.gayegypt.com/"}, {"url": "http://www.gayegypt.com/tanzania.html"}, {"url": "http://www.gayguatemala.com/"}, {"url": "http://www.gayhealth.com/"}, {"url": "http://www.gaylaxymag.com/latest-news/transgender-rights-bill-and-surrogacy-bill-lapse-as-rajya-sabha-adjourns-sine-die/"}, {"url": "http://www.gayly.ru/"}, {"url": "http://www.gaymexico.com.mx/"}, {"url": "http://www.gaytelaviv.com/"}, {"url": "http://www.gayvideo.ru/"}, {"url": "http://www.glas.org/"}, {"url": "http://www.glas.org/ahbab/"}, {"url": "http://www.glas.org/lazeeza.html"}, {"url": "http://www.glbtqarchive.com/"}, {"url": "http://www.glil.org/"}, {"url": "http://www.globalequality.org/newsroom/latest-news/1-in-the-news/186-the-facts-on-lgbt-rights-in-russia"}, {"url": "http://www.gmhc.org/"}, {"url": "http://www.goodasyou.net/"}, {"url": "http://www.gthai.net/"}, {"url": "http://www.gvantage.com/"}, {"url": "http://www.helem.net/"}, {"url": "http://www.houseofrainbow.org/"}, {"url": "http://www.ifge.org/"}, {"url": "http://www.kqcf.org/"}, {"url": "http://www.lambdaistanbul.org/"}, {"url": "http://www.lesben-gegen-gewalt.de/"}, {"url": "http://www.lesbi.ru/"}, {"url": "http://www.lesbian.org/"}, {"url": "http://www.lgbtglobalfaith.org/"}, {"url": "http://www.lgbtmuslimretreat.com/"}, {"url": "http://www.lgfoundation.org.uk/"}, {"url": "http://www.loveyourself.ph/"}, {"url": "http://www.marchadelorgullo.org.ar/"}, {"url": "http://www.mask.org.za/"}, {"url": "http://www.moscowpride.ru/"}, {"url": "http://www.newnownext.com/franchise/the-backlot/"}, {"url": "http://www.oneinstitute.org/"}, {"url": "http://www.out.org.za/"}, {"url": "http://www.panambi.org.py/"}, {"url": "http://www.parniplus.ru/"}, {"url": "http://www.pembehayat.org/"}, {"url": "http://www.pglo.net/"}, {"url": "http://www.queernet.org/"}, {"url": "http://www.rainbowfoundation.co.kr/"}, {"url": "http://www.richard-gay.com/"}, {"url": "http://www.runa.org.pe/"}, {"url": "http://www.russianopengames.ru/"}, {"url": "http://www.salganyc.org/"}, {"url": "http://www.samesexmarriage.ca/"}, {"url": "http://www.sapaan.org/"}, {"url": "http://www.siamboy.com/"}, {"url": "http://www.suarakita.org/"}, {"url": "http://www.thaigmc.com/"}, {"url": "http://www.theadvocatesforhumanrights.org/cameroon_upr_lgbti_2017"}, {"url": "http://www.turkgayclub.com/"}, {"url": "http://www.xgay.ru/"}, {"url": "http://xs.gay.ru/"}, {"url": "https://76crimes.com/2014/06/25/zambia-to-west-dont-watch-when-we-jail-lgbt-people/"}, {"url": "https://ahwaa.org/?lang=ar"}, {"url": "https://aseansogiecaucus.org/"}, {"url": "https://bedayaa.webs.com/"}, {"url": "https://boylovealone.blogspot.com/"}, {"url": "https://caracasqueer.blogspot.com/"}, {"url": "https://galz.org/"}, {"url": "https://gayguide.net/"}, {"url": "https://gaymenshealth.org/"}, {"url": "https://gender.do.am/"}, {"url": "https://hornet.com/"}, {"url": "https://humsafar.org/"}, {"url": "https://identidadydiversidad.adc.org.ar/"}, {"url": "https://igualdaddegenero.tedic.org/"}, {"url": "https://ilga.org/upr-cameroon"}, {"url": "https://ilga.org/upr-paraguay"}, {"url": "https://imaanlondon.wordpress.com/"}, {"url": "https://justiceforsisters.wordpress.com/"}, {"url": "https://kaleidoscopetrust.com/"}, {"url": "https://kwirmedia.com/"}, {"url": "https://lagablab.wordpress.com/"}, {"url": "https://lalgbtcenter.org/"}, {"url": "https://lgbtbar.org/"}, {"url": "https://lgbtnewsnow.org/"}, {"url": "https://lgbtvacationplanners.com/"}, {"url": "https://lgbtvoicetz.org/our-mission/"}, {"url": "https://londonqueermuslims.com/"}, {"url": "https://majal.org/ahwaa"}, {"url": "https://mexgay.com/"}, {"url": "https://mutsaween.wordpress.com/"}, {"url": "https://nostringsng.com/"}, {"url": "https://oblogdee.blog/"}, {"url": "https://outincolombiatravel.com/"}, {"url": "https://panafricailga.org/category/cameroon/"}, {"url": "https://paulshalala.blogspot.com/2015/05/investigation-reveals-zambias-600.html"}, {"url": "https://queeralliancenigeria.blogspot.com/"}, {"url": "https://queeralliancenigeria.wordpress.com/"}, {"url": "https://rainbowegypt.org/"}, {"url": "https://sexualminoritiesuganda.com/"}, {"url": "https://solidaritywithegyptlgbt.wordpress.com/"}, {"url": "https://tapcpr.org/"}, {"url": "https://thequeerarabs.com/"}, {"url": "https://transatsite.com/"}, {"url": "https://transequality.org/"}, {"url": "https://transnational-queer-underground.net/"}, {"url": "https://travesti-diva.io.ua/"}, {"url": "https://twitter.com/rulgbtnet"}, {"url": "https://www.aireana.org.py/"}, {"url": "https://www.beirutpride.org/"}, {"url": "https://www.bisialimifoundation.org/"}, {"url": "https://www.buzzarab.com/chat/gay-arabs/"}, {"url": "https://www.equaldex.com/region/mexico"}, {"url": "https://www.gaystarnews.com/"}, {"url": "https://www.globalgayz.com/"}, {"url": "https://www.hidayahlgbt.co.uk/"}, {"url": "https://www.homochat.org/ar/"}, {"url": "https://www.humandignitytrust.org/country-profile/cameroon/"}, {"url": "https://www.igualdadlgbt.org/"}, {"url": "https://www.kuchutimes.com/2016/04/smug-launches-publication-highlighting-human-rights-violations-of-lgbti-ugandans/"}, {"url": "https://www.lavaplace.com/"}, {"url": "https://www.lgbtarabic.com/"}, {"url": "https://www.mesahatfoundation.org/"}, {"url": "https://www.metrodate.com/gh/"}, {"url": "https://www.mombian.com/"}, {"url": "https://www.mykalimag.com/"}, {"url": "https://www.nostraightnews.com/"}, {"url": "https://outrightinternational.org/"}, {"url": "https://www.pinknews.co.uk/"}, {"url": "https://www.planetromeo.com/"}, {"url": "https://www.prideconnectionmexico.com/"}, {"url": "https://www.proudcampaign.ph/"}, {"url": "https://www.queer-refugees.hamburg/"}, {"url": "https://www.queerlapis.com/"}, {"url": "https://www.rainbow-street.org/"}, {"url": "https://www.salaamcanada.info/"}, {"url": "https://www.scruff.com/"}, {"url": "https://www.soyhomosensual.com/"}, {"url": "https://www.unfe.org/"}, {"url": "https://blogcritics.org"}, {"url": "http://bluesystem.info/"}, {"url": "https://caribeafirmativo.lgbt/"}, {"url": "https://empressmaruja.wordpress.com/"}, {"url": "https://equalitymyanmar.org/"}, {"url": "http://gay.kz/"}, {"url": "https://aglany.org/"}, {"url": "http://www.gaykenya.com/"}, {"url": "https://gaymorocco.yoo7.com/"}, {"url": "https://www.planetromeo.com/auth/signup"}, {"url": "https://gaycity.love/"}, {"url": "https://groups.yahoo.com/neo/groups/candies/info"}, {"url": "https://groups.yahoo.com/neo/groups/lesbianmuslims/info"}, {"url": "https://gukira.wordpress.com/"}, {"url": "https://kenyangayman.wordpress.com/"}, {"url": "https://kph.org.pl/"}, {"url": "https://lgbt.ie/"}, {"url": "http://www.lgbtph.org/"}, {"url": "https://love.gay.ru/en/search.phtml"}, {"url": "https://mingle2.com/online-dating/carriacou/ghana/gay-personals"}, {"url": "https://mnw.org.pl/"}, {"url": "https://naijalez.com/"}, {"url": "https://outragemag.com/"}, {"url": "https://www.rbth.com/lgbt-in-russia"}, {"url": "https://rwaho.wordpress.com/"}, {"url": "https://saudislgbt.tumblr.com/"}, {"url": "https://turkeygay.net/"}, {"url": "https://alturi.org/"}, {"url": "https://www.aswatgroup.org/"}, {"url": "https://chingusai.net/xe/main"}, {"url": "https://www.exgay.com/"}, {"url": "http://falgbt.url.ph/"}, {"url": "https://www.fridae.asia/"}, {"url": "https://www.fundacionreflejosdevenezuela.com/"}, {"url": "http://www.gaycambodianews.com/"}, {"url": "https://gaycationmagazine.com/"}, {"url": "https://gmad.org/"}, {"url": "https://www.grindr.com/"}, {"url": "https://www.istscare.org/"}, {"url": "https://www.kaosgl.org/"}, {"url": "https://www.baby-boy.fr/"}, {"url": "https://lgbti-ep.eu/"}, {"url": "https://www.pridemedia.com/"}, {"url": "http://lgbtpride.or.kr/"}, {"url": "https://oia.co.za/"}, {"url": "https://www.refworld.org/docid/53ecb2de4.html"}, {"url": "http://satrang.org/"}, {"url": "https://thegailygrind.com/"}, {"url": "https://www.transparentgender.com/"}, {"url": "https://www.utopia-asia.com/"}, {"url": "https://www.blued.com/en/index.html"}, {"url": "https://www.lgbtarabic.com/ar/"}, {"url": "https://www.mykalimag.com/en/home-page/"}] {} {} f t -349628539702 1 0 2023-07-19 11:08:59 web_connectivity [{"url": "https://www.reddit.com/"}, {"url": "https://edition.cnn.com/"}, {"url": "https://www.nytimes.com/"}, {"url": "https://news.google.com/"}, {"url": "https://www.theguardian.com/"}, {"url": "https://www.indiatimes.com/"}, {"url": "https://www.washingtonpost.com/"}, {"url": "https://news.yahoo.com/"}, {"url": "https://www.forbes.com/"}, {"url": "https://www.foxnews.com/"}, {"url": "https://timesofindia.indiatimes.com/"}, {"url": "https://www.cnbc.com/"}, {"url": "https://economictimes.indiatimes.com/"}, {"url": "https://www.bloomberg.com/"}, {"url": "https://www.wsj.com/"}, {"url": "https://www.bbc.com/"}, {"url": "https://www.dw.com/"}, {"url": "https://www.voanews.com/"}, {"url": "https://www.reuters.com/"}, {"url": "https://www.rt.com/"}, {"url": "http://www.chinadaily.com.cn/"}, {"url": "https://www.nbcnews.com/"}, {"url": "https://abcnews.go.com/"}, {"url": "https://nypost.com/"}, {"url": "https://www.cbsnews.com/"}, {"url": "https://www.aljazeera.net/"}, {"url": "https://time.com/"}, {"url": "https://www.theatlantic.com/"}, {"url": "https://www.alarabiya.net/"}, {"url": "https://www.alhurra.com/"}, {"url": "https://africatimes.com/"}, {"url": "https://allafrica.com/"}, {"url": "https://www.africanews.com/"}, {"url": "https://www.asiatimes.com/"}, {"url": "https://www.sina.com.cn/"}, {"url": "https://www.rambler.ru/"}, {"url": "https://www.lemonde.fr/"}, {"url": "https://www.rfa.org/"}, {"url": "https://www.rferl.org/"}, {"url": "https://elpais.com/"}, {"url": "https://www.elmundo.es/"}, {"url": "https://www.globo.com/"}, {"url": "https://www.irna.ir/"}, {"url": "https://www.euronews.com/"}, {"url": "http://ru.rfi.fr/"}, {"url": "https://ar.qantara.de/"}, {"url": "https://www.radiotelevisionmarti.com/"}, {"url": "https://www.radiosawa.com/"}] {} {} f t -361132445202 1 0 2023-08-08 12:26:15.453 Web Connectivity [{"url": "https://twitter.com/"}, {"url": "https://www.facebook.com/"}, {"url": "https://www.instagram.com/"}] {} {} t f -366386230002 1 0 2023-09-26 13:33:00.068 web_connectivity [{"url": "https://www.facebook.com/"}, {"url": "https://twitter.com/"}] {} {} t t -373393593802 1 0 2023-07-19 11:08:59 web_connectivity [{"url": "https://www.torproject.org/"}, {"url": "https://psiphon.ca/"}, {"url": "https://getlantern.org/"}, {"url": "https://www.tunnelbear.com/"}, {"url": "https://www.getoutline.org/"}, {"url": "https://www.hotspotshield.com/"}, {"url": "https://nordvpn.com/"}, {"url": "https://www.expressvpn.com/"}, {"url": "https://www.ipvanish.com/"}, {"url": "https://www.privateinternetaccess.com/"}, {"url": "https://www.purevpn.com/"}, {"url": "https://www.cyberghostvpn.com/"}, {"url": "https://mullvad.net/"}, {"url": "https://windscribe.com/"}, {"url": "https://strongvpn.com/"}, {"url": "https://www.vyprvpn.com/"}, {"url": "https://surfshark.com/"}, {"url": "https://goosevpn.com/"}, {"url": "https://buffered.com/"}, {"url": "https://ultrasurf.us/"}, {"url": "http://anonymouse.org/"}] {} {} f t -424988552202 1 0 2023-07-19 11:09:01 web_connectivity [{"url": "https://www.womenonwaves.org/"}, {"url": "https://www.womenonweb.org/"}, {"url": "http://www.womenonwaves.org/"}, {"url": "http://www.womenonweb.org/"}] {} {} f t -115983820800 1 0 2024-03-06 10:11:34 web_connectivity [] {} {} f f -156737021500 1 0 2024-03-06 10:23:33 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -381898594100 1 0 2024-03-06 10:26:47 web_connectivity [] {} {} f f -32447275800 1 0 2024-03-06 10:32:48 web_connectivity [] {} {} f f -95930853000 1 0 2024-03-06 10:32:59 web_connectivity [] {} {} f f -662549500 1 0 2024-03-06 10:39:27 web_connectivity [] {} {} f f -166744503000 1 0 2024-03-06 10:40:07 web_connectivity [] {} {} f f -164213156400 1 0 2024-03-06 11:13:31 web_connectivity [] {} {} f f -113145430800 1 0 2024-03-06 10:34:36 web_connectivity [] {} {} f f -201675761900 1 0 2024-03-07 12:52:26 web_connectivity [{"url": "https://twitter.com"}] {} {} f f -285684714800 1 0 2024-03-06 12:14:33 web_connectivity [] {} {} f f -181619554500 1 0 2024-03-06 10:42:08 web_connectivity [] {} {} f f -181619554500 2 0 2024-03-07 13:11:52 web_connectivity [{"url": "https://rumble.com"}] {} {} f f -421206342800 1 0 2024-03-07 13:00:22 web_connectivity [{"url": "https://twitter.com"}, {"url": "https://www.facebook.com"}] {} {} f f -51580415900 1 0 2024-03-07 18:17:18 facebook_messenger [] {} {} f f -51580415900 1 1 2024-03-07 18:17:18 signal [] {} {} f f -194411950900 1 0 2024-03-06 10:42:26 web_connectivity [] {} {} f f -181619554500 3 0 2024-03-08 13:41:33 web_connectivity [{"url": "https://rumble.com"}, {"url": "https://twitter.com"}] {} {} f f -\. - - --- --- Name: alembic_version alembic_version_pkc; Type: CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.alembic_version - ADD CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num); - - --- --- Name: oonirun_nettest oonirun_nettest_pkey; Type: CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.oonirun_nettest - ADD CONSTRAINT oonirun_nettest_pkey PRIMARY KEY (oonirun_link_id, revision, nettest_index); - - --- --- Name: oonirun tmp__oonirun_new_pkey; Type: CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.oonirun - ADD CONSTRAINT tmp__oonirun_new_pkey PRIMARY KEY (oonirun_link_id); - - --- --- Name: oonirun_nettest fk_oonirun_nettest_oonirun_link_id; Type: FK CONSTRAINT; Schema: public; Owner: oonipg --- - -ALTER TABLE ONLY public.oonirun_nettest - ADD CONSTRAINT fk_oonirun_nettest_oonirun_link_id FOREIGN KEY (oonirun_link_id) REFERENCES public.oonirun(oonirun_link_id); - - --- --- PostgreSQL database dump complete --- - diff --git a/ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py b/ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py deleted file mode 100644 index 6a1fa951..00000000 --- a/ooniapi/services/oonimeasurements/tests/gen-db-fixtures.py +++ /dev/null @@ -1,86 +0,0 @@ -import csv - -TABLE_CREATE_QUERIES = """ -CREATE TABLE fastpath ( - measurement_uid TEXT, - report_id TEXT, - input TEXT, - probe_cc TEXT, - probe_asn INTEGER, - test_name TEXT, - test_start_time TEXT, - measurement_start_time TEXT, - filename TEXT, - scores TEXT, - platform TEXT, - anomaly TEXT, - confirmed TEXT, - msm_failure TEXT, - domain TEXT, - software_name TEXT, - software_version TEXT, - control_failure TEXT, - blocking_general REAL, - is_ssl_expected INTEGER, - page_len INTEGER, - page_len_ratio REAL, - server_cc TEXT, - server_asn INTEGER, - server_as_name TEXT, - update_time TEXT, - test_version TEXT, - architecture TEXT, - engine_name TEXT, - engine_version TEXT, - test_runtime REAL, - blocking_type TEXT, - test_helper_address TEXT, - test_helper_type TEXT, - ooni_run_link_id TEXT -); - -CREATE TABLE citizenlab ( - domain TEXT, - url TEXT, - cc TEXT, - category_code TEXT -); -""" - -# Dumps for fastpath.csv and citizenlab.csv can be generated running the following commands on the backend-fsn host: -""" -clickhouse-client -q "SELECT * FROM fastpath WHERE measurement_start_time > '2024-01-01' AND measurement_start_time < '2024-01-10' ORDER BY measurement_start_time LIMIT 1000 INTO OUTFILE '20240110-fastpath.csv' FORMAT CSVWithNames" -clickhouse-client -q "SELECT * FROM citizenlab INTO OUTFILE '20240110-citizenlab.csv' FORMAT CSVWithNames" -""" -FASTPATH_CSV = "fastpath.csv" -CITIZENLAB_CSV = "citizenlab.csv" - -OUTPUT_SQL = "db-fixtures.sql" - -with open(OUTPUT_SQL, "w") as f: - f.write(TABLE_CREATE_QUERIES) - - input_set = set() - # Process fastpath.csv - with open(FASTPATH_CSV) as fastpath_csv: - reader = csv.DictReader(fastpath_csv) - for row in reader: - input_set.add(row["input"]) - columns = ",".join(row.keys()) - values = ",".join([f"'{v.strip()}'" for v in row.values()]) - query = f"INSERT INTO fastpath ({columns}) VALUES ({values});\n" - f.write(query) - - # Process citizenlab.csv - with open(CITIZENLAB_CSV) as citizenlab_csv: - reader = csv.DictReader(citizenlab_csv) - for row in reader: - if row["url"] not in input_set: - continue - row["cc"] = row["cc"][:2] - columns = ",".join(row.keys()) - values = ",".join([f"'{v.strip()}'" for v in row.values()]) - query = f"INSERT INTO citizenlab ({columns}) VALUES ({values});\n" - f.write(query) - -print("SQL file generated successfully!") diff --git a/ooniapi/services/oonimeasurements/tests/integ/__init__.py b/ooniapi/services/oonimeasurements/tests/integ/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ooniapi/services/oonimeasurements/tests/integ/conftest.py b/ooniapi/services/oonimeasurements/tests/integ/conftest.py deleted file mode 100644 index 10cbb46d..00000000 --- a/ooniapi/services/oonimeasurements/tests/integ/conftest.py +++ /dev/null @@ -1,91 +0,0 @@ -from pathlib import Path -import time -from typing import Iterable -from fastapi.testclient import TestClient -import pytest -import docker -from docker.models.containers import Container as DockerContainer - -from clickhouse_driver import Client as Clickhouse - -from ...config import settings -from ...main import app - -THIS_DIR = Path(__file__).parent.parent.resolve() - - -def run_clickhouse_sql_scripts(clickhouse_url): - click = Clickhouse.from_url(clickhouse_url) - tables = click.execute("SHOW TABLES") - assert isinstance(tables, Iterable) - for row in tables: - if row[0] == "fastpath": - return - - for fn in [ - "clickhouse_1_schema.sql", - "clickhouse_2_fixtures.sql", - "2_fastpath_fixtures.sql", - ]: - sql_f = THIS_DIR / "fixtures" / fn - print(f"[+] running {sql_f} on {clickhouse_url}") - sql_no_comment = "\n".join( - filter(lambda x: not x.startswith("--"), sql_f.read_text().split("\n")) - ) - queries = sql_no_comment.split(";") - for q in queries: - q = q.strip() - if not q: - continue - click.execute(q) - - -@pytest.fixture(autouse=True, scope="session") -def clickhouse(docker_clickhouse_url): - settings.clickhouse_url = docker_clickhouse_url - run_clickhouse_sql_scripts(settings.clickhouse_url) - return docker_clickhouse_url - - -@pytest.fixture(scope="session") -def docker_clickhouse_url(): - client = docker.from_env() - image_name = "yandex/clickhouse-server:21.8-alpine" - print(f"\n[+] pulling clickhouse image from {image_name}") - client.images.pull(image_name) - # Run a container with a random port mapping for ClickHouse default port 9000 - container = client.containers.run( - image_name, - ports={ - "9000/tcp": None - }, # This maps port 9000 inside the container to a random host port - detach=True, - ) - assert isinstance(container, DockerContainer) - - # Wait for the container to be ready - time.sleep(10) - - # Obtain the port mapping - container.reload() - assert isinstance(container.attrs, dict) - host_port = container.attrs["NetworkSettings"]["Ports"]["9000/tcp"][0]["HostPort"] - - # Construct the connection string - connection_string = f"clickhouse://localhost:{host_port}" - - yield connection_string - - # Cleanup after the test session is done - container.stop() - container.remove() - - -@pytest.yield_fixture -def client(): - """ - Overriding the `client` fixture from pytest_flask to fix this bug: - https://github.com/pytest-dev/pytest-flask/issues/42 - """ - client = TestClient(app) - yield client diff --git a/ooniapi/services/oonimeasurements/tests/integ/test_measurements.py b/ooniapi/services/oonimeasurements/tests/integ/test_measurements.py deleted file mode 100644 index 9abb19ec..00000000 --- a/ooniapi/services/oonimeasurements/tests/integ/test_measurements.py +++ /dev/null @@ -1,1087 +0,0 @@ -""" -Integration test for API - -Warning: this test runs against a real database -See README.adoc - -Lint using: - black -t py37 -l 100 --fast ooniapi/tests/integ/test_integration.py - -Test using: - pytest-3 -k test_aggregation -""" - -from datetime import datetime, timedelta -from hashlib import shake_128 -from urllib.parse import urlencode -import time - -from dateutil.parser import parse as parse_date -import pytest - -FASTPATH_MSM_ID_PREFIX = "temp-fid-" -# The flask app is created in tests/conftest.py - - -@pytest.fixture() -def fastpath_dup_rid_input(app): - """ - Access DB directly. Fetch > 1 measurements from fastpath that share the same - report_id and input - Returns (rid, input, count) - """ - # Too slow - sql = """ - SELECT report_id, input - from fastpath - group by report_id, input - HAVING count(*) > 1 - LIMIT 1 - """ - with app.app_context(): - for row in app.db_session.execute(sql): - return (row[0], row[1]) - - -@pytest.fixture() -def fastpath_rid_input(app): - """Access DB directly. Get a fresh msmt - Returns (rid, input) - """ - sql = """ - SELECT report_id, input, test_start_time FROM fastpath - WHERE input IS NOT NULL - ORDER BY measurement_start_time DESC - LIMIT 1 - """ - rid, inp, test_start_time = dbquery(app, sql)[0:3] - assert rid.strip() - assert inp.strip() - - return (rid, inp, test_start_time) - - -def dbquery(app, sql, **query_params): - """Access DB directly, returns row as tuple.""" - with app.app_context(): - q = app.db_session.execute(sql, query_params) - return q.fetchone() - - -def api(client, subpath, **kw): - url = f"/api/v1/{subpath}" - if kw: - assert "?" not in url - url += "?" + urlencode(kw) - - response = client.get(url) - assert response.status_code == 200 - assert response.is_json - return response.json - - -# # rate limiting / quotas # # - - -@pytest.fixture() -def enable_rate_limits(app): - # undo what the fixture "disable_rate_limits" in conftest.py where needed - # also reset the database before each test - app.limiter._disabled = False - app.limiter._limiter._lmdb.purge_databases() - - -@pytest.fixture() -def lower_rate_limits(app, enable_rate_limits): - # Access the rate limiter buckets directly - limits = app.limiter._limiter._ipaddr_limits - old, limits[0] = limits[0], 1 - yield - limits[0] = old - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_basic(client, app, enable_rate_limits): - # test started with emtpy database - with app.app_context(): - summary = app.limiter._limiter.get_lowest_daily_quotas_summary() - assert summary == [] - - # Simulate a forwarded client with a different ipaddr - # In production the API sits behind Nginx - headers = {"X-Real-IP": "1.2.3.4"} - limit = 4000 - paths = ( - "/stats", - "/files", - "/files/by_date", - "/api/_/test_names", - "/api/_/test_names", - ) - for p in paths: - resp = client.get(p, headers=headers) - assert int(resp.headers["X-RateLimit-Remaining"]) < limit - - with app.app_context(): - summary = app.limiter._limiter.get_lowest_daily_quotas_summary() - - assert len(summary) == 1 - ipa_str, remaining = summary[0] - assert remaining < 4000 - assert remaining > 3998 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_for_explorer(client, enable_rate_limits): - # Special ipaddr: no rate limiting. No header is set by the server - headers = {"X-Real-IP": "37.218.242.149"} - resp = client.get("/stats", headers=headers) - assert resp.status_code == 301 - assert "X-RateLimit-Remaining" not in resp.headers - - resp = client.get("/stats", headers=headers) - assert resp.status_code == 301 - assert "X-RateLimit-Remaining" not in resp.headers - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_spin(client, enable_rate_limits): - # Simulate a forwarded client with a different ipaddr - # In production the API sits behind Nginx - limit = 4000 - headers = {"X-Real-IP": "1.2.3.4"} - end_time = time.monotonic() + 1.1 - while time.monotonic() < end_time: - resp = client.get("/stats", headers=headers) - assert abs(time.monotonic() - end_time) < 0.2 - r = int(resp.headers["X-RateLimit-Remaining"]) - assert r in (limit - 1, limit - 2) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_summary(client, enable_rate_limits): - url = "quotas_summary" - response = privapi(client, url) - assert response == [] - response = privapi(client, url) - assert len(response) == 1 - assert int(response[0][1]) == 3998 # quota remaining in seconds - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_spin_to_zero(client, lower_rate_limits): - headers = {"X-Real-IP": "1.2.3.4"} - end_time = time.monotonic() + 2 - while time.monotonic() < end_time: - resp = client.get("/_/test_names", headers=headers) - if resp.status_code == 429: - return # we reached the limit - - assert 0, "429 was never received" - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_redirects_and_rate_limit_spin_to_zero_unmetered(client, lower_rate_limits): - headers = {"X-Real-IP": "1.2.3.4"} - end_time = time.monotonic() + 2 - while time.monotonic() < end_time: - resp = client.get("/health", headers=headers) - assert resp.status_code == 200, "Unmetered page should always work" - - -# # list_files # # - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_files_pagination(client): - url = "files?limit=1&since=2019-12-01&until=2019-12-02" - ret = api(client, url) - results = ret["results"] - assert len(results) == 1 - assert sorted(results[0].keys()) == [ - "download_url", - "index", - "probe_asn", - "probe_cc", - "test_name", - "test_start_time", - ] - assert ret["metadata"] == { - "count": 13273, - "current_page": 1, - "limit": 1, - "next_url": "https://api.ooni.io/api/v1/files?limit=1&since=2019-12-01&until=2019-12-02&offset=1", - "offset": 0, - "pages": 13273, - } - url = "files?limit=1&since=2019-12-01&until=2019-12-02&offset=1" - ret = api(client, url) - results = ret["results"] - assert len(results) == 1 - assert ret["metadata"] == { - "count": 13273, - "current_page": 2, - "limit": 1, - "next_url": "https://api.ooni.io/api/v1/files?limit=1&since=2019-12-01&until=2019-12-02&offset=2", - "offset": 1, - "pages": 13273, - } - - -@pytest.mark.skip(reason="DROP") -def test_list_files_asn(client): - url = "files?limit=1&since=2019-12-01&until=2019-12-02&probe_asn=AS45595" - results = api(client, url)["results"] - assert len(results) == 1 - assert results[0]["probe_asn"] == "AS45595" - - -@pytest.mark.skip(reason="DROP") -def test_list_files_asn_only_number(client): - url = "files?limit=1&since=2019-12-01&until=2019-12-02&probe_asn=45595" - results = api(client, url)["results"] - assert len(results) == 1 - assert results[0]["probe_asn"] == "AS45595" - - -@pytest.mark.skip(reason="DROP") -def test_list_files_range_cc(client): - url = "files?limit=1000&since=2019-12-01&until=2019-12-02&probe_cc=IR" - ret = api(client, url) - results = ret["results"] - assert len(results) == 215 - assert ret["metadata"] == { - "count": 215, - "current_page": 1, - "limit": 1000, - "next_url": None, - "offset": 0, - "pages": 1, - } - - -@pytest.mark.skip(reason="DROP") -def test_list_files_range_cc_asn(client): - url = "files?limit=1000&since=2019-12-01&until=2019-12-02&probe_cc=IR&probe_asn=AS44375" - results = api(client, url)["results"] - assert len(results) == 7 - - -# # get_measurement_meta # # - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_basic(client): - rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" - inp = "https://www.backtrack-linux.org/" - response = api(client, f"measurement_meta?report_id={rid}&input={inp}") - assert response == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": inp, - "measurement_start_time": "2021-07-09T00:55:13Z", - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "probe_asn": 4818, - "probe_cc": "MY", - "report_id": rid, - "scores": '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}', - "test_name": "web_connectivity", - "test_start_time": "2021-07-09T00:43:40Z", - "category_code": "", - } - # TODO implement - # "platform": None, - # "software_name": "ooniprobe-android", - # "software_version": "2.2.0" - # "analysis": {"blocking": "http-diff",}, - # "network_name": "Fidget Unlimited", - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_invalid_rid(client): - response = client.get(f"/api/v1/measurement_meta?report_id=BOGUS") - assert b"Invalid report_id" in response.data - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_not_found(client): - url = "/api/v1/measurement_meta?report_id=20200712T100000Z_AS9999_BOGUSsYKWBS2S0hdzXf7rhUusKfYP5cQM9HwAdZRPmUfroVoCn" - resp = client.get(url) - # TODO: is this a bug? - assert resp.status_code == 200 - assert resp.json == {} - - -@pytest.mark.skip(reason="TODO: fix this test") -def FIXME_MISSING_MSMT____test_get_measurement_meta_input_none_from_fp(client): - rid = "20200712T143743Z_AS27775_17Eq6sYKWBS2S0hdzXf7rhUusKfYP5cQM9HwAdZRPmUfroVoCn" - # input is None - response = api(client, f"measurement_meta?report_id={rid}") - assert response == { - "anomaly": False, - "category_code": None, - "confirmed": False, - "failure": False, - "fp_measurement_id": "temp-fid-00ae35a61dadc20e014d9d544525b823", - "input": None, - "measurement_id": "temp-fid-00ae35a61dadc20e014d9d544525b823", - "measurement_start_time": "2020-07-20T19:36:23Z", - "mr_measurement_id": None, - "platform": "android", - "probe_asn": 27775, - "probe_cc": "SR", - "report_id": rid, - "scores": "{}", - "software_name": None, - "software_version": None, - "test_name": "ndt", - "test_start_time": "2020-07-20 19:35:55", - } - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_input_none_from_fp(client): - rid = "20210709T000017Z_httpinvalidrequestline_CH_3303_n1_8mr2M3dzkoFmmjIU" - # input is None - response = api(client, f"measurement_meta?report_id={rid}") - assert response == { - "anomaly": False, - "category_code": None, - "confirmed": False, - "failure": False, - "input": "", - "measurement_start_time": "2021-07-09T00:00:18Z", - "measurement_uid": "20210709000024.440526_CH_httpinvalidrequestline_3937f817503ed4ea", - "probe_asn": 3303, - "probe_cc": "CH", - "report_id": rid, - "scores": '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', - "test_name": "http_invalid_request_line", - "test_start_time": "2021-07-09T00:00:16Z", - } - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_full(client): - rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" - inp = "https://www.backtrack-linux.org/" - response = api(client, f"measurement_meta?report_id={rid}&input={inp}&full=True") - data = response.pop("raw_measurement") - assert response == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": inp, - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "measurement_start_time": "2021-07-09T00:55:13Z", - "probe_asn": 4818, - "probe_cc": "MY", - "scores": '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}', - "report_id": rid, - "test_name": "web_connectivity", - "test_start_time": "2021-07-09T00:43:40Z", - "category_code": "", - } - assert data - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_by_uid(client): - uid = "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874" - response = api(client, f"measurement_meta?measurement_uid={uid}") - assert response == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": "https://www.backtrack-linux.org/", - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "measurement_start_time": "2021-07-09T00:55:13Z", - "probe_asn": 4818, - "probe_cc": "MY", - "scores": '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}', - "report_id": "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3", - "test_name": "web_connectivity", - "test_start_time": "2021-07-09T00:43:40Z", - "category_code": "", - } - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_by_uid_full(client): - uid = "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874" - response = api(client, f"measurement_meta?measurement_uid={uid}&full=True") - data = response.pop("raw_measurement") - assert response == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": "https://www.backtrack-linux.org/", - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "measurement_start_time": "2021-07-09T00:55:13Z", - "probe_asn": 4818, - "probe_cc": "MY", - "scores": '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}', - "report_id": "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3", - "test_name": "web_connectivity", - "test_start_time": "2021-07-09T00:43:40Z", - "category_code": "", - } - assert data - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_raw_measurement(client): - rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" - inp = "https://www.backtrack-linux.org/" - r = api(client, f"raw_measurement?report_id={rid}&input={inp}") - assert len(r.keys()) == 20 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_body_by_uid(client): - uid = "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874" - r = api(client, f"raw_measurement?measurement_uid={uid}") - assert len(r.keys()) == 20 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_body_by_uid_missing(client): - uid = "20210709005529.664022_MY_webconnectivity_bogusbogusbogusb" - r = api(client, f"raw_measurement?measurement_uid={uid}") - assert len(r.keys()) == 0 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_meta_duplicate_in_fp(client, fastpath_dup_rid_input): - rid, inp = fastpath_dup_rid_input - response = api(client, f"measurement_meta?report_id={rid}&input={inp}") - # TODO FIXME count duplicates and verify - assert response["input"] == inp - assert response["scores"] != "{}" # from faspath - - -@pytest.mark.skip(reason="WIP") -def test_get_measurement_meta_meek(client): - rid = "20210709T000901Z_meekfrontedrequeststest_AR_16814_n1_eFZoc0FuSddWPvmL" - inp = "{a0.awsstatic.com,d2cly7j4zqgua7.cloudfront.net}" - response = api(client, f"measurement_meta?report_id={rid}&input={inp}&full=True") - data = response.pop("raw_measurement") - assert response == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": inp, - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "measurement_start_time": "2021-07-09T00:55:13Z", - "probe_asn": 4818, - "probe_cc": "MY", - "scores": '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"blocking_type":"http-failure"}}', - "report_id": rid, - "test_name": "web_connectivity", - "test_start_time": "2021-07-09T00:43:40Z", - "category_code": "", - } - assert data - - -# https://explorer.ooni.org/measurement/20210622T144545Z_riseupvpn_MM_133384_n1_VJkB5EObudGDpy9Y -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_raw_measurement_input_null_bug(client): - # PR#254 - rid = "20210622T144545Z_riseupvpn_MM_133384_n1_VJkB5EObudGDpy9Y" - r = api(client, f"raw_measurement?report_id={rid}") - assert "test_keys" in r - - -# # list_measurements # # - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_one(client): - rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" - inp = "https://www.backtrack-linux.org/" - response = api(client, f"measurements?report_id={rid}&input={inp}") - assert response["metadata"]["count"] == 1, jd(response) - r = response["results"][0] - assert r == { - "anomaly": True, - "confirmed": False, - "failure": False, - "input": inp, - "measurement_start_time": "2021-07-09T00:55:13Z", - "measurement_uid": "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "measurement_url": "https://api.ooni.io/api/v1/raw_measurement?measurement_uid=20210709005529.664022_MY_webconnectivity_68e5bea1060d1874", - "probe_asn": "AS4818", - "probe_cc": "MY", - "report_id": rid, - "scores": { - "analysis": {"blocking_type": "http-failure"}, - "blocking_country": 0.0, - "blocking_general": 1.0, - "blocking_global": 0.0, - "blocking_isp": 0.0, - "blocking_local": 0.0, - }, - "test_name": "web_connectivity", - } - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_search(client): - # Used by Explorer search - response = api( - client, - f"measurements?domain=www.humanrightsfirst.org&until=2021-07-15&limit=50", - ) - assert len(response["results"]) == 7, jd(response) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_search_cc(client): - # Used by Explorer search - response = api( - client, - f"measurements?probe_cc=CA&domain=twitter.com&until=2021-07-15&limit=50", - ) - assert len(response["results"]) == 3, jd(response) - - -# # Test slow list_measurements queries with order_by = None -# pytest-3 ooniapi/tests/integ/test_integration.py \ -# -k test_list_measurements_slow --durations=40 -s -x - - -# These are some of the parameters exposed by Explorer Search -@pytest.mark.parametrize("probe_cc", ("US", None)) -@pytest.mark.parametrize("since", ("2021-01-01", None)) -@pytest.mark.parametrize("test_name", ("web_connectivity", None)) -@pytest.mark.parametrize("anomaly", ("true", None)) -@pytest.mark.parametrize("domain", ("twitter.com", None)) -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_slow_order_by_complete( - domain, anomaly, test_name, since, probe_cc, log, client -): - d = dict( - probe_cc=probe_cc, - since=since, - test_name=test_name, - anomaly=anomaly, - domain=domain, - until="2021-07-11", - ) - d = {k: v for k, v in d.items() if v is not None} - url = "measurements?" + urlencode(d) - response = api(client, url) - - -@pytest.mark.parametrize( - "f", ("probe_cc=YT", "probe_asn=AS3352", "test_name=web_connectivity") -) -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_slow_order_by_group_1(f, log, client): - # filter on probe_cc or probe_asn or test_name - # order by --> "test_start_time" - url = f"measurements?until=2019-01-01&{f}" - log.info(url) - response = api(client, url) - - -@pytest.mark.parametrize("f", ("anomaly=true", "domain=twitter.com")) -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_slow_order_by_group_2(f, log, client): - # anomaly or confirmed or failure or input or domain or category_code - # order by --> "measurement_start_time" - url = f"measurements?until=2019-01-01&{f}" - log.info(url) - response = api(client, url) - - -# This is the hard case. When mixing one filter that applies to the -# measurements table and one on the results table, there is no way to do -# an order_by that avoids heavy scans - - -@pytest.mark.parametrize("f1", ("probe_cc=YT", "test_name=web_connectivity")) -@pytest.mark.parametrize("f2", ("anomaly=true", "domain=twitter.com")) -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_slow_order_by_group_3(f1, f2, log, client): - url = f"measurements?until=2019-01-01&{f1}&{f2}" - log.info(url) - response = api(client, url) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_pagination_old(client, log): - # Ensure answers stay consistent across calls - using old data - # https://github.com/ooni/api/issues/49 - url = "measurements?probe_cc=RU&test_name=web_connectivity&limit=100&offset=5000&since=2018-12-24&until=2018-12-25" - j = None - for n in range(3): - log.info(f"{'-' * 20} Cycle {n} {'-' * 20}") - new = api(client, url) - del new["metadata"]["query_time"] - if j is not None: - assert j == new - j = new - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_pagination_new(client, log): - # Ensure answers stay consistent across calls - using fresh data - # https://github.com/ooni/api/issues/49 - since = (datetime.utcnow().date() - timedelta(days=1)).strftime("%Y-%m-%d") - until = datetime.utcnow().date().strftime("%Y-%m-%d") - url = f"measurements?probe_cc=RU&test_name=web_connectivity&limit=100&offset=5000&since={since}&until={until}" - j = None - for n in range(3): - log.info(f"{'-' * 20} Cycle {n} {'-' * 20}") - new = api(client, url) - del new["metadata"]["query_time"] - if j is not None: - assert j == new - j = new - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_arbitrary_test_name(client): - resp = api(client, "measurements?test_name=BOGUS1234") - resp["metadata"].pop("query_time") - assert resp == { - "metadata": { - "count": 0, - "current_page": 1, - "limit": 100, - "next_url": None, - "offset": 0, - "pages": 0, - }, - "results": [], - } - - -def today_range(): - """Return since/until pair to extract fresh fastpath entries""" - since = datetime.utcnow().date() - until = since + timedelta(days=1) - return since, until - - -@pytest.mark.parametrize("anomaly", (True, False)) -@pytest.mark.parametrize("confirmed", (True, False)) -@pytest.mark.parametrize("failure", (True, False)) -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_flags_fastpath( - anomaly, confirmed, failure, client, log -): - """Test filtering by anomaly/confirmed/msm_failure using the cartesian product - - SELECT COUNT(*), anomaly, confirmed, msm_failure AS failure - FROM fastpath - WHERE measurement_start_time > '2021-07-09' - AND measurement_start_time <= '2021-07-10' - GROUP BY anomaly, confirmed, failure - ORDER BY anomaly, confirmed, failure ASC; - ┌─count()─┬─anomaly─┬─confirmed─┬─failure─┐ - │ 8538 │ f │ f │ f │ - │ 705 │ f │ f │ t │ - │ 688 │ t │ f │ f │ - │ 13 │ t │ f │ t │ - │ 42 │ t │ t │ f │ - │ 2 │ t │ t │ t │ - └─────────┴─────────┴───────────┴─────────┘ - """ - p = f"measurements?since=2021-07-09&until=2021-07-10&anomaly={anomaly}" - p += f"&confirmed={confirmed}&failure={failure}&limit=100" - p = p.lower() - log.info("Calling %s", p) - response = api(client, p) - for r in response["results"]: - assert r["anomaly"] == anomaly, r - assert r["confirmed"] == confirmed, r - assert r["failure"] == failure, r - - i = anomaly * 4 + confirmed * 2 + failure * 1 - thresholds = [100, 100, 0, 0, 100, 13, 42, 2] - assert len(response["results"]) == thresholds[i], len(response["results"]) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_probe_asn(client): - p = "measurements?probe_asn=AS5089&since=2019-12-8&until=2021-12-11&limit=50" - response = api(client, p) - assert len(response["results"]) == 50 - for r in response["results"]: - assert r["probe_asn"] == "AS5089" - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_failure_true_fastpath(client): - p = f"measurements?failure=true&since=2021-07-09&until=2021-07-10&limit=50" - response = api(client, p) - assert len(response["results"]) > 0 - for r in response["results"]: - assert r["failure"] == True, r - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_failure_false_fastpath(client): - p = f"measurements?failure=false&since=2021-07-09&until=2021-07-10&limit=50" - response = api(client, p) - assert len(response["results"]) > 0 - for r in response["results"]: - assert r["failure"] == False, r - - -@pytest.mark.skip(reason="TODO: fix this test") -def notest_list_measurements_paging_1(client): - url = "measurements?test_name=dnscheck&since=2021-07-09&until=2021-07-10" - resp = api(client, url) - meta = resp["metadata"] - assert meta["count"] == 0 - assert meta["current_page"] == 1 - assert meta["next_url"] == None - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_paging_2(client): - url = "measurements?since=2021-07-09&until=2021-07-10&test_name=web_connectivity&probe_cc=IE" - for pagenum in range(1, 999): - resp = api(client, url) - meta = resp["metadata"] - if pagenum < 4: - assert meta["current_page"] == pagenum, url - assert meta["offset"] == (pagenum - 1) * 100, url - assert meta["count"] == -1, (url, meta) - assert meta["pages"] == -1, (url, meta) - assert meta["next_url"].startswith("https://api.ooni.io/api/v1/") - else: # last page - assert meta["current_page"] == pagenum, url - assert meta["offset"] == 300, url - assert meta["count"] == 300, (url, meta) # is this ok? - assert meta["next_url"] is None - assert meta["pages"] == 3 # in this a bug? - break - - url = meta["next_url"].split("/", 5)[-1] # fetch next url - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_category_code(client): - # requires `citizenlab` to be populated - p = "measurements?since=2021-7-9&until=2021-7-10&category_code=NEWS" - r = api(client, p) - urls = set(e["input"] for e in r["results"]) - assert "http://ncac.org/" in urls - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_order(client): - # https://github.com/ooni/backend/issues/587 - url = "measurements?since=2021-07-09&until=2021-07-10" - resp = api(client, url) - assert len(resp["results"]) == 100 - start_times = [parse_date(r["measurement_start_time"]) for r in resp["results"]] - assert start_times == sorted(start_times, reverse=True) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_stunreachability(client): - # https://github.com/ooni/backend/issues/573 - url = "measurements?test_name=stunreachability&since=2021-07-09&until=2021-07-10" - resp = api(client, url) - assert len(resp["results"]) > 2 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_AS0(client): - resp = client.get("/api/v1/measurements?probe_asn=AS0") - assert resp.status_code == 403 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_ZZ(client): - resp = client.get("/api/v1/measurements?probe_cc=ZZ") - assert resp.status_code == 403 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_software_version_nomatch(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&software_version=9.9.9" - resp = api(client, url) - assert len(resp["results"]) == 0 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_software_version_empty_value(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&software_version=" - resp = api(client, url) - assert len(resp["results"]) == 100 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_software_version_single(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&software_version=0.7.1" - resp = api(client, url) - assert len(resp["results"]) == 28 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_software_version_multiple(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&software_version=0.7.1,0.8.1" - resp = api(client, url) - assert len(resp["results"]) == 31 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_software_version(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&software_version=3.9.2" - resp = api(client, url) - assert len(resp["results"]) == 100 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_test_version(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&test_version=0.4.0" - resp = api(client, url) - assert len(resp["results"]) == 100 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_filter_engine_version(client): - # https://github.com/ooni/backend/issues/615 - url = "measurements?since=2021-07-09&until=2021-07-10&engine_version=3.9.2" - resp = api(client, url) - assert len(resp["results"]) == 100 - - -## get_measurement ## - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_found(client): - uid = "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874" - response = api(client, f"measurement/{uid}") - assert response["measurement_start_time"] == "2021-07-09 00:55:13" - assert response["probe_asn"] == "AS4818" - assert response["probe_cc"] == "MY" - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_found_dload(client): - uid = "20210709005529.664022_MY_webconnectivity_68e5bea1060d1874" - response = api(client, f"measurement/{uid}?download=true") - assert response["measurement_start_time"] == "2021-07-09 00:55:13" - assert response["probe_asn"] == "AS4818" - assert response["probe_cc"] == "MY" - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_missing_pipeline(client): - url = "measurement/temp-id-999999999999999999" - response = client.get(f"/api/v1/{url}") - assert response.status_code == 400 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_get_measurement_2(log, client, fastpath_rid_input): - """Simulate Explorer behavior - Get a measurement from the fastpath table - """ - # Get a real rid/inp directly from the database - rid, inp, test_start_time = fastpath_rid_input - - # This has collisions with data from the traditional pipeline - p = f"measurements?report_id={rid}&input={inp}" - log.info("Calling API on %s", p) - response = api(client, p) - assert response["metadata"]["count"] > 0, jd(response) - assert len(response["results"]) == 1, jd(response) - pick = response["results"][0] - url_substr = "measurement/{}".format(FASTPATH_MSM_ID_PREFIX) - assert url_substr in pick["measurement_url"] - assert "anomaly" in pick, pick.keys() - assert pick["scores"] != {} - assert "blocking_general" in pick["scores"] - - url = pick["measurement_url"] - relurl = url[27:] - log.info("Calling API on %r", relurl) - msm = api(client, relurl) - - # Assure the correct msmt was received - msm = api(client, relurl) - for f in ("probe_asn", "probe_cc", "report_id", "input", "test_name"): - # (measurement_start_time differs in the timezone letter) - assert msm[f] == pick[f], "%r field: %r != %r" % (f, msm[f], pick[f]) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_355_confirmed(client): - # Use RU to have enough msmt - p = "measurements?probe_cc=RU&limit=50&confirmed=true&since=2019-12-23&until=2019-12-24" - response = api(client, p) - for r in response["results"]: - assert r["confirmed"] == True, r - assert len(response["results"]) == 50 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_355_anomaly(client): - # https://github.com/ooni/explorer/issues/355 - p = "measurements?probe_cc=RU&limit=50&anomaly=true&since=2019-12-23&until=2019-12-24" - response = api(client, p) - for r in response["results"]: - assert r["anomaly"] == True, r - assert len(response["results"]) == 50 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_142_twitter(client): - # we can assume there's always enough data - ts = datetime.utcnow().date().strftime("%Y-%m-%d") - p = "measurements?domain=twitter.com&until=%s&limit=20" % ts - response = api(client, p) - rows = tuple(response["results"]) - assert len(rows) == 20 - for r in rows: - assert "twitter" in r["input"], r - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_278_with_input_none(client): - url = "measurements?report_id=20190113T202156Z_AS327931_CgoC3KbgM6zKajvIIt1AxxybJ1HbjwwWJjsJnlxy9rpcGY54VH" - response = api(client, url) - assert response["metadata"]["count"] == 1, jd(response) - assert response["results"] == [ - { - "anomaly": False, - "confirmed": False, - "failure": False, - "input": None, - "measurement_id": "temp-id-263478291", - "measurement_start_time": "2019-02-22T20:21:59Z", - "measurement_url": "https://api.ooni.io/api/v1/measurement/temp-id-263478291", - "probe_asn": "AS327931", - "probe_cc": "DZ", - "report_id": "20190113T202156Z_AS327931_CgoC3KbgM6zKajvIIt1AxxybJ1HbjwwWJjsJnlxy9rpcGY54VH", - "scores": {}, - "test_name": "ndt", - } - ] - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_278_with_input_not_none(client): - # Fetch web_connectivity by report_id only. Expect 25 hits where input - # is backfilled from domain_input - url = "measurements?report_id=20190221T235955Z_AS8346_GMKlfxcvS7Xcy2vPgzmOIxeJXnLdGmWVcZ18vrelaTQ4YDAUHo" - response = api(client, url) - assert response["metadata"]["count"] == 25, jd(response) - for r in response["results"]: - assert r["input"].startswith("http") - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_bug_list_measurements_anomaly_coalesce(client): - # list_measurements coalesce was giving priority to mr_table over fastpath - url = "measurements?report_id=20200222T165239Z_AS24691_5WcQoZyep2HktNd8UvKf1Ka4C3WPyOc9AQP79zoJ7oPgyDwSWh" - response = api(client, url) - assert len(response["results"]) == 1 - r = response["results"][0] - assert r["scores"]["blocking_general"] > 0.5 - assert not r["scores"]["analysis"]["whatsapp_endpoints_accessible"] - - assert not r["confirmed"] - assert not r["failure"] - assert r["anomaly"] - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_external_order_by(client): - # The last order-by on the rows from pipeline + fastpath - today = datetime.utcnow().date() - until = today + timedelta(days=1) - url = f"measurements?until={until}&probe_cc=TR" - response = api(client, url) - last = max(r["measurement_start_time"] for r in response["results"]) - # Ensure that the newest msmt is from today (hence fastpath) - assert last[:10] == str(today) - assert len(response["results"]) == 100, jd(response) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_list_measurements_bug_probe1034(client): - rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" - url = f"measurements?report_id={rid}" - r = client.get(f"/api/v1/{url}", headers={"user-agent": "okhttp"}) - assert r.status_code == 200 - assert r.is_json - response = r.json - assert "results" in response - assert len(response["results"]) == 1, repr(response) - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_slow_inexistent_domain(client): - # time-unbounded query, filtering by a domain never monitored - p = "measurements?domain=meow.com&until=2019-12-11&limit=50" - response = api(client, p) - rows = tuple(response["results"]) - assert len(rows) == 0 - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_slow_domain_unbounded(client): - # time-unbounded query, filtering by a popular domain - p = "measurements?domain=twitter.com&until=2019-12-11&limit=50" - response = api(client, p) - rows = tuple(response["results"]) - assert rows - - -@pytest.mark.skip(reason="TODO: fix this test") -def test_slow_domain_bounded(client): - p = "measurements?domain=twitter.com&since=2019-12-8&until=2019-12-11&limit=50" - response = api(client, p) - assert len(response["results"]) == 48 - - -## files_download ## - - -@pytest.mark.skip(reason="legacy") -def test_files_download_found(client): - url = "files/download/2019-06-06/20190606T115021Z-IE-AS5466-ndt-20190606T115024Z_AS5466_fBGQoRbWb034yaEVz25JzTTge72KzFhcXValcZmIaQ8HWqy4Y1-0.2.0-probe.json" - response = client.get(url) - assert response.status_code == 200 - assert response.is_streamed - data = response.get_data() - assert shake_128(data).hexdigest(3) == "3e50c7" - - -@pytest.mark.skip(reason="legacy") -def test_files_download_found_legacy(client): - url = "files/download/20190606T115021Z-IE-AS5466-ndt-20190606T115024Z_AS5466_fBGQoRbWb034yaEVz25JzTTge72KzFhcXValcZmIaQ8HWqy4Y1-0.2.0-probe.json" - response = client.get(url) - assert response.status_code == 302 - - -@pytest.mark.skip(reason="legacy") -def test_files_download_missing(client): - url = "files/download/2019-06-06/bogus-probe.json" - response = client.get(url) - assert response.status_code == 404 - - -@pytest.mark.skip(reason="legacy") -def test_files_download_missing_legacy(client): - url = "files/download/without-slash-bogus-probe.json" - response = client.get(url) - assert response.status_code == 404 diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql new file mode 100644 index 00000000..7245f4ae --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_init_tables.sql @@ -0,0 +1,66 @@ +-- Create tables for Clickhouse integ tests + +CREATE TABLE IF NOT EXISTS default.fastpath +( + `measurement_uid` String, + `report_id` String, + `input` String, + `probe_cc` String, + `probe_asn` UInt32, + `test_name` String, + `test_start_time` DateTime, + `measurement_start_time` DateTime, + `filename` String, + `scores` String, + `platform` String, + `anomaly` String, + `confirmed` String, + `msm_failure` String, + `domain` String, + `software_name` String, + `software_version` String, + `control_failure` String, + `blocking_general` Float32, + `is_ssl_expected` Int8, + `page_len` Int32, + `page_len_ratio` Float32, + `server_cc` String, + `server_asn` Int8, + `server_as_name` String, + `update_time` DateTime64(3) MATERIALIZED now64(), + `test_version` String, + `test_runtime` Float32, + `architecture` String, + `engine_name` String, + `engine_version` String, + `blocking_type` String, + `test_helper_address` LowCardinality(String), + `test_helper_type` LowCardinality(String), + `ooni_run_link_id` Nullable(UInt64) +) +ENGINE = ReplacingMergeTree +ORDER BY (measurement_start_time, report_id, input) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS default.citizenlab +( + `domain` String, + `url` String, + `cc` FixedString(32), + `category_code` String +) +ENGINE = ReplacingMergeTree +ORDER BY (domain, url, cc, category_code) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS default.jsonl +( + `report_id` String, + `input` String, + `s3path` String, + `linenum` Int32, + `measurement_uid` String +) +ENGINE = MergeTree +ORDER BY (report_id, input) +SETTINGS index_granularity = 8192; diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql new file mode 100644 index 00000000..ab289832 --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_citizenlab.sql @@ -0,0 +1,641 @@ +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ushmm.org', 'https://www.ushmm.org/', 'ZZ', 'CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cabofrio.rj.gov.br', 'http://www.cabofrio.rj.gov.br/', 'BR', 'CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ncac.org', 'http://ncac.org/', 'ZZ', 'NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ncac.org', 'https://ncac.org/', 'ZZ', 'NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.facebook.com','http://www.facebook.com/saakashvilimikheil','ge','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.facebook.com','http://www.facebook.com/somsakjeam/videos/1283095981743678/','th','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('facebook.com','http://facebook.com/','ua','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('facebook.com','https://facebook.com/watch','jo','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','http://twitter.com/ghonim','kw','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','http://twitter.com/ghonim','so','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','by','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','kz','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','tm','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','uz','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('200rf.com','https://200rf.com/','ru','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('24tv.ua','https://24tv.ua/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('4genderjustice.org','https://4genderjustice.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('666games.net','http://666games.net/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('6ott-mit-uns.livejournal.com','https://6ott-mit-uns.livejournal.com/','ru','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('76crimes.com','https://76crimes.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('abpr2.railfan.net','http://abpr2.railfan.net/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('abs.twimg.com','https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('addons.mozilla.org','https://addons.mozilla.org/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('adium.im','https://adium.im/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('adultfriendfinder.com','https://adultfriendfinder.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('advox.globalvoices.org','https://advox.globalvoices.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('alt.com','https://alt.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('api.protonvpn.ch','https://api.protonvpn.ch/tests/ping','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('arc-international.net','https://arc-international.net/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('archive.is','https://archive.is/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('archive.org','https://archive.org/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('asiatimes.com','https://asiatimes.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('assets.gitlab-static.net','https://assets.gitlab-static.net/robots.txt','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('atlasofsurveillance.org','https://atlasofsurveillance.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('avishanx.com','https://avishanx.com/','ir','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bahai-library.com','https://bahai-library.com/','ir','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','ch','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','lt','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','th','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','tz','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('blogsome.com','http://blogsome.com/','my','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('caranovanocongresso.blogs.sapo.pt','http://caranovanocongresso.blogs.sapo.pt/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('caranovanocongresso.blogspot.com','http://caranovanocongresso.blogspot.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carcari.blogspot.com','http://carcari.blogspot.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carcoucari.blogspot.com','http://carcoucari.blogspot.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cargadecavalaria.blogspot.com','http://cargadecavalaria.blogspot.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carineroos.wordpress.com','http://carineroos.wordpress.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carlosbaia.blogspot.com.br','http://carlosbaia.blogspot.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carnegieendowment.org','https://carnegieendowment.org/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carolinabrauer.wordpress.com','http://carolinabrauer.wordpress.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cdn.fbsbx.com','https://cdn.fbsbx.com/favicon.ico','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cf-ipfs.com','https://cf-ipfs.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('chat-gpt.org','https://chat-gpt.org/','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('chat.openai.com','https://chat.openai.com/','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cid.contact','https://cid.contact/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cinemata.org','https://cinemata.org/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('circumcision.org','https://circumcision.org/','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('citizenlab.ca','https://citizenlab.ca/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cloudflare-ipfs.com','https://cloudflare-ipfs.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cocaine.org','https://cocaine.org/','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('coronavirus.app','https://coronavirus.app/','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('coronavirus.jhu.edu','https://coronavirus.jhu.edu/map.html','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('covidtracking.com','https://covidtracking.com/','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('creativecommons.org','https://creativecommons.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cryptome.org','https://cryptome.org/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cryptpad.fr','https://cryptpad.fr/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cwtch.im','https://cwtch.im/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cyber.harvard.edu','https://cyber.harvard.edu/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dailystormer.name','https://dailystormer.name/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dartcenter.org','https://dartcenter.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ddosecrets.com','https://ddosecrets.com/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.indymedia.org','https://de.indymedia.org/','de','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','at','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','ch','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','de','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','fr','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('delta.chat','https://delta.chat/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('disclose.ngo','https://disclose.ngo/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discord.com','https://discord.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discord.gg','http://discord.gg/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discordapp.com','https://discordapp.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('disqus.com','https://disqus.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dit.whatsapp.net','https://dit.whatsapp.net/robots.txt','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dl.acm.org','https://dl.acm.org/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dn42.dev','https://dn42.dev/Home','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('doubleclick.net','https://doubleclick.net/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('download.cnet.com','https://download.cnet.com/Freegate/3000-2085_4-10415391.html','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('duckduckgo.com','https://duckduckgo.com/','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dweb.link','https://dweb.link/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('edge-chat.instagram.com','https://edge-chat.instagram.com/robots.txt','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('element.io','https://element.io/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('external.xx.fbcdn.net','https://external.xx.fbcdn.net/robots.txt','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('firebaseinstallations.googleapis.com','https://firebaseinstallations.googleapis.com/robots.txt','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('freedns.afraid.org','https://freedns.afraid.org/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('fteproxy.org','http://fteproxy.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gaymenshealth.org','https://gaymenshealth.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gaytoday.com','http://gaytoday.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('genderandaids.unwomen.org','https://genderandaids.unwomen.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ghost.org','https://ghost.org/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('git.io','https://git.io/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.com','https://github.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.com','https://github.com/twbs/bootstrap/releases/download/v4.5.3/bootstrap-4.5.3-examples.zip','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.githubassets.com','https://github.githubassets.com/images/modules/open_graph/github-logo.png','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gitlab.com','https://gitlab.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('goaskalice.columbia.edu','https://goaskalice.columbia.edu/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gospelhomeblog.blogspot.com.br','http://gospelhomeblog.blogspot.com.br/','br','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gostodesermulher.com','http://gostodesermulher.com/search/como-provocar-um-aborto-caseiro-as-16-semanas/','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('greenhost.net','https://greenhost.net/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gynuity.org','http://gynuity.org/downloads/polbrf_misoprostol_selfguide_po.pdf','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('health4beauty.net','http://health4beauty.net/cytotec-misoprostol.html','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('howtouseabortionpill.org','http://howtouseabortionpill.org/','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('i.ytimg.com','https://i.ytimg.com/vi/JBRf3nEqfZ8/hq720.jpg','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('iarf.net','https://iarf.net/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icao.maps.arcgis.com','https://icao.maps.arcgis.com/apps/opsdashboard/index.html#/977dde48256b489fb48fa98e724721e8','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icao.maps.arcgis.com','https://icao.maps.arcgis.com/apps/webappviewer3d/index.html?id=d9d3f8fa9a23425c8f0889baab626186','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icq.com','https://icq.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ifacetimeapp.com','https://ifacetimeapp.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('im0-tub-com.yandex.net','https://im0-tub-com.yandex.net/i?id=462f375c96139f1e41d919b44be2d780&n=13&exp=1','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('impactlab.org','https://impactlab.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('indymedia.org','https://indymedia.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('instagram.fsaw1-13.fna.fbcdn.net','https://instagram.fsaw1-13.fna.fbcdn.net/favicon.ico','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('instinctmagazine.com','http://instinctmagazine.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('internationalfamilyequalityday.org','https://internationalfamilyequalityday.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ipi.media','https://ipi.media/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('itgetsbetter.org','https://itgetsbetter.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('itgirls.com.br','http://itgirls.com.br/','br','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('iyfglobal.org','https://iyfglobal.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ja.wikipedia.org','https://ja.wikipedia.org/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('jezebel.com','https://jezebel.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('juliapetit.com.br','http://juliapetit.com.br/category/home/','br','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kaleidoscopetrust.com','https://kaleidoscopetrust.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kamikadze-d.net','https://kamikadze-d.net/','ru','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kassimahmad.blogspot.com','https://kassimahmad.blogspot.com/','my','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kat.am','https://kat.am/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('katcr.to','https://katcr.to/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('keadilanrakyat.org','https://keadilanrakyat.org/','my','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kedahlaniie.blogspot.com','https://kedahlaniie.blogspot.com/','my','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('khilafah.net','http://khilafah.net/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('khms0.google.com','https://khms0.google.com/kh/v=908?x=81&y=103&z=8','ZZ','CTRL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kickasstorrents.to','https://kickasstorrents.to/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kiwifarms.net','https://kiwifarms.net/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kolektiva.social','https://kolektiva.social/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('komas.org','https://komas.org/','my','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kremlin.ru','http://kremlin.ru/','ZZ','GOVT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('krishna.com','https://krishna.com/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kryss.network','https://kryss.network/','my','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ktemoc.blogspot.com','https://ktemoc.blogspot.com/','my','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('laborrights.org','https://laborrights.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lalgbtcenter.org','https://lalgbtcenter.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lambdalegal.org','http://lambdalegal.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('leap.se','https://leap.se/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbt.foundation','https://lgbt.foundation/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtmap.org','http://lgbtmap.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtnewsnow.org','https://lgbtnewsnow.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtvacationplanners.com','https://lgbtvacationplanners.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lh3.googleusercontent.com','https://lh3.googleusercontent.com/favicon.ico','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.fun','https://libgen.fun/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.gs','http://libgen.gs/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.is','https://libgen.is/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.lc','http://libgen.lc/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.li','http://libgen.li/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.life','https://libgen.life/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.me','https://libgen.me/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.rs','http://libgen.rs/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.space','https://libgen.space/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('liveuamap.com','https://liveuamap.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('locals.com','https://locals.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lokinet.org','https://lokinet.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.ru','https://mail.ru/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.yahoo.com','https://mail.yahoo.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.yandex.ru','https://mail.yandex.ru/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('makeamazonpay.com','https://makeamazonpay.com/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('maps.gstatic.com','https://maps.gstatic.com/tactile/basepage/grid-image.png','ZZ','CTRL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('marcelo.marques.zip.net','http://marcelo.marques.zip.net/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mart.ibon.com.tw','https://mart.ibon.com.tw/mart/','tw','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask-api.icloud.com','https://mask-api.icloud.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask-h2.icloud.com','https://mask-h2.icloud.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask.icloud.com','https://mask.icloud.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('massbrowser.cs.umass.edu','https://massbrowser.cs.umass.edu/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mastodon.xyz','https://mastodon.xyz/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix-client.matrix.org','https://matrix-client.matrix.org/_matrix/client/versions','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix.org','https://matrix.org/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix.to','https://matrix.to/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mattermost.com','https://mattermost.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media-arn2-1.cdn.whatsapp.net','https://media-arn2-1.cdn.whatsapp.net/robots.txt','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media.fevn1-1.fna.whatsapp.net','https://media.fevn1-1.fna.whatsapp.net/robots.txt','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media0.giphy.com','https://media0.giphy.com/robots.txt','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media3.giphy.com','https://media3.giphy.com/robots.txt','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('medium.com','https://medium.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('meet.jit.si','https://meet.jit.si/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('messages.google.com','https://messages.google.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mmg.whatsapp.net','https://mmg.whatsapp.net/robots.txt','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mobile.element.io','https://mobile.element.io/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('movabletype.com','https://movabletype.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mozilla.cloudflare-dns.com','https://mozilla.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mulheresmarias.blogspot.com','http://mulheresmarias.blogspot.com/2004/09/como-fazer-um-aborto.html','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('multimedia.scmp.com','https://multimedia.scmp.com/infographics/news/china/article/3047038/wuhan-virus/index.html?src=article-launcher','ZZ','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mural.codigofonte.net','http://mural.codigofonte.net/exibir.php?id=41767','br','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('myabortionstory.tumblr.com','https://myabortionstory.tumblr.com/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mytrans.com.tw','https://mytrans.com.tw/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('networkcheck.kde.org','http://networkcheck.kde.org/','ZZ','CTRL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ozgurgelecek17.net','https://ozgurgelecek17.net/','tr','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ozguruz20.org','https://ozguruz20.org/','tr','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('plus.im','https://plus.im/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('portal.odp.org.tr','http://portal.odp.org.tr/','tr','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('preview.redd.it','https://preview.redd.it/robots.txt','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('prism-break.org','https://prism-break.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('riseup.net','https://riseup.net/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('risk-inform.ru','https://risk-inform.ru/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('romb.tv','https://romb.tv/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('rosdep.org','https://rosdep.org/','ru','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('rsf.org','https://rsf.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('s.pinimg.com','https://s.pinimg.com/robots.txt','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sandbox.cryptpad.info','https://sandbox.cryptpad.info/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('scontent-vie1-1.cdninstagram.com','https://scontent-vie1-1.cdninstagram.com/favicon.ico','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secfirst.org','https://secfirst.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secondlife.com','https://secondlife.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secure.avaaz.org','https://secure.avaaz.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('securevpn.com','https://securevpn.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('securevpn.im','https://securevpn.im/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('seed4.me','https://seed4.me/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('semnasem.org','https://semnasem.org/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('severrea.github.io','https://severrea.github.io/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('severrea.gitlab.io','https://severrea.gitlab.io/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shadowsocks.org','https://shadowsocks.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shereilam.persianblog.ir','https://shereilam.persianblog.ir/','iq','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shtab.navalny.com','https://shtab.navalny.com/','ru','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sibrea.github.io','https://sibrea.github.io/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sibrea.gitlab.io','https://sibrea.gitlab.io/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('signal.org','https://signal.org/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('simple.wikipedia.org','https://simple.wikipedia.org/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('skat.media','https://skat.media/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('skiff.com','https://skiff.com/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('slashdot.org','https://slashdot.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('slate.com','https://slate.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('smex.org','http://smex.org/','iq','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('smotrim.ru','http://smotrim.ru/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sobesednik.ru','https://sobesednik.ru/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sota.vision','https://sota.vision/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sotkurdistan.net','http://sotkurdistan.net/','iq','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('soundcloud.com','https://soundcloud.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sputniknews.com','https://sputniknews.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('spys.one','https://spys.one/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('stealthchat.com','http://stealthchat.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('stolitomsonvpn.com','https://stolitomsonvpn.com/','ru','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('substack.com','https://substack.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('t.me','https://t.me/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tamtam.chat','https://tamtam.chat/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tdov.org','http://tdov.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('te-st.org','https://te-st.org/','ru','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('telegram.me','https://telegram.me/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('test-ipv6.com','https://test-ipv6.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('th.bing.com','https://th.bing.com/th/id/OIP.YnpdokcN_OQYn3n41AlqZAHaEe?w=250&h=180&c=7&o=5&pid=1.7','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('theintercept.com','https://theintercept.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('timcast.com','https://timcast.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('timesofindia.indiatimes.com','https://timesofindia.indiatimes.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transequality.org','https://transequality.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transparencyreport.google.com','https://transparencyreport.google.com/','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transsexual.org','http://transsexual.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('triller.co','https://triller.co/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tse1.explicit.bing.net','https://tse1.explicit.bing.net/robots.txt','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/Ahmed_Mansoor/','ae','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/UAE_Council/','ae','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/gamaleid/','ae','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/gamaleid/','qa','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/ghonim/','ae','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/ghonim/','qa','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/uae_detainees/','ae','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tyt.com','https://tyt.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('upload.twitter.com','https://upload.twitter.com/robots.txt','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vector.im','https://vector.im/_matrix/identity/api/v1','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vesselthefilm.com','https://vesselthefilm.com/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('video-preview.s3.yandex.net','https://video-preview.s3.yandex.net/robots.txt','ZZ','SRCH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vortex-team.org','https://vortex-team.org/','ir','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('watech.ir','http://watech.ir/','ir','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('womenhelp.org','https://womenhelp.org/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.2zaar.net','http://www.2zaar.net/','ir','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.4chan.org','https://www.4chan.org/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.acaoeducativa.org.br','http://www.acaoeducativa.org.br/','br','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.alahrambeverages.com','http://www.alahrambeverages.com/age-gate/?ref=1088','eg','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.amazon.com','https://www.amazon.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.americannaziparty.com','https://www.americannaziparty.com/','ZZ','HATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.apple.com','http://www.apple.com/library/test/success.html','ZZ','CTRL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.asiafinest.com','http://www.asiafinest.com/','my','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.atheists.org','https://www.atheists.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.benarnews.org','https://www.benarnews.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.benedelman.org','https://www.benedelman.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.betternet.co','https://www.betternet.co/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bfm.ru','https://www.bfm.ru/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bicommunitynews.co.uk','http://www.bicommunitynews.co.uk/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.binance.com','https://www.binance.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blackhat.be','http://www.blackhat.be/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blackseanews.net','https://www.blackseanews.net/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blockchain.com','https://www.blockchain.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blogmarcosmontinely.com.br','http://www.blogmarcosmontinely.com.br/index.php/resultado-oficial-do-sorte-grande/78-noticia/4863-manifestacoes-pro-dilma-viram-ato-contra-michel-temer','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blogtalkradio.com','http://www.blogtalkradio.com/olavo','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bnaibrith.org','https://www.bnaibrith.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bonbast.com','https://www.bonbast.com/','ir','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasil247.com','https://www.brasil247.com/pt/247/poder/237195/Dilma-ao-247-A-primeira-tenta%C3%A7%C3%A3o-de-um-golpe-%C3%A9-calar-a-diverg%C3%AAncia.htm','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasildefato.com.br','http://www.brasildefato.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasilpost.com.br','http://www.brasilpost.com.br/2016/05/15/mulheres-contra-temer-manifestacao_n_9986034.html','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasilpost.com.br','http://www.brasilpost.com.br/rafael-rodrigues/michel-temer-e-os-retroce_b_10905444.html?utm_hp_ref=pais','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bravoporn.com','https://www.bravoporn.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.breakfastnopantheon.blogspot.com','http://www.breakfastnopantheon.blogspot.com/','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.breastcancerfoundation.org.my','https://www.breastcancerfoundation.org.my/','my','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brookings.edu','https://www.brookings.edu/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.btselem.org','https://www.btselem.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.business-humanrights.org','https://www.business-humanrights.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.carbonbrief.org','https://www.carbonbrief.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.casinocity.com','http://www.casinocity.com/','my','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cato.org','https://www.cato.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ccc.de','https://www.ccc.de/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cearaagora.com.br','http://www.cearaagora.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cedenpa.org.br','http://www.cedenpa.org.br/','br','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cfr.org','https://www.cfr.org/backgrounder/same-sex-marriage-global-comparisons','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.chantelle.com','http://www.chantelle.com/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.chinadaily.com.cn','https://www.chinadaily.com.cn/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.christianity.com','https://www.christianity.com/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.christianitytoday.com','https://www.christianitytoday.com/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cna.com.tw','http://www.cna.com.tw/','tw','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coinbase.com','https://www.coinbase.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coinex.com','https://www.coinex.com/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coquette.com','http://www.coquette.com/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cpj.org','https://www.cpj.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.crisisgroup.org','https://www.crisisgroup.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cseindia.org','https://www.cseindia.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.csmonitor.com','https://www.csmonitor.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.currenttime.tv','https://www.currenttime.tv/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cwcmf.org.tw','http://www.cwcmf.org.tw/joomla/','tw','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cyber-pirates.org','http://www.cyber-pirates.org/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dailymail.co.uk','http://www.dailymail.co.uk/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dailysabah.com','https://www.dailysabah.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ae','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','bh','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','dz','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','eg','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','iq','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ps','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','qa','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','sa','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','sd','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','tn','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ye','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.darknet.org.uk','https://www.darknet.org.uk/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.date.com','http://www.date.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.datemenow.com.tw','https://www.datemenow.com.tw/','tw','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dating.com','https://www.dating.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.debian.org','https://www.debian.org/','ZZ','MISC'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.delicates.co.uk','http://www.delicates.co.uk/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.democracynow.org','https://www.democracynow.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.destructoid.com','https://www.destructoid.com/','ir','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dextroverse.org','https://www.dextroverse.org/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.digitrain.ir','http://www.digitrain.ir/','ir','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dreamhost.com','https://www.dreamhost.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.drugsdata.org','https://www.drugsdata.org/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.drugsense.org','http://www.drugsense.org/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.earthrights.org','https://www.earthrights.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ebaumsworld.com','http://www.ebaumsworld.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ecequality.org','http://www.ecequality.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.echr.coe.int','https://www.echr.coe.int/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.economist.com','https://www.economist.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eea.europa.eu','https://www.eea.europa.eu/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eharmony.com','https://www.eharmony.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eln-voces.com','https://www.eln-voces.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eluniversal.com','https://www.eluniversal.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.episcopalrelief.org','https://www.episcopalrelief.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.erowid.org','https://www.erowid.org/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.europacasino.com','http://www.europacasino.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eveonline.com','https://www.eveonline.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ewg.org','https://www.ewg.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.exploit-db.com','https://www.exploit-db.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.f-list.net','https://www.f-list.net/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.facebook.com','https://www.facebook.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.feedtheminds.org','http://www.feedtheminds.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.figleaves.com','http://www.figleaves.com/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.freespeechcoalition.com','https://www.freespeechcoalition.com/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.freshpair.com','http://www.freshpair.com/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.frontlinedefenders.org','https://www.frontlinedefenders.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ft.com','https://www.ft.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ftchinese.com','http://www.ftchinese.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.fuckingfreemovies.com','http://www.fuckingfreemovies.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.fup.org.br','http://www.fup.org.br/ultimas-noticias/item/19499-precisamos-de-um-novo-o-petroleo-e-nosso-afirma-o-coordenador-da-fup','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gamespot.com','https://www.gamespot.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gay.com','http://www.gay.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gayegypt.com','http://www.gayegypt.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gayscape.com','http://www.gayscape.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gaystarnews.com','https://www.gaystarnews.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gazetadopovo.com.br','http://www.gazetadopovo.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gazetadopovo.com.br','https://www.gazetadopovo.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goarch.org','https://www.goarch.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goldenpalace.com','https://www.goldenpalace.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goldenrivieracasino.com','http://www.goldenrivieracasino.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gotgayporn.com','http://www.gotgayporn.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.granma.cu','http://www.granma.cu/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.grindr.com','https://www.grindr.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.guerrillagirls.com','https://www.guerrillagirls.com/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.guildwars.com','https://www.guildwars.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.haaretz.com','https://www.haaretz.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.haaretz.com','https://www.haaretz.com/life/five-films-you-have-to-see-at-tel-aviv-s-international-lgbt-film-festival-1.7338266','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.habbo.com','https://www.habbo.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hackforums.net','http://www.hackforums.net/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hackhull.com','http://www.hackhull.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hacktivismo.com','http://www.hacktivismo.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hanes.com','http://www.hanes.com/','ZZ','PROV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.harkatulmujahideen.org','http://www.harkatulmujahideen.org/','ZZ','MILX'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.helvetas.ch','https://www.helvetas.ch/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.herdict.org','https://www.herdict.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.heritage.org','http://www.heritage.org/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hidemyass.com','https://www.hidemyass.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hizb-ut-tahrir.org','http://www.hizb-ut-tahrir.org/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hotspotshield.com','https://www.hotspotshield.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrc.org','http://www.hrc.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrea.org','http://www.hrea.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrusa.org','http://www.hrusa.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrw.org','https://www.hrw.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.huffpost.com','https://www.huffpost.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.humanrightsconnected.org','https://www.humanrightsconnected.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.humanrightsfirst.org','http://www.humanrightsfirst.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hustler.com','http://www.hustler.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hyphanet.org','https://www.hyphanet.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iaffe.org','http://www.iaffe.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.icc-cpi.int','https://www.icc-cpi.int/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.icij.org','https://www.icij.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ictj.org','https://www.ictj.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idea.int','http://www.idea.int/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idrc.ca','https://www.idrc.ca/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idsoftware.com','http://www.idsoftware.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifad.org','https://www.ifad.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifc.org','https://www.ifc.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifex.org','http://www.ifex.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifge.org','http://www.ifge.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifj.org','http://www.ifj.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ihf-hr.org','http://www.ihf-hr.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ihrc.org','http://www.ihrc.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iias.asia','https://www.iias.asia/','mm','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iias.asia','https://www.iias.asia/','th','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ijm.org','http://www.ijm.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ilga-europe.org','https://www.ilga-europe.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ilo.org','http://www.ilo.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.indiatimes.com','http://www.indiatimes.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.inetprivacy.com','http://www.inetprivacy.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.infomigrants.net','https://www.infomigrants.net/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.infowars.com','https://www.infowars.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.instagram.com','https://www.instagram.com/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.internationalrivers.org','https://www.internationalrivers.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.inxporn.com','http://www.inxporn.com/','br','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipair.com','https://www.ipair.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipcc.ch','http://www.ipcc.ch/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipetitions.com','https://www.ipetitions.com/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipredator.se','https://www.ipredator.se/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.irna.ir','http://www.irna.ir/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.isiswomen.org','http://www.isiswomen.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.islamhoy.org','http://www.islamhoy.org/principal/Latinoamerica/peru.htm','pe','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.islaminstitut.de','https://www.islaminstitut.de/en/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ivpn.net','https://www.ivpn.net/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jesuitasperu.org','http://www.jesuitasperu.org/','pe','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jmarshall.com','http://www.jmarshall.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jmarshall.com','http://www.jmarshall.com/tools/cgiproxy/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.judaismconversion.org','http://www.judaismconversion.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.last.fm','http://www.last.fm/music/+free','br','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lesbiansubmission.com','http://www.lesbiansubmission.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.libgen.tw','https://www.libgen.tw/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lightmatterphotography.com','http://www.lightmatterphotography.com/','br','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.limetorrents.lol','https://www.limetorrents.lol/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lingo.com','http://www.lingo.com/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linguamatix.com','http://www.linguamatix.com/','my','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linkedin.com','http://www.linkedin.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linkedin.com','https://www.linkedin.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linphone.org','http://www.linphone.org/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.literotica.com','https://www.literotica.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lushstories.com','https://www.lushstories.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.marijuana.com','http://www.marijuana.com/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.marxists.org','https://www.marxists.org/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.meetme.com','https://www.meetme.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.meetup.com','https://www.meetup.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.metacafe.com','http://www.metacafe.com/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.metal-archives.com','http://www.metal-archives.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.microsoft.com','https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mnr.gov.cn','http://www.mnr.gov.cn/','ZZ','GOVT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mofosex.com','http://www.mofosex.com/','br','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mozilla.org','https://www.mozilla.org/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mp3.com','https://www.mp3.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.muhammadanism.com','http://www.muhammadanism.com/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.musixmatch.com','https://www.musixmatch.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.myxvids.com','http://www.myxvids.com/','br','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nato.int','http://www.nato.int/','ZZ','IGO'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.navy.mil','http://www.navy.mil/','ZZ','GOVT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nazi-lauck-nsdapao.com','http://www.nazi-lauck-nsdapao.com/','ZZ','HATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.no-ip.com','http://www.no-ip.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nsa.gov','https://www.nsa.gov/','ZZ','GOVT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.oic-oci.org','http://www.oic-oci.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.oicc.org','http://www.oicc.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.okcupid.com','https://www.okcupid.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.online-dating.org','http://www.online-dating.org/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.onlinedating.com','http://www.onlinedating.com/','ZZ','DATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.opec.org','http://www.opec.org/','ZZ','IGO'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.opioids.com','https://www.opioids.com/','ZZ','ALDR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.osce.org','http://www.osce.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ou.org','https://www.ou.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.out.com','https://www.out.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.overdrive.com','https://www.overdrive.com/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.owl.ru','http://www.owl.ru/','ru','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.owl.ru','http://www.owl.ru/','ua','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pacom.mil','http://www.pacom.mil/','ZZ','GOVT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.paganpride.org','http://www.paganpride.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pap.pl','https://www.pap.pl/','pl','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pap.pl','https://www.pap.pl/','ru','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.partypoker.com','http://www.partypoker.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.partypoker.net','https://www.partypoker.net/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pcusa.org','http://www.pcusa.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pdhre.org','http://www.pdhre.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.peacefire.org','http://www.peacefire.org/circumventor/simple-circumventor-instructions.html','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.people.com.cn','http://www.people.com.cn/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.peta.org','https://www.peta.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.phenoelit.org','http://www.phenoelit.org/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pinkcupid.com','https://www.pinkcupid.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pinknews.co.uk','https://www.pinknews.co.uk/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.piratenpartei.ch','https://www.piratenpartei.ch/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.piratpartiet.se','http://www.piratpartiet.se/','ZZ','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pixnet.net','https://www.pixnet.net/blog','tw','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.plancpills.org','https://www.plancpills.org/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.planetromeo.com','https://www.planetromeo.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.plannedparenthood.org','https://www.plannedparenthood.org/learn/sexual-orientation-gender','us','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pogo.com','http://www.pogo.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.poker.com','http://www.poker.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pokerstars.com','http://www.pokerstars.com/','ZZ','GMB'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.politico.com','https://www.politico.com/','us','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pornhub.com','http://www.pornhub.com/','ZZ','PORN'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pravda.ru','http://www.pravda.ru/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pridemedia.com','https://www.pridemedia.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.primeirapagina-to.com.br','http://www.primeirapagina-to.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacidade.digital','https://www.privacidade.digital/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacyinternational.org','https://www.privacyinternational.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacytools.io','https://www.privacytools.io/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privateinternetaccess.com','https://www.privateinternetaccess.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.qhtyzx.com','http://www.qhtyzx.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.quantico.marines.mil','http://www.quantico.marines.mil/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.queernet.org','http://www.queernet.org/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.queerty.com','https://www.queerty.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.radicalparty.org','http://www.radicalparty.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rambler.ru','https://www.rambler.ru/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rarbg.to','https://www.rarbg.to/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rbf.org','http://www.rbf.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.refugeesinternational.org','https://www.refugeesinternational.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.religioustolerance.org','http://www.religioustolerance.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.reprolegalhelpline.org','https://www.reprolegalhelpline.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.repubblica.com','http://www.repubblica.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rescue.org','https://www.rescue.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.reuters.com','https://www.reuters.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rfa.org','https://www.rfa.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rferl.org','https://www.rferl.org/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.riftgame.com','http://www.riftgame.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.roblox.com','https://www.roblox.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rockstargames.com','http://www.rockstargames.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.runescape.com','http://www.runescape.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.sbs.com.au','https://www.sbs.com.au/language/coronavirus?cid=infocus','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.schwarzreport.org','http://www.schwarzreport.org/','ZZ','CULTR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.scribd.com','https://www.scribd.com/','ZZ','FILE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.scruff.com','https://www.scruff.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.securityfocus.com','http://www.securityfocus.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.securitytracker.com','http://www.securitytracker.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.shinto.org','http://www.shinto.org/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.sida.se','http://www.sida.se/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.solicitorsfromhell.com','http://www.solicitorsfromhell.com/','ZZ','HATE'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.square-enix.com','http://www.square-enix.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.thegeekdiary.com','https://www.thegeekdiary.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.thehacktivist.com','http://www.thehacktivist.com/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tialsoft.com','http://www.tialsoft.com/multitranse/','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tops.co.th','https://www.tops.co.th/th/','th','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tot.co.th','https://www.tot.co.th/','th','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.trendsmap.com','https://www.trendsmap.com/','ZZ','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkhukuksitesi.com','http://www.turkhukuksitesi.com/kadin_haklari.php/','tr','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkiyegazetesi.com.tr','http://www.turkiyegazetesi.com.tr/','tr','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkiyeittifakipartisi.org.tr','https://www.turkiyeittifakipartisi.org.tr/','tr','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tusev.org.tr','https://www.tusev.org.tr/tr/','tr','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.uber.com','https://www.uber.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ubisoft.com','https://www.ubisoft.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ultimate-anonymity.com','http://www.ultimate-anonymity.com/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unescap.org','https://www.unescap.org/','my','IGO'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unescap.org','https://www.unescap.org/','th','IGO'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unfpa.org','http://www.unfpa.org/swp','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unlockme.co.uk','http://www.unlockme.co.uk/','ZZ','HACK'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.uproxy.org','https://www.uproxy.org/','ZZ','ANON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.urduvoa.com','https://www.urduvoa.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usaid.gov','https://www.usaid.gov/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usatoday.com','https://www.usatoday.com/story/news/nation/2017/10/19/cities-made-strides-2017-lgbt-rights-report-shows/777907001/','us','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usj.com.my','http://www.usj.com.my/','my','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vamosbien.com','https://www.vamosbien.com/','ve','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venelogia.com','https://www.venelogia.com/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venevision.com','http://www.venevision.com/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelaaldia.com','https://www.venezuelaaldia.com/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelaawareness.com','http://www.venezuelaawareness.com/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelasinlimites.org','https://www.venezuelasinlimites.org/','ve','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ventevenezuela.org','http://www.ventevenezuela.org/','ve','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vesti.ru','http://www.vesti.ru/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vesti.ru','https://www.vesti.ru/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viasatprovider.com','https://www.viasatprovider.com/','us','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vice.com','https://www.vice.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vidaagro.com.ve','http://www.vidaagro.com.ve/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viomundo.com.br','http://www.viomundo.com.br/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viomundo.com.br','http://www.viomundo.com.br/denuncias/pedro-serrano-decisao-do-mpf-mata-impeachment-na-questao-das-pedaladas-mesmo-sendo-politico-julgamento-de-dilma-nao-pode-atropelar-justica.html','br','POLR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voanews.com','http://www.voanews.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voanoticias.com','https://www.voanoticias.com/','ve','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voltdata.info','http://www.voltdata.info/','br','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.walmart.com','https://www.walmart.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.warchild.org','http://www.warchild.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.warhammeronline.com','http://www.warhammeronline.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.washingtonpost.com','https://www.washingtonpost.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.washingtontimes.com','http://www.washingtontimes.com/','ZZ','NEWS'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.watsons.co.th','https://www.watsons.co.th/','th','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webbox.com','http://www.webbox.com/index.php','ZZ','COMT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webmaster.or.th','https://www.webmaster.or.th/','th','MISC'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webportal.com.my','http://www.webportal.com.my/','my','GRP'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.weforum.org','https://www.weforum.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.well.com','http://www.well.com/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.well.com','http://www.well.com/user/queerjhd/','ZZ','LGBT'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.westernunion.com','https://www.westernunion.com/','ZZ','COMM'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wftucentral.org','http://www.wftucentral.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','mm','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','my','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','th','PUBH'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wikidata.org','https://www.wikidata.org/','ZZ','MISC'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wikimedia.org','https://www.wikimedia.org/','ZZ','MISC'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wilpf.org','https://www.wilpf.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.witness.org','https://www.witness.org/','ZZ','HUMR'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.worldbank.org','http://www.worldbank.org/','ZZ','ECON'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.worldwildlife.org','https://www.worldwildlife.org/','ZZ','ENV'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xbox.com','http://www.xbox.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xbox.com','https://www.xbox.com/','ZZ','GAME'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xenu.net','https://www.xenu.net/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ymca.int','http://www.ymca.int/','ZZ','REL'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.yola.com','https://www.yola.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.youtube.com','https://www.youtube.com/','ZZ','MMED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zanzu.nl','https://www.zanzu.nl/','ZZ','XED'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zenhub.com','https://www.zenhub.com/','ru','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zoomshare.com','https://www.zoomshare.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('xda-developers.com','https://xda-developers.com/','ZZ','HOST'); +INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('xe.com','https://xe.com/','ZZ','COMM'); diff --git a/ooniapi/services/oonimeasurements/tests/fixtures/2_fastpath_fixtures.sql b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql similarity index 92% rename from ooniapi/services/oonimeasurements/tests/fixtures/2_fastpath_fixtures.sql rename to ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql index 5e6d6368..b15b86e7 100644 --- a/ooniapi/services/oonimeasurements/tests/fixtures/2_fastpath_fixtures.sql +++ b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_fastpath.sql @@ -1,5 +1,4 @@ - -INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000002.422281_TW_webconnectivity_55232ce980f563d5','20231231T221448Z_webconnectivity_TW_18182_n1_zBswL4awQc3qUCnU','https://vesselthefilm.com/','TW','18182','web_connectivity','2023-12-31 22:14:47','2024-01-01 00:00:01','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"fingerprints":[{"name":"cp.fp_x_redirect_just","scope":"fp","location_found":"body","confidence_no_fp":5,"expected_countries":[]}]}','linux','f','f','f','vesselthefilm.com','ooniprobe-cli','3.20.0','','0','0','0','0','','0','','0.4.3','amd64','ooniprobe-engine','3.20.0','3.259042','','https://1.th.ooni.org','https','\N'); +-- INSERT INTOstpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000002.422281_TW_webconnectivity_55232ce980f563d5','20231231T221448Z_webconnectivity_TW_18182_n1_zBswL4awQc3qUCnU','https://vesselthefilm.com/','TW','18182','web_connectivity','2023-12-31 22:14:47','2024-01-01 00:00:01','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"fingerprints":[{"name":"cp.fp_x_redirect_just","scope":"fp","location_found":"body","confidence_no_fp":5,"expected_countries":[]}]}','linux','f','f','f','vesselthefilm.com','ooniprobe-cli','3.20.0','','0','0','0','0','','0','','0.4.3','amd64','ooniprobe-engine','3.20.0','3.259042','','https://1.th.ooni.org','https','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000001.568391_SE_dnscheck_2c4c15529c6e2e15','20231231T232727Z_dnscheck_SE_44034_n1_YS3hnP0H0QoU4c7X','dot://dns.switch.ch/dns-query','SE','44034','dnscheck','2023-12-31 23:27:28','2024-01-01 00:00:01','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','dns.switch.ch','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','15.037774','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000002.759261_US_torsf_cdcd0913c665a955','20231231T234808Z_torsf_US_20001_n1_pxzHhtg9PKsm9FsG','','US','20001','torsf','2023-12-31 23:48:09','2024-01-01 00:00:01','','{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','t','f','f','','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.5.0','arm64','ooniprobe-engine','3.19.1','600.0654','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000005.268959_BR_webconnectivity_ae21d56f6bbe9576','20231231T235005Z_webconnectivity_BR_262907_n1_oBk3JJKE37H53Dfw','http://www.pravda.ru/','BR','262907','web_connectivity','2023-12-31 23:50:02','2024-01-01 00:00:01','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}','android','f','f','f','www.pravda.ru','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.4.3','arm64','ooniprobe-engine','3.19.1','0.99705654','','https://2.th.ooni.org','https','\N'); @@ -999,633 +998,3 @@ INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_na INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000059.576253_IT_dnscheck_4f526339965409f2','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://9.9.9.9:853/','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','9.9.9.9:853','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.19385019','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000059.817152_IT_dnscheck_3d7b742209d4f326','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://dns.quad9.net/','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','dns.quad9.net','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.18705674','','','','\N'); INSERT INTO fastpath (measurement_uid,report_id,input,probe_cc,probe_asn,test_name,test_start_time,measurement_start_time,filename,scores,platform,anomaly,confirmed,msm_failure,domain,software_name,software_version,control_failure,blocking_general,is_ssl_expected,page_len,page_len_ratio,server_cc,server_asn,server_as_name,test_version,architecture,engine_name,engine_version,test_runtime,blocking_type,test_helper_address,test_helper_type,ooni_run_link_id) VALUES ('20240101000100.128817_IT_dnscheck_d544bc263686846c','20240101T000056Z_dnscheck_IT_3269_n1_GsBrOjKnxGQSIRbo','dot://family.cloudflare-dns.com/dns-query','IT','3269','dnscheck','2024-01-01 00:00:59','2024-01-01 00:01:02','','{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}','android','f','f','t','family.cloudflare-dns.com','ooniprobe-android-unattended','3.8.5.1','','0','0','0','0','','0','','0.9.2','arm64','ooniprobe-engine','3.19.1','0.09800492','','','','\N'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','by','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','kz','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','tm','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('15minut.org','https://15minut.org/','uz','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('200rf.com','https://200rf.com/','ru','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('24tv.ua','https://24tv.ua/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('4genderjustice.org','https://4genderjustice.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('666games.net','http://666games.net/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('6ott-mit-uns.livejournal.com','https://6ott-mit-uns.livejournal.com/','ru','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('76crimes.com','https://76crimes.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('abpr2.railfan.net','http://abpr2.railfan.net/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('abs.twimg.com','https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('addons.mozilla.org','https://addons.mozilla.org/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('adium.im','https://adium.im/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('adultfriendfinder.com','https://adultfriendfinder.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('advox.globalvoices.org','https://advox.globalvoices.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('alt.com','https://alt.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('api.protonvpn.ch','https://api.protonvpn.ch/tests/ping','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('arc-international.net','https://arc-international.net/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('archive.is','https://archive.is/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('archive.org','https://archive.org/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('asiatimes.com','https://asiatimes.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('assets.gitlab-static.net','https://assets.gitlab-static.net/robots.txt','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('atlasofsurveillance.org','https://atlasofsurveillance.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('avishanx.com','https://avishanx.com/','ir','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bahai-library.com','https://bahai-library.com/','ir','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','ch','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','lt','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','th','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('bet365.com','http://bet365.com/','tz','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('blogsome.com','http://blogsome.com/','my','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('caranovanocongresso.blogs.sapo.pt','http://caranovanocongresso.blogs.sapo.pt/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('caranovanocongresso.blogspot.com','http://caranovanocongresso.blogspot.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carcari.blogspot.com','http://carcari.blogspot.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carcoucari.blogspot.com','http://carcoucari.blogspot.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cargadecavalaria.blogspot.com','http://cargadecavalaria.blogspot.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carineroos.wordpress.com','http://carineroos.wordpress.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carlosbaia.blogspot.com.br','http://carlosbaia.blogspot.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carnegieendowment.org','https://carnegieendowment.org/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('carolinabrauer.wordpress.com','http://carolinabrauer.wordpress.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cdn.fbsbx.com','https://cdn.fbsbx.com/favicon.ico','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cf-ipfs.com','https://cf-ipfs.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('chat-gpt.org','https://chat-gpt.org/','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('chat.openai.com','https://chat.openai.com/','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cid.contact','https://cid.contact/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cinemata.org','https://cinemata.org/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('circumcision.org','https://circumcision.org/','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('citizenlab.ca','https://citizenlab.ca/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cloudflare-ipfs.com','https://cloudflare-ipfs.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cocaine.org','https://cocaine.org/','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('coronavirus.app','https://coronavirus.app/','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('coronavirus.jhu.edu','https://coronavirus.jhu.edu/map.html','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('covidtracking.com','https://covidtracking.com/','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('creativecommons.org','https://creativecommons.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cryptome.org','https://cryptome.org/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cryptpad.fr','https://cryptpad.fr/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cwtch.im','https://cwtch.im/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('cyber.harvard.edu','https://cyber.harvard.edu/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dailystormer.name','https://dailystormer.name/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dartcenter.org','https://dartcenter.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ddosecrets.com','https://ddosecrets.com/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.indymedia.org','https://de.indymedia.org/','de','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','at','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','ch','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','de','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('de.rt.com','https://de.rt.com/','fr','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('delta.chat','https://delta.chat/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('disclose.ngo','https://disclose.ngo/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discord.com','https://discord.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discord.gg','http://discord.gg/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('discordapp.com','https://discordapp.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('disqus.com','https://disqus.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dit.whatsapp.net','https://dit.whatsapp.net/robots.txt','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dl.acm.org','https://dl.acm.org/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dn42.dev','https://dn42.dev/Home','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('doubleclick.net','https://doubleclick.net/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('download.cnet.com','https://download.cnet.com/Freegate/3000-2085_4-10415391.html','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('duckduckgo.com','https://duckduckgo.com/','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('dweb.link','https://dweb.link/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('edge-chat.instagram.com','https://edge-chat.instagram.com/robots.txt','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('element.io','https://element.io/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('external.xx.fbcdn.net','https://external.xx.fbcdn.net/robots.txt','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('firebaseinstallations.googleapis.com','https://firebaseinstallations.googleapis.com/robots.txt','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('freedns.afraid.org','https://freedns.afraid.org/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('fteproxy.org','http://fteproxy.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gaymenshealth.org','https://gaymenshealth.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gaytoday.com','http://gaytoday.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('genderandaids.unwomen.org','https://genderandaids.unwomen.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ghost.org','https://ghost.org/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('git.io','https://git.io/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.com','https://github.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.com','https://github.com/twbs/bootstrap/releases/download/v4.5.3/bootstrap-4.5.3-examples.zip','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('github.githubassets.com','https://github.githubassets.com/images/modules/open_graph/github-logo.png','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gitlab.com','https://gitlab.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('goaskalice.columbia.edu','https://goaskalice.columbia.edu/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gospelhomeblog.blogspot.com.br','http://gospelhomeblog.blogspot.com.br/','br','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gostodesermulher.com','http://gostodesermulher.com/search/como-provocar-um-aborto-caseiro-as-16-semanas/','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('greenhost.net','https://greenhost.net/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('gynuity.org','http://gynuity.org/downloads/polbrf_misoprostol_selfguide_po.pdf','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('health4beauty.net','http://health4beauty.net/cytotec-misoprostol.html','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('howtouseabortionpill.org','http://howtouseabortionpill.org/','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('i.ytimg.com','https://i.ytimg.com/vi/JBRf3nEqfZ8/hq720.jpg','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('iarf.net','https://iarf.net/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icao.maps.arcgis.com','https://icao.maps.arcgis.com/apps/opsdashboard/index.html#/977dde48256b489fb48fa98e724721e8','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icao.maps.arcgis.com','https://icao.maps.arcgis.com/apps/webappviewer3d/index.html?id=d9d3f8fa9a23425c8f0889baab626186','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('icq.com','https://icq.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ifacetimeapp.com','https://ifacetimeapp.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('im0-tub-com.yandex.net','https://im0-tub-com.yandex.net/i?id=462f375c96139f1e41d919b44be2d780&n=13&exp=1','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('impactlab.org','https://impactlab.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('indymedia.org','https://indymedia.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('instagram.fsaw1-13.fna.fbcdn.net','https://instagram.fsaw1-13.fna.fbcdn.net/favicon.ico','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('instinctmagazine.com','http://instinctmagazine.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('internationalfamilyequalityday.org','https://internationalfamilyequalityday.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ipi.media','https://ipi.media/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('itgetsbetter.org','https://itgetsbetter.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('itgirls.com.br','http://itgirls.com.br/','br','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('iyfglobal.org','https://iyfglobal.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ja.wikipedia.org','https://ja.wikipedia.org/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('jezebel.com','https://jezebel.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('juliapetit.com.br','http://juliapetit.com.br/category/home/','br','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kaleidoscopetrust.com','https://kaleidoscopetrust.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kamikadze-d.net','https://kamikadze-d.net/','ru','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kassimahmad.blogspot.com','https://kassimahmad.blogspot.com/','my','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kat.am','https://kat.am/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('katcr.to','https://katcr.to/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('keadilanrakyat.org','https://keadilanrakyat.org/','my','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kedahlaniie.blogspot.com','https://kedahlaniie.blogspot.com/','my','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('khilafah.net','http://khilafah.net/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('khms0.google.com','https://khms0.google.com/kh/v=908?x=81&y=103&z=8','ZZ','CTRL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kickasstorrents.to','https://kickasstorrents.to/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kiwifarms.net','https://kiwifarms.net/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kolektiva.social','https://kolektiva.social/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('komas.org','https://komas.org/','my','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kremlin.ru','http://kremlin.ru/','ZZ','GOVT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('krishna.com','https://krishna.com/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('kryss.network','https://kryss.network/','my','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ktemoc.blogspot.com','https://ktemoc.blogspot.com/','my','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('laborrights.org','https://laborrights.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lalgbtcenter.org','https://lalgbtcenter.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lambdalegal.org','http://lambdalegal.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('leap.se','https://leap.se/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbt.foundation','https://lgbt.foundation/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtmap.org','http://lgbtmap.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtnewsnow.org','https://lgbtnewsnow.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lgbtvacationplanners.com','https://lgbtvacationplanners.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lh3.googleusercontent.com','https://lh3.googleusercontent.com/favicon.ico','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.fun','https://libgen.fun/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.gs','http://libgen.gs/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.is','https://libgen.is/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.lc','http://libgen.lc/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.li','http://libgen.li/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.life','https://libgen.life/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.me','https://libgen.me/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.rs','http://libgen.rs/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('libgen.space','https://libgen.space/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('liveuamap.com','https://liveuamap.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('locals.com','https://locals.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('lokinet.org','https://lokinet.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.ru','https://mail.ru/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.yahoo.com','https://mail.yahoo.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mail.yandex.ru','https://mail.yandex.ru/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('makeamazonpay.com','https://makeamazonpay.com/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('maps.gstatic.com','https://maps.gstatic.com/tactile/basepage/grid-image.png','ZZ','CTRL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('marcelo.marques.zip.net','http://marcelo.marques.zip.net/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mart.ibon.com.tw','https://mart.ibon.com.tw/mart/','tw','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask-api.icloud.com','https://mask-api.icloud.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask-h2.icloud.com','https://mask-h2.icloud.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mask.icloud.com','https://mask.icloud.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('massbrowser.cs.umass.edu','https://massbrowser.cs.umass.edu/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mastodon.xyz','https://mastodon.xyz/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix-client.matrix.org','https://matrix-client.matrix.org/_matrix/client/versions','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix.org','https://matrix.org/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('matrix.to','https://matrix.to/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mattermost.com','https://mattermost.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media-arn2-1.cdn.whatsapp.net','https://media-arn2-1.cdn.whatsapp.net/robots.txt','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media.fevn1-1.fna.whatsapp.net','https://media.fevn1-1.fna.whatsapp.net/robots.txt','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media0.giphy.com','https://media0.giphy.com/robots.txt','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('media3.giphy.com','https://media3.giphy.com/robots.txt','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('medium.com','https://medium.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('meet.jit.si','https://meet.jit.si/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('messages.google.com','https://messages.google.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mmg.whatsapp.net','https://mmg.whatsapp.net/robots.txt','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mobile.element.io','https://mobile.element.io/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('movabletype.com','https://movabletype.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mozilla.cloudflare-dns.com','https://mozilla.cloudflare-dns.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mulheresmarias.blogspot.com','http://mulheresmarias.blogspot.com/2004/09/como-fazer-um-aborto.html','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('multimedia.scmp.com','https://multimedia.scmp.com/infographics/news/china/article/3047038/wuhan-virus/index.html?src=article-launcher','ZZ','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mural.codigofonte.net','http://mural.codigofonte.net/exibir.php?id=41767','br','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('myabortionstory.tumblr.com','https://myabortionstory.tumblr.com/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('mytrans.com.tw','https://mytrans.com.tw/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('networkcheck.kde.org','http://networkcheck.kde.org/','ZZ','CTRL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ozgurgelecek17.net','https://ozgurgelecek17.net/','tr','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('ozguruz20.org','https://ozguruz20.org/','tr','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('plus.im','https://plus.im/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('portal.odp.org.tr','http://portal.odp.org.tr/','tr','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('preview.redd.it','https://preview.redd.it/robots.txt','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('prism-break.org','https://prism-break.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('riseup.net','https://riseup.net/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('risk-inform.ru','https://risk-inform.ru/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('romb.tv','https://romb.tv/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('rosdep.org','https://rosdep.org/','ru','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('rsf.org','https://rsf.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('s.pinimg.com','https://s.pinimg.com/robots.txt','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sandbox.cryptpad.info','https://sandbox.cryptpad.info/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('scontent-vie1-1.cdninstagram.com','https://scontent-vie1-1.cdninstagram.com/favicon.ico','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secfirst.org','https://secfirst.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secondlife.com','https://secondlife.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('secure.avaaz.org','https://secure.avaaz.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('securevpn.com','https://securevpn.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('securevpn.im','https://securevpn.im/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('seed4.me','https://seed4.me/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('semnasem.org','https://semnasem.org/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('severrea.github.io','https://severrea.github.io/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('severrea.gitlab.io','https://severrea.gitlab.io/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shadowsocks.org','https://shadowsocks.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shereilam.persianblog.ir','https://shereilam.persianblog.ir/','iq','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('shtab.navalny.com','https://shtab.navalny.com/','ru','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sibrea.github.io','https://sibrea.github.io/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sibrea.gitlab.io','https://sibrea.gitlab.io/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('signal.org','https://signal.org/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('simple.wikipedia.org','https://simple.wikipedia.org/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('skat.media','https://skat.media/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('skiff.com','https://skiff.com/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('slashdot.org','https://slashdot.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('slate.com','https://slate.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('smex.org','http://smex.org/','iq','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('smotrim.ru','http://smotrim.ru/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sobesednik.ru','https://sobesednik.ru/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sota.vision','https://sota.vision/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sotkurdistan.net','http://sotkurdistan.net/','iq','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('soundcloud.com','https://soundcloud.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('sputniknews.com','https://sputniknews.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('spys.one','https://spys.one/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('stealthchat.com','http://stealthchat.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('stolitomsonvpn.com','https://stolitomsonvpn.com/','ru','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('substack.com','https://substack.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('t.me','https://t.me/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tamtam.chat','https://tamtam.chat/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tdov.org','http://tdov.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('te-st.org','https://te-st.org/','ru','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('telegram.me','https://telegram.me/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('test-ipv6.com','https://test-ipv6.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('th.bing.com','https://th.bing.com/th/id/OIP.YnpdokcN_OQYn3n41AlqZAHaEe?w=250&h=180&c=7&o=5&pid=1.7','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('theintercept.com','https://theintercept.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('timcast.com','https://timcast.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('timesofindia.indiatimes.com','https://timesofindia.indiatimes.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transequality.org','https://transequality.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transparencyreport.google.com','https://transparencyreport.google.com/','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('transsexual.org','http://transsexual.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('triller.co','https://triller.co/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tse1.explicit.bing.net','https://tse1.explicit.bing.net/robots.txt','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/Ahmed_Mansoor/','ae','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/UAE_Council/','ae','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/gamaleid/','ae','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/gamaleid/','qa','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/ghonim/','ae','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/ghonim/','qa','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('twitter.com','https://twitter.com/uae_detainees/','ae','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('tyt.com','https://tyt.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('upload.twitter.com','https://upload.twitter.com/robots.txt','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vector.im','https://vector.im/_matrix/identity/api/v1','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vesselthefilm.com','https://vesselthefilm.com/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('video-preview.s3.yandex.net','https://video-preview.s3.yandex.net/robots.txt','ZZ','SRCH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('vortex-team.org','https://vortex-team.org/','ir','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('watech.ir','http://watech.ir/','ir','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('womenhelp.org','https://womenhelp.org/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.2zaar.net','http://www.2zaar.net/','ir','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.4chan.org','https://www.4chan.org/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.acaoeducativa.org.br','http://www.acaoeducativa.org.br/','br','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.alahrambeverages.com','http://www.alahrambeverages.com/age-gate/?ref=1088','eg','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.amazon.com','https://www.amazon.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.americannaziparty.com','https://www.americannaziparty.com/','ZZ','HATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.apple.com','http://www.apple.com/library/test/success.html','ZZ','CTRL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.asiafinest.com','http://www.asiafinest.com/','my','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.atheists.org','https://www.atheists.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.benarnews.org','https://www.benarnews.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.benedelman.org','https://www.benedelman.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.betternet.co','https://www.betternet.co/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bfm.ru','https://www.bfm.ru/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bicommunitynews.co.uk','http://www.bicommunitynews.co.uk/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.binance.com','https://www.binance.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blackhat.be','http://www.blackhat.be/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blackseanews.net','https://www.blackseanews.net/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blockchain.com','https://www.blockchain.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blogmarcosmontinely.com.br','http://www.blogmarcosmontinely.com.br/index.php/resultado-oficial-do-sorte-grande/78-noticia/4863-manifestacoes-pro-dilma-viram-ato-contra-michel-temer','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.blogtalkradio.com','http://www.blogtalkradio.com/olavo','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bnaibrith.org','https://www.bnaibrith.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bonbast.com','https://www.bonbast.com/','ir','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasil247.com','https://www.brasil247.com/pt/247/poder/237195/Dilma-ao-247-A-primeira-tenta%C3%A7%C3%A3o-de-um-golpe-%C3%A9-calar-a-diverg%C3%AAncia.htm','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasildefato.com.br','http://www.brasildefato.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasilpost.com.br','http://www.brasilpost.com.br/2016/05/15/mulheres-contra-temer-manifestacao_n_9986034.html','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brasilpost.com.br','http://www.brasilpost.com.br/rafael-rodrigues/michel-temer-e-os-retroce_b_10905444.html?utm_hp_ref=pais','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.bravoporn.com','https://www.bravoporn.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.breakfastnopantheon.blogspot.com','http://www.breakfastnopantheon.blogspot.com/','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.breastcancerfoundation.org.my','https://www.breastcancerfoundation.org.my/','my','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.brookings.edu','https://www.brookings.edu/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.btselem.org','https://www.btselem.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.business-humanrights.org','https://www.business-humanrights.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.carbonbrief.org','https://www.carbonbrief.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.casinocity.com','http://www.casinocity.com/','my','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cato.org','https://www.cato.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ccc.de','https://www.ccc.de/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cearaagora.com.br','http://www.cearaagora.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cedenpa.org.br','http://www.cedenpa.org.br/','br','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cfr.org','https://www.cfr.org/backgrounder/same-sex-marriage-global-comparisons','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.chantelle.com','http://www.chantelle.com/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.chinadaily.com.cn','https://www.chinadaily.com.cn/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.christianity.com','https://www.christianity.com/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.christianitytoday.com','https://www.christianitytoday.com/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cna.com.tw','http://www.cna.com.tw/','tw','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coinbase.com','https://www.coinbase.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coinex.com','https://www.coinex.com/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.coquette.com','http://www.coquette.com/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cpj.org','https://www.cpj.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.crisisgroup.org','https://www.crisisgroup.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cseindia.org','https://www.cseindia.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.csmonitor.com','https://www.csmonitor.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.currenttime.tv','https://www.currenttime.tv/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cwcmf.org.tw','http://www.cwcmf.org.tw/joomla/','tw','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.cyber-pirates.org','http://www.cyber-pirates.org/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dailymail.co.uk','http://www.dailymail.co.uk/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dailysabah.com','https://www.dailysabah.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ae','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','bh','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','dz','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','eg','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','iq','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ps','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','qa','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','sa','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','sd','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','tn','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.daloah.com','https://www.daloah.com/','ye','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.darknet.org.uk','https://www.darknet.org.uk/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.date.com','http://www.date.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.datemenow.com.tw','https://www.datemenow.com.tw/','tw','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dating.com','https://www.dating.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.debian.org','https://www.debian.org/','ZZ','MISC'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.delicates.co.uk','http://www.delicates.co.uk/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.democracynow.org','https://www.democracynow.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.destructoid.com','https://www.destructoid.com/','ir','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dextroverse.org','https://www.dextroverse.org/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.digitrain.ir','http://www.digitrain.ir/','ir','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.dreamhost.com','https://www.dreamhost.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.drugsdata.org','https://www.drugsdata.org/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.drugsense.org','http://www.drugsense.org/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.earthrights.org','https://www.earthrights.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ebaumsworld.com','http://www.ebaumsworld.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ecequality.org','http://www.ecequality.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.echr.coe.int','https://www.echr.coe.int/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.economist.com','https://www.economist.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eea.europa.eu','https://www.eea.europa.eu/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eharmony.com','https://www.eharmony.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eln-voces.com','https://www.eln-voces.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eluniversal.com','https://www.eluniversal.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.episcopalrelief.org','https://www.episcopalrelief.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.erowid.org','https://www.erowid.org/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.europacasino.com','http://www.europacasino.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.eveonline.com','https://www.eveonline.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ewg.org','https://www.ewg.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.exploit-db.com','https://www.exploit-db.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.f-list.net','https://www.f-list.net/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.facebook.com','https://www.facebook.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.feedtheminds.org','http://www.feedtheminds.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.figleaves.com','http://www.figleaves.com/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.freespeechcoalition.com','https://www.freespeechcoalition.com/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.freshpair.com','http://www.freshpair.com/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.frontlinedefenders.org','https://www.frontlinedefenders.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ft.com','https://www.ft.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ftchinese.com','http://www.ftchinese.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.fuckingfreemovies.com','http://www.fuckingfreemovies.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.fup.org.br','http://www.fup.org.br/ultimas-noticias/item/19499-precisamos-de-um-novo-o-petroleo-e-nosso-afirma-o-coordenador-da-fup','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gamespot.com','https://www.gamespot.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gay.com','http://www.gay.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gayegypt.com','http://www.gayegypt.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gayscape.com','http://www.gayscape.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gaystarnews.com','https://www.gaystarnews.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gazetadopovo.com.br','http://www.gazetadopovo.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gazetadopovo.com.br','https://www.gazetadopovo.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goarch.org','https://www.goarch.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goldenpalace.com','https://www.goldenpalace.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.goldenrivieracasino.com','http://www.goldenrivieracasino.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.gotgayporn.com','http://www.gotgayporn.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.granma.cu','http://www.granma.cu/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.grindr.com','https://www.grindr.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.guerrillagirls.com','https://www.guerrillagirls.com/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.guildwars.com','https://www.guildwars.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.haaretz.com','https://www.haaretz.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.haaretz.com','https://www.haaretz.com/life/five-films-you-have-to-see-at-tel-aviv-s-international-lgbt-film-festival-1.7338266','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.habbo.com','https://www.habbo.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hackforums.net','http://www.hackforums.net/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hackhull.com','http://www.hackhull.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hacktivismo.com','http://www.hacktivismo.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hanes.com','http://www.hanes.com/','ZZ','PROV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.harkatulmujahideen.org','http://www.harkatulmujahideen.org/','ZZ','MILX'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.helvetas.ch','https://www.helvetas.ch/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.herdict.org','https://www.herdict.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.heritage.org','http://www.heritage.org/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hidemyass.com','https://www.hidemyass.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hizb-ut-tahrir.org','http://www.hizb-ut-tahrir.org/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hotspotshield.com','https://www.hotspotshield.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrc.org','http://www.hrc.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrea.org','http://www.hrea.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrusa.org','http://www.hrusa.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hrw.org','https://www.hrw.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.huffpost.com','https://www.huffpost.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.humanrightsconnected.org','https://www.humanrightsconnected.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.humanrightsfirst.org','http://www.humanrightsfirst.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hustler.com','http://www.hustler.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.hyphanet.org','https://www.hyphanet.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iaffe.org','http://www.iaffe.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.icc-cpi.int','https://www.icc-cpi.int/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.icij.org','https://www.icij.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ictj.org','https://www.ictj.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idea.int','http://www.idea.int/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idrc.ca','https://www.idrc.ca/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.idsoftware.com','http://www.idsoftware.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifad.org','https://www.ifad.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifc.org','https://www.ifc.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifex.org','http://www.ifex.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifge.org','http://www.ifge.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ifj.org','http://www.ifj.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ihf-hr.org','http://www.ihf-hr.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ihrc.org','http://www.ihrc.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iias.asia','https://www.iias.asia/','mm','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.iias.asia','https://www.iias.asia/','th','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ijm.org','http://www.ijm.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ilga-europe.org','https://www.ilga-europe.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ilo.org','http://www.ilo.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.indiatimes.com','http://www.indiatimes.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.inetprivacy.com','http://www.inetprivacy.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.infomigrants.net','https://www.infomigrants.net/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.infowars.com','https://www.infowars.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.instagram.com','https://www.instagram.com/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.internationalrivers.org','https://www.internationalrivers.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.inxporn.com','http://www.inxporn.com/','br','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipair.com','https://www.ipair.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipcc.ch','http://www.ipcc.ch/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipetitions.com','https://www.ipetitions.com/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ipredator.se','https://www.ipredator.se/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.irna.ir','http://www.irna.ir/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.isiswomen.org','http://www.isiswomen.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.islamhoy.org','http://www.islamhoy.org/principal/Latinoamerica/peru.htm','pe','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.islaminstitut.de','https://www.islaminstitut.de/en/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ivpn.net','https://www.ivpn.net/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jesuitasperu.org','http://www.jesuitasperu.org/','pe','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jmarshall.com','http://www.jmarshall.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.jmarshall.com','http://www.jmarshall.com/tools/cgiproxy/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.judaismconversion.org','http://www.judaismconversion.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.last.fm','http://www.last.fm/music/+free','br','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lesbiansubmission.com','http://www.lesbiansubmission.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.libgen.tw','https://www.libgen.tw/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lightmatterphotography.com','http://www.lightmatterphotography.com/','br','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.limetorrents.lol','https://www.limetorrents.lol/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lingo.com','http://www.lingo.com/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linguamatix.com','http://www.linguamatix.com/','my','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linkedin.com','http://www.linkedin.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linkedin.com','https://www.linkedin.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.linphone.org','http://www.linphone.org/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.literotica.com','https://www.literotica.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.lushstories.com','https://www.lushstories.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.marijuana.com','http://www.marijuana.com/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.marxists.org','https://www.marxists.org/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.meetme.com','https://www.meetme.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.meetup.com','https://www.meetup.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.metacafe.com','http://www.metacafe.com/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.metal-archives.com','http://www.metal-archives.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.microsoft.com','https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mnr.gov.cn','http://www.mnr.gov.cn/','ZZ','GOVT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mofosex.com','http://www.mofosex.com/','br','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mozilla.org','https://www.mozilla.org/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.mp3.com','https://www.mp3.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.muhammadanism.com','http://www.muhammadanism.com/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.musixmatch.com','https://www.musixmatch.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.myxvids.com','http://www.myxvids.com/','br','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nato.int','http://www.nato.int/','ZZ','IGO'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.navy.mil','http://www.navy.mil/','ZZ','GOVT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nazi-lauck-nsdapao.com','http://www.nazi-lauck-nsdapao.com/','ZZ','HATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.no-ip.com','http://www.no-ip.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.nsa.gov','https://www.nsa.gov/','ZZ','GOVT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.oic-oci.org','http://www.oic-oci.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.oicc.org','http://www.oicc.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.okcupid.com','https://www.okcupid.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.online-dating.org','http://www.online-dating.org/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.onlinedating.com','http://www.onlinedating.com/','ZZ','DATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.opec.org','http://www.opec.org/','ZZ','IGO'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.opioids.com','https://www.opioids.com/','ZZ','ALDR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.osce.org','http://www.osce.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ou.org','https://www.ou.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.out.com','https://www.out.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.overdrive.com','https://www.overdrive.com/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.owl.ru','http://www.owl.ru/','ru','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.owl.ru','http://www.owl.ru/','ua','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pacom.mil','http://www.pacom.mil/','ZZ','GOVT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.paganpride.org','http://www.paganpride.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pap.pl','https://www.pap.pl/','pl','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pap.pl','https://www.pap.pl/','ru','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.partypoker.com','http://www.partypoker.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.partypoker.net','https://www.partypoker.net/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pcusa.org','http://www.pcusa.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pdhre.org','http://www.pdhre.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.peacefire.org','http://www.peacefire.org/circumventor/simple-circumventor-instructions.html','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.people.com.cn','http://www.people.com.cn/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.peta.org','https://www.peta.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.phenoelit.org','http://www.phenoelit.org/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pinkcupid.com','https://www.pinkcupid.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pinknews.co.uk','https://www.pinknews.co.uk/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.piratenpartei.ch','https://www.piratenpartei.ch/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.piratpartiet.se','http://www.piratpartiet.se/','ZZ','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pixnet.net','https://www.pixnet.net/blog','tw','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.plancpills.org','https://www.plancpills.org/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.planetromeo.com','https://www.planetromeo.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.plannedparenthood.org','https://www.plannedparenthood.org/learn/sexual-orientation-gender','us','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pogo.com','http://www.pogo.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.poker.com','http://www.poker.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pokerstars.com','http://www.pokerstars.com/','ZZ','GMB'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.politico.com','https://www.politico.com/','us','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pornhub.com','http://www.pornhub.com/','ZZ','PORN'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pravda.ru','http://www.pravda.ru/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.pridemedia.com','https://www.pridemedia.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.primeirapagina-to.com.br','http://www.primeirapagina-to.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacidade.digital','https://www.privacidade.digital/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacyinternational.org','https://www.privacyinternational.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privacytools.io','https://www.privacytools.io/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.privateinternetaccess.com','https://www.privateinternetaccess.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.qhtyzx.com','http://www.qhtyzx.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.quantico.marines.mil','http://www.quantico.marines.mil/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.queernet.org','http://www.queernet.org/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.queerty.com','https://www.queerty.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.radicalparty.org','http://www.radicalparty.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rambler.ru','https://www.rambler.ru/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rarbg.to','https://www.rarbg.to/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rbf.org','http://www.rbf.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.refugeesinternational.org','https://www.refugeesinternational.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.religioustolerance.org','http://www.religioustolerance.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.reprolegalhelpline.org','https://www.reprolegalhelpline.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.repubblica.com','http://www.repubblica.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rescue.org','https://www.rescue.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.reuters.com','https://www.reuters.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rfa.org','https://www.rfa.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rferl.org','https://www.rferl.org/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.riftgame.com','http://www.riftgame.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.roblox.com','https://www.roblox.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.rockstargames.com','http://www.rockstargames.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.runescape.com','http://www.runescape.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.sbs.com.au','https://www.sbs.com.au/language/coronavirus?cid=infocus','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.schwarzreport.org','http://www.schwarzreport.org/','ZZ','CULTR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.scribd.com','https://www.scribd.com/','ZZ','FILE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.scruff.com','https://www.scruff.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.securityfocus.com','http://www.securityfocus.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.securitytracker.com','http://www.securitytracker.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.shinto.org','http://www.shinto.org/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.sida.se','http://www.sida.se/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.solicitorsfromhell.com','http://www.solicitorsfromhell.com/','ZZ','HATE'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.square-enix.com','http://www.square-enix.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.thegeekdiary.com','https://www.thegeekdiary.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.thehacktivist.com','http://www.thehacktivist.com/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tialsoft.com','http://www.tialsoft.com/multitranse/','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tops.co.th','https://www.tops.co.th/th/','th','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tot.co.th','https://www.tot.co.th/','th','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.trendsmap.com','https://www.trendsmap.com/','ZZ','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkhukuksitesi.com','http://www.turkhukuksitesi.com/kadin_haklari.php/','tr','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkiyegazetesi.com.tr','http://www.turkiyegazetesi.com.tr/','tr','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.turkiyeittifakipartisi.org.tr','https://www.turkiyeittifakipartisi.org.tr/','tr','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.tusev.org.tr','https://www.tusev.org.tr/tr/','tr','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.uber.com','https://www.uber.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ubisoft.com','https://www.ubisoft.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ultimate-anonymity.com','http://www.ultimate-anonymity.com/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unescap.org','https://www.unescap.org/','my','IGO'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unescap.org','https://www.unescap.org/','th','IGO'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unfpa.org','http://www.unfpa.org/swp','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.unlockme.co.uk','http://www.unlockme.co.uk/','ZZ','HACK'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.uproxy.org','https://www.uproxy.org/','ZZ','ANON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.urduvoa.com','https://www.urduvoa.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usaid.gov','https://www.usaid.gov/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usatoday.com','https://www.usatoday.com/story/news/nation/2017/10/19/cities-made-strides-2017-lgbt-rights-report-shows/777907001/','us','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.usj.com.my','http://www.usj.com.my/','my','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vamosbien.com','https://www.vamosbien.com/','ve','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venelogia.com','https://www.venelogia.com/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venevision.com','http://www.venevision.com/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelaaldia.com','https://www.venezuelaaldia.com/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelaawareness.com','http://www.venezuelaawareness.com/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.venezuelasinlimites.org','https://www.venezuelasinlimites.org/','ve','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ventevenezuela.org','http://www.ventevenezuela.org/','ve','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vesti.ru','http://www.vesti.ru/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vesti.ru','https://www.vesti.ru/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viasatprovider.com','https://www.viasatprovider.com/','us','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vice.com','https://www.vice.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.vidaagro.com.ve','http://www.vidaagro.com.ve/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viomundo.com.br','http://www.viomundo.com.br/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.viomundo.com.br','http://www.viomundo.com.br/denuncias/pedro-serrano-decisao-do-mpf-mata-impeachment-na-questao-das-pedaladas-mesmo-sendo-politico-julgamento-de-dilma-nao-pode-atropelar-justica.html','br','POLR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voanews.com','http://www.voanews.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voanoticias.com','https://www.voanoticias.com/','ve','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.voltdata.info','http://www.voltdata.info/','br','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.walmart.com','https://www.walmart.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.warchild.org','http://www.warchild.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.warhammeronline.com','http://www.warhammeronline.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.washingtonpost.com','https://www.washingtonpost.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.washingtontimes.com','http://www.washingtontimes.com/','ZZ','NEWS'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.watsons.co.th','https://www.watsons.co.th/','th','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webbox.com','http://www.webbox.com/index.php','ZZ','COMT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webmaster.or.th','https://www.webmaster.or.th/','th','MISC'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.webportal.com.my','http://www.webportal.com.my/','my','GRP'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.weforum.org','https://www.weforum.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.well.com','http://www.well.com/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.well.com','http://www.well.com/user/queerjhd/','ZZ','LGBT'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.westernunion.com','https://www.westernunion.com/','ZZ','COMM'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wftucentral.org','http://www.wftucentral.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','mm','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','my','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.who.int','https://www.who.int/southeastasia/','th','PUBH'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wikidata.org','https://www.wikidata.org/','ZZ','MISC'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wikimedia.org','https://www.wikimedia.org/','ZZ','MISC'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.wilpf.org','https://www.wilpf.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.witness.org','https://www.witness.org/','ZZ','HUMR'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.worldbank.org','http://www.worldbank.org/','ZZ','ECON'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.worldwildlife.org','https://www.worldwildlife.org/','ZZ','ENV'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xbox.com','http://www.xbox.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xbox.com','https://www.xbox.com/','ZZ','GAME'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.xenu.net','https://www.xenu.net/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.ymca.int','http://www.ymca.int/','ZZ','REL'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.yola.com','https://www.yola.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.youtube.com','https://www.youtube.com/','ZZ','MMED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zanzu.nl','https://www.zanzu.nl/','ZZ','XED'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zenhub.com','https://www.zenhub.com/','ru','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('www.zoomshare.com','https://www.zoomshare.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('xda-developers.com','https://xda-developers.com/','ZZ','HOST'); -INSERT INTO citizenlab (domain,url,cc,category_code) VALUES ('xe.com','https://xe.com/','ZZ','COMM'); diff --git a/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql new file mode 100644 index 00000000..04db39a8 --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/migrations/clickhouse_populate_jsonl.sql @@ -0,0 +1 @@ +INSERT INTO jsonl (report_id, input, s3path, linenum) VALUES ('20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3', 'https://www.backtrack-linux.org/', 'raw/20210709/00/MY/webconnectivity/2021070900_MY_webconnectivity.n0.2.jsonl.gz', 35) diff --git a/ooniapi/services/oonimeasurements/tests/integ/test_aggregation.py b/ooniapi/services/oonimeasurements/tests/test_aggregation.py similarity index 94% rename from ooniapi/services/oonimeasurements/tests/integ/test_aggregation.py rename to ooniapi/services/oonimeasurements/tests/test_aggregation.py index d2a6450c..0b06a6ce 100644 --- a/ooniapi/services/oonimeasurements/tests/integ/test_aggregation.py +++ b/ooniapi/services/oonimeasurements/tests/test_aggregation.py @@ -137,7 +137,6 @@ def test_aggregation_no_axis_input_ipaddr(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_domain(client): # 0-dimensional data url = ( @@ -158,7 +157,6 @@ def test_aggregation_no_axis_filter_multi_domain(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_probe_asn(client): # 0-dimensional dat url = "aggregation?probe_asn=AS3303,AS8167&since=2021-07-09&until=2021-07-10" @@ -177,7 +175,6 @@ def test_aggregation_no_axis_filter_multi_probe_asn(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_probe_cc(client): # 0-dimensional data url = "aggregation?probe_cc=BR,GB&since=2021-07-09&until=2021-07-10" @@ -196,7 +193,6 @@ def test_aggregation_no_axis_filter_multi_probe_cc(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_test_name(client): # 0-dimensional data url = "aggregation?test_name=web_connectivity,whatsapp&since=2021-07-09&until=2021-07-10" @@ -215,7 +211,6 @@ def test_aggregation_no_axis_filter_multi_test_name(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_test_name_1_axis(client): # 1-dimensional: test_name url = "aggregation?test_name=web_connectivity,whatsapp&since=2021-07-09&until=2021-07-10&axis_x=test_name" @@ -245,7 +240,6 @@ def test_aggregation_no_axis_filter_multi_test_name_1_axis(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_no_axis_filter_multi_oonirun(client): # 0-dimensional data url = "aggregation?ooni_run_link_id=1234,2345&since=2021-07-09&until=2021-07-10" @@ -264,7 +258,7 @@ def test_aggregation_no_axis_filter_multi_oonirun(client): }, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") + def test_aggregation_x_axis_only(client): # 1 dimension: X url = "aggregation?probe_cc=CH&probe_asn=AS3303&since=2021-07-09&until=2021-07-11&time_grain=day&axis_x=measurement_start_day" @@ -312,7 +306,6 @@ def test_aggregation_x_axis_only_invalid_time_grain_too_large(client): assert r.json()["msg"] == exp -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_only_hour(client): # 1 dimension: X url = "aggregation?since=2021-07-09&until=2021-07-11&axis_x=measurement_start_day" @@ -343,7 +336,6 @@ def test_aggregation_x_axis_only_hour(client): assert r == expected, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_domain(client): # 1 dimension: X url = "aggregation?probe_cc=CH&probe_asn=AS3303&since=2021-07-09&until=2021-07-10&axis_x=domain" @@ -372,7 +364,6 @@ def test_aggregation_x_axis_without_since(client): assert r.status_code == 400 -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_y_axis_only_blocking_type(client): # 1 dimension: Y: blocking_type url = "aggregation?since=2021-07-09&until=2021-07-10&axis_y=blocking_type" @@ -388,7 +379,6 @@ def test_aggregation_y_axis_only_blocking_type(client): assert r == expected, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_only_probe_cc(client): # 1 dimension: X url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=probe_cc" @@ -397,7 +387,6 @@ def test_aggregation_x_axis_only_probe_cc(client): assert len(r["result"]) == 33 -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_only_category_code(client): # 1-dimensional data url = "aggregation?probe_cc=IE&category_code=HACK&since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day" @@ -425,7 +414,6 @@ def test_aggregation_x_axis_only_category_code(client): assert r == expected, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_only_csv(client): # 1-dimensional data url = "aggregation?probe_cc=BR&probe_asn=AS8167&since=2021-07-09&until=2021-07-10&format=CSV&axis_x=measurement_start_day" @@ -448,7 +436,6 @@ def test_aggregation_x_axis_only_csv(client): assert r.replace("\r", "") == expected -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_y_axis(client): # 2-dimensional data url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day&axis_y=probe_cc&test_name=web_connectivity" @@ -466,14 +453,12 @@ def test_aggregation_x_axis_y_axis_are_the_same(client): assert r.json() == {"msg": "Axis X and Y cannot be the same", "v": 0} -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_two_axis_too_big(client): url = "aggregation?since=2021-10-14&until=2021-10-15&test_name=web_connectivity&axis_x=measurement_start_day&axis_y=input" r = client.get(f"/api/v1/{url}") assert r.json() == {} -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_foo(client): url = "aggregation?test_name=web_connectivity&since=2021-07-09&axis_x=probe_cc&until=2021-07-10" r = api(client, url) @@ -487,7 +472,6 @@ def test_aggregation_foo(client): ] -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_only_csv_2d(client): # 2-dimensional data: day vs ASN dom = "www.cabofrio.rj.gov.br" @@ -546,7 +530,6 @@ def test_aggregation_x_axis_only_csv_2d(client): ] -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_x_axis_category_code(client): # 1d data over a special column: category_code url = ( @@ -558,7 +541,6 @@ def test_aggregation_x_axis_category_code(client): assert r["result"][:3] == aggreg_over_category_code_expected, fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_y_axis_category_code(client): # 1d data over a special column: category_code url = ( @@ -572,7 +554,6 @@ def test_aggregation_y_axis_category_code(client): assert r["result"][:3] == aggreg_over_category_code_expected, fjd(r) -@pytest.mark.skip("FIXME citizenlab") def test_aggregation_xy_axis_category_code(client): # 2d data over a special column: category_code url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=category_code&axis_y=category_code" @@ -584,7 +565,6 @@ def test_aggregation_xy_axis_category_code(client): assert r["result"][:3] == [], fjd(r) -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_psiphon(client): url = "aggregation?probe_cc=BR&since=2021-07-09&until=2021-07-10&test_name=psiphon" r = api(client, url) @@ -602,7 +582,6 @@ def test_aggregation_psiphon(client): } -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_input(client): url = "aggregation?since=2021-07-09&until=2021-07-10&input=http://www.cabofrio.rj.gov.br/" r = api(client, url) @@ -620,28 +599,24 @@ def test_aggregation_input(client): } -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_invalid_input(client): url = "aggregation?since=2021-07-09&until=2021-07-10&input=~!^{}" r = client.get(f"/api/v1/{url}") assert r.json() == {"msg": "Invalid characters in input field", "v": 0} -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_invalid_input_2(client): url = "aggregation?since=2021-07-09&until=2021-07-10&input=foo.org;" r = client.get(f"/api/v1/{url}") assert r.json() == {"msg": "Invalid characters in input field", "v": 0} -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_invalid_input_3(client): url = "aggregation?since=2021-07-09&until=2021-07-10&input=foo.org%3D%27" r = client.get(f"/api/v1/{url}") assert r.json() == {"msg": "Invalid characters in input field", "v": 0} -@pytest.mark.skip(reason="TODO: fix this test") def test_aggregation_bug_585(client): url = "aggregation?test_name=web_connectivity&since=2022-01-24&until=2022-02-24&axis_x=measurement_start_day&category_code=LGBT" r = api(client, url) diff --git a/ooniapi/services/oonimeasurements/tests/test_auth.py b/ooniapi/services/oonimeasurements/tests/test_auth.py deleted file mode 100644 index 7332c356..00000000 --- a/ooniapi/services/oonimeasurements/tests/test_auth.py +++ /dev/null @@ -1,22 +0,0 @@ -import time - -from oonidataapi.utils import decode_jwt, create_jwt - - -def test_decode_jwt(): - key = "DUMMYKEY" - now = int(time.time()) - payload = { - "nbf": now, - "iat": now, - "exp": now + 2 * 60, - "aud": "user_auth", - "account_id": 0, - "login_time": now, - "role": "user", - } - token = create_jwt(payload, key) - d = decode_jwt(token=token, key=key, audience="user_auth") - assert d["role"] == "user" - assert d["aud"] == "user_auth" - assert d["account_id"] == 0 diff --git a/ooniapi/services/oonimeasurements/tests/test_integration.py b/ooniapi/services/oonimeasurements/tests/test_integration.py new file mode 100644 index 00000000..9b3b395c --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/test_integration.py @@ -0,0 +1,40 @@ +import os +import time +import random + +from multiprocessing import Process + +import httpx +import pytest +import uvicorn + + +LISTEN_PORT = random.randint(30_000, 42_000) + + +@pytest.fixture +def server(clickhouse_server): + os.environ["CLICKHOUSE_URL"] = clickhouse_server + proc = Process( + target=uvicorn.run, + args=("oonimeasurements.main:app"), + kwargs={"host": "127.0.0.1", "port": LISTEN_PORT, "log_level": "info"}, + daemon=True, + ) + + proc.start() + # Give it as second to start + time.sleep(1) + yield + proc.kill() + # Note: coverage is not being calculated properly + # TODO(art): https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html + proc.join() + + +@pytest.mark.skip("TODO(decfox): fix integration test") +def test_integration(server): + with httpx.Client(base_url=f"http://127.0.0.1:{LISTEN_PORT}") as client: + r = client.get("/version") + assert r.status_code == 200 + # TODO(decfox): add dedicated endpoint for integration test diff --git a/ooniapi/services/oonimeasurements/tests/test_main.py b/ooniapi/services/oonimeasurements/tests/test_main.py new file mode 100644 index 00000000..3c237892 --- /dev/null +++ b/ooniapi/services/oonimeasurements/tests/test_main.py @@ -0,0 +1,35 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from oonimeasurements.main import lifespan, app + + +def test_health_good(client): + r = client.get("health") + j = r.json() + assert j["status"] == "ok", j + assert len(j["errors"]) == 0, j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("health") + j = r.json() + assert j["detail"] == "health check failed", j + assert r.status_code == 400 + + +def test_metrics(client): + r = client.get("/metrics") + + +@pytest.mark.asyncio +async def test_lifecycle(): + async with lifespan(app): + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password="super_secure") + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/oonimeasurements/tests/test_measurements.py b/ooniapi/services/oonimeasurements/tests/test_measurements.py index feec64a7..ab4fc518 100644 --- a/ooniapi/services/oonimeasurements/tests/test_measurements.py +++ b/ooniapi/services/oonimeasurements/tests/test_measurements.py @@ -1,81 +1,623 @@ -import os -import re -import sqlite3 -import tempfile + import pytest -from fastapi.testclient import TestClient +from textwrap import dedent +from urllib.parse import urlencode +import json + + +def is_json(resp): + return resp.headers.get("content-type") == "application/json" + + +def fjd(o): + # non-indented JSON dump + return json.dumps(o, sort_keys=True) + -from ..dependencies import get_clickhouse_client -from ..main import app +def api(client, subpath, **kw): + url = f"/api/v1/{subpath}" + if kw: + assert "?" not in url + url += "?" + urlencode(kw) -THIS_DIR = os.path.dirname(__file__) + response = client.get(url) + assert response.status_code == 200, response.data + assert is_json(response) + return response.json() -@pytest.fixture(name="clickhouse") -def clickhouse_fixture(): - fd, path = tempfile.mkstemp() - print(f"created sqlite file {path}") +EXPECTED_RESULT_KEYS = [ + "anomaly_count", + "confirmed_count", + "failure_count", + "measurement_count", + "ok_count", +] - conn = sqlite3.connect(path, check_same_thread=False) - statements = [] - for fn in ["1_schema.sql", "2_fastpath_fixtures.sql"]: - with open(os.path.join(THIS_DIR, "fixtures", fn)) as in_file: - sql = in_file.read() - statements += sql.split(";") - cur = conn.cursor() - for statement in statements: - if statement.strip(): - cur.execute(statement) - conn.commit() +def test_aggregation_no_axis_with_caching(client): + # 0-dimensional data + url = "aggregation?probe_cc=IT&probe_asn=AS3269&since=2024-01-01&until=2024-02-01" + resp = client.get(f"/api/v1/{url}") + assert resp.status_code == 200, resp + j = resp.json() + assert j["dimension_count"] == 0 + assert j["v"] == 0 + assert set(j["result"].keys()) == set(EXPECTED_RESULT_KEYS) + + assert j["result"]["measurement_count"] > 0 + assert j["result"]["ok_count"] > 0 + + h = dict(resp.headers) + # FIXME: caching is currently disabled + # assert h["Cache-Control"] == "max-age=86400" + + +def test_aggregation_no_axis_csv(client): + # 0-dimensional data + url = "aggregation?probe_cc=IT&probe_asn=AS3269&since=2024-01-01&until=2024-02-01&format=CSV" + r = client.get(f"/api/v1/{url}") + assert not is_json(r) + assert ( + r.text.split("\r")[0] + == "anomaly_count,confirmed_count,failure_count,measurement_count,ok_count" + ) + assert "text/csv" in r.headers.get("content-type") + assert "Content-Disposition" not in r.headers # not a download - def replace_template_params(sql): - sql = re.sub(r"%\((.+?)\)s", r":\1", sql) - return sql - class MockClick: - def __init__(self, conn): - self.conn = conn +def test_aggregation_no_axis_csv_dload(client): + # 0-dimensional data + url = "aggregation?probe_cc=IT&probe_asn=AS3269&since=2024-01-01&until=2024-02-01&format=CSV&download=true" + r = client.get(f"/api/v1/{url}") + assert not is_json(r) + assert "text/csv" in r.headers.get("content-type") + exp = "attachment; filename=ooni-aggregate-data.csv" + assert r.headers["Content-Disposition"] == exp - def execute(self, sql, query_params=(), *arg, **kwargs): - cursor = conn.cursor() - sql = replace_template_params(sql) - cursor.execute(sql, query_params) - rows = cursor.fetchall() - colnames = [description[0] for description in cursor.description] - return rows, [(cn, None) for cn in colnames] - yield MockClick(conn) - conn.close() - os.close(fd) - os.remove(path) +def test_aggregation_no_axis_domain(client): + # 0-dimensional data + url = "aggregation?probe_cc=DE&domain=de.rt.com&since=2024-01-01&until=2024-02-01" + r = client.get(f"/api/v1/{url}") + j = r.json() + assert j["dimension_count"] == 0 + assert j["v"] == 0 + assert set(j["result"].keys()) == set(EXPECTED_RESULT_KEYS) + assert j["result"]["measurement_count"] > 0 -@pytest.fixture(name="client") -def client_fixture(clickhouse): - def get_clickhouse_override(): - return clickhouse - app.dependency_overrides[get_clickhouse_client] = get_clickhouse_override +def test_aggregation_no_axis_domain_ipaddr(client): + # 0-dimensional data + url = "aggregation?domain=8.8.8.8&since=2024-01-01&until=2024-02-01" + r = client.get(f"/api/v1/{url}") + j = r.json() + assert j["dimension_count"] == 0 + assert j["v"] == 0 + assert set(j["result"].keys()) == set(EXPECTED_RESULT_KEYS) - client = TestClient(app) - yield client - app.dependency_overrides.clear() + assert j["result"]["measurement_count"] > 0 -def test_list_measurements(client, clickhouse): - response = client.get("/api/v1/measurements?since=2024-01-01&until=2024-02-01") - assert response.status_code == 200 - j = response.json() - assert len(j["results"]) == 100 +def test_aggregation_no_axis_filter_by_category_code(client): + # 0-dimensional data + url = "aggregation?probe_cc=IT&since=2024-01-01&until=2024-02-01" + r = client.get(f"/api/v1/{url}") + j_nofilter = r.json() - response = client.get( - "/api/v1/measurements?probe_cc=IT&since=2024-01-01&until=2024-02-01" + url = "aggregation?probe_cc=IT&category_code=REL&since=2024-01-01&until=2024-02-01" + r = client.get(f"/api/v1/{url}") + j = r.json() + assert j["dimension_count"] == 0 + assert j["v"] == 0 + assert set(j["result"].keys()) == set(EXPECTED_RESULT_KEYS) + + assert j["result"]["measurement_count"] > 0 + assert j["result"]["ok_count"] > 0 + assert j_nofilter["result"]["measurement_count"] > j["result"]["measurement_count"] + + +def test_aggregation_no_axis_input_ipaddr(client): + # 0-dimensional data + url = "aggregation?input=109.105.109.146:22&since=2021-07-08&until=2021-07-10" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 2, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 2, + "ok_count": 0, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_domain(client): + # 0-dimensional data + url = ( + "aggregation?domain=twitter.com,facebook.com&since=2021-07-09&until=2021-07-10" ) - assert response.status_code == 200 - j = response.json() - for res in j["results"]: - assert res["probe_cc"] == "IT" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 1, + "measurement_count": 16, + "ok_count": 15, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_probe_asn(client): + # 0-dimensional dat + url = "aggregation?probe_asn=AS3303,AS8167&since=2021-07-09&until=2021-07-10" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 187, + "confirmed_count": 0, + "failure_count": 5, + "measurement_count": 1689, + "ok_count": 1497, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_probe_cc(client): + # 0-dimensional data + url = "aggregation?probe_cc=BR,GB&since=2021-07-09&until=2021-07-10" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 123, + "confirmed_count": 0, + "failure_count": 113, + "measurement_count": 2435, + "ok_count": 2199, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_test_name(client): + # 0-dimensional data + url = "aggregation?test_name=web_connectivity,whatsapp&since=2021-07-09&until=2021-07-10" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 319, + "confirmed_count": 42, + "failure_count": 340, + "measurement_count": 8547, + "ok_count": 7846, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_test_name_1_axis(client): + # 1-dimensional: test_name + url = "aggregation?test_name=web_connectivity,whatsapp&since=2021-07-09&until=2021-07-10&axis_x=test_name" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 1, + "result": [ + { + "anomaly_count": 317, + "confirmed_count": 42, + "failure_count": 339, + "measurement_count": 8488, + "ok_count": 7790, + "test_name": "web_connectivity", + }, + { + "anomaly_count": 2, + "confirmed_count": 0, + "failure_count": 1, + "measurement_count": 59, + "ok_count": 56, + "test_name": "whatsapp", + }, + ], + "v": 0, + }, fjd(r) + + +def test_aggregation_no_axis_filter_multi_oonirun(client): + # 0-dimensional data + url = "aggregation?ooni_run_link_id=1234,2345&since=2021-07-09&until=2021-07-10" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 0, + "ok_count": 0, + }, + "v": 0, + }, fjd(r) + + +def test_aggregation_x_axis_only(client): + # 1 dimension: X + url = "aggregation?probe_cc=CH&probe_asn=AS3303&since=2021-07-09&until=2021-07-11&time_grain=day&axis_x=measurement_start_day" + r = api(client, url) + r.pop("db_stats", None) + expected = { + "dimension_count": 1, + "result": [ + { + "anomaly_count": 187, + "confirmed_count": 0, + "failure_count": 5, + "measurement_count": 1689, + "measurement_start_day": "2021-07-09", + "ok_count": 1497, + }, + ], + "v": 0, + } + assert r == expected, fjd(r) + + +def test_aggregation_x_axis_only_invalid_range(client): + # 1 dimension: X + url = "aggregation?since=2022-07-09&until=2021-07-11&time_grain=day&axis_x=measurement_start_day" + r = client.get(f"/api/v1/{url}") + assert r.status_code == 400 + + +def test_aggregation_x_axis_only_invalid_time_grain_too_small(client): + # 1 dimension: X + url = "aggregation?since=2020-07-09&until=2022-07-11&time_grain=hour&axis_x=measurement_start_day" + r = client.get(f"/api/v1/{url}") + assert r.status_code == 400 + exp = "Choose time_grain between day, week, month, year, auto for the given time range" + assert r.json()["msg"] == exp + + +def test_aggregation_x_axis_only_invalid_time_grain_too_large(client): + # 1 dimension: X + url = "aggregation?since=2022-07-09&until=2022-07-11&time_grain=year&axis_x=measurement_start_day" + r = client.get(f"/api/v1/{url}") + assert r.status_code == 400 + exp = "Choose time_grain between hour, day, auto for the given time range" + assert r.json()["msg"] == exp + + +def test_aggregation_x_axis_only_hour(client): + # 1 dimension: X + url = "aggregation?since=2021-07-09&until=2021-07-11&axis_x=measurement_start_day" + r = api(client, url) + r.pop("db_stats", None) + expected = { + "dimension_count": 1, + "result": [ + { + "anomaly_count": 686, + "confirmed_count": 42, + "failure_count": 777, + "measurement_count": 9990, + "measurement_start_day": "2021-07-09T00:00:00Z", + "ok_count": 8485, + }, + { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1, + "measurement_start_day": "2021-07-09T01:00:00Z", + "ok_count": 1, + }, + ], + "v": 0, + } + assert r == expected, fjd(r) + + +def test_aggregation_x_axis_domain(client): + # 1 dimension: X + url = "aggregation?probe_cc=CH&probe_asn=AS3303&since=2021-07-09&until=2021-07-10&axis_x=domain" + r = api(client, url) + r.pop("db_stats", None) + assert r["dimension_count"] == 1 + for x in r["result"]: + if x["domain"] == "www.theregister.co.uk": + assert x == { + "anomaly_count": 0, + "confirmed_count": 0, + "domain": "www.theregister.co.uk", + "failure_count": 0, + "measurement_count": 1, + "ok_count": 1, + } + return + + assert False, "Msmt not found" + + +def test_aggregation_x_axis_without_since(client): + # 1 dimension: X + url = "aggregation?probe_cc=CH&probe_asn=AS3303&until=2021-07-10&axis_x=measurement_start_day" + r = client.get(f"/api/v1/{url}") + assert r.status_code == 400 + + +@pytest.mark.skip(reason="TODO: fix this test") +def test_aggregation_y_axis_only_blocking_type(client): + # 1 dimension: Y: blocking_type + url = "aggregation?since=2021-07-09&until=2021-07-10&axis_y=blocking_type" + r = api(client, url) + r.pop("db_stats", None) + expected = { + "dimension_count": 1, + "result": [ + # FIXME + ], + "v": 0, + } + assert r == expected, fjd(r) + + +def test_aggregation_x_axis_only_probe_cc(client): + # 1 dimension: X + url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=probe_cc" + r = api(client, url) + assert r["dimension_count"] == 1 + assert len(r["result"]) == 33 + + +def test_aggregation_x_axis_only_category_code(client): + # 1-dimensional data + url = "aggregation?probe_cc=IE&category_code=HACK&since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day" + r = api(client, url) + expected = { + "dimension_count": 1, + "result": [ + { + "anomaly_count": 32, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1302, + "measurement_start_day": "2021-07-10", + }, + { + "anomaly_count": 13, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 1236, + "measurement_start_day": "2021-07-10", + }, + ], + "v": 0, + } + assert r == expected, fjd(r) + + +def test_aggregation_x_axis_only_csv(client): + # 1-dimensional data + url = "aggregation?probe_cc=BR&probe_asn=AS8167&since=2021-07-09&until=2021-07-10&format=CSV&axis_x=measurement_start_day" + r = api(client, url) + expected = dedent( + """\ + anomaly_count,confirmed_count,failure_count,measurement_count,measurement_start_day + 0,0,0,5,2021-07-10 + 1,0,0,37,2020-01-04 + 2,0,0,46,2020-01-08 + 2,0,0,26,2020-01-13 + 0,0,0,20,2020-01-16 + 2,0,0,87,2020-01-20 + 0,0,0,6,2020-01-21 + 6,0,0,87,2020-01-23 + 0,0,0,11,2020-01-26 + 0,0,0,25,2020-01-27 + """ + ) + assert r.replace("\r", "") == expected + + +def test_aggregation_x_axis_y_axis(client): + # 2-dimensional data + url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=measurement_start_day&axis_y=probe_cc&test_name=web_connectivity" + r = api(client, url) + + assert "error" not in r + assert r["dimension_count"] == 2 + assert len(r["result"]) == 2140 + + +def test_aggregation_x_axis_y_axis_are_the_same(client): + # 2-dimensional data + url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=probe_cc&axis_y=probe_cc&test_name=web_connectivity" + r = client.get(f"/api/v1/{url}") + assert r.json() == {"msg": "Axis X and Y cannot be the same", "v": 0} + + +def test_aggregation_two_axis_too_big(client): + url = "aggregation?since=2021-10-14&until=2021-10-15&test_name=web_connectivity&axis_x=measurement_start_day&axis_y=input" + r = client.get(f"/api/v1/{url}") + assert r.json() == {} + + +def test_aggregation_foo(client): + url = "aggregation?test_name=web_connectivity&since=2021-07-09&axis_x=probe_cc&until=2021-07-10" + r = api(client, url) + assert sorted(r["result"][0]) == [ + "anomaly_count", + "confirmed_count", + "failure_count", + "measurement_count", + "ok_count", + "probe_cc", + ] + + +def test_aggregation_x_axis_only_csv_2d(client): + # 2-dimensional data: day vs ASN + dom = "www.cabofrio.rj.gov.br" + url = f"aggregation?probe_cc=BR&domain={dom}&since=2021-07-09&until=2021-07-10&time_grain=day&axis_x=measurement_start_day&axis_y=probe_asn&format=CSV" + r = client.get(f"/api/v1/{url}") + assert r.status_code == 200 + assert not is_json(r) + expected = dedent( + """\ + anomaly_count,confirmed_count,failure_count,measurement_count,measurement_start_day,ok_count,probe_asn + 1,0,0,1,2021-07-09,0,18881 + 1,0,0,1,2021-07-09,0,28154 + 1,0,0,1,2021-07-09,0,28183 + 1,0,0,1,2021-07-09,0,28210 + 1,0,0,1,2021-07-09,0,28343 + 3,0,0,3,2021-07-09,0,28573 + 1,0,0,1,2021-07-09,0,53029 + 1,0,0,1,2021-07-09,0,53089 + 1,0,0,1,2021-07-09,0,53209 + 1,0,0,1,2021-07-09,0,262616 + 1,0,0,1,2021-07-09,0,262644 + 1,0,0,1,2021-07-09,0,262970 + 2,0,0,2,2021-07-09,0,262983 + 1,0,0,1,2021-07-09,0,264146 + 1,0,0,1,2021-07-09,0,264510 + 1,0,0,1,2021-07-09,0,264592 + 1,0,0,1,2021-07-09,0,268821 + 1,0,0,1,2021-07-09,0,269246 + """ + ) + assert r.data.decode().replace("\r", "") == expected + + +aggreg_over_category_code_expected = [ + { + "anomaly_count": 77, + "category_code": "ALDR", + "confirmed_count": 0, + "failure_count": 116, + "measurement_count": 250, + }, + { + "anomaly_count": 118, + "category_code": "ANON", + "confirmed_count": 0, + "failure_count": 184, + "measurement_count": 405, + }, + { + "anomaly_count": 35, + "category_code": "COMM", + "confirmed_count": 0, + "failure_count": 54, + "measurement_count": 107, + }, +] + + +def test_aggregation_x_axis_category_code(client): + # 1d data over a special column: category_code + url = ( + "aggregation?probe_cc=DE&since=2021-07-09&until=2021-07-10&axis_x=category_code" + ) + r = api(client, url) + assert r["dimension_count"] == 1, fjd(r) + # shortened to save space + assert r["result"][:3] == aggreg_over_category_code_expected, fjd(r) + + +def test_aggregation_y_axis_category_code(client): + # 1d data over a special column: category_code + url = ( + "aggregation?probe_cc=DE&since=2021-07-09&until=2021-07-10&axis_y=category_code" + ) + r = api(client, url) + assert "dimension_count" in r, fjd(r) + assert r["dimension_count"] == 1, fjd(r) + # shortened to save space. The query should be identical to + # test_aggregation_x_axis_category_code + assert r["result"][:3] == aggreg_over_category_code_expected, fjd(r) + + +def test_aggregation_xy_axis_category_code(client): + # 2d data over a special column: category_code + url = "aggregation?since=2021-07-09&until=2021-07-10&axis_x=category_code&axis_y=category_code" + r = api(client, url) + assert "dimension_count" in r, fjd(r) + assert r["dimension_count"] == 2, fjd(r) + # shortened to save space. The query should be identical to + # test_aggregation_x_axis_category_code + assert r["result"][:3] == [], fjd(r) + + +def test_aggregation_psiphon(client): + url = "aggregation?probe_cc=BR&since=2021-07-09&until=2021-07-10&test_name=psiphon" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 0, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 20, + "ok_count": 20, + }, + "v": 0, + } + + +def test_aggregation_input(client): + url = "aggregation?since=2021-07-09&until=2021-07-10&input=http://www.cabofrio.rj.gov.br/" + r = api(client, url) + r.pop("db_stats", None) + assert r == { + "dimension_count": 0, + "result": { + "anomaly_count": 21, + "confirmed_count": 0, + "failure_count": 0, + "measurement_count": 21, + "ok_count": 0, + }, + "v": 0, + } + + +def test_aggregation_invalid_input(client): + url = "aggregation?since=2021-07-09&until=2021-07-10&input=~!^{}" + r = client.get(f"/api/v1/{url}") + assert r.json() == {"msg": "Invalid characters in input field", "v": 0} + + +def test_aggregation_invalid_input_2(client): + url = "aggregation?since=2021-07-09&until=2021-07-10&input=foo.org;" + r = client.get(f"/api/v1/{url}") + assert r.json() == {"msg": "Invalid characters in input field", "v": 0} + + +def test_aggregation_invalid_input_3(client): + url = "aggregation?since=2021-07-09&until=2021-07-10&input=foo.org%3D%27" + r = client.get(f"/api/v1/{url}") + assert r.json() == {"msg": "Invalid characters in input field", "v": 0} + - app.dependency_overrides.clear() +def test_aggregation_bug_585(client): + url = "aggregation?test_name=web_connectivity&since=2022-01-24&until=2022-02-24&axis_x=measurement_start_day&category_code=LGBT" + r = api(client, url) + # TODO: figure out what this test should be validating and add some checks for it. From 469354d3183014c437c523b426c8f4e87ddf824f Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 17 May 2024 15:44:49 +0530 Subject: [PATCH 4/5] fix: eof error --- ooniapi/services/oonimeasurements/src/oonimeasurements/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py index 449f22b0..8a5f8833 100644 --- a/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py +++ b/ooniapi/services/oonimeasurements/src/oonimeasurements/main.py @@ -107,4 +107,4 @@ async def health( @app.get("/") async def root(): - return {"message": "Hello OONItarian"} \ No newline at end of file + return {"message": "Hello OONItarian"} From 4e873a3f83229b59a8d8eb258cc68f968b5b7e0a Mon Sep 17 00:00:00 2001 From: decfox Date: Sun, 11 Aug 2024 11:47:41 +0530 Subject: [PATCH 5/5] fix: EOF lines --- ooniapi/services/oonimeasurements/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/oonimeasurements/pyproject.toml b/ooniapi/services/oonimeasurements/pyproject.toml index fcb19189..751f0106 100644 --- a/ooniapi/services/oonimeasurements/pyproject.toml +++ b/ooniapi/services/oonimeasurements/pyproject.toml @@ -107,4 +107,4 @@ exclude_lines = [ "no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:", -] \ No newline at end of file +]