From d47efee7daed68f857341cd95c70ee0d141a245a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 6 Mar 2024 09:26:40 +0100 Subject: [PATCH 01/25] Add extra dep temporalio --- poetry.lock | 237 ++++++++++++++----------------------------------- pyproject.toml | 5 +- 2 files changed, 70 insertions(+), 172 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5b796d03..d82fbcf7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "altair" version = "4.2.0" description = "Altair: A declarative statistical visualization library for Python." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -27,7 +26,6 @@ dev = ["black", "docutils", "flake8", "ipython", "m2r", "mistune (<2.0.0)", "pyt name = "anyio" version = "4.0.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -49,7 +47,6 @@ trio = ["trio (>=0.22)"] name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" -category = "main" optional = true python-versions = "*" files = [ @@ -61,7 +58,6 @@ files = [ name = "argon2-cffi" version = "23.1.0" description = "Argon2 for Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -82,7 +78,6 @@ typing = ["mypy"] name = "argon2-cffi-bindings" version = "21.2.0" description = "Low-level CFFI bindings for Argon2" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -120,7 +115,6 @@ tests = ["pytest"] name = "arrow" version = "1.3.0" description = "Better dates & times for Python" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -134,13 +128,12 @@ types-python-dateutil = ">=2.8.10" [package.extras] doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] -test = ["dateparser (>=1.0.0,<2.0.0)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (>=3.0.0,<4.0.0)"] +test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] [[package]] name = "asttokens" version = "2.4.1" description = "Annotate AST trees with source code positions" -category = "main" optional = true python-versions = "*" files = [ @@ -159,7 +152,6 @@ test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] name = "async-lru" version = "2.0.4" description = "Simple LRU cache for asyncio" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -174,7 +166,6 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -192,7 +183,6 @@ tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy name = "babel" version = "2.13.1" description = "Internationalization utilities" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -211,7 +201,6 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" -category = "main" optional = true python-versions = "*" files = [ @@ -223,7 +212,6 @@ files = [ name = "backports.zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -252,7 +240,6 @@ tzdata = ["tzdata"] name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" -category = "main" optional = true python-versions = ">=3.6.0" files = [ @@ -271,7 +258,6 @@ lxml = ["lxml"] name = "black" version = "22.10.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -316,7 +302,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "bleach" version = "6.1.0" description = "An easy safelist-based HTML-sanitizing tool." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -335,7 +320,6 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] name = "bokeh" version = "2.4.3" description = "Interactive plots and applications in the browser from Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -356,7 +340,6 @@ typing-extensions = ">=3.10.0" name = "boto3" version = "1.26.20" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -376,7 +359,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.20" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -396,7 +378,6 @@ crt = ["awscrt (==0.15.3)"] name = "certifi" version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -408,7 +389,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -485,7 +465,6 @@ pycparser = "*" name = "charset-normalizer" version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -500,7 +479,6 @@ unicode-backport = ["unicodedata2"] name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -515,7 +493,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "click-loglevel" version = "0.5.0" description = "Log level parameter type for Click" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -530,7 +507,6 @@ click = ">=8.0" name = "clickhouse-driver" version = "0.2.5" description = "Python driver with native interface for ClickHouse" -category = "main" optional = false python-versions = ">=3.6, <4" files = [ @@ -643,7 +619,6 @@ zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] name = "cloudpickle" version = "2.2.1" description = "Extended pickling support for Python objects" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -655,7 +630,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -667,7 +641,6 @@ files = [ name = "comm" version = "0.1.4" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -687,7 +660,6 @@ typing = ["mypy (>=0.990)"] name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" -category = "dev" optional = false python-versions = "*" files = [ @@ -702,7 +674,6 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -768,7 +739,6 @@ toml = ["tomli"] name = "cryptography" version = "38.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -815,7 +785,6 @@ test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0 name = "dask" version = "2023.3.2" description = "Parallel PyData with Task Scheduling" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -852,7 +821,6 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest- name = "debugpy" version = "1.8.0" description = "An implementation of the Debug Adapter Protocol for Python" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -880,7 +848,6 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -892,7 +859,6 @@ files = [ name = "defusedxml" version = "0.7.1" description = "XML bomb protection for Python stdlib modules" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -904,7 +870,6 @@ files = [ name = "distributed" version = "2023.3.2" description = "Distributed scheduler for Dask" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -933,7 +898,6 @@ zict = ">=2.1.0" name = "entrypoints" version = "0.4" description = "Discover and load entry points from installed packages." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -945,7 +909,6 @@ files = [ name = "exceptiongroup" version = "1.0.4" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -960,7 +923,6 @@ test = ["pytest (>=6)"] name = "executing" version = "2.0.1" description = "Get the currently executing AST node of a frame, and other information" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -975,7 +937,6 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth name = "fastjsonschema" version = "2.18.1" description = "Fastest Python implementation of JSON schema" -category = "main" optional = true python-versions = "*" files = [ @@ -990,7 +951,6 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc name = "flask" version = "2.2.2" description = "A simple framework for building complex web applications." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1013,7 +973,6 @@ dotenv = ["python-dotenv"] name = "fqdn" version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" -category = "main" optional = true python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" files = [ @@ -1025,7 +984,6 @@ files = [ name = "fsspec" version = "2023.4.0" description = "File-system specification" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1061,7 +1019,6 @@ tqdm = ["tqdm"] name = "gprof2dot" version = "2022.7.29" description = "Generate a dot graph from the output of several profilers." -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -1073,7 +1030,6 @@ files = [ name = "heapdict" version = "1.0.1" description = "a heap with decrease-key and increase-key operations" -category = "main" optional = false python-versions = "*" files = [ @@ -1085,7 +1041,6 @@ files = [ name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1097,7 +1052,6 @@ files = [ name = "importlib-metadata" version = "5.1.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1117,7 +1071,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.10.1" description = "Read resources from Python packages" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1136,7 +1089,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = "*" files = [ @@ -1148,7 +1100,6 @@ files = [ name = "ipykernel" version = "6.26.0" description = "IPython Kernel for Jupyter" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1162,7 +1113,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -1182,7 +1133,6 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.12.3" description = "IPython: Productive Interactive Computing" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1222,7 +1172,6 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "isoduration" version = "20.11.0" description = "Operations with ISO 8601 durations" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1237,7 +1186,6 @@ arrow = ">=0.15.0" name = "itsdangerous" version = "2.1.2" description = "Safely pass data to untrusted environments and back." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1249,7 +1197,6 @@ files = [ name = "jedi" version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1269,7 +1216,6 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1287,7 +1233,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1299,7 +1244,6 @@ files = [ name = "json5" version = "0.9.14" description = "A Python implementation of the JSON5 data format." -category = "main" optional = true python-versions = "*" files = [ @@ -1314,7 +1258,6 @@ dev = ["hypothesis"] name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -1326,7 +1269,6 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1356,7 +1298,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jupyter-client" version = "8.5.0" description = "Jupyter protocol implementation and client libraries" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1366,7 +1307,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -1380,7 +1321,6 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-core" version = "5.5.0" description = "Jupyter core package. A base package on which Jupyter projects rely." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1401,7 +1341,6 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] name = "jupyter-events" version = "0.6.3" description = "Jupyter Event System library" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1426,7 +1365,6 @@ test = ["click", "coverage", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>= name = "jupyter-lsp" version = "2.2.0" description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1442,7 +1380,6 @@ jupyter-server = ">=1.1.2" name = "jupyter-server" version = "2.9.1" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1455,7 +1392,7 @@ anyio = ">=3.1.0" argon2-cffi = "*" jinja2 = "*" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" jupyter-events = ">=0.6.0" jupyter-server-terminals = "*" nbconvert = ">=6.4.4" @@ -1479,7 +1416,6 @@ test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-sc name = "jupyter-server-terminals" version = "0.4.4" description = "A Jupyter Server Extension Providing Terminals." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1499,7 +1435,6 @@ test = ["coverage", "jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-cov", name = "jupyterlab" version = "4.0.7" description = "JupyterLab computational environment" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1533,7 +1468,6 @@ test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-cons name = "jupyterlab-pygments" version = "0.2.2" description = "Pygments theme using JupyterLab CSS variables" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1545,7 +1479,6 @@ files = [ name = "jupyterlab-server" version = "2.24.0" description = "A set of server components for JupyterLab and JupyterLab like applications." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1572,7 +1505,6 @@ test = ["hatch", "ipykernel", "jupyterlab-server[openapi]", "openapi-spec-valida name = "locket" version = "1.0.0" description = "File-based locks for Python on Linux and Windows" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1584,7 +1516,6 @@ files = [ name = "lxml" version = "4.9.1" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ @@ -1670,7 +1601,6 @@ source = ["Cython (>=0.29.7)"] name = "lz4" version = "4.3.2" description = "LZ4 Bindings for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1720,7 +1650,6 @@ tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] name = "markupsafe" version = "2.1.1" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1770,7 +1699,6 @@ files = [ name = "mashumaro" version = "3.1.1" description = "Fast serialization framework on top of dataclasses" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1791,7 +1719,6 @@ yaml = ["pyyaml (>=3.13)"] name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -1806,7 +1733,6 @@ traitlets = "*" name = "maxminddb" version = "2.2.0" description = "Reader for the MaxMind DB format" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1817,7 +1743,6 @@ files = [ name = "memray" version = "1.4.1" description = "A memory profiler for Python applications" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -1857,7 +1782,6 @@ test = ["Cython", "greenlet", "ipython", "pytest", "pytest-cov"] name = "mistune" version = "3.0.2" description = "A sane and fast Markdown parser with useful plugins and renderers" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1869,7 +1793,6 @@ files = [ name = "msgpack" version = "1.0.4" description = "MessagePack serializer" -category = "main" optional = false python-versions = "*" files = [ @@ -1931,7 +1854,6 @@ files = [ name = "mypy" version = "0.991" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1982,7 +1904,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" optional = false python-versions = "*" files = [ @@ -1994,7 +1915,6 @@ files = [ name = "nbclient" version = "0.8.0" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." -category = "main" optional = true python-versions = ">=3.8.0" files = [ @@ -2004,7 +1924,7 @@ files = [ [package.dependencies] jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" nbformat = ">=5.1" traitlets = ">=5.4" @@ -2017,7 +1937,6 @@ test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>= name = "nbconvert" version = "7.10.0" description = "Converting Jupyter Notebooks" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -2056,7 +1975,6 @@ webpdf = ["playwright"] name = "nbformat" version = "5.9.2" description = "The Jupyter Notebook format" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -2078,7 +1996,6 @@ test = ["pep440", "pre-commit", "pytest", "testpath"] name = "nest-asyncio" version = "1.5.8" description = "Patch asyncio to allow nested event loops" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2090,7 +2007,6 @@ files = [ name = "notebook-shim" version = "0.2.3" description = "A shim layer for notebook traits and config" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2108,7 +2024,6 @@ test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync" name = "numpy" version = "1.24.2" description = "Fundamental package for array computing in Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2146,7 +2061,6 @@ files = [ name = "objprint" version = "0.2.2" description = "A library that can print Python objects in human readable format" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2158,7 +2072,6 @@ files = [ name = "orjson" version = "3.8.2" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2217,7 +2130,6 @@ files = [ name = "overrides" version = "7.4.0" description = "A decorator to automatically detect mismatch when overriding a method." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2229,7 +2141,6 @@ files = [ name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2244,7 +2155,6 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" name = "pandas" version = "2.0.0" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2278,8 +2188,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -2312,7 +2222,6 @@ xml = ["lxml (>=4.6.3)"] name = "pandocfilters" version = "1.5.0" description = "Utilities for writing pandoc filters in python" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2324,7 +2233,6 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2340,7 +2248,6 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "partd" version = "1.3.0" description = "Appendable key-value storage" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2359,7 +2266,6 @@ complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"] name = "pathspec" version = "0.10.2" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2371,7 +2277,6 @@ files = [ name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." -category = "main" optional = true python-versions = "*" files = [ @@ -2386,7 +2291,6 @@ ptyprocess = ">=0.5" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" -category = "main" optional = true python-versions = "*" files = [ @@ -2398,7 +2302,6 @@ files = [ name = "pillow" version = "9.5.0" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2478,7 +2381,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2490,7 +2392,6 @@ files = [ name = "platformdirs" version = "2.5.4" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2506,7 +2407,6 @@ test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2522,7 +2422,6 @@ testing = ["pytest", "pytest-benchmark"] name = "prometheus-client" version = "0.17.1" description = "Python client for the Prometheus monitoring system." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2537,7 +2436,6 @@ twisted = ["twisted"] name = "prompt-toolkit" version = "3.0.39" description = "Library for building powerful interactive command lines in Python" -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -2548,11 +2446,30 @@ files = [ [package.dependencies] wcwidth = "*" +[[package]] +name = "protobuf" +version = "4.25.3" +description = "" +optional = true +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, + {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, + {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, + {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, + {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, + {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, + {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, + {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, + {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, +] + [[package]] name = "psutil" version = "5.9.4" description = "Cross-platform lib for process and system monitoring in Python." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2579,7 +2496,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "main" optional = true python-versions = "*" files = [ @@ -2591,7 +2507,6 @@ files = [ name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" -category = "main" optional = true python-versions = "*" files = [ @@ -2606,7 +2521,6 @@ tests = ["pytest"] name = "py-cpuinfo" version = "9.0.0" description = "Get CPU info with pure Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -2618,7 +2532,6 @@ files = [ name = "pyarrow" version = "11.0.0" description = "Python library for Apache Arrow" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2656,7 +2569,6 @@ numpy = ">=1.16.6" name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2668,7 +2580,6 @@ files = [ name = "pygments" version = "2.13.0" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2683,7 +2594,6 @@ plugins = ["importlib-metadata"] name = "pyopenssl" version = "22.1.0" description = "Python wrapper module around the OpenSSL library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2702,7 +2612,6 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -2717,7 +2626,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyrsistent" version = "0.19.2" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2749,7 +2657,6 @@ files = [ name = "pytest" version = "7.2.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2773,7 +2680,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. name = "pytest-benchmark" version = "4.0.0" description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2794,7 +2700,6 @@ histogram = ["pygal", "pygaljs"] name = "pytest-cov" version = "4.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2813,7 +2718,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-profiling" version = "1.7.0" description = "Profiling plugin for py.test" -category = "dev" optional = false python-versions = "*" files = [ @@ -2833,7 +2737,6 @@ tests = ["pytest-virtualenv"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2848,7 +2751,6 @@ six = ">=1.5" name = "python-json-logger" version = "2.0.7" description = "A python library adding a json log formatter" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2860,7 +2762,6 @@ files = [ name = "pytz" version = "2022.6" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -2872,7 +2773,6 @@ files = [ name = "pytz-deprecation-shim" version = "0.1.0.post0" description = "Shims to make deprecation of pytz easier" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -2888,7 +2788,6 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "main" optional = true python-versions = "*" files = [ @@ -2912,7 +2811,6 @@ files = [ name = "pywinpty" version = "2.0.12" description = "Pseudo terminal support for Windows from Python." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -2928,7 +2826,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2978,7 +2875,6 @@ files = [ name = "pyzmq" version = "25.1.1" description = "Python bindings for 0MQ" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3084,7 +2980,6 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "requests" version = "2.28.1" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7, <4" files = [ @@ -3106,7 +3001,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3121,7 +3015,6 @@ six = "*" name = "rfc3986-validator" version = "0.1.1" description = "Pure python rfc3986 validator" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3133,7 +3026,6 @@ files = [ name = "rich" version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "dev" optional = false python-versions = ">=3.6.3,<4.0.0" files = [ @@ -3153,7 +3045,6 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -3171,7 +3062,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "send2trash" version = "1.8.2" description = "Send file to trash natively under Mac OS X, Windows and Linux" -category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -3188,7 +3078,6 @@ win32 = ["pywin32"] name = "setuptools" version = "68.2.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -3205,7 +3094,6 @@ testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jar name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -3217,7 +3105,6 @@ files = [ name = "snakeviz" version = "2.1.1" description = "A web-based viewer for Python profiler output" -category = "dev" optional = false python-versions = "*" files = [ @@ -3232,7 +3119,6 @@ tornado = ">=2.0" name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3244,7 +3130,6 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -category = "main" optional = false python-versions = "*" files = [ @@ -3256,7 +3141,6 @@ files = [ name = "soupsieve" version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -3268,7 +3152,6 @@ files = [ name = "stack-data" version = "0.6.3" description = "Extract data from python stack frames and tracebacks for informative displays" -category = "main" optional = true python-versions = "*" files = [ @@ -3288,7 +3171,6 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "statsd" version = "4.0.1" description = "A simple statsd client." -category = "main" optional = false python-versions = "*" files = [ @@ -3300,7 +3182,6 @@ files = [ name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3315,7 +3196,6 @@ widechars = ["wcwidth"] name = "tblib" version = "1.7.0" description = "Traceback serialization library." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3323,11 +3203,35 @@ files = [ {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, ] +[[package]] +name = "temporalio" +version = "1.5.1" +description = "Temporal.io Python SDK" +optional = true +python-versions = ">=3.8,<4.0" +files = [ + {file = "temporalio-1.5.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cd1f8930787c728e30ca2fecf86175cafd1781d97e3ee7cdf6e41915c566a835"}, + {file = "temporalio-1.5.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:2b3765e0b6b0ef0b670cf39720a80280fd35be2444633c715b741d2b5428ceb6"}, + {file = "temporalio-1.5.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47149204b6430c8553d5dd6dfe2fbc6830bf6fd8ab08463ee4c97885c68f3082"}, + {file = "temporalio-1.5.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1167f6fc31355170cdb4f5f7b89f0f7e36c54d0aecb0ee9aa611f73e32db7d78"}, + {file = "temporalio-1.5.1-cp38-abi3-win_amd64.whl", hash = "sha256:15d36d2038b0ac33511163619bea7ead6f10aca3db5bad4b9d464d3fa0f4ff48"}, + {file = "temporalio-1.5.1.tar.gz", hash = "sha256:4c7bbc8a3e8df1ffc0c7d213bdcad26ae055bdd615567ce1ca4bfa9f28f831b8"}, +] + +[package.dependencies] +protobuf = ">=3.20" +python-dateutil = {version = ">=2.8.2,<3.0.0", markers = "python_version < \"3.11\""} +types-protobuf = ">=3.20" +typing-extensions = ">=4.2.0,<5.0.0" + +[package.extras] +grpc = ["grpcio (>=1.59.0,<2.0.0)"] +opentelemetry = ["opentelemetry-api (>=1.11.1,<2.0.0)", "opentelemetry-sdk (>=1.11.1,<2.0.0)"] + [[package]] name = "terminado" version = "0.17.1" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3348,7 +3252,6 @@ test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] name = "tinycss2" version = "1.2.1" description = "A tiny CSS parser" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3367,7 +3270,6 @@ test = ["flake8", "isort", "pytest"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3379,7 +3281,6 @@ files = [ name = "toolz" version = "0.12.0" description = "List processing tools and functional utilities" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3391,7 +3292,6 @@ files = [ name = "tornado" version = "6.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -3412,7 +3312,6 @@ files = [ name = "tqdm" version = "4.64.1" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" files = [ @@ -3433,7 +3332,6 @@ telegram = ["requests"] name = "traitlets" version = "5.13.0" description = "Traitlets Python configuration system" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -3445,11 +3343,21 @@ files = [ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["argcomplete (>=3.0.3)", "mypy (>=1.6.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] +[[package]] +name = "types-protobuf" +version = "4.24.0.20240302" +description = "Typing stubs for protobuf" +optional = true +python-versions = ">=3.8" +files = [ + {file = "types-protobuf-4.24.0.20240302.tar.gz", hash = "sha256:f22c00cc0cea9722e71e14d389bba429af9e35a74a949719c167203a5abbe2e4"}, + {file = "types_protobuf-4.24.0.20240302-py3-none-any.whl", hash = "sha256:5c607990f50f14606c2edaf379f8acc7418fef1451b227aa3c6a8a2cbc6ff14a"}, +] + [[package]] name = "types-python-dateutil" version = "2.8.19.14" description = "Typing stubs for python-dateutil" -category = "main" optional = true python-versions = "*" files = [ @@ -3461,7 +3369,6 @@ files = [ name = "typing-extensions" version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3473,7 +3380,6 @@ files = [ name = "tzdata" version = "2022.7" description = "Provider of IANA time zone data" -category = "main" optional = false python-versions = ">=2" files = [ @@ -3485,7 +3391,6 @@ files = [ name = "tzlocal" version = "4.2" description = "tzinfo object for the local timezone" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3506,7 +3411,6 @@ test = ["pytest (>=4.3)", "pytest-mock (>=3.3)"] name = "uri-template" version = "1.3.0" description = "RFC 6570 URI Template Processor" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3521,7 +3425,6 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake name = "urllib3" version = "1.26.13" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -3538,7 +3441,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "viztracer" version = "0.15.6" description = "A debugging and profiling tool that can trace and visualize python code execution" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3590,7 +3492,6 @@ full = ["orjson", "rich"] name = "warcio" version = "1.7.4" description = "Streaming WARC (and ARC) IO library" -category = "main" optional = false python-versions = "*" files = [ @@ -3605,7 +3506,6 @@ six = "*" name = "wcwidth" version = "0.2.8" description = "Measures the displayed width of unicode strings in a terminal" -category = "main" optional = true python-versions = "*" files = [ @@ -3617,7 +3517,6 @@ files = [ name = "webcolors" version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3633,7 +3532,6 @@ tests = ["pytest", "pytest-cov"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" -category = "main" optional = true python-versions = "*" files = [ @@ -3645,7 +3543,6 @@ files = [ name = "websocket-client" version = "1.6.4" description = "WebSocket client for Python with low level API options" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -3662,7 +3559,6 @@ test = ["websockets"] name = "werkzeug" version = "2.2.2" description = "The comprehensive WSGI web application library." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3680,7 +3576,6 @@ watchdog = ["watchdog"] name = "zict" version = "2.2.0" description = "Mutable mapping tools" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3695,7 +3590,6 @@ heapdict = "*" name = "zipp" version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3708,10 +3602,11 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -dataviz = ["altair", "numpy", "pandas", "flask"] +datapipeline = ["temporalio"] +dataviz = ["altair", "flask", "numpy", "pandas"] research = ["jupyterlab"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "dd5586d165d8e6d333c44d10223f88126f09700de8509a9674adc28adbc584ee" +content-hash = "d9d5bf32044045c897156b1c51d94161503c05395239b74a31fc838d74de9e34" diff --git a/pyproject.toml b/pyproject.toml index 6e6b5f0d..cbe0011b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,13 +24,15 @@ tabulate = "^0.9.0" warcio = "^1.7.4" msgpack = "^1.0.4" statsd = "^4.0.1" +click-loglevel = "^0.5.0" + dask = {extras = ["complete"], version = "^2023.3.2"} altair = {version = "^4.2.0", optional = true} numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} flask = {version = "^2.2.2", optional = true} jupyterlab = {version = "^4.0.7", optional = true} -click-loglevel = "^0.5.0" +temporalio = {version = "^1.5.1", optional = true} [tool.poetry.dev-dependencies] pytest = ">=7.2" @@ -49,6 +51,7 @@ oonidata = 'oonidata.cli:cli' [tool.poetry.extras] dataviz = ["altair", "numpy", "pandas", "flask"] research = ["jupyterlab"] +datapipeline = ["temporalio"] [build-system] requires = ["poetry-core>=1.0.0"] From 584306be15f958566374949793672b35c8a0b57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 6 Mar 2024 09:26:49 +0100 Subject: [PATCH 02/25] Move workers into workflows --- oonidata/datapipeline/worker/main.py | 0 oonidata/{workers => datapipeline/workflows}/analysis.py | 0 oonidata/{workers => datapipeline/workflows}/ground_truths.py | 0 oonidata/{workers => datapipeline/workflows}/observations.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 oonidata/datapipeline/worker/main.py rename oonidata/{workers => datapipeline/workflows}/analysis.py (100%) rename oonidata/{workers => datapipeline/workflows}/ground_truths.py (100%) rename oonidata/{workers => datapipeline/workflows}/observations.py (100%) diff --git a/oonidata/datapipeline/worker/main.py b/oonidata/datapipeline/worker/main.py new file mode 100644 index 00000000..e69de29b diff --git a/oonidata/workers/analysis.py b/oonidata/datapipeline/workflows/analysis.py similarity index 100% rename from oonidata/workers/analysis.py rename to oonidata/datapipeline/workflows/analysis.py diff --git a/oonidata/workers/ground_truths.py b/oonidata/datapipeline/workflows/ground_truths.py similarity index 100% rename from oonidata/workers/ground_truths.py rename to oonidata/datapipeline/workflows/ground_truths.py diff --git a/oonidata/workers/observations.py b/oonidata/datapipeline/workflows/observations.py similarity index 100% rename from oonidata/workers/observations.py rename to oonidata/datapipeline/workflows/observations.py From ea54a88234291eb33aa8f1169e41b386e4f14aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 6 Mar 2024 12:42:16 +0100 Subject: [PATCH 03/25] Rewrite observation generation using temporal.io --- debug-temporal.sh | 12 + oonidata/cli/command.py | 295 ------------------ .../{worker/main.py => __init__.py} | 0 oonidata/datapipeline/run_worker.py | 31 ++ oonidata/datapipeline/workflows/__init__.py | 0 .../datapipeline/workflows/observations.py | 261 ++++++++-------- oonidata/workers/__init__.py | 4 - pyproject.toml | 1 + 8 files changed, 181 insertions(+), 423 deletions(-) create mode 100644 debug-temporal.sh rename oonidata/datapipeline/{worker/main.py => __init__.py} (100%) create mode 100644 oonidata/datapipeline/run_worker.py create mode 100644 oonidata/datapipeline/workflows/__init__.py diff --git a/debug-temporal.sh b/debug-temporal.sh new file mode 100644 index 00000000..bbf43acc --- /dev/null +++ b/debug-temporal.sh @@ -0,0 +1,12 @@ +# >>> json.dumps(asdict(ObservationsWorkflowParams(probe_cc=["IT"], start_day="2024-01-01", end_day="2024-01-02", clickhouse="clickhouse://localhost/", data_dir="/Users/art/repos/ooni/data/tests/data/", parallelism=10, fast_fail=False, test_name=["signal"]))) +# +# +INPUT_JSON="{\"probe_cc\": [\"IT\"], \"test_name\": [\"signal\"], \"start_day\": \"2024-01-01\", \"end_day\": \"2024-01-20\", \"clickhouse\": \"clickhouse://localhost/\", \"data_dir\": \"$(pwd)/tests/data/datadir/\", \"parallelism\": 10, \"fast_fail\": false, \"log_level\": 20}" + +echo $INPUT_JSON +temporal workflow start \ + --task-queue oonidatapipeline-task-queue \ + --type ObservationsWorkflow \ + --namespace default \ + --input "$INPUT_JSON" + diff --git a/oonidata/cli/command.py b/oonidata/cli/command.py index 60b664d2..62b556d7 100644 --- a/oonidata/cli/command.py +++ b/oonidata/cli/command.py @@ -16,13 +16,6 @@ from oonidata.db.connections import ClickhouseConnection from oonidata.db.create_tables import create_queries, list_all_table_diffs from oonidata.netinfo import NetinfoDB -from oonidata.workers import ( - start_fingerprint_hunter, - start_observation_maker, - start_ground_truth_builder, - start_response_archiver, -) -from oonidata.workers.analysis import start_analysis log = logging.getLogger("oonidata") @@ -125,294 +118,6 @@ def sync( ) -@cli.command() -@probe_cc_option -@test_name_option -@start_day_option -@end_day_option -@click.option("--clickhouse", type=str) -@click.option( - "--data-dir", - type=Path, - required=True, - help="data directory to store fingerprint and geoip databases", -) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count() + 2, - help="number of processes to use. Only works when writing to a database", -) -@click.option( - "--fast-fail", - is_flag=True, - help="should we fail immediately when we encounter an error?", -) -@click.option( - "--create-tables", - is_flag=True, - help="should we attempt to create the required clickhouse tables", -) -@click.option( - "--drop-tables", - is_flag=True, - help="should we drop tables before creating them", -) -def mkobs( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - clickhouse: Optional[str], - data_dir: Path, - parallelism: int, - fast_fail: bool, - create_tables: bool, - drop_tables: bool, -): - """ - Make observations for OONI measurements and write them into clickhouse or a CSV file - """ - if create_tables: - if not clickhouse: - click.echo("--clickhouse needs to be specified when creating tables") - return 1 - if drop_tables: - click.confirm( - "Are you sure you want to drop the tables before creation?", abort=True - ) - - with ClickhouseConnection(clickhouse) as db: - for query, table_name in create_queries: - if drop_tables: - db.execute(f"DROP TABLE IF EXISTS {table_name};") - db.execute(query) - - NetinfoDB(datadir=data_dir, download=True) - - start_observation_maker( - probe_cc=probe_cc, - test_name=test_name, - start_day=start_day, - end_day=end_day, - clickhouse=clickhouse, - data_dir=data_dir, - parallelism=parallelism, - fast_fail=fast_fail, - ) - - -@cli.command() -@probe_cc_option -@test_name_option -@start_day_option -@end_day_option -@click.option("--clickhouse", type=str, required=True) -@click.option( - "--data-dir", - type=Path, - required=True, - help="data directory to store fingerprint and geoip databases", -) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count(), - help="number of processes to use. Only works when writing to a database", -) -@click.option( - "--fast-fail", - is_flag=True, - help="should we fail immediately when we encounter an error?", -) -@click.option( - "--create-tables", - is_flag=True, - help="should we attempt to create the required clickhouse tables", -) -@click.option( - "--rebuild-ground-truths", - is_flag=True, - help="should we force the rebuilding of ground truths", -) -def mker( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - clickhouse: str, - data_dir: Path, - parallelism: int, - fast_fail: bool, - create_tables: bool, - rebuild_ground_truths: bool, -): - if create_tables: - with ClickhouseConnection(clickhouse) as db: - for query, table_name in create_queries: - click.echo(f"Running create query for {table_name}") - db.execute(query) - raise Exception("Run this via the analysis command") - - -@cli.command() -@probe_cc_option -@test_name_option -@start_day_option -@end_day_option -@click.option("--clickhouse", type=str, required=True) -@click.option( - "--data-dir", - type=Path, - required=True, - help="data directory to store fingerprint and geoip databases", -) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count() + 2, - help="number of processes to use. Only works when writing to a database", -) -@click.option( - "--fast-fail", - is_flag=True, - help="should we fail immediately when we encounter an error?", -) -@click.option( - "--create-tables", - is_flag=True, - help="should we attempt to create the required clickhouse tables", -) -@click.option( - "--rebuild-ground-truths", - is_flag=True, - help="should we force the rebuilding of ground truths", -) -def mkanalysis( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - clickhouse: str, - data_dir: Path, - parallelism: int, - fast_fail: bool, - create_tables: bool, - rebuild_ground_truths: bool, -): - if create_tables: - with ClickhouseConnection(clickhouse) as db: - for query, table_name in create_queries: - click.echo(f"Running create query for {table_name}") - db.execute(query) - - start_analysis( - probe_cc=probe_cc, - test_name=test_name, - start_day=start_day, - end_day=end_day, - clickhouse=clickhouse, - data_dir=data_dir, - parallelism=parallelism, - fast_fail=fast_fail, - rebuild_ground_truths=rebuild_ground_truths, - ) - - -@cli.command() -@start_day_option -@end_day_option -@click.option("--clickhouse", type=str, required=True) -@click.option( - "--data-dir", - type=Path, - required=True, - help="data directory to store fingerprint and geoip databases", -) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count() + 2, - help="number of processes to use. Only works when writing to a database", -) -def mkgt( - start_day: date, - end_day: date, - clickhouse: str, - data_dir: Path, - parallelism: int, -): - start_ground_truth_builder( - start_day=start_day, - end_day=end_day, - clickhouse=clickhouse, - data_dir=data_dir, - parallelism=parallelism, - ) - - -@cli.command() -@probe_cc_option -@test_name_option -@start_day_option -@end_day_option -@click.option("--clickhouse", type=str) -@click.option("--data-dir", type=Path, required=True) -@click.option("--archives-dir", type=Path, required=True) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count() + 2, - help="number of processes to use. Only works when writing to a database", -) -def mkbodies( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - clickhouse: str, - data_dir: Path, - archives_dir: Path, - parallelism: int, -): - """ - Make response body archives - """ - start_response_archiver( - probe_cc=probe_cc, - test_name=test_name, - start_day=start_day, - end_day=end_day, - data_dir=data_dir, - archives_dir=archives_dir, - clickhouse=clickhouse, - parallelism=parallelism, - ) - - -@cli.command() -@click.option( - "--data-dir", - type=Path, - required=True, - help="data directory to store fingerprint and geoip databases", -) -@click.option("--archives-dir", type=Path, required=True) -@click.option( - "--parallelism", - type=int, - default=multiprocessing.cpu_count() + 2, - help="number of processes to use", -) -def fphunt(data_dir: Path, archives_dir: Path, parallelism: int): - click.echo("🏹 starting the hunt for blockpage fingerprints!") - start_fingerprint_hunter( - archives_dir=archives_dir, - data_dir=data_dir, - parallelism=parallelism, - ) - - @cli.command() @click.option("--clickhouse", type=str) @click.option( diff --git a/oonidata/datapipeline/worker/main.py b/oonidata/datapipeline/__init__.py similarity index 100% rename from oonidata/datapipeline/worker/main.py rename to oonidata/datapipeline/__init__.py diff --git a/oonidata/datapipeline/run_worker.py b/oonidata/datapipeline/run_worker.py new file mode 100644 index 00000000..e097bc2d --- /dev/null +++ b/oonidata/datapipeline/run_worker.py @@ -0,0 +1,31 @@ +import asyncio + +import concurrent.futures + +from temporalio.client import Client +from temporalio.worker import Worker + +from .workflows.observations import ObservationsWorkflow +from .workflows.observations import make_observation_in_day + + +async def async_main(): + client = await Client.connect("localhost:7233") + with concurrent.futures.ThreadPoolExecutor(max_workers=100) as activity_executor: + worker = Worker( + client, + task_queue="oonidatapipeline-task-queue", + workflows=[ObservationsWorkflow], + activities=[make_observation_in_day], + activity_executor=activity_executor, + ) + + await worker.run() + + +def main(): + asyncio.run(async_main()) + + +if __name__ == "__main__": + main() diff --git a/oonidata/datapipeline/workflows/__init__.py b/oonidata/datapipeline/workflows/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oonidata/datapipeline/workflows/observations.py b/oonidata/datapipeline/workflows/observations.py index 23bdf925..49c00710 100644 --- a/oonidata/datapipeline/workflows/observations.py +++ b/oonidata/datapipeline/workflows/observations.py @@ -1,3 +1,4 @@ +import asyncio import pathlib import logging import dataclasses @@ -5,44 +6,66 @@ from typing import ( List, - Optional, Sequence, Tuple, ) +from temporalio import workflow, activity +from dataclasses import dataclass -import statsd +with workflow.unsafe.imports_passed_through(): + import clickhouse_driver -from dask.distributed import Client as DaskClient -from dask.distributed import progress as dask_progress -from dask.distributed import wait as dask_wait -from dask.distributed import as_completed +with workflow.unsafe.imports_passed_through(): + import statsd + from oonidata.analysis.datasources import load_measurement + from oonidata.datautils import PerfTimer + from oonidata.models.nettests import SupportedDataformats -from oonidata.analysis.datasources import load_measurement -from oonidata.datautils import PerfTimer -from oonidata.models.nettests import SupportedDataformats + from oonidata.netinfo import NetinfoDB -from oonidata.netinfo import NetinfoDB - -from oonidata.dataclient import ( - date_interval, - list_file_entries_batches, - stream_measurements, - ccs_set, -) -from oonidata.db.connections import ( - ClickhouseConnection, -) -from oonidata.transforms import measurement_to_observations -from oonidata.workers.common import ( - get_prev_range, - make_db_rows, - maybe_delete_prev_range, - optimize_all_tables, -) + from oonidata.dataclient import ( + date_interval, + list_file_entries_batches, + stream_measurements, + ccs_set, + ) + from oonidata.db.connections import ( + ClickhouseConnection, + ) + from oonidata.transforms import measurement_to_observations + from oonidata.workers.common import ( + get_prev_range, + make_db_rows, + maybe_delete_prev_range, + optimize_all_tables, + ) log = logging.getLogger("oonidata.processing") +@dataclass +class ObservationsWorkflowParams: + probe_cc: List[str] + test_name: List[str] + start_day: str + end_day: str + clickhouse: str + data_dir: str + parallelism: int + fast_fail: bool + log_level: int = logging.INFO + + +@dataclass +class MakeObservationsParams: + probe_cc: List[str] + test_name: List[str] + clickhouse: str + data_dir: str + fast_fail: bool + bucket_date: str + + def write_observations_to_db( msmt: SupportedDataformats, netinfodb: NetinfoDB, @@ -68,7 +91,7 @@ def make_observations_for_file_entry_batch( row_buffer_size: int, data_dir: pathlib.Path, bucket_date: str, - probe_cc: str, + probe_cc: List[str], fast_fail: bool, ): netinfodb = NetinfoDB(datadir=data_dir, download=False) @@ -124,20 +147,13 @@ def make_observations_for_file_entry_batch( return idx -def make_observation_in_day( - dask_client: DaskClient, - probe_cc: List[str], - test_name: List[str], - clickhouse: Optional[str], - data_dir: pathlib.Path, - fast_fail: bool, - day: date, -): +@activity.defn +def make_observation_in_day(params: MakeObservationsParams) -> dict: statsd_client = statsd.StatsClient("localhost", 8125) - bucket_date = day.strftime("%Y-%m-%d") + day = datetime.strptime(params.bucket_date, "%Y-%m-%d").date() - with ClickhouseConnection(clickhouse, row_buffer_size=10_000) as db: + with ClickhouseConnection(params.clickhouse, row_buffer_size=10_000) as db: prev_ranges = [] for table_name in ["obs_web"]: prev_ranges.append( @@ -146,9 +162,9 @@ def make_observation_in_day( get_prev_range( db=db, table_name=table_name, - bucket_date=bucket_date, - test_name=test_name, - probe_cc=probe_cc, + bucket_date=params.bucket_date, + test_name=params.test_name, + probe_cc=params.probe_cc, ), ) ) @@ -156,35 +172,27 @@ def make_observation_in_day( t = PerfTimer() total_t = PerfTimer() file_entry_batches, total_size = list_file_entries_batches( - probe_cc=probe_cc, - test_name=test_name, + probe_cc=params.probe_cc, + test_name=params.test_name, start_day=day, end_day=day + timedelta(days=1), ) log.info(f"running {len(file_entry_batches)} batches took {t.pretty}") - future_list = [] + total_msmt_count = 0 for batch in file_entry_batches: - t = dask_client.submit( - make_observations_for_file_entry_batch, + # TODO(art): add extra parallelism here + msmt_cnt = make_observations_for_file_entry_batch( batch, - clickhouse, + params.clickhouse, 10_000, - data_dir, - bucket_date, - probe_cc, - fast_fail, + pathlib.Path(params.data_dir), + params.bucket_date, + params.probe_cc, + params.fast_fail, ) - future_list.append(t) + total_msmt_count += msmt_cnt - log.debug("starting progress monitoring") - dask_progress(future_list) - total_msmt_count = 0 - for _, result in as_completed(future_list, with_results=True): - total_msmt_count += result # type: ignore - - log.debug("waiting on task_list") - dask_wait(future_list) mb_per_sec = round(total_size / total_t.s / 10**6, 1) msmt_per_sec = round(total_msmt_count / total_t.s) log.info( @@ -193,78 +201,83 @@ def make_observation_in_day( statsd_client.timing("oonidata.dataclient.daily.timed", total_t.ms) if len(prev_ranges) > 0: - with ClickhouseConnection(clickhouse, row_buffer_size=10_000) as db: + with ClickhouseConnection(params.clickhouse, row_buffer_size=10_000) as db: for table_name, pr in prev_ranges: maybe_delete_prev_range(db=db, prev_range=pr) - return total_size, total_msmt_count + return {"size": total_size, "measurement_count": total_msmt_count} -def start_observation_maker( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - clickhouse: Optional[str], - data_dir: pathlib.Path, - parallelism: int, - fast_fail: bool, - log_level: int = logging.INFO, -): - log.info("Optimizing all tables") - optimize_all_tables(clickhouse) +@workflow.defn +class ObservationsWorkflow: + @workflow.run + async def run(self, params: ObservationsWorkflowParams) -> dict: + log.info("Optimizing all tables") + optimize_all_tables(params.clickhouse) - dask_client = DaskClient( - threads_per_worker=2, - n_workers=parallelism, - ) + t_total = PerfTimer() + log.info( + f"Starting observation making on {params.probe_cc} ({params.start_day} - {params.end_day})" + ) + task_list = [] + start_day = datetime.strptime(params.start_day, "%Y-%m-%d").date() + end_day = datetime.strptime(params.end_day, "%Y-%m-%d").date() - t_total = PerfTimer() - total_size, total_msmt_count = 0, 0 - day_list = list(date_interval(start_day, end_day)) - # See: https://stackoverflow.com/questions/51099685/best-practices-in-setting-number-of-dask-workers + async with asyncio.TaskGroup() as tg: + for day in date_interval(start_day, end_day): + task = tg.create_task( + workflow.execute_activity( + make_observation_in_day, + MakeObservationsParams( + probe_cc=params.probe_cc, + test_name=params.test_name, + clickhouse=params.clickhouse, + data_dir=params.data_dir, + fast_fail=params.fast_fail, + bucket_date=day.strftime("%Y-%m-%d"), + ), + start_to_close_timeout=timedelta(minutes=30), + ) + ) + task_list.append(task) - log.info(f"Starting observation making on {probe_cc} ({start_day} - {end_day})") - for day in day_list: t = PerfTimer() - size, msmt_count = make_observation_in_day( - dask_client=dask_client, - probe_cc=probe_cc, - test_name=test_name, - clickhouse=clickhouse, - data_dir=data_dir, - fast_fail=fast_fail, - day=day, + # size, msmt_count = + total_size, total_msmt_count = 0, 0 + for task in task_list: + res = task.result() + + total_size += res["size"] + total_msmt_count += res["measurement_count"] + + # This needs to be adjusted once we get the the per entry concurrency working + # mb_per_sec = round(total_size / t.s / 10**6, 1) + # msmt_per_sec = round(total_msmt_count / t.s) + # log.info( + # f"finished processing {day} speed: {mb_per_sec}MB/s ({msmt_per_sec}msmt/s)" + # ) + + # with ClickhouseConnection(params.clickhouse) as db: + # db.execute( + # "INSERT INTO oonidata_processing_logs (key, timestamp, runtime_ms, bytes, msmt_count, comment) VALUES", + # [ + # [ + # "oonidata.bucket_processed", + # datetime.now(timezone.utc).replace(tzinfo=None), + # int(t.ms), + # total_size, + # total_msmt_count, + # day.strftime("%Y-%m-%d"), + # ] + # ], + # ) + + mb_per_sec = round(total_size / t_total.s / 10**6, 1) + msmt_per_sec = round(total_msmt_count / t_total.s) + log.info( + f"finished processing {params.start_day} - {params.end_day} speed: {mb_per_sec}MB/s ({msmt_per_sec}msmt/s)" ) - total_size += size - total_msmt_count += msmt_count - mb_per_sec = round(size / t.s / 10**6, 1) - msmt_per_sec = round(msmt_count / t.s) log.info( - f"finished processing {day} speed: {mb_per_sec}MB/s ({msmt_per_sec}msmt/s)" + f"{round(total_size/10**9, 2)}GB {total_msmt_count} msmts in {t_total.pretty}" ) - with ClickhouseConnection(clickhouse) as db: - db.execute( - "INSERT INTO oonidata_processing_logs (key, timestamp, runtime_ms, bytes, msmt_count, comment) VALUES", - [ - [ - "oonidata.bucket_processed", - datetime.now(timezone.utc).replace(tzinfo=None), - int(t.ms), - size, - msmt_count, - day.strftime("%Y-%m-%d"), - ] - ], - ) - - mb_per_sec = round(total_size / t_total.s / 10**6, 1) - msmt_per_sec = round(total_msmt_count / t_total.s) - log.info( - f"finished processing {start_day} - {end_day} speed: {mb_per_sec}MB/s ({msmt_per_sec}msmt/s)" - ) - log.info( - f"{round(total_size/10**9, 2)}GB {total_msmt_count} msmts in {t_total.pretty}" - ) - - dask_client.shutdown() + return {"size": total_size, "measurement_count": total_msmt_count} diff --git a/oonidata/workers/__init__.py b/oonidata/workers/__init__.py index f22d4557..e69de29b 100644 --- a/oonidata/workers/__init__.py +++ b/oonidata/workers/__init__.py @@ -1,4 +0,0 @@ -from .fingerprint_hunter import start_fingerprint_hunter -from .observations import start_observation_maker -from .ground_truths import start_ground_truth_builder -from .response_archiver import start_response_archiver diff --git a/pyproject.toml b/pyproject.toml index cbe0011b..7e585242 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ viztracer = "^0.15.6" [tool.poetry.scripts] oonidata = 'oonidata.cli:cli' +datapipeline = 'oonidata.datapipeline.run_worker:main' [tool.poetry.extras] dataviz = ["altair", "numpy", "pandas", "flask"] From de77282e3447bc4ab93a1356d04befb6371fd3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 17:17:12 +0100 Subject: [PATCH 04/25] Break up oonidata from oonipipeline --- .gitignore | 3 - .vscode/settings.json | 1 + oonidata/pyproject.toml | 152 + oonidata/{ => src/oonidata}/__init__.py | 0 oonidata/{ => src/oonidata}/__main__.py | 0 .../{ => src/oonidata}/analysis/__init__.py | 0 .../{ => src/oonidata}/analysis/control.py | 0 .../oonidata}/analysis/datasources.py | 0 .../{ => src/oonidata}/analysis/signal.py | 0 .../analysis/sql/tls_consistency_matview.sql | 77 + .../oonidata}/analysis/web_analysis.py | 0 .../analysis/website_experiment_results.py | 0 oonidata/{ => src/oonidata}/apiclient.py | 0 oonidata/{ => src/oonidata}/cli/__init__.py | 0 oonidata/{ => src/oonidata}/cli/command.py | 0 oonidata/{ => src/oonidata}/compat.py | 0 oonidata/{ => src/oonidata}/dataclient.py | 0 oonidata/{ => src/oonidata}/datautils.py | 0 .../oonidata/dataviz}/__init__.py | 0 .../oonidata}/dataviz/react/.gitignore | 0 .../oonidata}/dataviz/react/Readme.md | 0 .../oonidata}/dataviz/react/package.json | 0 .../oonidata}/dataviz/react/src/App.tsx | 0 .../react/src/components/DNSAnalysis.tsx | 0 .../dataviz/react/src/components/Dataviz.tsx | 0 .../oonidata}/dataviz/react/src/index.tsx | 0 .../oonidata}/dataviz/react/yarn.lock | 0 .../oonidata}/dataviz/static/.gitignore | 0 .../oonidata}/dataviz/templates/analysis.html | 0 .../oonidata}/dataviz/templates/react.html | 0 .../oonidata}/dataviz/templates/vega.html | 0 oonidata/{ => src/oonidata}/dataviz/theme.py | 0 oonidata/{ => src/oonidata}/dataviz/utils.py | 0 oonidata/{ => src/oonidata}/dataviz/viz.py | 0 oonidata/{ => src/oonidata}/dataviz/web.py | 0 .../workflows => src/oonidata/db}/__init__.py | 0 oonidata/{ => src/oonidata}/db/connections.py | 0 .../{ => src/oonidata}/db/create_tables.py | 0 .../{ => src/oonidata}/db/speed_estimate.py | 0 oonidata/{ => src/oonidata}/fingerprintdb.py | 0 .../oonidata/models}/__init__.py | 0 .../{ => src/oonidata}/models/analysis.py | 0 .../{ => src/oonidata}/models/base_model.py | 0 .../{ => src/oonidata}/models/dataformats.py | 0 .../oonidata}/models/experiment_result.py | 0 .../oonidata}/models/nettests/__init__.py | 0 .../models/nettests/base_measurement.py | 0 .../oonidata}/models/nettests/browser_web.py | 0 .../oonidata}/models/nettests/dnscheck.py | 0 .../models/nettests/facebook_messenger.py | 0 .../http_header_field_manipulation.py | 0 .../nettests/http_invalid_request_line.py | 0 .../oonidata}/models/nettests/signal.py | 0 .../models/nettests/stun_reachability.py | 0 .../oonidata}/models/nettests/telegram.py | 0 .../{ => src/oonidata}/models/nettests/tor.py | 0 .../oonidata}/models/nettests/urlgetter.py | 0 .../models/nettests/web_connectivity.py | 0 .../oonidata}/models/nettests/whatsapp.py | 0 .../{ => src/oonidata}/models/observations.py | 0 oonidata/{ => src/oonidata}/netinfo.py | 0 oonidata/{ => src/oonidata}/normalize.py | 0 .../{ => src/oonidata}/transforms/__init__.py | 0 .../oonidata/transforms/nettests}/__init__.py | 0 .../transforms/nettests/browser_web.py | 0 .../oonidata}/transforms/nettests/dnscheck.py | 0 .../transforms/nettests/facebook_messenger.py | 0 .../http_header_field_manipulation.py | 0 .../nettests/http_invalid_request_line.py | 0 .../nettests/measurement_transformer.py | 0 .../oonidata}/transforms/nettests/signal.py | 0 .../transforms/nettests/stun_reachability.py | 0 .../oonidata}/transforms/nettests/telegram.py | 0 .../oonidata}/transforms/nettests/tor.py | 0 .../transforms/nettests/urlgetter.py | 0 .../transforms/nettests/web_connectivity.py | 0 .../oonidata}/transforms/nettests/whatsapp.py | 0 .../oonidata/workers}/__init__.py | 0 oonidata/{ => src/oonidata}/workers/common.py | 0 .../oonidata}/workers/fingerprint_hunter.py | 0 .../oonidata}/workers/response_archiver.py | 0 .../nettests => tests}/__init__.py | 0 .../tests}/_sample_measurements.py | 0 {tests => oonidata/tests}/conftest.py | 0 oonidata/tests/data/.gitignore | 4 + {tests => oonidata/tests}/test_analysis.py | 0 {tests => oonidata/tests}/test_cli.py | 0 {tests => oonidata/tests}/test_ctrl.py | 0 {tests => oonidata/tests}/test_dataclient.py | 0 {tests => oonidata/tests}/test_dataformat.py | 0 {tests => oonidata/tests}/test_db.py | 0 .../tests}/test_experiment_results.py | 0 .../tests}/test_fingerprints.py | 0 {tests => oonidata/tests}/test_netinfo.py | 0 {tests => oonidata/tests}/test_normalize.py | 0 {tests => oonidata/tests}/test_scoring.py | 0 {tests => oonidata/tests}/test_transforms.py | 0 {tests => oonidata/tests}/test_workers.py | 0 .../debug-temporal.sh | 0 oonipipeline/pyproject.toml | 67 + oonipipeline/src/oonidata | 1 + oonipipeline/src/oonipipeline/__about__.py | 1 + .../src/oonipipeline}/__init__.py | 0 .../src/oonipipeline/main.py | 2 +- .../src/oonipipeline/workflows}/__init__.py | 0 .../src/oonipipeline}/workflows/analysis.py | 0 .../oonipipeline}/workflows/ground_truths.py | 0 .../oonipipeline}/workflows/observations.py | 0 poetry.lock | 3612 ----------------- pyproject.toml | 59 - 110 files changed, 304 insertions(+), 3675 deletions(-) create mode 100644 oonidata/pyproject.toml rename oonidata/{ => src/oonidata}/__init__.py (100%) rename oonidata/{ => src/oonidata}/__main__.py (100%) rename oonidata/{ => src/oonidata}/analysis/__init__.py (100%) rename oonidata/{ => src/oonidata}/analysis/control.py (100%) rename oonidata/{ => src/oonidata}/analysis/datasources.py (100%) rename oonidata/{ => src/oonidata}/analysis/signal.py (100%) create mode 100644 oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql rename oonidata/{ => src/oonidata}/analysis/web_analysis.py (100%) rename oonidata/{ => src/oonidata}/analysis/website_experiment_results.py (100%) rename oonidata/{ => src/oonidata}/apiclient.py (100%) rename oonidata/{ => src/oonidata}/cli/__init__.py (100%) rename oonidata/{ => src/oonidata}/cli/command.py (100%) rename oonidata/{ => src/oonidata}/compat.py (100%) rename oonidata/{ => src/oonidata}/dataclient.py (100%) rename oonidata/{ => src/oonidata}/datautils.py (100%) rename oonidata/{datapipeline => src/oonidata/dataviz}/__init__.py (100%) rename oonidata/{ => src/oonidata}/dataviz/react/.gitignore (100%) rename oonidata/{ => src/oonidata}/dataviz/react/Readme.md (100%) rename oonidata/{ => src/oonidata}/dataviz/react/package.json (100%) rename oonidata/{ => src/oonidata}/dataviz/react/src/App.tsx (100%) rename oonidata/{ => src/oonidata}/dataviz/react/src/components/DNSAnalysis.tsx (100%) rename oonidata/{ => src/oonidata}/dataviz/react/src/components/Dataviz.tsx (100%) rename oonidata/{ => src/oonidata}/dataviz/react/src/index.tsx (100%) rename oonidata/{ => src/oonidata}/dataviz/react/yarn.lock (100%) rename oonidata/{ => src/oonidata}/dataviz/static/.gitignore (100%) rename oonidata/{ => src/oonidata}/dataviz/templates/analysis.html (100%) rename oonidata/{ => src/oonidata}/dataviz/templates/react.html (100%) rename oonidata/{ => src/oonidata}/dataviz/templates/vega.html (100%) rename oonidata/{ => src/oonidata}/dataviz/theme.py (100%) rename oonidata/{ => src/oonidata}/dataviz/utils.py (100%) rename oonidata/{ => src/oonidata}/dataviz/viz.py (100%) rename oonidata/{ => src/oonidata}/dataviz/web.py (100%) rename oonidata/{datapipeline/workflows => src/oonidata/db}/__init__.py (100%) rename oonidata/{ => src/oonidata}/db/connections.py (100%) rename oonidata/{ => src/oonidata}/db/create_tables.py (100%) rename oonidata/{ => src/oonidata}/db/speed_estimate.py (100%) rename oonidata/{ => src/oonidata}/fingerprintdb.py (100%) rename oonidata/{dataviz => src/oonidata/models}/__init__.py (100%) rename oonidata/{ => src/oonidata}/models/analysis.py (100%) rename oonidata/{ => src/oonidata}/models/base_model.py (100%) rename oonidata/{ => src/oonidata}/models/dataformats.py (100%) rename oonidata/{ => src/oonidata}/models/experiment_result.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/__init__.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/base_measurement.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/browser_web.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/dnscheck.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/facebook_messenger.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/http_header_field_manipulation.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/http_invalid_request_line.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/signal.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/stun_reachability.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/telegram.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/tor.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/urlgetter.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/web_connectivity.py (100%) rename oonidata/{ => src/oonidata}/models/nettests/whatsapp.py (100%) rename oonidata/{ => src/oonidata}/models/observations.py (100%) rename oonidata/{ => src/oonidata}/netinfo.py (100%) rename oonidata/{ => src/oonidata}/normalize.py (100%) rename oonidata/{ => src/oonidata}/transforms/__init__.py (100%) rename oonidata/{db => src/oonidata/transforms/nettests}/__init__.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/browser_web.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/dnscheck.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/facebook_messenger.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/http_header_field_manipulation.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/http_invalid_request_line.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/measurement_transformer.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/signal.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/stun_reachability.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/telegram.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/tor.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/urlgetter.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/web_connectivity.py (100%) rename oonidata/{ => src/oonidata}/transforms/nettests/whatsapp.py (100%) rename oonidata/{models => src/oonidata/workers}/__init__.py (100%) rename oonidata/{ => src/oonidata}/workers/common.py (100%) rename oonidata/{ => src/oonidata}/workers/fingerprint_hunter.py (100%) rename oonidata/{ => src/oonidata}/workers/response_archiver.py (100%) rename oonidata/{transforms/nettests => tests}/__init__.py (100%) rename {tests => oonidata/tests}/_sample_measurements.py (100%) rename {tests => oonidata/tests}/conftest.py (100%) create mode 100644 oonidata/tests/data/.gitignore rename {tests => oonidata/tests}/test_analysis.py (100%) rename {tests => oonidata/tests}/test_cli.py (100%) rename {tests => oonidata/tests}/test_ctrl.py (100%) rename {tests => oonidata/tests}/test_dataclient.py (100%) rename {tests => oonidata/tests}/test_dataformat.py (100%) rename {tests => oonidata/tests}/test_db.py (100%) rename {tests => oonidata/tests}/test_experiment_results.py (100%) rename {tests => oonidata/tests}/test_fingerprints.py (100%) rename {tests => oonidata/tests}/test_netinfo.py (100%) rename {tests => oonidata/tests}/test_normalize.py (100%) rename {tests => oonidata/tests}/test_scoring.py (100%) rename {tests => oonidata/tests}/test_transforms.py (100%) rename {tests => oonidata/tests}/test_workers.py (100%) rename debug-temporal.sh => oonipipeline/debug-temporal.sh (100%) create mode 100644 oonipipeline/pyproject.toml create mode 120000 oonipipeline/src/oonidata create mode 100644 oonipipeline/src/oonipipeline/__about__.py rename {oonidata/workers => oonipipeline/src/oonipipeline}/__init__.py (100%) rename oonidata/datapipeline/run_worker.py => oonipipeline/src/oonipipeline/main.py (92%) rename {tests => oonipipeline/src/oonipipeline/workflows}/__init__.py (100%) rename {oonidata/datapipeline => oonipipeline/src/oonipipeline}/workflows/analysis.py (100%) rename {oonidata/datapipeline => oonipipeline/src/oonipipeline}/workflows/ground_truths.py (100%) rename {oonidata/datapipeline => oonipipeline/src/oonipipeline}/workflows/observations.py (100%) delete mode 100644 poetry.lock delete mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 5b15e9f9..5299ac8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ __pycache__ /.coverage /coverage.xml -/tests/data/datadir/* -/tests/data/raw_measurements/* -/tests/data/measurements/* /dist /datadir /output diff --git a/.vscode/settings.json b/.vscode/settings.json index c6934f91..1ff25cfc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "python.testing.pytestArgs": ["tests"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, + "python.defaultInterpreterPath": "${workspaceFolder}/oonipipeline/.venv", "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" } diff --git a/oonidata/pyproject.toml b/oonidata/pyproject.toml new file mode 100644 index 00000000..06e8594d --- /dev/null +++ b/oonidata/pyproject.toml @@ -0,0 +1,152 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "oonidata" +version = "0.2.3" +description = "" +authors = [{ name = "OONI", email = "contact@ooni.org"}] + +dependencies = [ +"boto3 ~= 1.24", +"PyYAML ~= 6.0", +"tqdm ~= 4.64", +"lz4 ~= 4.0", +"requests ~= 2.27", +"cryptography ~= 38.0.3", +"clickhouse-driver ~= 0.2", +"click ~= 8.0.0", +"lxml ~= 4.9", +"maxminddb ~= 2.2", +"orjson ~= 3.8", +"mashumaro ~= 3.0", +"pyOpenSSL ~= 22.1", + "fastapi ~= 0.108.0", +"tabulate ~= 0.9.0", +"warcio ~= 1.7.4", +"msgpack ~= 1.0.4", +"statsd ~= 4.0.1", +"click-loglevel ~= 0.5.0", +"dask ~= 2023.3.2", +"altair ~= 4.2.0", +"numpy ~= 1.23.5", +"pandas ~= 2.0.0", +"flask ~= 2.2.0", +"jupyterlab ~= 4.0.7", +"temporalio ~= 1.5.1", +] + +[tool.hatch.build.targets.sdist] + +[tool.hatch.build.targets.wheel] +packages = ["oonidata"] + + +[tool.hatch.envs.default] +dependencies = [ +"pytest", +"pytest-cov", +"black", +"snakeviz", +"mypy", +"pytest-benchmark", +"pytest-profiling", +"memray", +"viztracer" +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[tool.poetry] +name = "oonidata" +version = "0.2.3" +description = "" +authors = ["Arturo Filastò "] + +python = ">=3.8,<4" +boto3 = ">=1.24" +PyYAML = ">=6.0" +tqdm = ">=4.64" +lz4 = ">=4.0" +requests = ">=2.27" +cryptography = ">=38.0.3" +clickhouse-driver = ">=0.2" +click = ">=8.0.0" +lxml = ">=4.9" +maxminddb = ">=2.2" +orjson = ">=3.8" +mashumaro = ">=3.0" +pyOpenSSL = ">=22.1" +importlib-metadata = {version = ">=1.0", python = "<3.8"} +tabulate = "^0.9.0" +warcio = "^1.7.4" +msgpack = "^1.0.4" +statsd = "^4.0.1" +click-loglevel = "^0.5.0" + +dask = {extras = ["complete"], version = "^2023.3.2"} +altair = {version = "^4.2.0", optional = true} +numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} +pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} +flask = {version = "^2.2.2", optional = true} +jupyterlab = {version = "^4.0.7", optional = true} +temporalio = {version = "^1.5.1", optional = true} + + +[tool.poetry.dependencies] +python = ">=3.8,<4" +boto3 = ">=1.24" +PyYAML = ">=6.0" +tqdm = ">=4.64" +lz4 = ">=4.0" +requests = ">=2.27" +cryptography = ">=38.0.3" +clickhouse-driver = ">=0.2" +click = ">=8.0.0" +lxml = ">=4.9" +maxminddb = ">=2.2" +orjson = ">=3.8" +mashumaro = ">=3.0" +pyOpenSSL = ">=22.1" +importlib-metadata = {version = ">=1.0", python = "<3.8"} +tabulate = "^0.9.0" +warcio = "^1.7.4" +msgpack = "^1.0.4" +statsd = "^4.0.1" +click-loglevel = "^0.5.0" + +dask = {extras = ["complete"], version = "^2023.3.2"} +altair = {version = "^4.2.0", optional = true} +numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} +pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} +flask = {version = "^2.2.2", optional = true} +jupyterlab = {version = "^4.0.7", optional = true} +temporalio = {version = "^1.5.1", optional = true} + +[tool.poetry.dev-dependencies] +pytest = ">=7.2" +pytest-cov = ">=3.0.0" +black = ">=22.3.0" +snakeviz = ">=2.1.1" +mypy = ">=0.961" +pytest-benchmark = "^4.0.0" +pytest-profiling = "^1.7.0" +memray = "^1.4.1" +viztracer = "^0.15.6" + +[tool.poetry.scripts] +oonidata = 'oonidata.cli:cli' +datapipeline = 'oonidata.datapipeline.run_worker:main' + +[tool.poetry.extras] +dataviz = ["altair", "numpy", "pandas", "flask"] +research = ["jupyterlab"] +datapipeline = ["temporalio"] + + diff --git a/oonidata/__init__.py b/oonidata/src/oonidata/__init__.py similarity index 100% rename from oonidata/__init__.py rename to oonidata/src/oonidata/__init__.py diff --git a/oonidata/__main__.py b/oonidata/src/oonidata/__main__.py similarity index 100% rename from oonidata/__main__.py rename to oonidata/src/oonidata/__main__.py diff --git a/oonidata/analysis/__init__.py b/oonidata/src/oonidata/analysis/__init__.py similarity index 100% rename from oonidata/analysis/__init__.py rename to oonidata/src/oonidata/analysis/__init__.py diff --git a/oonidata/analysis/control.py b/oonidata/src/oonidata/analysis/control.py similarity index 100% rename from oonidata/analysis/control.py rename to oonidata/src/oonidata/analysis/control.py diff --git a/oonidata/analysis/datasources.py b/oonidata/src/oonidata/analysis/datasources.py similarity index 100% rename from oonidata/analysis/datasources.py rename to oonidata/src/oonidata/analysis/datasources.py diff --git a/oonidata/analysis/signal.py b/oonidata/src/oonidata/analysis/signal.py similarity index 100% rename from oonidata/analysis/signal.py rename to oonidata/src/oonidata/analysis/signal.py diff --git a/oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql b/oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql new file mode 100644 index 00000000..b419c294 --- /dev/null +++ b/oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql @@ -0,0 +1,77 @@ +-- The mat view can be queried like so: +-- SELECT +-- ip, +-- tls_server_name, +-- anyMerge(ip_cc) as ip_cc, +-- anyMerge(ip_asn) as ip_asn, +-- anyMerge(ip_as_cc) as ip_as_cc, +-- anyMerge(ip_as_org_name) as ip_as_org_name, +-- sumMerge(msmt_cnt) as msmt_cnt, +-- uniqMerge(vp_cnt) as vp_cnt, +-- maxMerge(is_ctrl_vp) as is_ctrl_vp +-- FROM tls_consistency_matview +-- GROUP BY ip, tls_server_name +CREATE MATERIALIZED VIEW tls_consistency_matview +( + ip String NOT NULL, + tls_server_name String NOT NULL, + -- Note: it seems like there are cases where this will change over time, so we are effectively picking any one of them + ip_cc AggregateFunction(any, Nullable(String)), + ip_asn AggregateFunction(any, Nullable(Int32)), + ip_as_cc AggregateFunction(any, Nullable(String)), + ip_as_org_name AggregateFunction(any, Nullable(String)), + msmt_cnt AggregateFunction(sum, UInt64), + vp_cnt AggregateFunction(uniq, UInt64), + is_ctrl_vp AggregateFunction(max, UInt8) +) +ENGINE = AggregatingMergeTree() +ORDER BY (ip, tls_server_name) +POPULATE +AS SELECT +IF( + obs_web_ctrl.ip IS NOT NULL, + obs_web_ctrl.ip, + ctrl.ip +) as ip, +IF( + obs_web_ctrl.tls_server_name IS NOT NULL, + obs_web_ctrl.tls_server_name, + ctrl.tls_server_name +) as tls_server_name, +anyState(ip_cc) as ip_cc, +anyState(ip_asn) as ip_asn, +anyState(ip_as_cc) as ip_as_cc, +anyState(ip_as_org_name) as ip_as_org_name, +sumState(obs_web_ctrl.msmt_cnt + ctrl.msmt_cnt) as msmt_cnt, +uniqState(obs_web_ctrl.vp_cnt + ctrl.vp_cnt) as vp_cnt, +maxState(IF(is_ctrl_vp = 1, 1, 0)) as is_ctrl_vp +FROM ( + SELECT + ip, ip_cc, ip_asn, ip_as_cc, ip_as_org_name, tls_server_name, + COUNT() as msmt_cnt, + COUNT(DISTINCT probe_cc, probe_asn) as vp_cnt + FROM obs_web + WHERE tls_server_name IS NOT NULL + AND tls_failure IS NULL + AND ip IS NOT NULL + GROUP BY ip, ip_cc, ip_asn, ip_as_cc, ip_as_org_name, tls_server_name +) as obs_web_ctrl +FULL OUTER JOIN +( + SELECT + ip, tls_server_name, + COUNT() as msmt_cnt, + 1 as vp_cnt, + 1 as is_ctrl_vp + FROM obs_web_ctrl + WHERE + tls_success = 1 + AND tls_server_name IS NOT NULL + AND ip IS NOT NULL + GROUP BY ip, tls_server_name +) as ctrl +ON ( + ctrl.ip = obs_web_ctrl.ip + AND ctrl.tls_server_name = obs_web_ctrl.tls_server_name +) +GROUP BY ip, tls_server_name \ No newline at end of file diff --git a/oonidata/analysis/web_analysis.py b/oonidata/src/oonidata/analysis/web_analysis.py similarity index 100% rename from oonidata/analysis/web_analysis.py rename to oonidata/src/oonidata/analysis/web_analysis.py diff --git a/oonidata/analysis/website_experiment_results.py b/oonidata/src/oonidata/analysis/website_experiment_results.py similarity index 100% rename from oonidata/analysis/website_experiment_results.py rename to oonidata/src/oonidata/analysis/website_experiment_results.py diff --git a/oonidata/apiclient.py b/oonidata/src/oonidata/apiclient.py similarity index 100% rename from oonidata/apiclient.py rename to oonidata/src/oonidata/apiclient.py diff --git a/oonidata/cli/__init__.py b/oonidata/src/oonidata/cli/__init__.py similarity index 100% rename from oonidata/cli/__init__.py rename to oonidata/src/oonidata/cli/__init__.py diff --git a/oonidata/cli/command.py b/oonidata/src/oonidata/cli/command.py similarity index 100% rename from oonidata/cli/command.py rename to oonidata/src/oonidata/cli/command.py diff --git a/oonidata/compat.py b/oonidata/src/oonidata/compat.py similarity index 100% rename from oonidata/compat.py rename to oonidata/src/oonidata/compat.py diff --git a/oonidata/dataclient.py b/oonidata/src/oonidata/dataclient.py similarity index 100% rename from oonidata/dataclient.py rename to oonidata/src/oonidata/dataclient.py diff --git a/oonidata/datautils.py b/oonidata/src/oonidata/datautils.py similarity index 100% rename from oonidata/datautils.py rename to oonidata/src/oonidata/datautils.py diff --git a/oonidata/datapipeline/__init__.py b/oonidata/src/oonidata/dataviz/__init__.py similarity index 100% rename from oonidata/datapipeline/__init__.py rename to oonidata/src/oonidata/dataviz/__init__.py diff --git a/oonidata/dataviz/react/.gitignore b/oonidata/src/oonidata/dataviz/react/.gitignore similarity index 100% rename from oonidata/dataviz/react/.gitignore rename to oonidata/src/oonidata/dataviz/react/.gitignore diff --git a/oonidata/dataviz/react/Readme.md b/oonidata/src/oonidata/dataviz/react/Readme.md similarity index 100% rename from oonidata/dataviz/react/Readme.md rename to oonidata/src/oonidata/dataviz/react/Readme.md diff --git a/oonidata/dataviz/react/package.json b/oonidata/src/oonidata/dataviz/react/package.json similarity index 100% rename from oonidata/dataviz/react/package.json rename to oonidata/src/oonidata/dataviz/react/package.json diff --git a/oonidata/dataviz/react/src/App.tsx b/oonidata/src/oonidata/dataviz/react/src/App.tsx similarity index 100% rename from oonidata/dataviz/react/src/App.tsx rename to oonidata/src/oonidata/dataviz/react/src/App.tsx diff --git a/oonidata/dataviz/react/src/components/DNSAnalysis.tsx b/oonidata/src/oonidata/dataviz/react/src/components/DNSAnalysis.tsx similarity index 100% rename from oonidata/dataviz/react/src/components/DNSAnalysis.tsx rename to oonidata/src/oonidata/dataviz/react/src/components/DNSAnalysis.tsx diff --git a/oonidata/dataviz/react/src/components/Dataviz.tsx b/oonidata/src/oonidata/dataviz/react/src/components/Dataviz.tsx similarity index 100% rename from oonidata/dataviz/react/src/components/Dataviz.tsx rename to oonidata/src/oonidata/dataviz/react/src/components/Dataviz.tsx diff --git a/oonidata/dataviz/react/src/index.tsx b/oonidata/src/oonidata/dataviz/react/src/index.tsx similarity index 100% rename from oonidata/dataviz/react/src/index.tsx rename to oonidata/src/oonidata/dataviz/react/src/index.tsx diff --git a/oonidata/dataviz/react/yarn.lock b/oonidata/src/oonidata/dataviz/react/yarn.lock similarity index 100% rename from oonidata/dataviz/react/yarn.lock rename to oonidata/src/oonidata/dataviz/react/yarn.lock diff --git a/oonidata/dataviz/static/.gitignore b/oonidata/src/oonidata/dataviz/static/.gitignore similarity index 100% rename from oonidata/dataviz/static/.gitignore rename to oonidata/src/oonidata/dataviz/static/.gitignore diff --git a/oonidata/dataviz/templates/analysis.html b/oonidata/src/oonidata/dataviz/templates/analysis.html similarity index 100% rename from oonidata/dataviz/templates/analysis.html rename to oonidata/src/oonidata/dataviz/templates/analysis.html diff --git a/oonidata/dataviz/templates/react.html b/oonidata/src/oonidata/dataviz/templates/react.html similarity index 100% rename from oonidata/dataviz/templates/react.html rename to oonidata/src/oonidata/dataviz/templates/react.html diff --git a/oonidata/dataviz/templates/vega.html b/oonidata/src/oonidata/dataviz/templates/vega.html similarity index 100% rename from oonidata/dataviz/templates/vega.html rename to oonidata/src/oonidata/dataviz/templates/vega.html diff --git a/oonidata/dataviz/theme.py b/oonidata/src/oonidata/dataviz/theme.py similarity index 100% rename from oonidata/dataviz/theme.py rename to oonidata/src/oonidata/dataviz/theme.py diff --git a/oonidata/dataviz/utils.py b/oonidata/src/oonidata/dataviz/utils.py similarity index 100% rename from oonidata/dataviz/utils.py rename to oonidata/src/oonidata/dataviz/utils.py diff --git a/oonidata/dataviz/viz.py b/oonidata/src/oonidata/dataviz/viz.py similarity index 100% rename from oonidata/dataviz/viz.py rename to oonidata/src/oonidata/dataviz/viz.py diff --git a/oonidata/dataviz/web.py b/oonidata/src/oonidata/dataviz/web.py similarity index 100% rename from oonidata/dataviz/web.py rename to oonidata/src/oonidata/dataviz/web.py diff --git a/oonidata/datapipeline/workflows/__init__.py b/oonidata/src/oonidata/db/__init__.py similarity index 100% rename from oonidata/datapipeline/workflows/__init__.py rename to oonidata/src/oonidata/db/__init__.py diff --git a/oonidata/db/connections.py b/oonidata/src/oonidata/db/connections.py similarity index 100% rename from oonidata/db/connections.py rename to oonidata/src/oonidata/db/connections.py diff --git a/oonidata/db/create_tables.py b/oonidata/src/oonidata/db/create_tables.py similarity index 100% rename from oonidata/db/create_tables.py rename to oonidata/src/oonidata/db/create_tables.py diff --git a/oonidata/db/speed_estimate.py b/oonidata/src/oonidata/db/speed_estimate.py similarity index 100% rename from oonidata/db/speed_estimate.py rename to oonidata/src/oonidata/db/speed_estimate.py diff --git a/oonidata/fingerprintdb.py b/oonidata/src/oonidata/fingerprintdb.py similarity index 100% rename from oonidata/fingerprintdb.py rename to oonidata/src/oonidata/fingerprintdb.py diff --git a/oonidata/dataviz/__init__.py b/oonidata/src/oonidata/models/__init__.py similarity index 100% rename from oonidata/dataviz/__init__.py rename to oonidata/src/oonidata/models/__init__.py diff --git a/oonidata/models/analysis.py b/oonidata/src/oonidata/models/analysis.py similarity index 100% rename from oonidata/models/analysis.py rename to oonidata/src/oonidata/models/analysis.py diff --git a/oonidata/models/base_model.py b/oonidata/src/oonidata/models/base_model.py similarity index 100% rename from oonidata/models/base_model.py rename to oonidata/src/oonidata/models/base_model.py diff --git a/oonidata/models/dataformats.py b/oonidata/src/oonidata/models/dataformats.py similarity index 100% rename from oonidata/models/dataformats.py rename to oonidata/src/oonidata/models/dataformats.py diff --git a/oonidata/models/experiment_result.py b/oonidata/src/oonidata/models/experiment_result.py similarity index 100% rename from oonidata/models/experiment_result.py rename to oonidata/src/oonidata/models/experiment_result.py diff --git a/oonidata/models/nettests/__init__.py b/oonidata/src/oonidata/models/nettests/__init__.py similarity index 100% rename from oonidata/models/nettests/__init__.py rename to oonidata/src/oonidata/models/nettests/__init__.py diff --git a/oonidata/models/nettests/base_measurement.py b/oonidata/src/oonidata/models/nettests/base_measurement.py similarity index 100% rename from oonidata/models/nettests/base_measurement.py rename to oonidata/src/oonidata/models/nettests/base_measurement.py diff --git a/oonidata/models/nettests/browser_web.py b/oonidata/src/oonidata/models/nettests/browser_web.py similarity index 100% rename from oonidata/models/nettests/browser_web.py rename to oonidata/src/oonidata/models/nettests/browser_web.py diff --git a/oonidata/models/nettests/dnscheck.py b/oonidata/src/oonidata/models/nettests/dnscheck.py similarity index 100% rename from oonidata/models/nettests/dnscheck.py rename to oonidata/src/oonidata/models/nettests/dnscheck.py diff --git a/oonidata/models/nettests/facebook_messenger.py b/oonidata/src/oonidata/models/nettests/facebook_messenger.py similarity index 100% rename from oonidata/models/nettests/facebook_messenger.py rename to oonidata/src/oonidata/models/nettests/facebook_messenger.py diff --git a/oonidata/models/nettests/http_header_field_manipulation.py b/oonidata/src/oonidata/models/nettests/http_header_field_manipulation.py similarity index 100% rename from oonidata/models/nettests/http_header_field_manipulation.py rename to oonidata/src/oonidata/models/nettests/http_header_field_manipulation.py diff --git a/oonidata/models/nettests/http_invalid_request_line.py b/oonidata/src/oonidata/models/nettests/http_invalid_request_line.py similarity index 100% rename from oonidata/models/nettests/http_invalid_request_line.py rename to oonidata/src/oonidata/models/nettests/http_invalid_request_line.py diff --git a/oonidata/models/nettests/signal.py b/oonidata/src/oonidata/models/nettests/signal.py similarity index 100% rename from oonidata/models/nettests/signal.py rename to oonidata/src/oonidata/models/nettests/signal.py diff --git a/oonidata/models/nettests/stun_reachability.py b/oonidata/src/oonidata/models/nettests/stun_reachability.py similarity index 100% rename from oonidata/models/nettests/stun_reachability.py rename to oonidata/src/oonidata/models/nettests/stun_reachability.py diff --git a/oonidata/models/nettests/telegram.py b/oonidata/src/oonidata/models/nettests/telegram.py similarity index 100% rename from oonidata/models/nettests/telegram.py rename to oonidata/src/oonidata/models/nettests/telegram.py diff --git a/oonidata/models/nettests/tor.py b/oonidata/src/oonidata/models/nettests/tor.py similarity index 100% rename from oonidata/models/nettests/tor.py rename to oonidata/src/oonidata/models/nettests/tor.py diff --git a/oonidata/models/nettests/urlgetter.py b/oonidata/src/oonidata/models/nettests/urlgetter.py similarity index 100% rename from oonidata/models/nettests/urlgetter.py rename to oonidata/src/oonidata/models/nettests/urlgetter.py diff --git a/oonidata/models/nettests/web_connectivity.py b/oonidata/src/oonidata/models/nettests/web_connectivity.py similarity index 100% rename from oonidata/models/nettests/web_connectivity.py rename to oonidata/src/oonidata/models/nettests/web_connectivity.py diff --git a/oonidata/models/nettests/whatsapp.py b/oonidata/src/oonidata/models/nettests/whatsapp.py similarity index 100% rename from oonidata/models/nettests/whatsapp.py rename to oonidata/src/oonidata/models/nettests/whatsapp.py diff --git a/oonidata/models/observations.py b/oonidata/src/oonidata/models/observations.py similarity index 100% rename from oonidata/models/observations.py rename to oonidata/src/oonidata/models/observations.py diff --git a/oonidata/netinfo.py b/oonidata/src/oonidata/netinfo.py similarity index 100% rename from oonidata/netinfo.py rename to oonidata/src/oonidata/netinfo.py diff --git a/oonidata/normalize.py b/oonidata/src/oonidata/normalize.py similarity index 100% rename from oonidata/normalize.py rename to oonidata/src/oonidata/normalize.py diff --git a/oonidata/transforms/__init__.py b/oonidata/src/oonidata/transforms/__init__.py similarity index 100% rename from oonidata/transforms/__init__.py rename to oonidata/src/oonidata/transforms/__init__.py diff --git a/oonidata/db/__init__.py b/oonidata/src/oonidata/transforms/nettests/__init__.py similarity index 100% rename from oonidata/db/__init__.py rename to oonidata/src/oonidata/transforms/nettests/__init__.py diff --git a/oonidata/transforms/nettests/browser_web.py b/oonidata/src/oonidata/transforms/nettests/browser_web.py similarity index 100% rename from oonidata/transforms/nettests/browser_web.py rename to oonidata/src/oonidata/transforms/nettests/browser_web.py diff --git a/oonidata/transforms/nettests/dnscheck.py b/oonidata/src/oonidata/transforms/nettests/dnscheck.py similarity index 100% rename from oonidata/transforms/nettests/dnscheck.py rename to oonidata/src/oonidata/transforms/nettests/dnscheck.py diff --git a/oonidata/transforms/nettests/facebook_messenger.py b/oonidata/src/oonidata/transforms/nettests/facebook_messenger.py similarity index 100% rename from oonidata/transforms/nettests/facebook_messenger.py rename to oonidata/src/oonidata/transforms/nettests/facebook_messenger.py diff --git a/oonidata/transforms/nettests/http_header_field_manipulation.py b/oonidata/src/oonidata/transforms/nettests/http_header_field_manipulation.py similarity index 100% rename from oonidata/transforms/nettests/http_header_field_manipulation.py rename to oonidata/src/oonidata/transforms/nettests/http_header_field_manipulation.py diff --git a/oonidata/transforms/nettests/http_invalid_request_line.py b/oonidata/src/oonidata/transforms/nettests/http_invalid_request_line.py similarity index 100% rename from oonidata/transforms/nettests/http_invalid_request_line.py rename to oonidata/src/oonidata/transforms/nettests/http_invalid_request_line.py diff --git a/oonidata/transforms/nettests/measurement_transformer.py b/oonidata/src/oonidata/transforms/nettests/measurement_transformer.py similarity index 100% rename from oonidata/transforms/nettests/measurement_transformer.py rename to oonidata/src/oonidata/transforms/nettests/measurement_transformer.py diff --git a/oonidata/transforms/nettests/signal.py b/oonidata/src/oonidata/transforms/nettests/signal.py similarity index 100% rename from oonidata/transforms/nettests/signal.py rename to oonidata/src/oonidata/transforms/nettests/signal.py diff --git a/oonidata/transforms/nettests/stun_reachability.py b/oonidata/src/oonidata/transforms/nettests/stun_reachability.py similarity index 100% rename from oonidata/transforms/nettests/stun_reachability.py rename to oonidata/src/oonidata/transforms/nettests/stun_reachability.py diff --git a/oonidata/transforms/nettests/telegram.py b/oonidata/src/oonidata/transforms/nettests/telegram.py similarity index 100% rename from oonidata/transforms/nettests/telegram.py rename to oonidata/src/oonidata/transforms/nettests/telegram.py diff --git a/oonidata/transforms/nettests/tor.py b/oonidata/src/oonidata/transforms/nettests/tor.py similarity index 100% rename from oonidata/transforms/nettests/tor.py rename to oonidata/src/oonidata/transforms/nettests/tor.py diff --git a/oonidata/transforms/nettests/urlgetter.py b/oonidata/src/oonidata/transforms/nettests/urlgetter.py similarity index 100% rename from oonidata/transforms/nettests/urlgetter.py rename to oonidata/src/oonidata/transforms/nettests/urlgetter.py diff --git a/oonidata/transforms/nettests/web_connectivity.py b/oonidata/src/oonidata/transforms/nettests/web_connectivity.py similarity index 100% rename from oonidata/transforms/nettests/web_connectivity.py rename to oonidata/src/oonidata/transforms/nettests/web_connectivity.py diff --git a/oonidata/transforms/nettests/whatsapp.py b/oonidata/src/oonidata/transforms/nettests/whatsapp.py similarity index 100% rename from oonidata/transforms/nettests/whatsapp.py rename to oonidata/src/oonidata/transforms/nettests/whatsapp.py diff --git a/oonidata/models/__init__.py b/oonidata/src/oonidata/workers/__init__.py similarity index 100% rename from oonidata/models/__init__.py rename to oonidata/src/oonidata/workers/__init__.py diff --git a/oonidata/workers/common.py b/oonidata/src/oonidata/workers/common.py similarity index 100% rename from oonidata/workers/common.py rename to oonidata/src/oonidata/workers/common.py diff --git a/oonidata/workers/fingerprint_hunter.py b/oonidata/src/oonidata/workers/fingerprint_hunter.py similarity index 100% rename from oonidata/workers/fingerprint_hunter.py rename to oonidata/src/oonidata/workers/fingerprint_hunter.py diff --git a/oonidata/workers/response_archiver.py b/oonidata/src/oonidata/workers/response_archiver.py similarity index 100% rename from oonidata/workers/response_archiver.py rename to oonidata/src/oonidata/workers/response_archiver.py diff --git a/oonidata/transforms/nettests/__init__.py b/oonidata/tests/__init__.py similarity index 100% rename from oonidata/transforms/nettests/__init__.py rename to oonidata/tests/__init__.py diff --git a/tests/_sample_measurements.py b/oonidata/tests/_sample_measurements.py similarity index 100% rename from tests/_sample_measurements.py rename to oonidata/tests/_sample_measurements.py diff --git a/tests/conftest.py b/oonidata/tests/conftest.py similarity index 100% rename from tests/conftest.py rename to oonidata/tests/conftest.py diff --git a/oonidata/tests/data/.gitignore b/oonidata/tests/data/.gitignore new file mode 100644 index 00000000..3dd48105 --- /dev/null +++ b/oonidata/tests/data/.gitignore @@ -0,0 +1,4 @@ +/datadir/* +/raw_measurements/* +/measurements/* +/s3-dump/* \ No newline at end of file diff --git a/tests/test_analysis.py b/oonidata/tests/test_analysis.py similarity index 100% rename from tests/test_analysis.py rename to oonidata/tests/test_analysis.py diff --git a/tests/test_cli.py b/oonidata/tests/test_cli.py similarity index 100% rename from tests/test_cli.py rename to oonidata/tests/test_cli.py diff --git a/tests/test_ctrl.py b/oonidata/tests/test_ctrl.py similarity index 100% rename from tests/test_ctrl.py rename to oonidata/tests/test_ctrl.py diff --git a/tests/test_dataclient.py b/oonidata/tests/test_dataclient.py similarity index 100% rename from tests/test_dataclient.py rename to oonidata/tests/test_dataclient.py diff --git a/tests/test_dataformat.py b/oonidata/tests/test_dataformat.py similarity index 100% rename from tests/test_dataformat.py rename to oonidata/tests/test_dataformat.py diff --git a/tests/test_db.py b/oonidata/tests/test_db.py similarity index 100% rename from tests/test_db.py rename to oonidata/tests/test_db.py diff --git a/tests/test_experiment_results.py b/oonidata/tests/test_experiment_results.py similarity index 100% rename from tests/test_experiment_results.py rename to oonidata/tests/test_experiment_results.py diff --git a/tests/test_fingerprints.py b/oonidata/tests/test_fingerprints.py similarity index 100% rename from tests/test_fingerprints.py rename to oonidata/tests/test_fingerprints.py diff --git a/tests/test_netinfo.py b/oonidata/tests/test_netinfo.py similarity index 100% rename from tests/test_netinfo.py rename to oonidata/tests/test_netinfo.py diff --git a/tests/test_normalize.py b/oonidata/tests/test_normalize.py similarity index 100% rename from tests/test_normalize.py rename to oonidata/tests/test_normalize.py diff --git a/tests/test_scoring.py b/oonidata/tests/test_scoring.py similarity index 100% rename from tests/test_scoring.py rename to oonidata/tests/test_scoring.py diff --git a/tests/test_transforms.py b/oonidata/tests/test_transforms.py similarity index 100% rename from tests/test_transforms.py rename to oonidata/tests/test_transforms.py diff --git a/tests/test_workers.py b/oonidata/tests/test_workers.py similarity index 100% rename from tests/test_workers.py rename to oonidata/tests/test_workers.py diff --git a/debug-temporal.sh b/oonipipeline/debug-temporal.sh similarity index 100% rename from debug-temporal.sh rename to oonipipeline/debug-temporal.sh diff --git a/oonipipeline/pyproject.toml b/oonipipeline/pyproject.toml new file mode 100644 index 00000000..a69b2f82 --- /dev/null +++ b/oonipipeline/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "oonipipeline" +dynamic = ["version"] +description = "" +authors = [{ name = "OONI", email = "contact@ooni.org" }] + +dependencies = [ + "boto3 ~= 1.24", + "PyYAML ~= 6.0", + "tqdm ~= 4.64", + "lz4 ~= 4.0", + "requests ~= 2.27", + "cryptography ~= 38.0.3", + "clickhouse-driver ~= 0.2", + "click ~= 8.0.0", + "lxml ~= 4.9", + "maxminddb ~= 2.2", + "orjson ~= 3.8", + "mashumaro ~= 3.0", + "pyOpenSSL ~= 22.1", + "fastapi ~= 0.108.0", + "tabulate ~= 0.9.0", + "warcio ~= 1.7.4", + "msgpack ~= 1.0.4", + "statsd ~= 4.0.1", + "click-loglevel ~= 0.5.0", + "dask ~= 2023.3.2", + "altair ~= 4.2.0", + "numpy ~= 1.23.5", + "pandas ~= 2.0.0", + "flask ~= 2.2.0", + "jupyterlab ~= 4.0.7", + "temporalio ~= 1.5.1", +] + +[tool.hatch.build.targets.sdist] + +[tool.hatch.build.targets.wheel] +packages = ["src/oonipipeline", "src/oonidata"] + + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "black", + "snakeviz", + "mypy", + "pytest-benchmark", + "pytest-profiling", + "memray", + "viztracer", +] +path = ".venv/" + +[tool.hatch.version] +path = "src/oonipipeline/__about__.py" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] diff --git a/oonipipeline/src/oonidata b/oonipipeline/src/oonidata new file mode 120000 index 00000000..7e24531f --- /dev/null +++ b/oonipipeline/src/oonidata @@ -0,0 +1 @@ +../../oonidata/src/oonidata \ No newline at end of file diff --git a/oonipipeline/src/oonipipeline/__about__.py b/oonipipeline/src/oonipipeline/__about__.py new file mode 100644 index 00000000..8f3f4b21 --- /dev/null +++ b/oonipipeline/src/oonipipeline/__about__.py @@ -0,0 +1 @@ +VERSION = "4.0.0dev1" diff --git a/oonidata/workers/__init__.py b/oonipipeline/src/oonipipeline/__init__.py similarity index 100% rename from oonidata/workers/__init__.py rename to oonipipeline/src/oonipipeline/__init__.py diff --git a/oonidata/datapipeline/run_worker.py b/oonipipeline/src/oonipipeline/main.py similarity index 92% rename from oonidata/datapipeline/run_worker.py rename to oonipipeline/src/oonipipeline/main.py index e097bc2d..ed0e4cfc 100644 --- a/oonidata/datapipeline/run_worker.py +++ b/oonipipeline/src/oonipipeline/main.py @@ -14,7 +14,7 @@ async def async_main(): with concurrent.futures.ThreadPoolExecutor(max_workers=100) as activity_executor: worker = Worker( client, - task_queue="oonidatapipeline-task-queue", + task_queue="oonipipeline-task-queue", workflows=[ObservationsWorkflow], activities=[make_observation_in_day], activity_executor=activity_executor, diff --git a/tests/__init__.py b/oonipipeline/src/oonipipeline/workflows/__init__.py similarity index 100% rename from tests/__init__.py rename to oonipipeline/src/oonipipeline/workflows/__init__.py diff --git a/oonidata/datapipeline/workflows/analysis.py b/oonipipeline/src/oonipipeline/workflows/analysis.py similarity index 100% rename from oonidata/datapipeline/workflows/analysis.py rename to oonipipeline/src/oonipipeline/workflows/analysis.py diff --git a/oonidata/datapipeline/workflows/ground_truths.py b/oonipipeline/src/oonipipeline/workflows/ground_truths.py similarity index 100% rename from oonidata/datapipeline/workflows/ground_truths.py rename to oonipipeline/src/oonipipeline/workflows/ground_truths.py diff --git a/oonidata/datapipeline/workflows/observations.py b/oonipipeline/src/oonipipeline/workflows/observations.py similarity index 100% rename from oonidata/datapipeline/workflows/observations.py rename to oonipipeline/src/oonipipeline/workflows/observations.py diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index d82fbcf7..00000000 --- a/poetry.lock +++ /dev/null @@ -1,3612 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "altair" -version = "4.2.0" -description = "Altair: A declarative statistical visualization library for Python." -optional = true -python-versions = ">=3.7" -files = [ - {file = "altair-4.2.0-py3-none-any.whl", hash = "sha256:0c724848ae53410c13fa28be2b3b9a9dcb7b5caa1a70f7f217bd663bb419935a"}, - {file = "altair-4.2.0.tar.gz", hash = "sha256:d87d9372e63b48cd96b2a6415f0cf9457f50162ab79dc7a31cd7e024dd840026"}, -] - -[package.dependencies] -entrypoints = "*" -jinja2 = "*" -jsonschema = ">=3.0" -numpy = "*" -pandas = ">=0.18" -toolz = "*" - -[package.extras] -dev = ["black", "docutils", "flake8", "ipython", "m2r", "mistune (<2.0.0)", "pytest", "recommonmark", "sphinx", "vega-datasets"] - -[[package]] -name = "anyio" -version = "4.0.0" -description = "High level compatibility layer for multiple asynchronous event loop implementations" -optional = true -python-versions = ">=3.8" -files = [ - {file = "anyio-4.0.0-py3-none-any.whl", hash = "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f"}, - {file = "anyio-4.0.0.tar.gz", hash = "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} -idna = ">=2.8" -sniffio = ">=1.1" - -[package.extras] -doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (>=0.22)"] - -[[package]] -name = "appnope" -version = "0.1.3" -description = "Disable App Nap on macOS >= 10.9" -optional = true -python-versions = "*" -files = [ - {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, - {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, -] - -[[package]] -name = "argon2-cffi" -version = "23.1.0" -description = "Argon2 for Python" -optional = true -python-versions = ">=3.7" -files = [ - {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, - {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, -] - -[package.dependencies] -argon2-cffi-bindings = "*" - -[package.extras] -dev = ["argon2-cffi[tests,typing]", "tox (>4)"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] -tests = ["hypothesis", "pytest"] -typing = ["mypy"] - -[[package]] -name = "argon2-cffi-bindings" -version = "21.2.0" -description = "Low-level CFFI bindings for Argon2" -optional = true -python-versions = ">=3.6" -files = [ - {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, - {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, -] - -[package.dependencies] -cffi = ">=1.0.1" - -[package.extras] -dev = ["cogapp", "pre-commit", "pytest", "wheel"] -tests = ["pytest"] - -[[package]] -name = "arrow" -version = "1.3.0" -description = "Better dates & times for Python" -optional = true -python-versions = ">=3.8" -files = [ - {file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"}, - {file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"}, -] - -[package.dependencies] -python-dateutil = ">=2.7.0" -types-python-dateutil = ">=2.8.10" - -[package.extras] -doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] -test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] - -[[package]] -name = "asttokens" -version = "2.4.1" -description = "Annotate AST trees with source code positions" -optional = true -python-versions = "*" -files = [ - {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, - {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, -] - -[package.dependencies] -six = ">=1.12.0" - -[package.extras] -astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] -test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] - -[[package]] -name = "async-lru" -version = "2.0.4" -description = "Simple LRU cache for asyncio" -optional = true -python-versions = ">=3.8" -files = [ - {file = "async-lru-2.0.4.tar.gz", hash = "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627"}, - {file = "async_lru-2.0.4-py3-none-any.whl", hash = "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "attrs" -version = "22.1.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.5" -files = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] - -[package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] - -[[package]] -name = "babel" -version = "2.13.1" -description = "Internationalization utilities" -optional = true -python-versions = ">=3.7" -files = [ - {file = "Babel-2.13.1-py3-none-any.whl", hash = "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"}, - {file = "Babel-2.13.1.tar.gz", hash = "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900"}, -] - -[package.dependencies] -pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} -setuptools = {version = "*", markers = "python_version >= \"3.12\""} - -[package.extras] -dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] - -[[package]] -name = "backcall" -version = "0.2.0" -description = "Specifications for callback functions passed in to an API" -optional = true -python-versions = "*" -files = [ - {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, - {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, -] - -[[package]] -name = "backports.zoneinfo" -version = "0.2.1" -description = "Backport of the standard library zoneinfo module" -optional = false -python-versions = ">=3.6" -files = [ - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win32.whl", hash = "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win32.whl", hash = "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-win32.whl", hash = "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"}, - {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"}, -] - -[package.extras] -tzdata = ["tzdata"] - -[[package]] -name = "beautifulsoup4" -version = "4.12.2" -description = "Screen-scraping library" -optional = true -python-versions = ">=3.6.0" -files = [ - {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, - {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, -] - -[package.dependencies] -soupsieve = ">1.2" - -[package.extras] -html5lib = ["html5lib"] -lxml = ["lxml"] - -[[package]] -name = "black" -version = "22.10.0" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.7" -files = [ - {file = "black-22.10.0-1fixedarch-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa"}, - {file = "black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef"}, - {file = "black-22.10.0-1fixedarch-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6"}, - {file = "black-22.10.0-1fixedarch-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d"}, - {file = "black-22.10.0-1fixedarch-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4"}, - {file = "black-22.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb"}, - {file = "black-22.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7"}, - {file = "black-22.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66"}, - {file = "black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae"}, - {file = "black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b"}, - {file = "black-22.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d"}, - {file = "black-22.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650"}, - {file = "black-22.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d"}, - {file = "black-22.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"}, - {file = "black-22.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87"}, - {file = "black-22.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395"}, - {file = "black-22.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0"}, - {file = "black-22.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383"}, - {file = "black-22.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de"}, - {file = "black-22.10.0-py3-none-any.whl", hash = "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458"}, - {file = "black-22.10.0.tar.gz", hash = "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "bleach" -version = "6.1.0" -description = "An easy safelist-based HTML-sanitizing tool." -optional = true -python-versions = ">=3.8" -files = [ - {file = "bleach-6.1.0-py3-none-any.whl", hash = "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"}, - {file = "bleach-6.1.0.tar.gz", hash = "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe"}, -] - -[package.dependencies] -six = ">=1.9.0" -webencodings = "*" - -[package.extras] -css = ["tinycss2 (>=1.1.0,<1.3)"] - -[[package]] -name = "bokeh" -version = "2.4.3" -description = "Interactive plots and applications in the browser from Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "bokeh-2.4.3-py3-none-any.whl", hash = "sha256:104d2f0a4ca7774ee4b11e545aa34ff76bf3e2ad6de0d33944361981b65da420"}, - {file = "bokeh-2.4.3.tar.gz", hash = "sha256:ef33801161af379665ab7a34684f2209861e3aefd5c803a21fbbb99d94874b03"}, -] - -[package.dependencies] -Jinja2 = ">=2.9" -numpy = ">=1.11.3" -packaging = ">=16.8" -pillow = ">=7.1.0" -PyYAML = ">=3.10" -tornado = ">=5.1" -typing-extensions = ">=3.10.0" - -[[package]] -name = "boto3" -version = "1.26.20" -description = "The AWS SDK for Python" -optional = false -python-versions = ">= 3.7" -files = [ - {file = "boto3-1.26.20-py3-none-any.whl", hash = "sha256:6b2e04ea415f375f607e6c633f7699e6a56bdc35c294116c5a6dc02da6aaa2ba"}, - {file = "boto3-1.26.20.tar.gz", hash = "sha256:0d61243999adaddee22855ada1fdf7fe3d89cc95821e82f3b3a9dbe56c1589ba"}, -] - -[package.dependencies] -botocore = ">=1.29.20,<1.30.0" -jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.6.0,<0.7.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.29.20" -description = "Low-level, data-driven core of boto 3." -optional = false -python-versions = ">= 3.7" -files = [ - {file = "botocore-1.29.20-py3-none-any.whl", hash = "sha256:7c26cb870d0cae26349df2926a7bf6277c1326db6d42c650807a519d50a83786"}, - {file = "botocore-1.29.20.tar.gz", hash = "sha256:806dab6358b0b44d7b283f133aadd26846f31fab12c97d348a1849b3b5a36c74"}, -] - -[package.dependencies] -jmespath = ">=0.7.1,<2.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<1.27" - -[package.extras] -crt = ["awscrt (==0.15.3)"] - -[[package]] -name = "certifi" -version = "2022.9.24" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, -] - -[[package]] -name = "cffi" -version = "1.15.1" -description = "Foreign Function Interface for Python calling C code." -optional = false -python-versions = "*" -files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] - -[package.dependencies] -pycparser = "*" - -[[package]] -name = "charset-normalizer" -version = "2.1.1" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.6.0" -files = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] - -[package.extras] -unicode-backport = ["unicodedata2"] - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "click-loglevel" -version = "0.5.0" -description = "Log level parameter type for Click" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-loglevel-0.5.0.tar.gz", hash = "sha256:fcc98a136a96479b4768494df25017114ba1cb525dac0bed619209b3578fd4f3"}, - {file = "click_loglevel-0.5.0-py3-none-any.whl", hash = "sha256:897070fd4bf5b503edb5a1ecc0551448647901f4fac83a01c9f00aa28ad86d60"}, -] - -[package.dependencies] -click = ">=8.0" - -[[package]] -name = "clickhouse-driver" -version = "0.2.5" -description = "Python driver with native interface for ClickHouse" -optional = false -python-versions = ">=3.6, <4" -files = [ - {file = "clickhouse-driver-0.2.5.tar.gz", hash = "sha256:403239e5820d07d35a380f1abcd45ea0397fe8059984abf0594783d1474d56f5"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:413c7116a0f35bfc12c22f616b4cc9673d74b293c90361c748be03a987c753c4"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc7d48fc72ddb5a24c59b521c7ff56e7bcc8520a9b4afe50d7efaec1aa4ae616"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5fbaef87d62bdf932c6d2f86c3a77d4675d309b9dd8c579b94776ec3da044bdf"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda807b08ed23c6388c734486dcd218adefe78cac298f08dbc3a62d27d20462a"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2eb78e361ff0f4f45b694e2d08e58c00329ee512471b23f53a23815dc34660"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c70a2f0c8bd8165e63788aa64b583a2c92ff73ca0a07b60ef7980250cee24a3d"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:77c541961f29f17880bece8c5cf22f52564579c086fe859568189ccb36cef7cb"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c0f5567e935b09ac6836e2a015306a6233538111be2a1067997fd7fdb865bed"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:36077623489c1558495cecfb682a777d33cb3f90b0d1e61558fe323d39cd736b"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ff24c5425b926eb18e434aa977f2ac1dfa6f907245fc1989633c07b67973843c"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d4fa637d702aca6993b5b03c20c910743e2a2e6a38661b8e9449b898d836b049"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-win32.whl", hash = "sha256:319c4ac6e063ca6786ca25aac86685841523d421f5483dc890589d221d36d562"}, - {file = "clickhouse_driver-0.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:f0cb4b9b1dd316611e4de858eb82a4f70fd1807cce9919aeed5b7bfd320ca3dc"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4605f2f197c07c6db91b9636d29e50322f095b006b6aca0f5f958bbbfe3a4381"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc5543eb45e556a45ed57328446971ad3f786f8230fbb96d736ad51ca6a0f085"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ab396a5ad5c54d463bac1b268373b7977d076c3c7ec474fc5fc908629b7e989"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:758c3d0c09d6c23f9014045aeadf3a2620c3763fc0b2875a1a8e213092afb92b"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe9781724eaf2a0cd85a1f1b2906867d548399f4ecc6b8b7460f20410eb036b5"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e8aa79ddfb67ae2b6bf844d1493ab83321c4f2052686c6aa93f6658c737bd12"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8092518f839e01edbbe74db4c3d902028f9f4af5410571a131dac40f190edd4"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:019da690c9b82034ff7ebe2a2ac2af87b715a2c010f9a65fefc08c4bd792b0d6"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:fc300a72f0a09c2c286a55723466315a1b307f94a2ed81d1b7802598cd451059"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:937b663c400233d5ddd8b50fc260c3e374b7803dd401cb194ff64ddce9825aac"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e577b20814021e6851a7a3c075b1b4498c07ee5c1863481a6a1558a85739e145"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-win32.whl", hash = "sha256:d0d8f06a9264e1279c0aefc5dd7a6c6cdc4ce2d3aac3565865250db92694c9e7"}, - {file = "clickhouse_driver-0.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:3a0741b32cf2ed019aba7c65e198d33a74d91b4ca67d5918860b7a32b7a5c158"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1edf78ff50587bde5775bfa98a17764d9fa2588e7841d63fadfca169277438d8"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e225d9261fbd35024b87b83c9d703331247d64583aedeb1210a700058422548e"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5db4eccf1f9ff685b6a92b3755cfea612f58868c513344f71927697e93e7103"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b4fe92b120faae255354d6cf4a45639236583add948401e2b72600be793a1a"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85e1b9618dbd0772497556e7b6dcf8063191c0e2ea9d7ba8e447e70aea0a88c8"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02c540813d644f1de73b0c033c10671b6c2ce97b46a233ac1936d71cc527d48f"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:d9d27f28fe2d3ca3e9ef995466b44ee7863de61a671e1e305853e235c4f4b36e"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:ee77062908490d111f65cc795cad9db2d3952630905f44ad17cb0b71374cffad"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:2a3e9d7dc9e58c7fee174516d00ad00b81fa6e4da56461d8ac206c28a16ad44c"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:d73c9dbc146d1456ea26c87f8b245f7ca69e5bc1883521fbc9074c3dd74a439a"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:14ce9ea8e4270307563bb95a1e39d8e9d6130cda926b33abbfb17d826b23acf0"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-win32.whl", hash = "sha256:d1cc49e8579b22d7fd2a8550b2da294bc21cc00189fa71ecf46b2f939011654c"}, - {file = "clickhouse_driver-0.2.5-cp36-cp36m-win_amd64.whl", hash = "sha256:e6693c1ee9c6c248fe31db716f4f0bdeff84ea8f1b4262c4d941f58d4a3a58d5"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e1d7b6d22dd5efd771aa925a586185a51a64eeb908d61d5bfdcbb1416608d89"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20c8eb194897c7826d82aa67ac2f92496214cdac3a3713df5c48d94fff494593"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0631980b28267f1137b08e6982f244597c0809478edeaa9d6b8046f38e1369ed"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86c4b3bf77d3f81394b05dc4795485a95076bbe85708d14de261d7f961b0eff5"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:704a68cec8ce9997d2a8d43678852cbcc734b77fbe8c9b4b6c3aee9e09fed9ee"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8884f12911202c7b22502d1224cffb84559c91f93bcb1deb6a28251f7a82e98e"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:45852b9c90fa75a70c2a98556ad0a031a2a1c40e5bffe6dcf252c8cbace1f2b4"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:780843107ad370fdf1742aa68196158e395345ef72bb10f12a63ab82ea6f823d"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f8ceb503bacd0234d28c2e99ad61ca0d64d3282d8531c3607a3d7a849aedaf01"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:fd9116181a645c9393e5ef9287fc8747ce25ce5a1d8d8cf079223de8bdb00236"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:efcb67627233f377f6057388b88b9f68a98f06588da24d073166fed9b5e4e875"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-win32.whl", hash = "sha256:6876827eaf7b1c9711a54a1241cc7be9417ba4885c89e6dd49205417e870a261"}, - {file = "clickhouse_driver-0.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:82835a6dca0016ac50d0b29eb02e9609f87f99af83e99fb0c5f15f893404c8bf"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e577d3e6d83cf71cf3dd29f5914448eb4d37adb61d9585a06642bd21c0b4bf6a"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e95ad1e7d8fa42b79b44a8fad5db1a92c6e09534f1a72209e0e6a00fdf2c148"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a61ec8e789f47933efb88c2d34af27e664a15f064d53411bfa2a56a34c1c8c39"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf671bac0d8f5e83736547c5ee3935ec859a77d5ce75cdfbbb5313e87cb3cb0b"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44f01d4a257b928e5981b8a0a0193c28a48ce3aed8b06f7a32f717ad2224836d"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bf7aad1d2b2c22dbccfaebcca55469558b0a754911b25e3a8506c4cc15cf01"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0e3762a866321af6a7ece560ed89c072e7bd6381fe642343e9d47c47ff191e23"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9f152bfe68af34468280f780b622a0c5dc0a5f19fd352542bf3491be4654cc13"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9954ca8b56e81f077d3f3845f88fabbf223eb4260b09691c3d957db7ff31f0d5"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5b011ee7340a73fda603e009f5f5463aabb4584d92aa93e96969908e9befb967"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ef2545823cfff38a61255651373b4705946accac38b5ffe817caaf9147a42ec4"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-win32.whl", hash = "sha256:0d1f4545e0de52774b512d943783529109b088b8716f16c75072a8af7b11ce22"}, - {file = "clickhouse_driver-0.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:420b8be690e678c86ba57e446c98716598dc29a128759cc50bb1a6dc75e6b619"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b75463226330fd5d37bb2590f47cfa43c6694efffe516d598bd663b81a1d09dc"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f40176af385fc8d69e48a1e66d79349b0cdc05aeb83a45bc655d3fa5a7c6c36"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e2a0e059b5b2886c2ada313799dafddab5052f3f2209f0429cc738c3d7e55f8"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd78c78de4a74ac9fc3bcce3916b735539d34caaefb3c0fdcb39fea763078367"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70a70e46df56d966aed8423d5870a6b7f6137e20bb8ec2572723efb4bd1c80ca"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78062269ae40b9ba2d2f5bb4318c705d5b9543d268982f4fbf39f80f1325207c"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ba975dbc89444ac69075007a97dfae5c7d51cd807372259e0051c723cc75fd21"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e61f72a6615477f3577714f5ba657be649e9c348a2d20864d16eb41b4446882d"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0254a800731469c00fa46d8f25cfab565703d07ae34ebe9bceb64ebfb0d8e99b"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:115abd0113e78dcc618b219c5539942792d121070b40f4e4b4bdbc03a23f6630"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c7311c46274f480f2bdc19d42151054eae8e3b9045624d4ca3182afd2fb7c5df"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-win32.whl", hash = "sha256:852b2d57890db87248c60f05809df3ce8a3737fc37b3908b23b6a914212cc2b9"}, - {file = "clickhouse_driver-0.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:55f989eba98bfcf28da1dcbf4b7bb8322cd4a98df7855a68d4090160da2058a1"}, - {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5f8585d6e1023779da471d37a3ea8623017f90115a85fef51d49a0c619010b44"}, - {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59c07f648edbff11fdb5fc57f5d56f585acc0381bfda2cbbbc2ea04f6adf1767"}, - {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd8e223a434fb7bffdd10e4541a3f756b58d15eac5e6bb701d0ad37719b20298"}, - {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d092c89725c1c616ba9c1624a1ea5abbfc13d671ef2197b6c075b2d051f8bba2"}, - {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f4587b47b8651f7c0576cfb374f318c758c420883fe8fbb97b2a6503ce525150"}, - {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6083afaa4bb2691e42c588e635cc7bdfae3060d0a627e5a49f6bda2cb7bb3468"}, - {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:585decb371c6b5cd3e9607c703925c49e96e7b872bbd676c71a0acf85397eeef"}, - {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08aa10e63a911513fbe4ea7168a9fe2c55ce47693094e0e49d3db98ab1da9b1d"}, - {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81c639203ae0c0491af6794dd88a7ee0329298cc76389e5475c89df732a649b4"}, - {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5afb33e39d2ac0765e6e5014d57056bf1a783b7938a6d231eeeb049fe1967c27"}, - {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:378d4a641842b891387d9a577eccbecdc0ac3dbafc29f7a6fa273cab2e3591f4"}, - {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6321f328d79c1986706db57109ec80ac7c78b3782f261509e5441ba0ff8fce18"}, - {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7ec4a6a8aba19c5b5945ad10ac45224a1203d00939d6382105f69c6367599ed"}, - {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac0d8c2f1d54fcca45393d6dbe40f0b11500ee07471671345361ca259ffd726"}, - {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f380c0c7c598c22ed7c9ee0f6ecc133af959ba359999b09ec42dbbc97e7482dc"}, -] - -[package.dependencies] -pytz = "*" -tzlocal = "*" - -[package.extras] -lz4 = ["clickhouse-cityhash (>=1.0.2.1)", "lz4", "lz4 (<=3.0.1)"] -numpy = ["numpy (>=1.12.0)", "pandas (>=0.24.0)"] -zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] - -[[package]] -name = "cloudpickle" -version = "2.2.1" -description = "Extended pickling support for Python objects" -optional = false -python-versions = ">=3.6" -files = [ - {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, - {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "comm" -version = "0.1.4" -description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." -optional = true -python-versions = ">=3.6" -files = [ - {file = "comm-0.1.4-py3-none-any.whl", hash = "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a"}, - {file = "comm-0.1.4.tar.gz", hash = "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15"}, -] - -[package.dependencies] -traitlets = ">=4" - -[package.extras] -lint = ["black (>=22.6.0)", "mdformat (>0.7)", "mdformat-gfm (>=0.3.5)", "ruff (>=0.0.156)"] -test = ["pytest"] -typing = ["mypy (>=0.990)"] - -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -optional = false -python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - -[[package]] -name = "coverage" -version = "6.5.0" -description = "Code coverage measurement for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, - {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, - {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, - {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, - {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, - {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, - {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, - {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, - {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, - {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, - {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, - {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, - {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, - {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, - {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, - {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, -] - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cryptography" -version = "38.0.4" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -optional = false -python-versions = ">=3.6" -files = [ - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, - {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, - {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, - {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, -] - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] - -[[package]] -name = "dask" -version = "2023.3.2" -description = "Parallel PyData with Task Scheduling" -optional = false -python-versions = ">=3.8" -files = [ - {file = "dask-2023.3.2-py3-none-any.whl", hash = "sha256:5e64763d62feb18afd3ad66f364e0b4f456f7ac92e894fcc87950af75029ecdf"}, - {file = "dask-2023.3.2.tar.gz", hash = "sha256:51009e92ba9a280bd417633d1ae84f3ed23a8940f0a19594a4b7797ef226fff4"}, -] - -[package.dependencies] -bokeh = {version = ">=2.4.2,<3", optional = true, markers = "extra == \"complete\""} -click = ">=7.0" -cloudpickle = ">=1.1.1" -distributed = {version = "2023.3.2", optional = true, markers = "extra == \"complete\""} -fsspec = ">=0.6.0" -importlib-metadata = ">=4.13.0" -jinja2 = {version = ">=2.10.3", optional = true, markers = "extra == \"complete\""} -lz4 = {version = ">=4.3.2", optional = true, markers = "extra == \"complete\""} -numpy = {version = ">=1.21", optional = true, markers = "extra == \"complete\""} -packaging = ">=20.0" -pandas = {version = ">=1.3", optional = true, markers = "extra == \"complete\""} -partd = ">=1.2.0" -pyarrow = {version = ">=7.0", optional = true, markers = "extra == \"complete\""} -pyyaml = ">=5.3.1" -toolz = ">=0.8.2" - -[package.extras] -array = ["numpy (>=1.21)"] -complete = ["bokeh (>=2.4.2,<3)", "distributed (==2023.3.2)", "jinja2 (>=2.10.3)", "lz4 (>=4.3.2)", "numpy (>=1.21)", "pandas (>=1.3)", "pyarrow (>=7.0)"] -dataframe = ["numpy (>=1.21)", "pandas (>=1.3)"] -diagnostics = ["bokeh (>=2.4.2,<3)", "jinja2 (>=2.10.3)"] -distributed = ["distributed (==2023.3.2)"] -test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest-xdist"] - -[[package]] -name = "debugpy" -version = "1.8.0" -description = "An implementation of the Debug Adapter Protocol for Python" -optional = true -python-versions = ">=3.8" -files = [ - {file = "debugpy-1.8.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb"}, - {file = "debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"}, - {file = "debugpy-1.8.0-cp310-cp310-win32.whl", hash = "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f"}, - {file = "debugpy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637"}, - {file = "debugpy-1.8.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e"}, - {file = "debugpy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6"}, - {file = "debugpy-1.8.0-cp311-cp311-win32.whl", hash = "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b"}, - {file = "debugpy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153"}, - {file = "debugpy-1.8.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd"}, - {file = "debugpy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f"}, - {file = "debugpy-1.8.0-cp38-cp38-win32.whl", hash = "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa"}, - {file = "debugpy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595"}, - {file = "debugpy-1.8.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8"}, - {file = "debugpy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332"}, - {file = "debugpy-1.8.0-cp39-cp39-win32.whl", hash = "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6"}, - {file = "debugpy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926"}, - {file = "debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4"}, - {file = "debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0"}, -] - -[[package]] -name = "decorator" -version = "5.1.1" -description = "Decorators for Humans" -optional = true -python-versions = ">=3.5" -files = [ - {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, - {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, -] - -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - -[[package]] -name = "distributed" -version = "2023.3.2" -description = "Distributed scheduler for Dask" -optional = false -python-versions = ">=3.8" -files = [ - {file = "distributed-2023.3.2-py3-none-any.whl", hash = "sha256:43b74f08ea4ea0696ae8d2f1ea53adcda998a2a159bd34dffd42e4e678ff5f8a"}, - {file = "distributed-2023.3.2.tar.gz", hash = "sha256:3f0b4ce73289583a27376672f67895e041c71d9a6e901b4f4cc4a95dd81ec349"}, -] - -[package.dependencies] -click = ">=8.0" -cloudpickle = ">=1.5.0" -dask = "2023.3.2" -jinja2 = ">=2.10.3" -locket = ">=1.0.0" -msgpack = ">=1.0.0" -packaging = ">=20.0" -psutil = ">=5.7.0" -pyyaml = ">=5.3.1" -sortedcontainers = ">=2.0.5" -tblib = ">=1.6.0" -toolz = ">=0.10.0" -tornado = ">=6.0.3" -urllib3 = ">=1.24.3" -zict = ">=2.1.0" - -[[package]] -name = "entrypoints" -version = "0.4" -description = "Discover and load entry points from installed packages." -optional = true -python-versions = ">=3.6" -files = [ - {file = "entrypoints-0.4-py3-none-any.whl", hash = "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"}, - {file = "entrypoints-0.4.tar.gz", hash = "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.0.4" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "executing" -version = "2.0.1" -description = "Get the currently executing AST node of a frame, and other information" -optional = true -python-versions = ">=3.5" -files = [ - {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, - {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, -] - -[package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] - -[[package]] -name = "fastjsonschema" -version = "2.18.1" -description = "Fastest Python implementation of JSON schema" -optional = true -python-versions = "*" -files = [ - {file = "fastjsonschema-2.18.1-py3-none-any.whl", hash = "sha256:aec6a19e9f66e9810ab371cc913ad5f4e9e479b63a7072a2cd060a9369e329a8"}, - {file = "fastjsonschema-2.18.1.tar.gz", hash = "sha256:06dc8680d937628e993fa0cd278f196d20449a1adc087640710846b324d422ea"}, -] - -[package.extras] -devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] - -[[package]] -name = "flask" -version = "2.2.2" -description = "A simple framework for building complex web applications." -optional = true -python-versions = ">=3.7" -files = [ - {file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"}, - {file = "Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b"}, -] - -[package.dependencies] -click = ">=8.0" -importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} -itsdangerous = ">=2.0" -Jinja2 = ">=3.0" -Werkzeug = ">=2.2.2" - -[package.extras] -async = ["asgiref (>=3.2)"] -dotenv = ["python-dotenv"] - -[[package]] -name = "fqdn" -version = "1.5.1" -description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" -optional = true -python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" -files = [ - {file = "fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"}, - {file = "fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f"}, -] - -[[package]] -name = "fsspec" -version = "2023.4.0" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -files = [ - {file = "fsspec-2023.4.0-py3-none-any.whl", hash = "sha256:f398de9b49b14e9d84d2c2d11b7b67121bc072fe97b930c4e5668ac3917d8307"}, - {file = "fsspec-2023.4.0.tar.gz", hash = "sha256:bf064186cd8808f0b2f6517273339ba0a0c8fb1b7048991c28bc67f58b8b67cd"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "gprof2dot" -version = "2022.7.29" -description = "Generate a dot graph from the output of several profilers." -optional = false -python-versions = ">=2.7" -files = [ - {file = "gprof2dot-2022.7.29-py2.py3-none-any.whl", hash = "sha256:f165b3851d3c52ee4915eb1bd6cca571e5759823c2cd0f71a79bda93c2dc85d6"}, - {file = "gprof2dot-2022.7.29.tar.gz", hash = "sha256:45b4d298bd36608fccf9511c3fd88a773f7a1abc04d6cd39445b11ba43133ec5"}, -] - -[[package]] -name = "heapdict" -version = "1.0.1" -description = "a heap with decrease-key and increase-key operations" -optional = false -python-versions = "*" -files = [ - {file = "HeapDict-1.0.1-py3-none-any.whl", hash = "sha256:6065f90933ab1bb7e50db403b90cab653c853690c5992e69294c2de2b253fc92"}, - {file = "HeapDict-1.0.1.tar.gz", hash = "sha256:8495f57b3e03d8e46d5f1b2cc62ca881aca392fd5cc048dc0aa2e1a6d23ecdb6"}, -] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] - -[[package]] -name = "importlib-metadata" -version = "5.1.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, - {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "importlib-resources" -version = "5.10.1" -description = "Read resources from Python packages" -optional = true -python-versions = ">=3.7" -files = [ - {file = "importlib_resources-5.10.1-py3-none-any.whl", hash = "sha256:c09b067d82e72c66f4f8eb12332f5efbebc9b007c0b6c40818108c9870adc363"}, - {file = "importlib_resources-5.10.1.tar.gz", hash = "sha256:32bb095bda29741f6ef0e5278c42df98d135391bee5f932841efc0041f748dc3"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -optional = false -python-versions = "*" -files = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] - -[[package]] -name = "ipykernel" -version = "6.26.0" -description = "IPython Kernel for Jupyter" -optional = true -python-versions = ">=3.8" -files = [ - {file = "ipykernel-6.26.0-py3-none-any.whl", hash = "sha256:3ba3dc97424b87b31bb46586b5167b3161b32d7820b9201a9e698c71e271602c"}, - {file = "ipykernel-6.26.0.tar.gz", hash = "sha256:553856658eb8430bbe9653ea041a41bff63e9606fc4628873fc92a6cf3abd404"}, -] - -[package.dependencies] -appnope = {version = "*", markers = "platform_system == \"Darwin\""} -comm = ">=0.1.1" -debugpy = ">=1.6.5" -ipython = ">=7.23.1" -jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -matplotlib-inline = ">=0.1" -nest-asyncio = "*" -packaging = "*" -psutil = "*" -pyzmq = ">=20" -tornado = ">=6.1" -traitlets = ">=5.4.0" - -[package.extras] -cov = ["coverage[toml]", "curio", "matplotlib", "pytest-cov", "trio"] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "trio"] -pyqt5 = ["pyqt5"] -pyside6 = ["pyside6"] -test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "ipython" -version = "8.12.3" -description = "IPython: Productive Interactive Computing" -optional = true -python-versions = ">=3.8" -files = [ - {file = "ipython-8.12.3-py3-none-any.whl", hash = "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c"}, - {file = "ipython-8.12.3.tar.gz", hash = "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363"}, -] - -[package.dependencies] -appnope = {version = "*", markers = "sys_platform == \"darwin\""} -backcall = "*" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -decorator = "*" -jedi = ">=0.16" -matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} -pickleshare = "*" -prompt-toolkit = ">=3.0.30,<3.0.37 || >3.0.37,<3.1.0" -pygments = ">=2.4.0" -stack-data = "*" -traitlets = ">=5" -typing-extensions = {version = "*", markers = "python_version < \"3.10\""} - -[package.extras] -all = ["black", "curio", "docrepr", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.21)", "pandas", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] -black = ["black"] -doc = ["docrepr", "ipykernel", "matplotlib", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["ipywidgets", "notebook"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] - -[[package]] -name = "isoduration" -version = "20.11.0" -description = "Operations with ISO 8601 durations" -optional = true -python-versions = ">=3.7" -files = [ - {file = "isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"}, - {file = "isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9"}, -] - -[package.dependencies] -arrow = ">=0.15.0" - -[[package]] -name = "itsdangerous" -version = "2.1.2" -description = "Safely pass data to untrusted environments and back." -optional = true -python-versions = ">=3.7" -files = [ - {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, - {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, -] - -[[package]] -name = "jedi" -version = "0.19.1" -description = "An autocompletion tool for Python that can be used for text editors." -optional = true -python-versions = ">=3.6" -files = [ - {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, - {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, -] - -[package.dependencies] -parso = ">=0.8.3,<0.9.0" - -[package.extras] -docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -optional = false -python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] - -[[package]] -name = "json5" -version = "0.9.14" -description = "A Python implementation of the JSON5 data format." -optional = true -python-versions = "*" -files = [ - {file = "json5-0.9.14-py2.py3-none-any.whl", hash = "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f"}, - {file = "json5-0.9.14.tar.gz", hash = "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"}, -] - -[package.extras] -dev = ["hypothesis"] - -[[package]] -name = "jsonpointer" -version = "2.4" -description = "Identify specific nodes in a JSON document (RFC 6901)" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" -files = [ - {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, -] - -[[package]] -name = "jsonschema" -version = "4.17.3" -description = "An implementation of JSON Schema validation for Python" -optional = true -python-versions = ">=3.7" -files = [ - {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, - {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, -] - -[package.dependencies] -attrs = ">=17.4.0" -fqdn = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -idna = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} -isoduration = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format-nongpl\""} -pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} -pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" -rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} -uri-template = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} -webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format-nongpl\""} - -[package.extras] -format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] - -[[package]] -name = "jupyter-client" -version = "8.5.0" -description = "Jupyter protocol implementation and client libraries" -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyter_client-8.5.0-py3-none-any.whl", hash = "sha256:c3877aac7257ec68d79b5c622ce986bd2a992ca42f6ddc9b4dd1da50e89f7028"}, - {file = "jupyter_client-8.5.0.tar.gz", hash = "sha256:e8754066510ce456358df363f97eae64b50860f30dc1fe8c6771440db3be9a63"}, -] - -[package.dependencies] -importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -python-dateutil = ">=2.8.2" -pyzmq = ">=23.0" -tornado = ">=6.2" -traitlets = ">=5.3" - -[package.extras] -docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] - -[[package]] -name = "jupyter-core" -version = "5.5.0" -description = "Jupyter core package. A base package on which Jupyter projects rely." -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyter_core-5.5.0-py3-none-any.whl", hash = "sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805"}, - {file = "jupyter_core-5.5.0.tar.gz", hash = "sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3"}, -] - -[package.dependencies] -platformdirs = ">=2.5" -pywin32 = {version = ">=300", markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} -traitlets = ">=5.3" - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] -test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "jupyter-events" -version = "0.6.3" -description = "Jupyter Event System library" -optional = true -python-versions = ">=3.7" -files = [ - {file = "jupyter_events-0.6.3-py3-none-any.whl", hash = "sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17"}, - {file = "jupyter_events-0.6.3.tar.gz", hash = "sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3"}, -] - -[package.dependencies] -jsonschema = {version = ">=3.2.0", extras = ["format-nongpl"]} -python-json-logger = ">=2.0.4" -pyyaml = ">=5.3" -rfc3339-validator = "*" -rfc3986-validator = ">=0.1.1" -traitlets = ">=5.3" - -[package.extras] -cli = ["click", "rich"] -docs = ["jupyterlite-sphinx", "myst-parser", "pydata-sphinx-theme", "sphinxcontrib-spelling"] -test = ["click", "coverage", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>=0.19.0)", "pytest-console-scripts", "pytest-cov", "rich"] - -[[package]] -name = "jupyter-lsp" -version = "2.2.0" -description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyter-lsp-2.2.0.tar.gz", hash = "sha256:8ebbcb533adb41e5d635eb8fe82956b0aafbf0fd443b6c4bfa906edeeb8635a1"}, - {file = "jupyter_lsp-2.2.0-py3-none-any.whl", hash = "sha256:9e06b8b4f7dd50300b70dd1a78c0c3b0c3d8fa68e0f2d8a5d1fbab62072aca3f"}, -] - -[package.dependencies] -importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-server = ">=1.1.2" - -[[package]] -name = "jupyter-server" -version = "2.9.1" -description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyter_server-2.9.1-py3-none-any.whl", hash = "sha256:21ad1a3d455d5a79ce4bef5201925cd17510c17898cf9d54e3ccfb6b12734948"}, - {file = "jupyter_server-2.9.1.tar.gz", hash = "sha256:9ba71be4b9c16e479e4c50c929f8ac4b1015baf90237a08681397a98c76c7e5e"}, -] - -[package.dependencies] -anyio = ">=3.1.0" -argon2-cffi = "*" -jinja2 = "*" -jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -jupyter-events = ">=0.6.0" -jupyter-server-terminals = "*" -nbconvert = ">=6.4.4" -nbformat = ">=5.3.0" -overrides = "*" -packaging = "*" -prometheus-client = "*" -pywinpty = {version = "*", markers = "os_name == \"nt\""} -pyzmq = ">=24" -send2trash = ">=1.8.2" -terminado = ">=0.8.3" -tornado = ">=6.2.0" -traitlets = ">=5.6.0" -websocket-client = "*" - -[package.extras] -docs = ["ipykernel", "jinja2", "jupyter-client", "jupyter-server", "myst-parser", "nbformat", "prometheus-client", "pydata-sphinx-theme", "send2trash", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-openapi (>=0.8.0)", "sphinxcontrib-spelling", "sphinxemoji", "tornado", "typing-extensions"] -test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-scripts", "pytest-jupyter[server] (>=0.4)", "pytest-timeout", "requests"] - -[[package]] -name = "jupyter-server-terminals" -version = "0.4.4" -description = "A Jupyter Server Extension Providing Terminals." -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyter_server_terminals-0.4.4-py3-none-any.whl", hash = "sha256:75779164661cec02a8758a5311e18bb8eb70c4e86c6b699403100f1585a12a36"}, - {file = "jupyter_server_terminals-0.4.4.tar.gz", hash = "sha256:57ab779797c25a7ba68e97bcfb5d7740f2b5e8a83b5e8102b10438041a7eac5d"}, -] - -[package.dependencies] -pywinpty = {version = ">=2.0.3", markers = "os_name == \"nt\""} -terminado = ">=0.8.3" - -[package.extras] -docs = ["jinja2", "jupyter-server", "mistune (<3.0)", "myst-parser", "nbformat", "packaging", "pydata-sphinx-theme", "sphinxcontrib-github-alt", "sphinxcontrib-openapi", "sphinxcontrib-spelling", "sphinxemoji", "tornado"] -test = ["coverage", "jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-cov", "pytest-jupyter[server] (>=0.5.3)", "pytest-timeout"] - -[[package]] -name = "jupyterlab" -version = "4.0.7" -description = "JupyterLab computational environment" -optional = true -python-versions = ">=3.8" -files = [ - {file = "jupyterlab-4.0.7-py3-none-any.whl", hash = "sha256:08683045117cc495531fdb39c22ababb9aaac6977a45e67cfad20046564c9c7c"}, - {file = "jupyterlab-4.0.7.tar.gz", hash = "sha256:48792efd9f962b2bcda1f87d72168ff122c288b1d97d32109e4a11b33dc862be"}, -] - -[package.dependencies] -async-lru = ">=1.0.0" -importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -importlib-resources = {version = ">=1.4", markers = "python_version < \"3.9\""} -ipykernel = "*" -jinja2 = ">=3.0.3" -jupyter-core = "*" -jupyter-lsp = ">=2.0.0" -jupyter-server = ">=2.4.0,<3" -jupyterlab-server = ">=2.19.0,<3" -notebook-shim = ">=0.2" -packaging = "*" -tomli = {version = "*", markers = "python_version < \"3.11\""} -tornado = ">=6.2.0" -traitlets = "*" - -[package.extras] -dev = ["black[jupyter] (==23.7.0)", "build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov", "ruff (==0.0.286)"] -docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", "pytest-check-links", "pytest-tornasync", "sphinx (>=1.8,<7.2.0)", "sphinx-copybutton"] -docs-screenshots = ["altair (==5.0.1)", "ipython (==8.14.0)", "ipywidgets (==8.0.6)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.0.post0)", "matplotlib (==3.7.1)", "nbconvert (>=7.0.0)", "pandas (==2.0.2)", "scipy (==1.10.1)", "vega-datasets (==0.9.0)"] -test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"] - -[[package]] -name = "jupyterlab-pygments" -version = "0.2.2" -description = "Pygments theme using JupyterLab CSS variables" -optional = true -python-versions = ">=3.7" -files = [ - {file = "jupyterlab_pygments-0.2.2-py2.py3-none-any.whl", hash = "sha256:2405800db07c9f770863bcf8049a529c3dd4d3e28536638bd7c1c01d2748309f"}, - {file = "jupyterlab_pygments-0.2.2.tar.gz", hash = "sha256:7405d7fde60819d905a9fa8ce89e4cd830e318cdad22a0030f7a901da705585d"}, -] - -[[package]] -name = "jupyterlab-server" -version = "2.24.0" -description = "A set of server components for JupyterLab and JupyterLab like applications." -optional = true -python-versions = ">=3.7" -files = [ - {file = "jupyterlab_server-2.24.0-py3-none-any.whl", hash = "sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876"}, - {file = "jupyterlab_server-2.24.0.tar.gz", hash = "sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07"}, -] - -[package.dependencies] -babel = ">=2.10" -importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jinja2 = ">=3.0.3" -json5 = ">=0.9.0" -jsonschema = ">=4.17.3" -jupyter-server = ">=1.21,<3" -packaging = ">=21.3" -requests = ">=2.28" - -[package.extras] -docs = ["autodoc-traits", "jinja2 (<3.2.0)", "mistune (<4)", "myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-copybutton", "sphinxcontrib-openapi (>0.8)"] -openapi = ["openapi-core (>=0.16.1,<0.17.0)", "ruamel-yaml"] -test = ["hatch", "ipykernel", "jupyterlab-server[openapi]", "openapi-spec-validator (>=0.5.1,<0.7.0)", "pytest (>=7.0)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter[server] (>=0.6.2)", "pytest-timeout", "requests-mock", "sphinxcontrib-spelling", "strict-rfc3339", "werkzeug"] - -[[package]] -name = "locket" -version = "1.0.0" -description = "File-based locks for Python on Linux and Windows" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"}, - {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"}, -] - -[[package]] -name = "lxml" -version = "4.9.1" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" -files = [ - {file = "lxml-4.9.1-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed"}, - {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc"}, - {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc"}, - {file = "lxml-4.9.1-cp27-cp27m-win32.whl", hash = "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3"}, - {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"}, - {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"}, - {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"}, - {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"}, - {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8"}, - {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d"}, - {file = "lxml-4.9.1-cp310-cp310-win32.whl", hash = "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7"}, - {file = "lxml-4.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b"}, - {file = "lxml-4.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d"}, - {file = "lxml-4.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3"}, - {file = "lxml-4.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29"}, - {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d"}, - {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318"}, - {file = "lxml-4.9.1-cp35-cp35m-win32.whl", hash = "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7"}, - {file = "lxml-4.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4"}, - {file = "lxml-4.9.1-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b"}, - {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf"}, - {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3"}, - {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391"}, - {file = "lxml-4.9.1-cp36-cp36m-win32.whl", hash = "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e"}, - {file = "lxml-4.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7"}, - {file = "lxml-4.9.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3"}, - {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca"}, - {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785"}, - {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785"}, - {file = "lxml-4.9.1-cp37-cp37m-win32.whl", hash = "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a"}, - {file = "lxml-4.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e"}, - {file = "lxml-4.9.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130"}, - {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715"}, - {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036"}, - {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387"}, - {file = "lxml-4.9.1-cp38-cp38-win32.whl", hash = "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94"}, - {file = "lxml-4.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345"}, - {file = "lxml-4.9.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91"}, - {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000"}, - {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25"}, - {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd"}, - {file = "lxml-4.9.1-cp39-cp39-win32.whl", hash = "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb"}, - {file = "lxml-4.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b"}, - {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2"}, - {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73"}, - {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c"}, - {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9"}, - {file = "lxml-4.9.1.tar.gz", hash = "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"}, -] - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=0.29.7)"] - -[[package]] -name = "lz4" -version = "4.3.2" -description = "LZ4 Bindings for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "lz4-4.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1c4c100d99eed7c08d4e8852dd11e7d1ec47a3340f49e3a96f8dfbba17ffb300"}, - {file = "lz4-4.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:edd8987d8415b5dad25e797043936d91535017237f72fa456601be1479386c92"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7c50542b4ddceb74ab4f8b3435327a0861f06257ca501d59067a6a482535a77"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f5614d8229b33d4a97cb527db2a1ac81308c6e796e7bdb5d1309127289f69d5"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f00a9ba98f6364cadda366ae6469b7b3568c0cced27e16a47ddf6b774169270"}, - {file = "lz4-4.3.2-cp310-cp310-win32.whl", hash = "sha256:b10b77dc2e6b1daa2f11e241141ab8285c42b4ed13a8642495620416279cc5b2"}, - {file = "lz4-4.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:86480f14a188c37cb1416cdabacfb4e42f7a5eab20a737dac9c4b1c227f3b822"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c2df117def1589fba1327dceee51c5c2176a2b5a7040b45e84185ce0c08b6a3"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1f25eb322eeb24068bb7647cae2b0732b71e5c639e4e4026db57618dcd8279f0"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8df16c9a2377bdc01e01e6de5a6e4bbc66ddf007a6b045688e285d7d9d61d1c9"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f571eab7fec554d3b1db0d666bdc2ad85c81f4b8cb08906c4c59a8cad75e6e22"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7211dc8f636ca625abc3d4fb9ab74e5444b92df4f8d58ec83c8868a2b0ff643d"}, - {file = "lz4-4.3.2-cp311-cp311-win32.whl", hash = "sha256:867664d9ca9bdfce840ac96d46cd8838c9ae891e859eb98ce82fcdf0e103a947"}, - {file = "lz4-4.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a6a46889325fd60b8a6b62ffc61588ec500a1883db32cddee9903edfba0b7584"}, - {file = "lz4-4.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a85b430138882f82f354135b98c320dafb96fc8fe4656573d95ab05de9eb092"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d5c93f8badacfa0456b660285e394e65023ef8071142e0dcbd4762166e1be0"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b50f096a6a25f3b2edca05aa626ce39979d63c3b160687c8c6d50ac3943d0ba"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:200d05777d61ba1ff8d29cb51c534a162ea0b4fe6d3c28be3571a0a48ff36080"}, - {file = "lz4-4.3.2-cp37-cp37m-win32.whl", hash = "sha256:edc2fb3463d5d9338ccf13eb512aab61937be50aa70734bcf873f2f493801d3b"}, - {file = "lz4-4.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:83acfacab3a1a7ab9694333bcb7950fbeb0be21660d236fd09c8337a50817897"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a9eec24ec7d8c99aab54de91b4a5a149559ed5b3097cf30249b665689b3d402"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:31d72731c4ac6ebdce57cd9a5cabe0aecba229c4f31ba3e2c64ae52eee3fdb1c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83903fe6db92db0be101acedc677aa41a490b561567fe1b3fe68695b2110326c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926b26db87ec8822cf1870efc3d04d06062730ec3279bbbd33ba47a6c0a5c673"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e05afefc4529e97c08e65ef92432e5f5225c0bb21ad89dee1e06a882f91d7f5e"}, - {file = "lz4-4.3.2-cp38-cp38-win32.whl", hash = "sha256:ad38dc6a7eea6f6b8b642aaa0683253288b0460b70cab3216838747163fb774d"}, - {file = "lz4-4.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:7e2dc1bd88b60fa09b9b37f08553f45dc2b770c52a5996ea52b2b40f25445676"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:edda4fb109439b7f3f58ed6bede59694bc631c4b69c041112b1b7dc727fffb23"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ca83a623c449295bafad745dcd399cea4c55b16b13ed8cfea30963b004016c9"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5ea0e788dc7e2311989b78cae7accf75a580827b4d96bbaf06c7e5a03989bd5"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a98b61e504fb69f99117b188e60b71e3c94469295571492a6468c1acd63c37ba"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4931ab28a0d1c133104613e74eec1b8bb1f52403faabe4f47f93008785c0b929"}, - {file = "lz4-4.3.2-cp39-cp39-win32.whl", hash = "sha256:ec6755cacf83f0c5588d28abb40a1ac1643f2ff2115481089264c7630236618a"}, - {file = "lz4-4.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:4caedeb19e3ede6c7a178968b800f910db6503cb4cb1e9cc9221157572139b49"}, - {file = "lz4-4.3.2.tar.gz", hash = "sha256:e1431d84a9cfb23e6773e72078ce8e65cad6745816d4cbf9ae67da5ea419acda"}, -] - -[package.extras] -docs = ["sphinx (>=1.6.0)", "sphinx-bootstrap-theme"] -flake8 = ["flake8"] -tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] - -[[package]] -name = "markupsafe" -version = "2.1.1" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, - {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, -] - -[[package]] -name = "mashumaro" -version = "3.1.1" -description = "Fast serialization framework on top of dataclasses" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mashumaro-3.1.1-py3-none-any.whl", hash = "sha256:a0b3e3343da8f996456529ec1d2910210b012831e384935f73cfe4adfae010f2"}, - {file = "mashumaro-3.1.1.tar.gz", hash = "sha256:3d053abe717c386ae6844f6eb26fee91cf3a41c5d85526c962e1e1df68baf1da"}, -] - -[package.dependencies] -typing-extensions = ">=4.0.1" - -[package.extras] -msgpack = ["msgpack (>=0.5.6)"] -orjson = ["orjson"] -toml = ["tomli (>=1.1.0)", "tomli-w (>=1.0)"] -yaml = ["pyyaml (>=3.13)"] - -[[package]] -name = "matplotlib-inline" -version = "0.1.6" -description = "Inline Matplotlib backend for Jupyter" -optional = true -python-versions = ">=3.5" -files = [ - {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, - {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, -] - -[package.dependencies] -traitlets = "*" - -[[package]] -name = "maxminddb" -version = "2.2.0" -description = "Reader for the MaxMind DB format" -optional = false -python-versions = ">=3.6" -files = [ - {file = "maxminddb-2.2.0.tar.gz", hash = "sha256:e37707ec4fab115804670e0fb7aedb4b57075a8b6f80052bdc648d3c005184e5"}, -] - -[[package]] -name = "memray" -version = "1.4.1" -description = "A memory profiler for Python applications" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "memray-1.4.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:a5329dceb805134bef685a1c338d75fe7e0dcf307f6952cd518472c7406feaef"}, - {file = "memray-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4520a8e01067a3d0077ed32be2ae9e9fb0222cf177713db673151373984fa067"}, - {file = "memray-1.4.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:835e038677e68ba84ec1226ba71c9b8df5f0d261aa7760c26963e11f6e09e2c3"}, - {file = "memray-1.4.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6727cb477cbd3011f2ba41b0b6cbeeb9a9a8faf526af738edc48c7cd255d217d"}, - {file = "memray-1.4.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:bac4835b742aa2d9f3df0e26b88b515bdf45279fbdc2f82c96431c6f2efd1405"}, - {file = "memray-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:449d1cc93a5e589357c9cf319f84bd30b47e2ab0b2c6d52285ec35eeb72108c6"}, - {file = "memray-1.4.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b5d6d50c406eeb42b369b0a41cd81b130246e1b6c4b99fdc72d9d3b78830202"}, - {file = "memray-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9874ca3383a73b63f75e450e98a12084984f739729200e3ba3b2dc323ea245e7"}, - {file = "memray-1.4.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a4dfb8466ab929a8ec2925124b4251545e50470027330e6d83791a2e97181473"}, - {file = "memray-1.4.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e83c25ec3643bded948a264969197b60cce45c69464d431d36859d71ed31db3f"}, - {file = "memray-1.4.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:417451084e826040f42517accaa11b468fb70b0816a44156d567af9e04aa8e0a"}, - {file = "memray-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f0d35654227e840da338c1a1b4bf64b8ec47533de84d08787f4cc19b94acf3a1"}, - {file = "memray-1.4.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9a3a83ecef10efb7b5af0a7f97f60cd9c24c589bbe6693e4dd0efd01ec9d19e4"}, - {file = "memray-1.4.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2bb4e7172f0cb60364d23280b85436a25020616a3ce32667156538eebcdefac"}, - {file = "memray-1.4.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:c38dedfc5a75b15a02e121bed8eae8333eed9ae9dbcbae50489b3d9ab6a04e5b"}, - {file = "memray-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ef4ffc3a78c1e00888b5766ad693c30f692694b62a6349e4593b8f5655c4bb7"}, - {file = "memray-1.4.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:aa0f52a5c30308bba4e81c78cbfe2981db181ce65f44cac609c9a2cfb0117d6e"}, - {file = "memray-1.4.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40d7ce415e48990deba70a5e0e3da45dd3c2ec5cb13fcbd66918371d1d47bae6"}, - {file = "memray-1.4.1.tar.gz", hash = "sha256:afae6056a12260d4411a862d235f62b6607b46bcd08ba455d3f1eaba7f323450"}, -] - -[package.dependencies] -jinja2 = "*" -rich = ">=11.2.0" - -[package.extras] -benchmark = ["asv"] -dev = ["Cython", "asv", "black", "bump2version", "check-manifest", "flake8", "furo", "greenlet", "ipython", "isort", "mypy", "pytest", "pytest-cov", "sphinx", "sphinx-argparse", "towncrier"] -docs = ["IPython", "bump2version", "furo", "sphinx", "sphinx-argparse", "towncrier"] -lint = ["black", "check-manifest", "flake8", "isort", "mypy"] -test = ["Cython", "greenlet", "ipython", "pytest", "pytest-cov"] - -[[package]] -name = "mistune" -version = "3.0.2" -description = "A sane and fast Markdown parser with useful plugins and renderers" -optional = true -python-versions = ">=3.7" -files = [ - {file = "mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205"}, - {file = "mistune-3.0.2.tar.gz", hash = "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"}, -] - -[[package]] -name = "msgpack" -version = "1.0.4" -description = "MessagePack serializer" -optional = false -python-versions = "*" -files = [ - {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250"}, - {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88"}, - {file = "msgpack-1.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:002b5c72b6cd9b4bafd790f364b8480e859b4712e91f43014fe01e4f957b8467"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35bc0faa494b0f1d851fd29129b2575b2e26d41d177caacd4206d81502d4c6a6"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4733359808c56d5d7756628736061c432ded018e7a1dff2d35a02439043321aa"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb514ad14edf07a1dbe63761fd30f89ae79b42625731e1ccf5e1f1092950eaa6"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c23080fdeec4716aede32b4e0ef7e213c7b1093eede9ee010949f2a418ced6ba"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:49565b0e3d7896d9ea71d9095df15b7f75a035c49be733051c34762ca95bbf7e"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca0f1644d6b5a73eb3e74d4d64d5d8c6c3d577e753a04c9e9c87d07692c58db"}, - {file = "msgpack-1.0.4-cp310-cp310-win32.whl", hash = "sha256:0dfe3947db5fb9ce52aaea6ca28112a170db9eae75adf9339a1aec434dc954ef"}, - {file = "msgpack-1.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dea20515f660aa6b7e964433b1808d098dcfcabbebeaaad240d11f909298075"}, - {file = "msgpack-1.0.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e83f80a7fec1a62cf4e6c9a660e39c7f878f603737a0cdac8c13131d11d97f52"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c11a48cf5e59026ad7cb0dc29e29a01b5a66a3e333dc11c04f7e991fc5510a9"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1276e8f34e139aeff1c77a3cefb295598b504ac5314d32c8c3d54d24fadb94c9"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c9566f2c39ccced0a38d37c26cc3570983b97833c365a6044edef3574a00c08"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:fcb8a47f43acc113e24e910399376f7277cf8508b27e5b88499f053de6b115a8"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:76ee788122de3a68a02ed6f3a16bbcd97bc7c2e39bd4d94be2f1821e7c4a64e6"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0a68d3ac0104e2d3510de90a1091720157c319ceeb90d74f7b5295a6bee51bae"}, - {file = "msgpack-1.0.4-cp36-cp36m-win32.whl", hash = "sha256:85f279d88d8e833ec015650fd15ae5eddce0791e1e8a59165318f371158efec6"}, - {file = "msgpack-1.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:c1683841cd4fa45ac427c18854c3ec3cd9b681694caf5bff04edb9387602d661"}, - {file = "msgpack-1.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a75dfb03f8b06f4ab093dafe3ddcc2d633259e6c3f74bb1b01996f5d8aa5868c"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9667bdfdf523c40d2511f0e98a6c9d3603be6b371ae9a238b7ef2dc4e7a427b0"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11184bc7e56fd74c00ead4f9cc9a3091d62ecb96e97653add7a879a14b003227"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac5bd7901487c4a1dd51a8c58f2632b15d838d07ceedaa5e4c080f7190925bff"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1e91d641d2bfe91ba4c52039adc5bccf27c335356055825c7f88742c8bb900dd"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2a2df1b55a78eb5f5b7d2a4bb221cd8363913830145fad05374a80bf0877cb1e"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:545e3cf0cf74f3e48b470f68ed19551ae6f9722814ea969305794645da091236"}, - {file = "msgpack-1.0.4-cp37-cp37m-win32.whl", hash = "sha256:2cc5ca2712ac0003bcb625c96368fd08a0f86bbc1a5578802512d87bc592fe44"}, - {file = "msgpack-1.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:eba96145051ccec0ec86611fe9cf693ce55f2a3ce89c06ed307de0e085730ec1"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7760f85956c415578c17edb39eed99f9181a48375b0d4a94076d84148cf67b2d"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:449e57cc1ff18d3b444eb554e44613cffcccb32805d16726a5494038c3b93dab"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d603de2b8d2ea3f3bcb2efe286849aa7a81531abc52d8454da12f46235092bcb"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f5d88c99f64c456413d74a975bd605a9b0526293218a3b77220a2c15458ba9"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916c78f33602ecf0509cc40379271ba0f9ab572b066bd4bdafd7434dee4bc6e"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81fc7ba725464651190b196f3cd848e8553d4d510114a954681fd0b9c479d7e1"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5b5b962221fa2c5d3a7f8133f9abffc114fe218eb4365e40f17732ade576c8e"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:77ccd2af37f3db0ea59fb280fa2165bf1b096510ba9fe0cc2bf8fa92a22fdb43"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b17be2478b622939e39b816e0aa8242611cc8d3583d1cd8ec31b249f04623243"}, - {file = "msgpack-1.0.4-cp38-cp38-win32.whl", hash = "sha256:2bb8cdf50dd623392fa75525cce44a65a12a00c98e1e37bf0fb08ddce2ff60d2"}, - {file = "msgpack-1.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:26b8feaca40a90cbe031b03d82b2898bf560027160d3eae1423f4a67654ec5d6"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:462497af5fd4e0edbb1559c352ad84f6c577ffbbb708566a0abaaa84acd9f3ae"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2999623886c5c02deefe156e8f869c3b0aaeba14bfc50aa2486a0415178fce55"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f0029245c51fd9473dc1aede1160b0a29f4a912e6b1dd353fa6d317085b219da"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed6f7b854a823ea44cf94919ba3f727e230da29feb4a99711433f25800cf747f"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df96d6eaf45ceca04b3f3b4b111b86b33785683d682c655063ef8057d61fd92"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4192b1ab40f8dca3f2877b70e63799d95c62c068c84dc028b40a6cb03ccd0f"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0e3590f9fb9f7fbc36df366267870e77269c03172d086fa76bb4eba8b2b46624"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1576bd97527a93c44fa856770197dec00d223b0b9f36ef03f65bac60197cedf8"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:63e29d6e8c9ca22b21846234913c3466b7e4ee6e422f205a2988083de3b08cae"}, - {file = "msgpack-1.0.4-cp39-cp39-win32.whl", hash = "sha256:fb62ea4b62bfcb0b380d5680f9a4b3f9a2d166d9394e9bbd9666c0ee09a3645c"}, - {file = "msgpack-1.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4d5834a2a48965a349da1c5a79760d94a1a0172fbb5ab6b5b33cbf8447e109ce"}, - {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, -] - -[[package]] -name = "mypy" -version = "0.991" -description = "Optional static typing for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, - {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, - {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, - {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, - {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, - {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, - {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, - {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, - {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, - {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, - {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, - {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, - {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, - {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, - {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, - {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, - {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, - {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, - {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, - {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, - {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, - {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, - {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, - {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, -] - -[package.dependencies] -mypy-extensions = ">=0.4.3" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=3.10" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -install-types = ["pip"] -python2 = ["typed-ast (>=1.4.0,<2)"] -reports = ["lxml"] - -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -optional = false -python-versions = "*" -files = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] - -[[package]] -name = "nbclient" -version = "0.8.0" -description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." -optional = true -python-versions = ">=3.8.0" -files = [ - {file = "nbclient-0.8.0-py3-none-any.whl", hash = "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548"}, - {file = "nbclient-0.8.0.tar.gz", hash = "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"}, -] - -[package.dependencies] -jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -nbformat = ">=5.1" -traitlets = ">=5.4" - -[package.extras] -dev = ["pre-commit"] -docs = ["autodoc-traits", "mock", "moto", "myst-parser", "nbclient[test]", "sphinx (>=1.7)", "sphinx-book-theme", "sphinxcontrib-spelling"] -test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>=7.0.0)", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov (>=4.0)", "testpath", "xmltodict"] - -[[package]] -name = "nbconvert" -version = "7.10.0" -description = "Converting Jupyter Notebooks" -optional = true -python-versions = ">=3.8" -files = [ - {file = "nbconvert-7.10.0-py3-none-any.whl", hash = "sha256:8cf1d95e569730f136feb85e4bba25bdcf3a63fefb122d854ddff6771c0ac933"}, - {file = "nbconvert-7.10.0.tar.gz", hash = "sha256:4bedff08848626be544de193b7594d98a048073f392178008ff4f171f5e21d26"}, -] - -[package.dependencies] -beautifulsoup4 = "*" -bleach = "!=5.0.0" -defusedxml = "*" -importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} -jinja2 = ">=3.0" -jupyter-core = ">=4.7" -jupyterlab-pygments = "*" -markupsafe = ">=2.0" -mistune = ">=2.0.3,<4" -nbclient = ">=0.5.0" -nbformat = ">=5.7" -packaging = "*" -pandocfilters = ">=1.4.1" -pygments = ">=2.4.1" -tinycss2 = "*" -traitlets = ">=5.1" - -[package.extras] -all = ["nbconvert[docs,qtpdf,serve,test,webpdf]"] -docs = ["ipykernel", "ipython", "myst-parser", "nbsphinx (>=0.2.12)", "pydata-sphinx-theme", "sphinx (==5.0.2)", "sphinxcontrib-spelling"] -qtpdf = ["nbconvert[qtpng]"] -qtpng = ["pyqtwebengine (>=5.15)"] -serve = ["tornado (>=6.1)"] -test = ["flaky", "ipykernel", "ipywidgets (>=7)", "pytest", "pytest-dependency"] -webpdf = ["playwright"] - -[[package]] -name = "nbformat" -version = "5.9.2" -description = "The Jupyter Notebook format" -optional = true -python-versions = ">=3.8" -files = [ - {file = "nbformat-5.9.2-py3-none-any.whl", hash = "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9"}, - {file = "nbformat-5.9.2.tar.gz", hash = "sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192"}, -] - -[package.dependencies] -fastjsonschema = "*" -jsonschema = ">=2.6" -jupyter-core = "*" -traitlets = ">=5.1" - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["pep440", "pre-commit", "pytest", "testpath"] - -[[package]] -name = "nest-asyncio" -version = "1.5.8" -description = "Patch asyncio to allow nested event loops" -optional = true -python-versions = ">=3.5" -files = [ - {file = "nest_asyncio-1.5.8-py3-none-any.whl", hash = "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"}, - {file = "nest_asyncio-1.5.8.tar.gz", hash = "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb"}, -] - -[[package]] -name = "notebook-shim" -version = "0.2.3" -description = "A shim layer for notebook traits and config" -optional = true -python-versions = ">=3.7" -files = [ - {file = "notebook_shim-0.2.3-py3-none-any.whl", hash = "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7"}, - {file = "notebook_shim-0.2.3.tar.gz", hash = "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"}, -] - -[package.dependencies] -jupyter-server = ">=1.8,<3" - -[package.extras] -test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync"] - -[[package]] -name = "numpy" -version = "1.24.2" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, -] - -[[package]] -name = "objprint" -version = "0.2.2" -description = "A library that can print Python objects in human readable format" -optional = false -python-versions = ">=3.6" -files = [ - {file = "objprint-0.2.2-py3-none-any.whl", hash = "sha256:9f50bb3b7cbe95b6d22c79e4f08a5273e3f7e5ff5ce35e146f6e7854abd276a0"}, - {file = "objprint-0.2.2.tar.gz", hash = "sha256:90f2f3c19dfda1b5eb50d87d82ccbc20511ca4f02c4878553c56edc711e39689"}, -] - -[[package]] -name = "orjson" -version = "3.8.2" -description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -optional = false -python-versions = ">=3.7" -files = [ - {file = "orjson-3.8.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:43e69b360c2851b45c7dbab3b95f7fa8469df73fab325a683f7389c4db63aa71"}, - {file = "orjson-3.8.2-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:64c5da5c9679ef3d85e9bbcbb62f4ccdc1f1975780caa20f2ec1e37b4da6bd36"}, - {file = "orjson-3.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c632a2157fa9ec098d655287e9e44809615af99837c49f53d96bfbca453c5bd"}, - {file = "orjson-3.8.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f63da6309c282a2b58d4a846f0717f6440356b4872838b9871dc843ed1fe2b38"}, - {file = "orjson-3.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c9be25c313ba2d5478829d949165445c3bd36c62e07092b4ba8dbe5426574d1"}, - {file = "orjson-3.8.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4bcce53e9e088f82633f784f79551fcd7637943ab56c51654aaf9d4c1d5cfa54"}, - {file = "orjson-3.8.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:33edb5379c6e6337f9383c85fe4080ce3aa1057cc2ce29345b7239461f50cbd6"}, - {file = "orjson-3.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:da35d347115758bbc8bfaf39bb213c42000f2a54e3f504c84374041d20835cd6"}, - {file = "orjson-3.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d755d94a90a941b91b4d39a6b02e289d8ba358af2d1a911edf266be7942609dc"}, - {file = "orjson-3.8.2-cp310-none-win_amd64.whl", hash = "sha256:7ea96923e26390b2142602ebb030e2a4db9351134696e0b219e5106bddf9b48e"}, - {file = "orjson-3.8.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:a0d89de876e6f1cef917a2338378a60a98584e1c2e1c67781e20b6ed1c512478"}, - {file = "orjson-3.8.2-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:8d47e7592fe938aec898eb22ea4946298c018133df084bc78442ff18e2c6347c"}, - {file = "orjson-3.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3d9f1043f618d0c64228aab9711e5bd822253c50b6c56223951e32b51f81d62"}, - {file = "orjson-3.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed10600e8b08f1e87b656ad38ab316191ce94f2c9adec57035680c0dc9e93c81"}, - {file = "orjson-3.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99c49e49a04bf61fee7aaea6d92ac2b1fcf6507aea894bbdf3fbb25fe792168c"}, - {file = "orjson-3.8.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1463674f8efe6984902473d7b5ce3edf444c1fcd09dc8aa4779638a28fb9ca01"}, - {file = "orjson-3.8.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c1ef75f1d021d817e5c60a42da0b4b7e3123b1b37415260b8415666ddacc7cd7"}, - {file = "orjson-3.8.2-cp311-none-win_amd64.whl", hash = "sha256:b6007e1ac8564b13b2521720929e8bb3ccd3293d9fdf38f28728dcc06db6248f"}, - {file = "orjson-3.8.2-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:a02c13ae523221576b001071354380e277346722cc6b7fdaacb0fd6db5154b3e"}, - {file = "orjson-3.8.2-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:fa2e565cf8ffdb37ce1887bd1592709ada7f701e61aa4b1e710be94b0aecbab4"}, - {file = "orjson-3.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1d8864288f7c5fccc07b43394f83b721ddc999f25dccfb5d0651671a76023f5"}, - {file = "orjson-3.8.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1874c05d0bb994601fa2d51605cb910d09343c6ebd36e84a573293523fab772a"}, - {file = "orjson-3.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:349387ed6989e5db22e08c9af8d7ca14240803edc50de451d48d41a0e7be30f6"}, - {file = "orjson-3.8.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:4e42b19619d6e97e201053b865ca4e62a48da71165f4081508ada8e1b91c6a30"}, - {file = "orjson-3.8.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:bc112c17e607c59d1501e72afb44226fa53d947d364aed053f0c82d153e29616"}, - {file = "orjson-3.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6fda669211f2ed1fc2c8130187ec90c96b4f77b6a250004e666d2ef8ed524e5f"}, - {file = "orjson-3.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:aebd4e80fea0f20578fd0452908b9206a6a0d5ae9f5c99b6e665bbcd989e56cd"}, - {file = "orjson-3.8.2-cp37-none-win_amd64.whl", hash = "sha256:9f3cd0394eb6d265beb2a1572b5663bc910883ddbb5cdfbcb660f5a0444e7fd8"}, - {file = "orjson-3.8.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:74e7d54d11b3da42558d69a23bf92c2c48fabf69b38432d5eee2c5b09cd4c433"}, - {file = "orjson-3.8.2-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:8cbadc9be748a823f9c743c7631b1ee95d3925a9c0b21de4e862a1d57daa10ec"}, - {file = "orjson-3.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07d5a8c69a2947d9554a00302734fe3d8516415c8b280963c92bc1033477890"}, - {file = "orjson-3.8.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6b364ea01d1b71b9f97bf97af9eb79ebee892df302e127a9e2e4f8eaa74d6b98"}, - {file = "orjson-3.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b98a8c825a59db94fbe8e0cce48618624c5a6fb1436467322d90667c08a0bf80"}, - {file = "orjson-3.8.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:ab63103f60b516c0fce9b62cb4773f689a82ab56e19ef2387b5a3182f80c0d78"}, - {file = "orjson-3.8.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:73ab3f4288389381ae33ab99f914423b69570c88d626d686764634d5e0eeb909"}, - {file = "orjson-3.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2ab3fd8728e12c36e20c6d9d70c9e15033374682ce5acb6ed6a08a80dacd254d"}, - {file = "orjson-3.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cde11822cf71a7f0daaa84223249b2696a2b6cda7fa587e9fd762dff1a8848e4"}, - {file = "orjson-3.8.2-cp38-none-win_amd64.whl", hash = "sha256:b14765ea5aabfeab1a194abfaa0be62c9fee6480a75ac8c6974b4eeede3340b4"}, - {file = "orjson-3.8.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:6068a27d59d989d4f2864c2fc3440eb7126a0cfdfaf8a4ad136b0ffd932026ae"}, - {file = "orjson-3.8.2-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:6bf36fa759a1b941fc552ad76b2d7fb10c1d2a20c056be291ea45eb6ae1da09b"}, - {file = "orjson-3.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f436132e62e647880ca6988974c8e3165a091cb75cbed6c6fd93e931630c22fa"}, - {file = "orjson-3.8.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3ecd8936259a5920b52a99faf62d4efeb9f5e25a0aacf0cce1e9fa7c37af154f"}, - {file = "orjson-3.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c13114b345cda33644f64e92fe5d8737828766cf02fbbc7d28271a95ea546832"}, - {file = "orjson-3.8.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6e43cdc3ddf96bdb751b748b1984b701125abacca8fc2226b808d203916e8cba"}, - {file = "orjson-3.8.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ee39071da2026b11e4352d6fc3608a7b27ee14bc699fd240f4e604770bc7a255"}, - {file = "orjson-3.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1c3833976ebbeb3b5b6298cb22e23bf18453f6b80802103b7d08f7dd8a61611d"}, - {file = "orjson-3.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b9a34519d3d70935e1cd3797fbed8fbb6f61025182bea0140ca84d95b6f8fbe5"}, - {file = "orjson-3.8.2-cp39-none-win_amd64.whl", hash = "sha256:2734086d9a3dd9591c4be7d05aff9beccc086796d3f243685e56b7973ebac5bc"}, - {file = "orjson-3.8.2.tar.gz", hash = "sha256:a2fb95a45031ccf278e44341027b3035ab99caa32aa173279b1f0a06324f434b"}, -] - -[[package]] -name = "overrides" -version = "7.4.0" -description = "A decorator to automatically detect mismatch when overriding a method." -optional = true -python-versions = ">=3.6" -files = [ - {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, - {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, -] - -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.6" -files = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - -[[package]] -name = "pandas" -version = "2.0.0" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pandas-2.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bbb2c5e94d6aa4e632646a3bacd05c2a871c3aa3e85c9bec9be99cb1267279f2"}, - {file = "pandas-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b5337c87c4e963f97becb1217965b6b75c6fe5f54c4cf09b9a5ac52fc0bd03d3"}, - {file = "pandas-2.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ded51f7e3dd9b4f8b87f2ceb7bd1a8df2491f7ee72f7074c6927a512607199e"}, - {file = "pandas-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c858de9e9fc422d25e67e1592a6e6135d7bcf9a19fcaf4d0831a0be496bf21"}, - {file = "pandas-2.0.0-cp310-cp310-win32.whl", hash = "sha256:2d1d138848dd71b37e3cbe7cd952ff84e2ab04d8988972166e18567dcc811245"}, - {file = "pandas-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:d08e41d96bc4de6f500afe80936c68fce6099d5a434e2af7c7fd8e7c72a3265d"}, - {file = "pandas-2.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24472cfc7ced511ac90608728b88312be56edc8f19b9ed885a7d2e47ffaf69c0"}, - {file = "pandas-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ffb14f50c74ee541610668137830bb93e9dfa319b1bef2cedf2814cd5ac9c70"}, - {file = "pandas-2.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c24c7d12d033a372a9daf9ff2c80f8b0af6f98d14664dbb0a4f6a029094928a7"}, - {file = "pandas-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8318de0f886e4dcb8f9f36e45a3d6a6c3d1cfdc508354da85e739090f0222991"}, - {file = "pandas-2.0.0-cp311-cp311-win32.whl", hash = "sha256:57c34b79c13249505e850d0377b722961b99140f81dafbe6f19ef10239f6284a"}, - {file = "pandas-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:8f987ec26e96a8490909bc5d98c514147236e49830cba7df8690f6087c12bbae"}, - {file = "pandas-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b3ba8f5dd470d8bfbc4259829589f4a32881151c49e36384d9eb982b35a12020"}, - {file = "pandas-2.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcd471c9d9f60926ab2f15c6c29164112f458acb42280365fbefa542d0c2fc74"}, - {file = "pandas-2.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9253edfd015520ce77a9343eb7097429479c039cd3ebe81d7810ea11b4b24695"}, - {file = "pandas-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977326039bd1ded620001a1889e2ed4798460a6bc5a24fbaebb5f07a41c32a55"}, - {file = "pandas-2.0.0-cp38-cp38-win32.whl", hash = "sha256:78425ca12314b23356c28b16765639db10ebb7d8983f705d6759ff7fe41357fa"}, - {file = "pandas-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d93b7fcfd9f3328072b250d6d001dcfeec5d3bb66c1b9c8941e109a46c0c01a8"}, - {file = "pandas-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:425705cee8be54db2504e8dd2a730684790b15e5904b750c367611ede49098ab"}, - {file = "pandas-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a4f789b7c012a608c08cda4ff0872fd979cb18907a37982abe884e6f529b8793"}, - {file = "pandas-2.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bb9d840bf15656805f6a3d87eea9dcb7efdf1314a82adcf7f00b820427c5570"}, - {file = "pandas-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0778ab54c8f399d83d98ffb674d11ec716449956bc6f6821891ab835848687f2"}, - {file = "pandas-2.0.0-cp39-cp39-win32.whl", hash = "sha256:70db5c278bbec0306d32bf78751ff56b9594c05a5098386f6c8a563659124f91"}, - {file = "pandas-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:4f3320bb55f34af4193020158ef8118ee0fb9aec7cc47d2084dbfdd868a0a24f"}, - {file = "pandas-2.0.0.tar.gz", hash = "sha256:cda9789e61b44463c1c4fe17ef755de77bcd13b09ba31c940d20f193d63a5dc8"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, -] -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.1" - -[package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] - -[[package]] -name = "pandocfilters" -version = "1.5.0" -description = "Utilities for writing pandoc filters in python" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pandocfilters-1.5.0-py2.py3-none-any.whl", hash = "sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f"}, - {file = "pandocfilters-1.5.0.tar.gz", hash = "sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38"}, -] - -[[package]] -name = "parso" -version = "0.8.3" -description = "A Python Parser" -optional = true -python-versions = ">=3.6" -files = [ - {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, - {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, -] - -[package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] - -[[package]] -name = "partd" -version = "1.3.0" -description = "Appendable key-value storage" -optional = false -python-versions = ">=3.7" -files = [ - {file = "partd-1.3.0-py3-none-any.whl", hash = "sha256:6393a0c898a0ad945728e34e52de0df3ae295c5aff2e2926ba7cc3c60a734a15"}, - {file = "partd-1.3.0.tar.gz", hash = "sha256:ce91abcdc6178d668bcaa431791a5a917d902341cb193f543fe445d494660485"}, -] - -[package.dependencies] -locket = "*" -toolz = "*" - -[package.extras] -complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"] - -[[package]] -name = "pathspec" -version = "0.10.2" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pathspec-0.10.2-py3-none-any.whl", hash = "sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5"}, - {file = "pathspec-0.10.2.tar.gz", hash = "sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"}, -] - -[[package]] -name = "pexpect" -version = "4.8.0" -description = "Pexpect allows easy control of interactive console applications." -optional = true -python-versions = "*" -files = [ - {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, - {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, -] - -[package.dependencies] -ptyprocess = ">=0.5" - -[[package]] -name = "pickleshare" -version = "0.7.5" -description = "Tiny 'shelve'-like database with concurrency support" -optional = true -python-versions = "*" -files = [ - {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, - {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, -] - -[[package]] -name = "pillow" -version = "9.5.0" -description = "Python Imaging Library (Fork)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, - {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a"}, - {file = "Pillow-9.5.0-cp310-cp310-win32.whl", hash = "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44"}, - {file = "Pillow-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"}, - {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"}, - {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"}, - {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"}, - {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"}, - {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"}, - {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47"}, - {file = "Pillow-9.5.0-cp37-cp37m-win32.whl", hash = "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7"}, - {file = "Pillow-9.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f"}, - {file = "Pillow-9.5.0-cp38-cp38-win32.whl", hash = "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc"}, - {file = "Pillow-9.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865"}, - {file = "Pillow-9.5.0-cp39-cp39-win32.whl", hash = "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964"}, - {file = "Pillow-9.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799"}, - {file = "Pillow-9.5.0.tar.gz", hash = "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1"}, -] - -[package.extras] -docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "pkgutil-resolve-name" -version = "1.3.10" -description = "Resolve a name to an object." -optional = true -python-versions = ">=3.6" -files = [ - {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, - {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, -] - -[[package]] -name = "platformdirs" -version = "2.5.4" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.7" -files = [ - {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, - {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, -] - -[package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "prometheus-client" -version = "0.17.1" -description = "Python client for the Prometheus monitoring system." -optional = true -python-versions = ">=3.6" -files = [ - {file = "prometheus_client-0.17.1-py3-none-any.whl", hash = "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101"}, - {file = "prometheus_client-0.17.1.tar.gz", hash = "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091"}, -] - -[package.extras] -twisted = ["twisted"] - -[[package]] -name = "prompt-toolkit" -version = "3.0.39" -description = "Library for building powerful interactive command lines in Python" -optional = true -python-versions = ">=3.7.0" -files = [ - {file = "prompt_toolkit-3.0.39-py3-none-any.whl", hash = "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"}, - {file = "prompt_toolkit-3.0.39.tar.gz", hash = "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac"}, -] - -[package.dependencies] -wcwidth = "*" - -[[package]] -name = "protobuf" -version = "4.25.3" -description = "" -optional = true -python-versions = ">=3.8" -files = [ - {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, - {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, - {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, - {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, - {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, - {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, - {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, - {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, - {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, -] - -[[package]] -name = "psutil" -version = "5.9.4" -description = "Cross-platform lib for process and system monitoring in Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, - {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"}, - {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"}, - {file = "psutil-5.9.4-cp27-cp27m-win32.whl", hash = "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad"}, - {file = "psutil-5.9.4-cp27-cp27m-win_amd64.whl", hash = "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94"}, - {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24"}, - {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7"}, - {file = "psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7"}, - {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1"}, - {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08"}, - {file = "psutil-5.9.4-cp36-abi3-win32.whl", hash = "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff"}, - {file = "psutil-5.9.4-cp36-abi3-win_amd64.whl", hash = "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"}, - {file = "psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"}, - {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, -] - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -description = "Run a subprocess in a pseudo terminal" -optional = true -python-versions = "*" -files = [ - {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, - {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, -] - -[[package]] -name = "pure-eval" -version = "0.2.2" -description = "Safely evaluate AST nodes without side effects" -optional = true -python-versions = "*" -files = [ - {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, - {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, -] - -[package.extras] -tests = ["pytest"] - -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -description = "Get CPU info with pure Python" -optional = false -python-versions = "*" -files = [ - {file = "py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690"}, - {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, -] - -[[package]] -name = "pyarrow" -version = "11.0.0" -description = "Python library for Apache Arrow" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyarrow-11.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2"}, - {file = "pyarrow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba"}, - {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85"}, - {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689"}, - {file = "pyarrow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c"}, - {file = "pyarrow-11.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca"}, - {file = "pyarrow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56"}, - {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841"}, - {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5"}, - {file = "pyarrow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2"}, - {file = "pyarrow-11.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc"}, - {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00"}, - {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e"}, - {file = "pyarrow-11.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06"}, - {file = "pyarrow-11.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"}, - {file = "pyarrow-11.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97"}, - {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c"}, - {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70"}, - {file = "pyarrow-11.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835"}, - {file = "pyarrow-11.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841"}, - {file = "pyarrow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854"}, - {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418"}, - {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b"}, - {file = "pyarrow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4"}, - {file = "pyarrow-11.0.0.tar.gz", hash = "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d"}, -] - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] - -[[package]] -name = "pygments" -version = "2.13.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.6" -files = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyopenssl" -version = "22.1.0" -description = "Python wrapper module around the OpenSSL library" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pyOpenSSL-22.1.0-py3-none-any.whl", hash = "sha256:b28437c9773bb6c6958628cf9c3bebe585de661dba6f63df17111966363dd15e"}, - {file = "pyOpenSSL-22.1.0.tar.gz", hash = "sha256:7a83b7b272dd595222d672f5ce29aa030f1fb837630ef229f62e72e395ce8968"}, -] - -[package.dependencies] -cryptography = ">=38.0.0,<39" - -[package.extras] -docs = ["sphinx (!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] -test = ["flaky", "pretend", "pytest (>=3.0.1)"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pyrsistent" -version = "0.19.2" -description = "Persistent/Functional/Immutable data structures" -optional = true -python-versions = ">=3.7" -files = [ - {file = "pyrsistent-0.19.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a"}, - {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a"}, - {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed"}, - {file = "pyrsistent-0.19.2-cp310-cp310-win32.whl", hash = "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41"}, - {file = "pyrsistent-0.19.2-cp310-cp310-win_amd64.whl", hash = "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-win32.whl", hash = "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2"}, - {file = "pyrsistent-0.19.2-cp37-cp37m-win_amd64.whl", hash = "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73"}, - {file = "pyrsistent-0.19.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308"}, - {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584"}, - {file = "pyrsistent-0.19.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb"}, - {file = "pyrsistent-0.19.2-cp38-cp38-win32.whl", hash = "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a"}, - {file = "pyrsistent-0.19.2-cp38-cp38-win_amd64.whl", hash = "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab"}, - {file = "pyrsistent-0.19.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"}, - {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95"}, - {file = "pyrsistent-0.19.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e"}, - {file = "pyrsistent-0.19.2-cp39-cp39-win32.whl", hash = "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b"}, - {file = "pyrsistent-0.19.2-cp39-cp39-win_amd64.whl", hash = "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291"}, - {file = "pyrsistent-0.19.2-py3-none-any.whl", hash = "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0"}, - {file = "pyrsistent-0.19.2.tar.gz", hash = "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2"}, -] - -[[package]] -name = "pytest" -version = "7.2.0" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] - -[package.dependencies] -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-benchmark" -version = "4.0.0" -description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-benchmark-4.0.0.tar.gz", hash = "sha256:fb0785b83efe599a6a956361c0691ae1dbb5318018561af10f3e915caa0048d1"}, - {file = "pytest_benchmark-4.0.0-py3-none-any.whl", hash = "sha256:fdb7db64e31c8b277dff9850d2a2556d8b60bcb0ea6524e36e28ffd7c87f71d6"}, -] - -[package.dependencies] -py-cpuinfo = "*" -pytest = ">=3.8" - -[package.extras] -aspect = ["aspectlib"] -elasticsearch = ["elasticsearch"] -histogram = ["pygal", "pygaljs"] - -[[package]] -name = "pytest-cov" -version = "4.0.0" -description = "Pytest plugin for measuring coverage." -optional = false -python-versions = ">=3.6" -files = [ - {file = "pytest-cov-4.0.0.tar.gz", hash = "sha256:996b79efde6433cdbd0088872dbc5fb3ed7fe1578b68cdbba634f14bb8dd0470"}, - {file = "pytest_cov-4.0.0-py3-none-any.whl", hash = "sha256:2feb1b751d66a8bd934e5edfa2e961d11309dc37b73b0eabe73b5945fee20f6b"}, -] - -[package.dependencies] -coverage = {version = ">=5.2.1", extras = ["toml"]} -pytest = ">=4.6" - -[package.extras] -testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] - -[[package]] -name = "pytest-profiling" -version = "1.7.0" -description = "Profiling plugin for py.test" -optional = false -python-versions = "*" -files = [ - {file = "pytest-profiling-1.7.0.tar.gz", hash = "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29"}, - {file = "pytest_profiling-1.7.0-py2.py3-none-any.whl", hash = "sha256:999cc9ac94f2e528e3f5d43465da277429984a1c237ae9818f8cfd0b06acb019"}, -] - -[package.dependencies] -gprof2dot = "*" -pytest = "*" -six = "*" - -[package.extras] -tests = ["pytest-virtualenv"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-json-logger" -version = "2.0.7" -description = "A python library adding a json log formatter" -optional = true -python-versions = ">=3.6" -files = [ - {file = "python-json-logger-2.0.7.tar.gz", hash = "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c"}, - {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"}, -] - -[[package]] -name = "pytz" -version = "2022.6" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, - {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, -] - -[[package]] -name = "pytz-deprecation-shim" -version = "0.1.0.post0" -description = "Shims to make deprecation of pytz easier" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -files = [ - {file = "pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl", hash = "sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6"}, - {file = "pytz_deprecation_shim-0.1.0.post0.tar.gz", hash = "sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d"}, -] - -[package.dependencies] -"backports.zoneinfo" = {version = "*", markers = "python_version >= \"3.6\" and python_version < \"3.9\""} -tzdata = {version = "*", markers = "python_version >= \"3.6\""} - -[[package]] -name = "pywin32" -version = "306" -description = "Python for Window Extensions" -optional = true -python-versions = "*" -files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, - {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, - {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, - {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, - {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, - {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, - {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, -] - -[[package]] -name = "pywinpty" -version = "2.0.12" -description = "Pseudo terminal support for Windows from Python." -optional = true -python-versions = ">=3.8" -files = [ - {file = "pywinpty-2.0.12-cp310-none-win_amd64.whl", hash = "sha256:21319cd1d7c8844fb2c970fb3a55a3db5543f112ff9cfcd623746b9c47501575"}, - {file = "pywinpty-2.0.12-cp311-none-win_amd64.whl", hash = "sha256:853985a8f48f4731a716653170cd735da36ffbdc79dcb4c7b7140bce11d8c722"}, - {file = "pywinpty-2.0.12-cp312-none-win_amd64.whl", hash = "sha256:1617b729999eb6713590e17665052b1a6ae0ad76ee31e60b444147c5b6a35dca"}, - {file = "pywinpty-2.0.12-cp38-none-win_amd64.whl", hash = "sha256:189380469ca143d06e19e19ff3fba0fcefe8b4a8cc942140a6b863aed7eebb2d"}, - {file = "pywinpty-2.0.12-cp39-none-win_amd64.whl", hash = "sha256:7520575b6546db23e693cbd865db2764097bd6d4ef5dc18c92555904cd62c3d4"}, - {file = "pywinpty-2.0.12.tar.gz", hash = "sha256:8197de460ae8ebb7f5d1701dfa1b5df45b157bb832e92acba316305e18ca00dd"}, -] - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - -[[package]] -name = "pyzmq" -version = "25.1.1" -description = "Python bindings for 0MQ" -optional = true -python-versions = ">=3.6" -files = [ - {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:381469297409c5adf9a0e884c5eb5186ed33137badcbbb0560b86e910a2f1e76"}, - {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:955215ed0604dac5b01907424dfa28b40f2b2292d6493445dd34d0dfa72586a8"}, - {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:985bbb1316192b98f32e25e7b9958088431d853ac63aca1d2c236f40afb17c83"}, - {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:afea96f64efa98df4da6958bae37f1cbea7932c35878b185e5982821bc883369"}, - {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76705c9325d72a81155bb6ab48d4312e0032bf045fb0754889133200f7a0d849"}, - {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:77a41c26205d2353a4c94d02be51d6cbdf63c06fbc1295ea57dad7e2d3381b71"}, - {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:12720a53e61c3b99d87262294e2b375c915fea93c31fc2336898c26d7aed34cd"}, - {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:57459b68e5cd85b0be8184382cefd91959cafe79ae019e6b1ae6e2ba8a12cda7"}, - {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:292fe3fc5ad4a75bc8df0dfaee7d0babe8b1f4ceb596437213821f761b4589f9"}, - {file = "pyzmq-25.1.1-cp310-cp310-win32.whl", hash = "sha256:35b5ab8c28978fbbb86ea54958cd89f5176ce747c1fb3d87356cf698048a7790"}, - {file = "pyzmq-25.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:11baebdd5fc5b475d484195e49bae2dc64b94a5208f7c89954e9e354fc609d8f"}, - {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:d20a0ddb3e989e8807d83225a27e5c2eb2260eaa851532086e9e0fa0d5287d83"}, - {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e1c1be77bc5fb77d923850f82e55a928f8638f64a61f00ff18a67c7404faf008"}, - {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d89528b4943d27029a2818f847c10c2cecc79fa9590f3cb1860459a5be7933eb"}, - {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90f26dc6d5f241ba358bef79be9ce06de58d477ca8485e3291675436d3827cf8"}, - {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2b92812bd214018e50b6380ea3ac0c8bb01ac07fcc14c5f86a5bb25e74026e9"}, - {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:2f957ce63d13c28730f7fd6b72333814221c84ca2421298f66e5143f81c9f91f"}, - {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:047a640f5c9c6ade7b1cc6680a0e28c9dd5a0825135acbd3569cc96ea00b2505"}, - {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7f7e58effd14b641c5e4dec8c7dab02fb67a13df90329e61c869b9cc607ef752"}, - {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c2910967e6ab16bf6fbeb1f771c89a7050947221ae12a5b0b60f3bca2ee19bca"}, - {file = "pyzmq-25.1.1-cp311-cp311-win32.whl", hash = "sha256:76c1c8efb3ca3a1818b837aea423ff8a07bbf7aafe9f2f6582b61a0458b1a329"}, - {file = "pyzmq-25.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:44e58a0554b21fc662f2712814a746635ed668d0fbc98b7cb9d74cb798d202e6"}, - {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:e1ffa1c924e8c72778b9ccd386a7067cddf626884fd8277f503c48bb5f51c762"}, - {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1af379b33ef33757224da93e9da62e6471cf4a66d10078cf32bae8127d3d0d4a"}, - {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cff084c6933680d1f8b2f3b4ff5bbb88538a4aac00d199ac13f49d0698727ecb"}, - {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2400a94f7dd9cb20cd012951a0cbf8249e3d554c63a9c0cdfd5cbb6c01d2dec"}, - {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d81f1ddae3858b8299d1da72dd7d19dd36aab654c19671aa8a7e7fb02f6638a"}, - {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:255ca2b219f9e5a3a9ef3081512e1358bd4760ce77828e1028b818ff5610b87b"}, - {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a882ac0a351288dd18ecae3326b8a49d10c61a68b01419f3a0b9a306190baf69"}, - {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:724c292bb26365659fc434e9567b3f1adbdb5e8d640c936ed901f49e03e5d32e"}, - {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ca1ed0bb2d850aa8471387882247c68f1e62a4af0ce9c8a1dbe0d2bf69e41fb"}, - {file = "pyzmq-25.1.1-cp312-cp312-win32.whl", hash = "sha256:b3451108ab861040754fa5208bca4a5496c65875710f76789a9ad27c801a0075"}, - {file = "pyzmq-25.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:eadbefd5e92ef8a345f0525b5cfd01cf4e4cc651a2cffb8f23c0dd184975d787"}, - {file = "pyzmq-25.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:db0b2af416ba735c6304c47f75d348f498b92952f5e3e8bff449336d2728795d"}, - {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c133e93b405eb0d36fa430c94185bdd13c36204a8635470cccc200723c13bb"}, - {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:273bc3959bcbff3f48606b28229b4721716598d76b5aaea2b4a9d0ab454ec062"}, - {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cbc8df5c6a88ba5ae385d8930da02201165408dde8d8322072e3e5ddd4f68e22"}, - {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:18d43df3f2302d836f2a56f17e5663e398416e9dd74b205b179065e61f1a6edf"}, - {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:73461eed88a88c866656e08f89299720a38cb4e9d34ae6bf5df6f71102570f2e"}, - {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:34c850ce7976d19ebe7b9d4b9bb8c9dfc7aac336c0958e2651b88cbd46682123"}, - {file = "pyzmq-25.1.1-cp36-cp36m-win32.whl", hash = "sha256:d2045d6d9439a0078f2a34b57c7b18c4a6aef0bee37f22e4ec9f32456c852c71"}, - {file = "pyzmq-25.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:458dea649f2f02a0b244ae6aef8dc29325a2810aa26b07af8374dc2a9faf57e3"}, - {file = "pyzmq-25.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cff25c5b315e63b07a36f0c2bab32c58eafbe57d0dce61b614ef4c76058c115"}, - {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1579413ae492b05de5a6174574f8c44c2b9b122a42015c5292afa4be2507f28"}, - {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3d0a409d3b28607cc427aa5c30a6f1e4452cc44e311f843e05edb28ab5e36da0"}, - {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21eb4e609a154a57c520e3d5bfa0d97e49b6872ea057b7c85257b11e78068222"}, - {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:034239843541ef7a1aee0c7b2cb7f6aafffb005ede965ae9cbd49d5ff4ff73cf"}, - {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f8115e303280ba09f3898194791a153862cbf9eef722ad8f7f741987ee2a97c7"}, - {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a5d26fe8f32f137e784f768143728438877d69a586ddeaad898558dc971a5ae"}, - {file = "pyzmq-25.1.1-cp37-cp37m-win32.whl", hash = "sha256:f32260e556a983bc5c7ed588d04c942c9a8f9c2e99213fec11a031e316874c7e"}, - {file = "pyzmq-25.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:abf34e43c531bbb510ae7e8f5b2b1f2a8ab93219510e2b287a944432fad135f3"}, - {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:87e34f31ca8f168c56d6fbf99692cc8d3b445abb5bfd08c229ae992d7547a92a"}, - {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c9c6c9b2c2f80747a98f34ef491c4d7b1a8d4853937bb1492774992a120f475d"}, - {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5619f3f5a4db5dbb572b095ea3cb5cc035335159d9da950830c9c4db2fbb6995"}, - {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5a34d2395073ef862b4032343cf0c32a712f3ab49d7ec4f42c9661e0294d106f"}, - {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f0e6b78220aba09815cd1f3a32b9c7cb3e02cb846d1cfc526b6595f6046618"}, - {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3669cf8ee3520c2f13b2e0351c41fea919852b220988d2049249db10046a7afb"}, - {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2d163a18819277e49911f7461567bda923461c50b19d169a062536fffe7cd9d2"}, - {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:df27ffddff4190667d40de7beba4a950b5ce78fe28a7dcc41d6f8a700a80a3c0"}, - {file = "pyzmq-25.1.1-cp38-cp38-win32.whl", hash = "sha256:a382372898a07479bd34bda781008e4a954ed8750f17891e794521c3e21c2e1c"}, - {file = "pyzmq-25.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:52533489f28d62eb1258a965f2aba28a82aa747202c8fa5a1c7a43b5db0e85c1"}, - {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:03b3f49b57264909aacd0741892f2aecf2f51fb053e7d8ac6767f6c700832f45"}, - {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:330f9e188d0d89080cde66dc7470f57d1926ff2fb5576227f14d5be7ab30b9fa"}, - {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2ca57a5be0389f2a65e6d3bb2962a971688cbdd30b4c0bd188c99e39c234f414"}, - {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d457aed310f2670f59cc5b57dcfced452aeeed77f9da2b9763616bd57e4dbaae"}, - {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c56d748ea50215abef7030c72b60dd723ed5b5c7e65e7bc2504e77843631c1a6"}, - {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8f03d3f0d01cb5a018debeb412441996a517b11c5c17ab2001aa0597c6d6882c"}, - {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:820c4a08195a681252f46926de10e29b6bbf3e17b30037bd4250d72dd3ddaab8"}, - {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17ef5f01d25b67ca8f98120d5fa1d21efe9611604e8eb03a5147360f517dd1e2"}, - {file = "pyzmq-25.1.1-cp39-cp39-win32.whl", hash = "sha256:04ccbed567171579ec2cebb9c8a3e30801723c575601f9a990ab25bcac6b51e2"}, - {file = "pyzmq-25.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e61f091c3ba0c3578411ef505992d356a812fb200643eab27f4f70eed34a29ef"}, - {file = "pyzmq-25.1.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ade6d25bb29c4555d718ac6d1443a7386595528c33d6b133b258f65f963bb0f6"}, - {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0c95ddd4f6e9fca4e9e3afaa4f9df8552f0ba5d1004e89ef0a68e1f1f9807c7"}, - {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48e466162a24daf86f6b5ca72444d2bf39a5e58da5f96370078be67c67adc978"}, - {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abc719161780932c4e11aaebb203be3d6acc6b38d2f26c0f523b5b59d2fc1996"}, - {file = "pyzmq-25.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1ccf825981640b8c34ae54231b7ed00271822ea1c6d8ba1090ebd4943759abf5"}, - {file = "pyzmq-25.1.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c2f20ce161ebdb0091a10c9ca0372e023ce24980d0e1f810f519da6f79c60800"}, - {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:deee9ca4727f53464daf089536e68b13e6104e84a37820a88b0a057b97bba2d2"}, - {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa8d6cdc8b8aa19ceb319aaa2b660cdaccc533ec477eeb1309e2a291eaacc43a"}, - {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:019e59ef5c5256a2c7378f2fb8560fc2a9ff1d315755204295b2eab96b254d0a"}, - {file = "pyzmq-25.1.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b9af3757495c1ee3b5c4e945c1df7be95562277c6e5bccc20a39aec50f826cd0"}, - {file = "pyzmq-25.1.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:548d6482dc8aadbe7e79d1b5806585c8120bafa1ef841167bc9090522b610fa6"}, - {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:057e824b2aae50accc0f9a0570998adc021b372478a921506fddd6c02e60308e"}, - {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2243700cc5548cff20963f0ca92d3e5e436394375ab8a354bbea2b12911b20b0"}, - {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79986f3b4af059777111409ee517da24a529bdbd46da578b33f25580adcff728"}, - {file = "pyzmq-25.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:11d58723d44d6ed4dd677c5615b2ffb19d5c426636345567d6af82be4dff8a55"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:49d238cf4b69652257db66d0c623cd3e09b5d2e9576b56bc067a396133a00d4a"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fedbdc753827cf014c01dbbee9c3be17e5a208dcd1bf8641ce2cd29580d1f0d4"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc16ac425cc927d0a57d242589f87ee093884ea4804c05a13834d07c20db203c"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11c1d2aed9079c6b0c9550a7257a836b4a637feb334904610f06d70eb44c56d2"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e8a701123029cc240cea61dd2d16ad57cab4691804143ce80ecd9286b464d180"}, - {file = "pyzmq-25.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:61706a6b6c24bdece85ff177fec393545a3191eeda35b07aaa1458a027ad1304"}, - {file = "pyzmq-25.1.1.tar.gz", hash = "sha256:259c22485b71abacdfa8bf79720cd7bcf4b9d128b30ea554f01ae71fdbfdaa23"}, -] - -[package.dependencies] -cffi = {version = "*", markers = "implementation_name == \"pypy\""} - -[[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.7, <4" -files = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "rfc3339-validator" -version = "0.1.4" -description = "A pure python RFC3339 validator" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, - {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, -] - -[package.dependencies] -six = "*" - -[[package]] -name = "rfc3986-validator" -version = "0.1.1" -description = "Pure python rfc3986 validator" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9"}, - {file = "rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"}, -] - -[[package]] -name = "rich" -version = "12.6.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.6.3,<4.0.0" -files = [ - {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, - {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, -] - -[package.dependencies] -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -optional = false -python-versions = ">= 3.7" -files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "send2trash" -version = "1.8.2" -description = "Send file to trash natively under Mac OS X, Windows and Linux" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "Send2Trash-1.8.2-py3-none-any.whl", hash = "sha256:a384719d99c07ce1eefd6905d2decb6f8b7ed054025bb0e618919f945de4f679"}, - {file = "Send2Trash-1.8.2.tar.gz", hash = "sha256:c132d59fa44b9ca2b1699af5c86f57ce9f4c5eb56629d5d55fbb7a35f84e2312"}, -] - -[package.extras] -nativelib = ["pyobjc-framework-Cocoa", "pywin32"] -objc = ["pyobjc-framework-Cocoa"] -win32 = ["pywin32"] - -[[package]] -name = "setuptools" -version = "68.2.2" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = true -python-versions = ">=3.8" -files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "snakeviz" -version = "2.1.1" -description = "A web-based viewer for Python profiler output" -optional = false -python-versions = "*" -files = [ - {file = "snakeviz-2.1.1-py2.py3-none-any.whl", hash = "sha256:931142dc927101c9a4b6e89bc0577ff1a3d1886b483a04e6af70c31d2c3dce19"}, - {file = "snakeviz-2.1.1.tar.gz", hash = "sha256:0d96c006304f095cb4b3fb7ed98bb866ca35a7ca4ab9020bbc27d295ee4c94d9"}, -] - -[package.dependencies] -tornado = ">=2.0" - -[[package]] -name = "sniffio" -version = "1.3.0" -description = "Sniff out which async library your code is running under" -optional = true -python-versions = ">=3.7" -files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] - -[[package]] -name = "sortedcontainers" -version = "2.4.0" -description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -optional = false -python-versions = "*" -files = [ - {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, - {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, -] - -[[package]] -name = "soupsieve" -version = "2.5" -description = "A modern CSS selector implementation for Beautiful Soup." -optional = true -python-versions = ">=3.8" -files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, -] - -[[package]] -name = "stack-data" -version = "0.6.3" -description = "Extract data from python stack frames and tracebacks for informative displays" -optional = true -python-versions = "*" -files = [ - {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, - {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, -] - -[package.dependencies] -asttokens = ">=2.1.0" -executing = ">=1.2.0" -pure-eval = "*" - -[package.extras] -tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] - -[[package]] -name = "statsd" -version = "4.0.1" -description = "A simple statsd client." -optional = false -python-versions = "*" -files = [ - {file = "statsd-4.0.1-py2.py3-none-any.whl", hash = "sha256:c2676519927f7afade3723aca9ca8ea986ef5b059556a980a867721ca69df093"}, - {file = "statsd-4.0.1.tar.gz", hash = "sha256:99763da81bfea8daf6b3d22d11aaccb01a8d0f52ea521daab37e758a4ca7d128"}, -] - -[[package]] -name = "tabulate" -version = "0.9.0" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, - {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, -] - -[package.extras] -widechars = ["wcwidth"] - -[[package]] -name = "tblib" -version = "1.7.0" -description = "Traceback serialization library." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "tblib-1.7.0-py2.py3-none-any.whl", hash = "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"}, - {file = "tblib-1.7.0.tar.gz", hash = "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c"}, -] - -[[package]] -name = "temporalio" -version = "1.5.1" -description = "Temporal.io Python SDK" -optional = true -python-versions = ">=3.8,<4.0" -files = [ - {file = "temporalio-1.5.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cd1f8930787c728e30ca2fecf86175cafd1781d97e3ee7cdf6e41915c566a835"}, - {file = "temporalio-1.5.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:2b3765e0b6b0ef0b670cf39720a80280fd35be2444633c715b741d2b5428ceb6"}, - {file = "temporalio-1.5.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47149204b6430c8553d5dd6dfe2fbc6830bf6fd8ab08463ee4c97885c68f3082"}, - {file = "temporalio-1.5.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1167f6fc31355170cdb4f5f7b89f0f7e36c54d0aecb0ee9aa611f73e32db7d78"}, - {file = "temporalio-1.5.1-cp38-abi3-win_amd64.whl", hash = "sha256:15d36d2038b0ac33511163619bea7ead6f10aca3db5bad4b9d464d3fa0f4ff48"}, - {file = "temporalio-1.5.1.tar.gz", hash = "sha256:4c7bbc8a3e8df1ffc0c7d213bdcad26ae055bdd615567ce1ca4bfa9f28f831b8"}, -] - -[package.dependencies] -protobuf = ">=3.20" -python-dateutil = {version = ">=2.8.2,<3.0.0", markers = "python_version < \"3.11\""} -types-protobuf = ">=3.20" -typing-extensions = ">=4.2.0,<5.0.0" - -[package.extras] -grpc = ["grpcio (>=1.59.0,<2.0.0)"] -opentelemetry = ["opentelemetry-api (>=1.11.1,<2.0.0)", "opentelemetry-sdk (>=1.11.1,<2.0.0)"] - -[[package]] -name = "terminado" -version = "0.17.1" -description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." -optional = true -python-versions = ">=3.7" -files = [ - {file = "terminado-0.17.1-py3-none-any.whl", hash = "sha256:8650d44334eba354dd591129ca3124a6ba42c3d5b70df5051b6921d506fdaeae"}, - {file = "terminado-0.17.1.tar.gz", hash = "sha256:6ccbbcd3a4f8a25a5ec04991f39a0b8db52dfcd487ea0e578d977e6752380333"}, -] - -[package.dependencies] -ptyprocess = {version = "*", markers = "os_name != \"nt\""} -pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""} -tornado = ">=6.1.0" - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] - -[[package]] -name = "tinycss2" -version = "1.2.1" -description = "A tiny CSS parser" -optional = true -python-versions = ">=3.7" -files = [ - {file = "tinycss2-1.2.1-py3-none-any.whl", hash = "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847"}, - {file = "tinycss2-1.2.1.tar.gz", hash = "sha256:8cff3a8f066c2ec677c06dbc7b45619804a6938478d9d73c284b29d14ecb0627"}, -] - -[package.dependencies] -webencodings = ">=0.4" - -[package.extras] -doc = ["sphinx", "sphinx_rtd_theme"] -test = ["flake8", "isort", "pytest"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "toolz" -version = "0.12.0" -description = "List processing tools and functional utilities" -optional = false -python-versions = ">=3.5" -files = [ - {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, - {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, -] - -[[package]] -name = "tornado" -version = "6.2" -description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -optional = false -python-versions = ">= 3.7" -files = [ - {file = "tornado-6.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72"}, - {file = "tornado-6.2-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9"}, - {file = "tornado-6.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac"}, - {file = "tornado-6.2-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75"}, - {file = "tornado-6.2-cp37-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e"}, - {file = "tornado-6.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8"}, - {file = "tornado-6.2-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b"}, - {file = "tornado-6.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca"}, - {file = "tornado-6.2-cp37-abi3-win32.whl", hash = "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23"}, - {file = "tornado-6.2-cp37-abi3-win_amd64.whl", hash = "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b"}, - {file = "tornado-6.2.tar.gz", hash = "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13"}, -] - -[[package]] -name = "tqdm" -version = "4.64.1" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -files = [ - {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"}, - {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - -[[package]] -name = "traitlets" -version = "5.13.0" -description = "Traitlets Python configuration system" -optional = true -python-versions = ">=3.8" -files = [ - {file = "traitlets-5.13.0-py3-none-any.whl", hash = "sha256:baf991e61542da48fe8aef8b779a9ea0aa38d8a54166ee250d5af5ecf4486619"}, - {file = "traitlets-5.13.0.tar.gz", hash = "sha256:9b232b9430c8f57288c1024b34a8f0251ddcc47268927367a0dd3eeaca40deb5"}, -] - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["argcomplete (>=3.0.3)", "mypy (>=1.6.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] - -[[package]] -name = "types-protobuf" -version = "4.24.0.20240302" -description = "Typing stubs for protobuf" -optional = true -python-versions = ">=3.8" -files = [ - {file = "types-protobuf-4.24.0.20240302.tar.gz", hash = "sha256:f22c00cc0cea9722e71e14d389bba429af9e35a74a949719c167203a5abbe2e4"}, - {file = "types_protobuf-4.24.0.20240302-py3-none-any.whl", hash = "sha256:5c607990f50f14606c2edaf379f8acc7418fef1451b227aa3c6a8a2cbc6ff14a"}, -] - -[[package]] -name = "types-python-dateutil" -version = "2.8.19.14" -description = "Typing stubs for python-dateutil" -optional = true -python-versions = "*" -files = [ - {file = "types-python-dateutil-2.8.19.14.tar.gz", hash = "sha256:1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b"}, - {file = "types_python_dateutil-2.8.19.14-py3-none-any.whl", hash = "sha256:f977b8de27787639986b4e28963263fd0e5158942b3ecef91b9335c130cb1ce9"}, -] - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] - -[[package]] -name = "tzdata" -version = "2022.7" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -files = [ - {file = "tzdata-2022.7-py2.py3-none-any.whl", hash = "sha256:2b88858b0e3120792a3c0635c23daf36a7d7eeeca657c323da299d2094402a0d"}, - {file = "tzdata-2022.7.tar.gz", hash = "sha256:fe5f866eddd8b96e9fcba978f8e503c909b19ea7efda11e52e39494bad3a7bfa"}, -] - -[[package]] -name = "tzlocal" -version = "4.2" -description = "tzinfo object for the local timezone" -optional = false -python-versions = ">=3.6" -files = [ - {file = "tzlocal-4.2-py3-none-any.whl", hash = "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745"}, - {file = "tzlocal-4.2.tar.gz", hash = "sha256:ee5842fa3a795f023514ac2d801c4a81d1743bbe642e3940143326b3a00addd7"}, -] - -[package.dependencies] -"backports.zoneinfo" = {version = "*", markers = "python_version < \"3.9\""} -pytz-deprecation-shim = "*" -tzdata = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -devenv = ["black", "pyroma", "pytest-cov", "zest.releaser"] -test = ["pytest (>=4.3)", "pytest-mock (>=3.3)"] - -[[package]] -name = "uri-template" -version = "1.3.0" -description = "RFC 6570 URI Template Processor" -optional = true -python-versions = ">=3.7" -files = [ - {file = "uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7"}, - {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"}, -] - -[package.extras] -dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake8-commas", "flake8-comprehensions", "flake8-continuation", "flake8-datetimez", "flake8-docstrings", "flake8-import-order", "flake8-literal", "flake8-modern-annotations", "flake8-noqa", "flake8-pyproject", "flake8-requirements", "flake8-typechecking-import", "flake8-use-fstring", "mypy", "pep8-naming", "types-PyYAML"] - -[[package]] -name = "urllib3" -version = "1.26.13" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "viztracer" -version = "0.15.6" -description = "A debugging and profiling tool that can trace and visualize python code execution" -optional = false -python-versions = ">=3.7" -files = [ - {file = "viztracer-0.15.6-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:73fbd8e6f7ebe7e1eb78dd6bedbec7b8a217aa9d4910d880268de26ec8089590"}, - {file = "viztracer-0.15.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:917b6709d4880ecd4419fb627290e27026a3e01b45bd965f02b147579f093893"}, - {file = "viztracer-0.15.6-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.manylinux_2_24_i686.whl", hash = "sha256:92253c2d83b30527ac423dc5be2f4a0bc68473548e81d91367cc98c915589824"}, - {file = "viztracer-0.15.6-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4598cf49fa4f58f476996df00af9b13c0300fd5e236629214cae4a409839a66"}, - {file = "viztracer-0.15.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f50c0f33d6396c19923d91f797baff8b7a3b8a08c3419164cc9e2a58af5bae4"}, - {file = "viztracer-0.15.6-cp310-cp310-win32.whl", hash = "sha256:d9646c4f2ee4cde6f643d7060b5dd5d23e79e3689c9b1c55d3afa5a60d4a3572"}, - {file = "viztracer-0.15.6-cp310-cp310-win_amd64.whl", hash = "sha256:dc65dcb8284d9b632c9abb2ab1185fba547a1eb419b33848a062927e9e1ddb89"}, - {file = "viztracer-0.15.6-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:96ce96695583ca845d6d4b1bede44e2330a5fe672541ba066d723b215e4ba0d2"}, - {file = "viztracer-0.15.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a783a30fe759adc08ce482d6d3c94df1fd89ef4906b2b77ff3c43a6982365e7"}, - {file = "viztracer-0.15.6-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.manylinux_2_24_i686.whl", hash = "sha256:95590ae898e8388bd4961d977977afa496fbe35f378f4f2f3b869d2e62ff0edb"}, - {file = "viztracer-0.15.6-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1ee3d8cf988f38018c6820d55292eb21781b95ec8bec29e9afd1d61dab28a13"}, - {file = "viztracer-0.15.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9d077b5415eb01792fb53362ab6142beb35914c655dae3d98df6000637267d1"}, - {file = "viztracer-0.15.6-cp311-cp311-win32.whl", hash = "sha256:b7136f84f626676e18573dae0913525880cf156d16b9ddc5e7ec806d092c03d4"}, - {file = "viztracer-0.15.6-cp311-cp311-win_amd64.whl", hash = "sha256:b393f15a66618c87fb7eeb2e072c35aa25c4b9449b37f82e3f00067bdd10b20b"}, - {file = "viztracer-0.15.6-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:4c579da3430a29e6bebb43f52487690aa120d0c3680e4039fc0db0a606cb4355"}, - {file = "viztracer-0.15.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4210c39e8c40bbfdeda96d402f99ffdd6b078e8ab3d2137829f4833c5876bc27"}, - {file = "viztracer-0.15.6-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.manylinux_2_24_i686.whl", hash = "sha256:221362a11cd11942cd7c741335f5a805bd9293a0e762c36949bcc760fc1b5bca"}, - {file = "viztracer-0.15.6-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a702dfcbc9c6ad2a52f0cc1eb95f93620cdc2101637ae43d636d107e73df818"}, - {file = "viztracer-0.15.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08570e77d57263e3cac4279f5e60f280c1250cb04b1245e37da0cd4479abe9ef"}, - {file = "viztracer-0.15.6-cp37-cp37m-win32.whl", hash = "sha256:e817ffe06d5a3cfd85abc98bdce6b03561069f3200f5e329cfca003366bfa83f"}, - {file = "viztracer-0.15.6-cp37-cp37m-win_amd64.whl", hash = "sha256:33bb4a5990cb183c7150c435249cf4481974b9698cfebcd71fdba24de3458932"}, - {file = "viztracer-0.15.6-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:35dce1ee0963baa9c6f032306f1c93f826ac3ac685f8d5d1e1749e5da927e270"}, - {file = "viztracer-0.15.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e46b5a5d057d549fcf5e0de830595286c8800dfa419c0d648f11fb1f5f7ac5d"}, - {file = "viztracer-0.15.6-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.manylinux_2_24_i686.whl", hash = "sha256:12eb68546fa995b08f576d6dd97cd8b55c5f7bb45e0886652890ffebf2ffa781"}, - {file = "viztracer-0.15.6-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c50a6e36cae6cbc571bd761dbbcba14e14069eafadc38745fd1fbae14a2a3538"}, - {file = "viztracer-0.15.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abd349608ee9efaf2cb3d0dc6c18a42ca3288ba32e0a4e8838afc9965e11e59a"}, - {file = "viztracer-0.15.6-cp38-cp38-win32.whl", hash = "sha256:a5b8112ac24443345058b1bfe91b78cedea59a51cf8ac05d8912d642d88ee5e5"}, - {file = "viztracer-0.15.6-cp38-cp38-win_amd64.whl", hash = "sha256:41117c0b8f418b1694cc95b48c6f1a28fd1936efe6d4e2982475bfe194d46d1b"}, - {file = "viztracer-0.15.6-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:9df88a236685a3b368a3c33176b1e660c792fed67f4dfb29970c61b96769e88a"}, - {file = "viztracer-0.15.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fef6e892356becfdf6c9d4b43ee6ff3349fecad0f2e0be068326e227e71c98"}, - {file = "viztracer-0.15.6-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.manylinux_2_24_i686.whl", hash = "sha256:b7a0c216fd5eb29a6f8e0b3f546270b29738666a57df44fa7ccf0d7b31445099"}, - {file = "viztracer-0.15.6-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7054ed15a10f8052c5f85d21e712df5cd6ed53c7b6c1d2aac9d482fee42b9743"}, - {file = "viztracer-0.15.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca356949ca34fc4462ae467fcd281ea96722442f67824214f5868d30551bc96b"}, - {file = "viztracer-0.15.6-cp39-cp39-win32.whl", hash = "sha256:a0f927339a30442b0cf87c37e573c8b0423374f8664ec0daab29052fb9134d4a"}, - {file = "viztracer-0.15.6-cp39-cp39-win_amd64.whl", hash = "sha256:c0bb60a012e130413ce2d0d7452349869e0bc09e8fc39053df60215b218915b8"}, - {file = "viztracer-0.15.6.tar.gz", hash = "sha256:cfdbb2ae1a00f4903ec80ac1f9fadaf8f777f3d61082b98abd993795bd3da282"}, -] - -[package.dependencies] -objprint = ">=0.1.3" - -[package.extras] -full = ["orjson", "rich"] - -[[package]] -name = "warcio" -version = "1.7.4" -description = "Streaming WARC (and ARC) IO library" -optional = false -python-versions = "*" -files = [ - {file = "warcio-1.7.4-py2.py3-none-any.whl", hash = "sha256:ced1a162d76434d56abd81b37ac152821d1a11e1db835ead5d649f58068c2203"}, - {file = "warcio-1.7.4.tar.gz", hash = "sha256:e1889dad9ecac654de5b0973247f335a55827b1b14a8203772d18c749143ea51"}, -] - -[package.dependencies] -six = "*" - -[[package]] -name = "wcwidth" -version = "0.2.8" -description = "Measures the displayed width of unicode strings in a terminal" -optional = true -python-versions = "*" -files = [ - {file = "wcwidth-0.2.8-py2.py3-none-any.whl", hash = "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704"}, - {file = "wcwidth-0.2.8.tar.gz", hash = "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"}, -] - -[[package]] -name = "webcolors" -version = "1.13" -description = "A library for working with the color formats defined by HTML and CSS." -optional = true -python-versions = ">=3.7" -files = [ - {file = "webcolors-1.13-py3-none-any.whl", hash = "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf"}, - {file = "webcolors-1.13.tar.gz", hash = "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"}, -] - -[package.extras] -docs = ["furo", "sphinx", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-notfound-page", "sphinxext-opengraph"] -tests = ["pytest", "pytest-cov"] - -[[package]] -name = "webencodings" -version = "0.5.1" -description = "Character encoding aliases for legacy web content" -optional = true -python-versions = "*" -files = [ - {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, - {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, -] - -[[package]] -name = "websocket-client" -version = "1.6.4" -description = "WebSocket client for Python with low level API options" -optional = true -python-versions = ">=3.8" -files = [ - {file = "websocket-client-1.6.4.tar.gz", hash = "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df"}, - {file = "websocket_client-1.6.4-py3-none-any.whl", hash = "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24"}, -] - -[package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - -[[package]] -name = "werkzeug" -version = "2.2.2" -description = "The comprehensive WSGI web application library." -optional = true -python-versions = ">=3.7" -files = [ - {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, - {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, -] - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "zict" -version = "2.2.0" -description = "Mutable mapping tools" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zict-2.2.0-py2.py3-none-any.whl", hash = "sha256:dabcc8c8b6833aa3b6602daad50f03da068322c1a90999ff78aed9eecc8fa92c"}, - {file = "zict-2.2.0.tar.gz", hash = "sha256:d7366c2e2293314112dcf2432108428a67b927b00005619feefc310d12d833f3"}, -] - -[package.dependencies] -heapdict = "*" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, - {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[extras] -datapipeline = ["temporalio"] -dataviz = ["altair", "flask", "numpy", "pandas"] -research = ["jupyterlab"] - -[metadata] -lock-version = "2.0" -python-versions = ">=3.8,<4" -content-hash = "d9d5bf32044045c897156b1c51d94161503c05395239b74a31fc838d74de9e34" diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 7e585242..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,59 +0,0 @@ -[tool.poetry] -name = "oonidata" -version = "0.2.3" -description = "" -authors = ["Arturo Filastò "] - -[tool.poetry.dependencies] -python = ">=3.8,<4" -boto3 = ">=1.24" -PyYAML = ">=6.0" -tqdm = ">=4.64" -lz4 = ">=4.0" -requests = ">=2.27" -cryptography = ">=38.0.3" -clickhouse-driver = ">=0.2" -click = ">=8.0.0" -lxml = ">=4.9" -maxminddb = ">=2.2" -orjson = ">=3.8" -mashumaro = ">=3.0" -pyOpenSSL = ">=22.1" -importlib-metadata = {version = ">=1.0", python = "<3.8"} -tabulate = "^0.9.0" -warcio = "^1.7.4" -msgpack = "^1.0.4" -statsd = "^4.0.1" -click-loglevel = "^0.5.0" - -dask = {extras = ["complete"], version = "^2023.3.2"} -altair = {version = "^4.2.0", optional = true} -numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} -pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} -flask = {version = "^2.2.2", optional = true} -jupyterlab = {version = "^4.0.7", optional = true} -temporalio = {version = "^1.5.1", optional = true} - -[tool.poetry.dev-dependencies] -pytest = ">=7.2" -pytest-cov = ">=3.0.0" -black = ">=22.3.0" -snakeviz = ">=2.1.1" -mypy = ">=0.961" -pytest-benchmark = "^4.0.0" -pytest-profiling = "^1.7.0" -memray = "^1.4.1" -viztracer = "^0.15.6" - -[tool.poetry.scripts] -oonidata = 'oonidata.cli:cli' -datapipeline = 'oonidata.datapipeline.run_worker:main' - -[tool.poetry.extras] -dataviz = ["altair", "numpy", "pandas", "flask"] -research = ["jupyterlab"] -datapipeline = ["temporalio"] - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" From e89d69799af206069049cbdb44d80589403041e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 17:50:13 +0100 Subject: [PATCH 05/25] Separate pipeline code from data code --- oonidata/pyproject.toml | 165 +++++------------- oonidata/src/oonidata/__about__.py | 1 + oonidata/src/oonidata/__init__.py | 6 - oonidata/src/oonidata/cli/command.py | 52 +----- oonidata/src/oonidata/dataclient.py | 22 ++- .../normalize_yamlooni.py} | 2 +- .../src/oonidata/{__main__.py => main.py} | 0 oonidata/tests/conftest.py | 53 +----- oonidata/tests/test_cli.py | 119 ------------- oonidata/tests/test_dataformat.py | 2 +- .../src/oonipipeline}/analysis/__init__.py | 0 .../src/oonipipeline}/analysis/control.py | 0 .../src/oonipipeline}/analysis/datasources.py | 16 -- .../src/oonipipeline}/analysis/signal.py | 2 +- .../analysis/sql/tls_consistency_matview.sql | 0 .../oonipipeline}/analysis/web_analysis.py | 2 +- .../analysis/website_experiment_results.py | 0 .../src/oonipipeline}/dataviz/__init__.py | 0 .../oonipipeline}/dataviz/react/.gitignore | 0 .../src/oonipipeline}/dataviz/react/Readme.md | 0 .../oonipipeline}/dataviz/react/package.json | 0 .../oonipipeline}/dataviz/react/src/App.tsx | 0 .../react/src/components/DNSAnalysis.tsx | 0 .../dataviz/react/src/components/Dataviz.tsx | 0 .../oonipipeline}/dataviz/react/src/index.tsx | 0 .../src/oonipipeline}/dataviz/react/yarn.lock | 0 .../oonipipeline}/dataviz/static/.gitignore | 0 .../dataviz/templates/analysis.html | 0 .../dataviz/templates/react.html | 0 .../oonipipeline}/dataviz/templates/vega.html | 0 .../src/oonipipeline}/dataviz/theme.py | 0 .../src/oonipipeline}/dataviz/utils.py | 0 .../src/oonipipeline}/dataviz/viz.py | 0 .../src/oonipipeline}/dataviz/web.py | 2 +- .../src/oonipipeline}/db/__init__.py | 0 .../src/oonipipeline}/db/connections.py | 0 .../src/oonipipeline}/db/create_tables.py | 0 .../src/oonipipeline}/db/speed_estimate.py | 0 .../src/oonipipeline}/fingerprintdb.py | 0 .../src/oonipipeline/models}/__init__.py | 0 .../src/oonipipeline}/models/analysis.py | 0 .../oonipipeline}/models/experiment_result.py | 0 .../src/oonipipeline}/models/observations.py | 0 .../oonipipeline/to-port-workers}/__init__.py | 0 .../oonipipeline/to-port-workers}/common.py | 0 .../to-port-workers}/fingerprint_hunter.py | 2 +- .../to-port-workers}/response_archiver.py | 2 +- .../src/oonipipeline}/transforms/__init__.py | 0 .../transforms/nettests/__init__.py | 0 .../transforms/nettests/browser_web.py | 0 .../transforms/nettests/dnscheck.py | 0 .../transforms/nettests/facebook_messenger.py | 0 .../http_header_field_manipulation.py | 0 .../nettests/http_invalid_request_line.py | 0 .../nettests/measurement_transformer.py | 0 .../transforms/nettests/signal.py | 0 .../transforms/nettests/stun_reachability.py | 0 .../transforms/nettests/telegram.py | 0 .../oonipipeline}/transforms/nettests/tor.py | 0 .../transforms/nettests/urlgetter.py | 0 .../transforms/nettests/web_connectivity.py | 0 .../transforms/nettests/whatsapp.py | 0 .../src/oonipipeline/workflows/analysis.py | 2 +- oonipipeline/tests/__init__.py | 0 oonipipeline/tests/conftest.py | 107 ++++++++++++ .../tests/test_analysis.py | 0 {oonidata => oonipipeline}/tests/test_ctrl.py | 0 {oonidata => oonipipeline}/tests/test_db.py | 0 .../tests/test_experiment_results.py | 0 .../tests/test_fingerprints.py | 0 .../tests/test_netinfo.py | 0 .../tests/test_scoring.py | 0 .../tests/test_transforms.py | 0 .../tests/test_workers.py | 0 .../tests/to_port_test_full_workflow.py | 115 ++++++++++++ 75 files changed, 295 insertions(+), 377 deletions(-) create mode 100644 oonidata/src/oonidata/__about__.py rename oonidata/src/oonidata/{normalize.py => legacy/normalize_yamlooni.py} (99%) rename oonidata/src/oonidata/{__main__.py => main.py} (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/__init__.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/control.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/datasources.py (77%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/signal.py (99%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/sql/tls_consistency_matview.sql (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/web_analysis.py (99%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/analysis/website_experiment_results.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/__init__.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/.gitignore (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/Readme.md (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/package.json (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/src/App.tsx (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/src/components/DNSAnalysis.tsx (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/src/components/Dataviz.tsx (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/src/index.tsx (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/react/yarn.lock (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/static/.gitignore (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/templates/analysis.html (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/templates/react.html (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/templates/vega.html (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/theme.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/utils.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/viz.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/dataviz/web.py (98%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/db/__init__.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/db/connections.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/db/create_tables.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/db/speed_estimate.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/fingerprintdb.py (100%) rename {oonidata/src/oonidata/transforms/nettests => oonipipeline/src/oonipipeline/models}/__init__.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/models/analysis.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/models/experiment_result.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/models/observations.py (100%) rename {oonidata/src/oonidata/workers => oonipipeline/src/oonipipeline/to-port-workers}/__init__.py (100%) rename {oonidata/src/oonidata/workers => oonipipeline/src/oonipipeline/to-port-workers}/common.py (100%) rename {oonidata/src/oonidata/workers => oonipipeline/src/oonipipeline/to-port-workers}/fingerprint_hunter.py (97%) rename {oonidata/src/oonidata/workers => oonipipeline/src/oonipipeline/to-port-workers}/response_archiver.py (99%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/__init__.py (100%) create mode 100644 oonipipeline/src/oonipipeline/transforms/nettests/__init__.py rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/browser_web.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/dnscheck.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/facebook_messenger.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/http_header_field_manipulation.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/http_invalid_request_line.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/measurement_transformer.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/signal.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/stun_reachability.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/telegram.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/tor.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/urlgetter.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/web_connectivity.py (100%) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/transforms/nettests/whatsapp.py (100%) create mode 100644 oonipipeline/tests/__init__.py create mode 100644 oonipipeline/tests/conftest.py rename {oonidata => oonipipeline}/tests/test_analysis.py (100%) rename {oonidata => oonipipeline}/tests/test_ctrl.py (100%) rename {oonidata => oonipipeline}/tests/test_db.py (100%) rename {oonidata => oonipipeline}/tests/test_experiment_results.py (100%) rename {oonidata => oonipipeline}/tests/test_fingerprints.py (100%) rename {oonidata => oonipipeline}/tests/test_netinfo.py (100%) rename {oonidata => oonipipeline}/tests/test_scoring.py (100%) rename {oonidata => oonipipeline}/tests/test_transforms.py (100%) rename {oonidata => oonipipeline}/tests/test_workers.py (100%) create mode 100644 oonipipeline/tests/to_port_test_full_workflow.py diff --git a/oonidata/pyproject.toml b/oonidata/pyproject.toml index 06e8594d..f700860c 100644 --- a/oonidata/pyproject.toml +++ b/oonidata/pyproject.toml @@ -4,149 +4,64 @@ build-backend = "hatchling.build" [project] name = "oonidata" -version = "0.2.3" +dynamic = ["version"] description = "" -authors = [{ name = "OONI", email = "contact@ooni.org"}] +authors = [{ name = "OONI", email = "contact@ooni.org" }] dependencies = [ -"boto3 ~= 1.24", -"PyYAML ~= 6.0", -"tqdm ~= 4.64", -"lz4 ~= 4.0", -"requests ~= 2.27", -"cryptography ~= 38.0.3", -"clickhouse-driver ~= 0.2", -"click ~= 8.0.0", -"lxml ~= 4.9", -"maxminddb ~= 2.2", -"orjson ~= 3.8", -"mashumaro ~= 3.0", -"pyOpenSSL ~= 22.1", + "boto3 ~= 1.24", + "PyYAML ~= 6.0", + "tqdm ~= 4.64", + "lz4 ~= 4.0", + "requests ~= 2.27", + "cryptography ~= 38.0.3", + "clickhouse-driver ~= 0.2", + "click ~= 8.0.0", + "lxml ~= 4.9", + "maxminddb ~= 2.2", + "orjson ~= 3.8", + "mashumaro ~= 3.0", + "pyOpenSSL ~= 22.1", "fastapi ~= 0.108.0", -"tabulate ~= 0.9.0", -"warcio ~= 1.7.4", -"msgpack ~= 1.0.4", -"statsd ~= 4.0.1", -"click-loglevel ~= 0.5.0", -"dask ~= 2023.3.2", -"altair ~= 4.2.0", -"numpy ~= 1.23.5", -"pandas ~= 2.0.0", -"flask ~= 2.2.0", -"jupyterlab ~= 4.0.7", -"temporalio ~= 1.5.1", + "tabulate ~= 0.9.0", + "warcio ~= 1.7.4", + "msgpack ~= 1.0.4", + "statsd ~= 4.0.1", + "click-loglevel ~= 0.5.0", + "dask ~= 2023.3.2", + "altair ~= 4.2.0", + "numpy ~= 1.23.5", + "pandas ~= 2.0.0", + "flask ~= 2.2.0", + "jupyterlab ~= 4.0.7", + "temporalio ~= 1.5.1", ] [tool.hatch.build.targets.sdist] [tool.hatch.build.targets.wheel] -packages = ["oonidata"] +packages = ["src/oonidata"] [tool.hatch.envs.default] dependencies = [ -"pytest", -"pytest-cov", -"black", -"snakeviz", -"mypy", -"pytest-benchmark", -"pytest-profiling", -"memray", -"viztracer" + "pytest", + "pytest-cov", + "black", + "snakeviz", + "mypy", + "pytest-benchmark", + "pytest-profiling", + "memray", + "viztracer", ] path = ".venv/" +[tool.hatch.version] +path = "src/oonidata/__about__.py" + [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" cov-report = ["coverage report"] cov = ["test-cov", "cov-report"] - -[tool.poetry] -name = "oonidata" -version = "0.2.3" -description = "" -authors = ["Arturo Filastò "] - -python = ">=3.8,<4" -boto3 = ">=1.24" -PyYAML = ">=6.0" -tqdm = ">=4.64" -lz4 = ">=4.0" -requests = ">=2.27" -cryptography = ">=38.0.3" -clickhouse-driver = ">=0.2" -click = ">=8.0.0" -lxml = ">=4.9" -maxminddb = ">=2.2" -orjson = ">=3.8" -mashumaro = ">=3.0" -pyOpenSSL = ">=22.1" -importlib-metadata = {version = ">=1.0", python = "<3.8"} -tabulate = "^0.9.0" -warcio = "^1.7.4" -msgpack = "^1.0.4" -statsd = "^4.0.1" -click-loglevel = "^0.5.0" - -dask = {extras = ["complete"], version = "^2023.3.2"} -altair = {version = "^4.2.0", optional = true} -numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} -pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} -flask = {version = "^2.2.2", optional = true} -jupyterlab = {version = "^4.0.7", optional = true} -temporalio = {version = "^1.5.1", optional = true} - - -[tool.poetry.dependencies] -python = ">=3.8,<4" -boto3 = ">=1.24" -PyYAML = ">=6.0" -tqdm = ">=4.64" -lz4 = ">=4.0" -requests = ">=2.27" -cryptography = ">=38.0.3" -clickhouse-driver = ">=0.2" -click = ">=8.0.0" -lxml = ">=4.9" -maxminddb = ">=2.2" -orjson = ">=3.8" -mashumaro = ">=3.0" -pyOpenSSL = ">=22.1" -importlib-metadata = {version = ">=1.0", python = "<3.8"} -tabulate = "^0.9.0" -warcio = "^1.7.4" -msgpack = "^1.0.4" -statsd = "^4.0.1" -click-loglevel = "^0.5.0" - -dask = {extras = ["complete"], version = "^2023.3.2"} -altair = {version = "^4.2.0", optional = true} -numpy = {version = "^1.23.5", optional = true, python = ">=3.8"} -pandas = {version = "^2.0.0", optional = true, python = ">=3.8"} -flask = {version = "^2.2.2", optional = true} -jupyterlab = {version = "^4.0.7", optional = true} -temporalio = {version = "^1.5.1", optional = true} - -[tool.poetry.dev-dependencies] -pytest = ">=7.2" -pytest-cov = ">=3.0.0" -black = ">=22.3.0" -snakeviz = ">=2.1.1" -mypy = ">=0.961" -pytest-benchmark = "^4.0.0" -pytest-profiling = "^1.7.0" -memray = "^1.4.1" -viztracer = "^0.15.6" - -[tool.poetry.scripts] -oonidata = 'oonidata.cli:cli' -datapipeline = 'oonidata.datapipeline.run_worker:main' - -[tool.poetry.extras] -dataviz = ["altair", "numpy", "pandas", "flask"] -research = ["jupyterlab"] -datapipeline = ["temporalio"] - - diff --git a/oonidata/src/oonidata/__about__.py b/oonidata/src/oonidata/__about__.py new file mode 100644 index 00000000..ae50f1f6 --- /dev/null +++ b/oonidata/src/oonidata/__about__.py @@ -0,0 +1 @@ +VERSION = "0.2.3" diff --git a/oonidata/src/oonidata/__init__.py b/oonidata/src/oonidata/__init__.py index 91258bf2..e69de29b 100644 --- a/oonidata/src/oonidata/__init__.py +++ b/oonidata/src/oonidata/__init__.py @@ -1,6 +0,0 @@ -try: - import importlib.metadata as importlib_metadata -except ModuleNotFoundError: - import importlib_metadata # type: ignore it's only for py <=3.8 - -__version__ = importlib_metadata.version(__name__) diff --git a/oonidata/src/oonidata/cli/command.py b/oonidata/src/oonidata/cli/command.py index 62b556d7..8e160f71 100644 --- a/oonidata/src/oonidata/cli/command.py +++ b/oonidata/src/oonidata/cli/command.py @@ -1,20 +1,17 @@ import logging -import multiprocessing -from pathlib import Path import sys + +from pathlib import Path from typing import List, Optional from datetime import date, timedelta, datetime -from typing import List, Optional import click from click_loglevel import LogLevel -from oonidata import __version__ +from oonidata.__about__ import VERSION from oonidata.dataclient import ( sync_measurements, ) -from oonidata.db.connections import ClickhouseConnection -from oonidata.db.create_tables import create_queries, list_all_table_diffs from oonidata.netinfo import NetinfoDB @@ -72,7 +69,7 @@ def _parse_csv(ctx, param, s: Optional[str]) -> List[str]: help="Set logging level", show_default=True, ) -@click.version_option(__version__) +@click.version_option(VERSION) def cli(error_log_file: Path, log_level: int): log.addHandler(logging.StreamHandler(sys.stderr)) log.setLevel(log_level) @@ -118,46 +115,5 @@ def sync( ) -@cli.command() -@click.option("--clickhouse", type=str) -@click.option( - "--create-tables", - is_flag=True, - help="should we attempt to create the required clickhouse tables", -) -@click.option( - "--drop-tables", - is_flag=True, - help="should we drop tables before creating them", -) -def checkdb( - clickhouse: Optional[str], - create_tables: bool, - drop_tables: bool, -): - """ - Check if the database tables require migrations. If the create-tables flag - is not specified, it will not perform any operations. - """ - - if create_tables: - if not clickhouse: - click.echo("--clickhouse needs to be specified when creating tables") - return 1 - if drop_tables: - click.confirm( - "Are you sure you want to drop the tables before creation?", abort=True - ) - - with ClickhouseConnection(clickhouse) as db: - for query, table_name in create_queries: - if drop_tables: - db.execute(f"DROP TABLE IF EXISTS {table_name};") - db.execute(query) - - with ClickhouseConnection(clickhouse) as db: - list_all_table_diffs(db) - - if __name__ == "__main__": cli() diff --git a/oonidata/src/oonidata/dataclient.py b/oonidata/src/oonidata/dataclient.py index 22078ad5..551677a4 100644 --- a/oonidata/src/oonidata/dataclient.py +++ b/oonidata/src/oonidata/dataclient.py @@ -24,11 +24,13 @@ from tqdm.contrib.logging import tqdm_logging_redirect +from .models.nettests import NETTEST_MODELS +from .models.nettests.base_measurement import BaseMeasurement +from .models.nettests import SupportedDataformats -from oonidata.datautils import PerfTimer - -from oonidata.datautils import trim_measurement, trivial_id -from oonidata.normalize import iter_yaml_msmt_normalized +from .datautils import PerfTimer +from .datautils import trim_measurement, trivial_id +from .legacy.normalize_yamlooni import iter_yaml_msmt_normalized LEGACY_BUCKET_NAME = "ooni-data" MC_BUCKET_NAME = "ooni-data-eu-fra" @@ -820,3 +822,15 @@ def cb_list_fe(p): pbar.set_description( f"downloaded {download_count}/{len(to_download_fe)}" ) + + +def load_measurement( + msmt: Optional[dict] = None, msmt_path: Optional[Path] = None +) -> SupportedDataformats: + if msmt_path: + with msmt_path.open() as in_file: + msmt = orjson.loads(in_file.read()) + + assert msmt, "either msmt or msmt_path should be set" + dc = NETTEST_MODELS.get(msmt["test_name"], BaseMeasurement) + return dc.from_dict(msmt) diff --git a/oonidata/src/oonidata/normalize.py b/oonidata/src/oonidata/legacy/normalize_yamlooni.py similarity index 99% rename from oonidata/src/oonidata/normalize.py rename to oonidata/src/oonidata/legacy/normalize_yamlooni.py index 0c334c0a..9fa7c534 100644 --- a/oonidata/src/oonidata/normalize.py +++ b/oonidata/src/oonidata/legacy/normalize_yamlooni.py @@ -1,5 +1,5 @@ # -# Normalize YAML reports +# Normalize YAMLooni reports # from datetime import datetime diff --git a/oonidata/src/oonidata/__main__.py b/oonidata/src/oonidata/main.py similarity index 100% rename from oonidata/src/oonidata/__main__.py rename to oonidata/src/oonidata/main.py diff --git a/oonidata/tests/conftest.py b/oonidata/tests/conftest.py index cb9c036b..edf155a9 100644 --- a/oonidata/tests/conftest.py +++ b/oonidata/tests/conftest.py @@ -6,10 +6,7 @@ import pytest import orjson -from oonidata.db.connections import ClickhouseConnection -from oonidata.fingerprintdb import FingerprintDB -from oonidata.netinfo import NetinfoDB from oonidata.dataclient import sync_measurements from oonidata.apiclient import get_measurement_dict_by_uid @@ -24,19 +21,6 @@ def datadir(): return DATA_DIR -@pytest.fixture -def fingerprintdb(datadir): - return FingerprintDB( - datadir=datadir, - download=True, - ) - - -@pytest.fixture -def netinfodb(): - return NetinfoDB(datadir=DATA_DIR, download=True, max_age_seconds=60 * 60 * 24) - - @pytest.fixture def raw_measurements(): output_dir = FIXTURE_PATH / "raw_measurements" @@ -53,14 +37,14 @@ def raw_measurements(): @pytest.fixture -def measurements(): +def measurements(should_download=False): measurement_dir = FIXTURE_PATH / "measurements" measurement_dir.mkdir(parents=True, exist_ok=True) sampled_measurements = {} for msmt_uid in SAMPLE_MEASUREMENTS: sampled_measurements[msmt_uid] = measurement_dir / f"{msmt_uid}.json" - if sampled_measurements[msmt_uid].exists(): + if sampled_measurements[msmt_uid].exists() or not should_download: continue msmt = get_measurement_dict_by_uid(msmt_uid) with sampled_measurements[msmt_uid].open("wb") as out_file: @@ -71,36 +55,3 @@ def measurements(): @pytest.fixture def cli_runner(): return CliRunner() - - -from oonidata.db.create_tables import create_queries - - -def create_db_for_fixture(): - try: - with ClickhouseConnection(conn_url="clickhouse://localhost/") as db: - db.execute("CREATE DATABASE IF NOT EXISTS testing_oonidata") - except: - pytest.skip("no database connection") - - db = ClickhouseConnection(conn_url="clickhouse://localhost/testing_oonidata") - try: - db.execute("SELECT 1") - except: - pytest.skip("no database connection") - for query, _ in create_queries: - db.execute(query) - return db - - -@pytest.fixture -def db_notruncate(): - return create_db_for_fixture() - - -@pytest.fixture -def db(): - db = create_db_for_fixture() - for _, table_name in create_queries: - db.execute(f"TRUNCATE TABLE {table_name};") - return db diff --git a/oonidata/tests/test_cli.py b/oonidata/tests/test_cli.py index bf3e5158..3086e165 100644 --- a/oonidata/tests/test_cli.py +++ b/oonidata/tests/test_cli.py @@ -1,9 +1,7 @@ from pathlib import Path import time -import pytest from oonidata.cli import cli -from oonidata.db.connections import ClickhouseConnection def test_sync(cli_runner, tmp_path: Path): @@ -37,120 +35,3 @@ def wait_for_mutations(db, table_name): if len(res) == 0: # type: ignore break time.sleep(1) - - -def test_full_workflow( - db, cli_runner, fingerprintdb, netinfodb, datadir, tmp_path: Path -): - result = cli_runner.invoke( - cli, - [ - "mkobs", - "--probe-cc", - "BA", - "--start-day", - "2022-10-20", - "--end-day", - "2022-10-21", - "--test-name", - "web_connectivity", - "--create-tables", - "--data-dir", - datadir, - "--clickhouse", - db.clickhouse_url, - # "--archives-dir", - # tmp_path.absolute(), - ], - ) - assert result.exit_code == 0 - # assert len(list(tmp_path.glob("*.warc.gz"))) == 1 - res = db.execute( - "SELECT COUNT(DISTINCT(measurement_uid)) FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" - ) - assert res[0][0] == 200 # type: ignore - res = db.execute( - "SELECT COUNT() FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" - ) - obs_count = res[0][0] # type: ignore - - result = cli_runner.invoke( - cli, - [ - "mkobs", - "--probe-cc", - "BA", - "--start-day", - "2022-10-20", - "--end-day", - "2022-10-21", - "--test-name", - "web_connectivity", - "--create-tables", - "--data-dir", - datadir, - "--clickhouse", - db.clickhouse_url, - ], - ) - assert result.exit_code == 0 - - # Wait for the mutation to finish running - wait_for_mutations(db, "obs_web") - res = db.execute( - "SELECT COUNT() FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" - ) - # By re-running it against the same date, we should still get the same observation count - assert res[0][0] == obs_count # type: ignore - - result = cli_runner.invoke( - cli, - [ - "mkgt", - "--start-day", - "2022-10-20", - "--end-day", - "2022-10-21", - "--data-dir", - datadir, - "--clickhouse", - "clickhouse://localhost/testing_oonidata", - ], - ) - assert result.exit_code == 0 - - # result = cli_runner.invoke( - # cli, - # [ - # "fphunt", - # "--data-dir", - # datadir, - # "--archives-dir", - # tmp_path.absolute(), - # ], - # ) - # assert result.exit_code == 0 - - result = cli_runner.invoke( - cli, - [ - "mkanalysis", - "--probe-cc", - "BA", - "--start-day", - "2022-10-20", - "--end-day", - "2022-10-21", - "--test-name", - "web_connectivity", - "--data-dir", - datadir, - "--clickhouse", - db.clickhouse_url, - ], - ) - assert result.exit_code == 0 - res = db.execute( - "SELECT COUNT(DISTINCT(measurement_uid)) FROM measurement_experiment_result WHERE measurement_uid LIKE '20221020%' AND location_network_cc = 'BA'" - ) - assert res[0][0] == 200 # type: ignore diff --git a/oonidata/tests/test_dataformat.py b/oonidata/tests/test_dataformat.py index f8fb198e..9127011d 100644 --- a/oonidata/tests/test_dataformat.py +++ b/oonidata/tests/test_dataformat.py @@ -1,7 +1,7 @@ from base64 import b64encode from copy import deepcopy -from oonidata.analysis.datasources import load_measurement +from oonidata.dataclient import load_measurement from oonidata.models.dataformats import HTTPTransaction from oonidata.models.nettests.telegram import Telegram from oonidata.models.nettests.web_connectivity import WebConnectivity diff --git a/oonidata/src/oonidata/analysis/__init__.py b/oonipipeline/src/oonipipeline/analysis/__init__.py similarity index 100% rename from oonidata/src/oonidata/analysis/__init__.py rename to oonipipeline/src/oonipipeline/analysis/__init__.py diff --git a/oonidata/src/oonidata/analysis/control.py b/oonipipeline/src/oonipipeline/analysis/control.py similarity index 100% rename from oonidata/src/oonidata/analysis/control.py rename to oonipipeline/src/oonipipeline/analysis/control.py diff --git a/oonidata/src/oonidata/analysis/datasources.py b/oonipipeline/src/oonipipeline/analysis/datasources.py similarity index 77% rename from oonidata/src/oonidata/analysis/datasources.py rename to oonipipeline/src/oonipipeline/analysis/datasources.py index 3fb5bec0..7548fa56 100644 --- a/oonidata/src/oonidata/analysis/datasources.py +++ b/oonipipeline/src/oonipipeline/analysis/datasources.py @@ -1,12 +1,8 @@ import dataclasses from datetime import date, timedelta from typing import Generator, List, Optional -from pathlib import Path -import orjson from oonidata.db.connections import ClickhouseConnection -from oonidata.models.nettests import NETTEST_MODELS, SupportedDataformats -from oonidata.models.nettests.base_measurement import BaseMeasurement from oonidata.models.observations import WebObservation @@ -58,15 +54,3 @@ def iter_web_observations( if len(obs_group) > 0: yield obs_group - - -def load_measurement( - msmt: Optional[dict] = None, msmt_path: Optional[Path] = None -) -> SupportedDataformats: - if msmt_path: - with msmt_path.open() as in_file: - msmt = orjson.loads(in_file.read()) - - assert msmt, "either msmt or msmt_path should be set" - dc = NETTEST_MODELS.get(msmt["test_name"], BaseMeasurement) - return dc.from_dict(msmt) diff --git a/oonidata/src/oonidata/analysis/signal.py b/oonipipeline/src/oonipipeline/analysis/signal.py similarity index 99% rename from oonidata/src/oonidata/analysis/signal.py rename to oonipipeline/src/oonipipeline/analysis/signal.py index 16075d8c..1991aca4 100644 --- a/oonidata/src/oonidata/analysis/signal.py +++ b/oonipipeline/src/oonipipeline/analysis/signal.py @@ -7,7 +7,7 @@ fp_to_scope, iter_experiment_results, ) -from oonidata.fingerprintdb import FingerprintDB +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB from oonidata.models.observations import WebObservation diff --git a/oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql b/oonipipeline/src/oonipipeline/analysis/sql/tls_consistency_matview.sql similarity index 100% rename from oonidata/src/oonidata/analysis/sql/tls_consistency_matview.sql rename to oonipipeline/src/oonipipeline/analysis/sql/tls_consistency_matview.sql diff --git a/oonidata/src/oonidata/analysis/web_analysis.py b/oonipipeline/src/oonipipeline/analysis/web_analysis.py similarity index 99% rename from oonidata/src/oonidata/analysis/web_analysis.py rename to oonipipeline/src/oonipipeline/analysis/web_analysis.py index 3921ba65..522407ab 100644 --- a/oonidata/src/oonidata/analysis/web_analysis.py +++ b/oonipipeline/src/oonipipeline/analysis/web_analysis.py @@ -17,7 +17,7 @@ ) from oonidata.models.analysis import WebAnalysis -from oonidata.fingerprintdb import FingerprintDB +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB from oonidata.models.observations import WebControlObservation, WebObservation import logging diff --git a/oonidata/src/oonidata/analysis/website_experiment_results.py b/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py similarity index 100% rename from oonidata/src/oonidata/analysis/website_experiment_results.py rename to oonipipeline/src/oonipipeline/analysis/website_experiment_results.py diff --git a/oonidata/src/oonidata/dataviz/__init__.py b/oonipipeline/src/oonipipeline/dataviz/__init__.py similarity index 100% rename from oonidata/src/oonidata/dataviz/__init__.py rename to oonipipeline/src/oonipipeline/dataviz/__init__.py diff --git a/oonidata/src/oonidata/dataviz/react/.gitignore b/oonipipeline/src/oonipipeline/dataviz/react/.gitignore similarity index 100% rename from oonidata/src/oonidata/dataviz/react/.gitignore rename to oonipipeline/src/oonipipeline/dataviz/react/.gitignore diff --git a/oonidata/src/oonidata/dataviz/react/Readme.md b/oonipipeline/src/oonipipeline/dataviz/react/Readme.md similarity index 100% rename from oonidata/src/oonidata/dataviz/react/Readme.md rename to oonipipeline/src/oonipipeline/dataviz/react/Readme.md diff --git a/oonidata/src/oonidata/dataviz/react/package.json b/oonipipeline/src/oonipipeline/dataviz/react/package.json similarity index 100% rename from oonidata/src/oonidata/dataviz/react/package.json rename to oonipipeline/src/oonipipeline/dataviz/react/package.json diff --git a/oonidata/src/oonidata/dataviz/react/src/App.tsx b/oonipipeline/src/oonipipeline/dataviz/react/src/App.tsx similarity index 100% rename from oonidata/src/oonidata/dataviz/react/src/App.tsx rename to oonipipeline/src/oonipipeline/dataviz/react/src/App.tsx diff --git a/oonidata/src/oonidata/dataviz/react/src/components/DNSAnalysis.tsx b/oonipipeline/src/oonipipeline/dataviz/react/src/components/DNSAnalysis.tsx similarity index 100% rename from oonidata/src/oonidata/dataviz/react/src/components/DNSAnalysis.tsx rename to oonipipeline/src/oonipipeline/dataviz/react/src/components/DNSAnalysis.tsx diff --git a/oonidata/src/oonidata/dataviz/react/src/components/Dataviz.tsx b/oonipipeline/src/oonipipeline/dataviz/react/src/components/Dataviz.tsx similarity index 100% rename from oonidata/src/oonidata/dataviz/react/src/components/Dataviz.tsx rename to oonipipeline/src/oonipipeline/dataviz/react/src/components/Dataviz.tsx diff --git a/oonidata/src/oonidata/dataviz/react/src/index.tsx b/oonipipeline/src/oonipipeline/dataviz/react/src/index.tsx similarity index 100% rename from oonidata/src/oonidata/dataviz/react/src/index.tsx rename to oonipipeline/src/oonipipeline/dataviz/react/src/index.tsx diff --git a/oonidata/src/oonidata/dataviz/react/yarn.lock b/oonipipeline/src/oonipipeline/dataviz/react/yarn.lock similarity index 100% rename from oonidata/src/oonidata/dataviz/react/yarn.lock rename to oonipipeline/src/oonipipeline/dataviz/react/yarn.lock diff --git a/oonidata/src/oonidata/dataviz/static/.gitignore b/oonipipeline/src/oonipipeline/dataviz/static/.gitignore similarity index 100% rename from oonidata/src/oonidata/dataviz/static/.gitignore rename to oonipipeline/src/oonipipeline/dataviz/static/.gitignore diff --git a/oonidata/src/oonidata/dataviz/templates/analysis.html b/oonipipeline/src/oonipipeline/dataviz/templates/analysis.html similarity index 100% rename from oonidata/src/oonidata/dataviz/templates/analysis.html rename to oonipipeline/src/oonipipeline/dataviz/templates/analysis.html diff --git a/oonidata/src/oonidata/dataviz/templates/react.html b/oonipipeline/src/oonipipeline/dataviz/templates/react.html similarity index 100% rename from oonidata/src/oonidata/dataviz/templates/react.html rename to oonipipeline/src/oonipipeline/dataviz/templates/react.html diff --git a/oonidata/src/oonidata/dataviz/templates/vega.html b/oonipipeline/src/oonipipeline/dataviz/templates/vega.html similarity index 100% rename from oonidata/src/oonidata/dataviz/templates/vega.html rename to oonipipeline/src/oonipipeline/dataviz/templates/vega.html diff --git a/oonidata/src/oonidata/dataviz/theme.py b/oonipipeline/src/oonipipeline/dataviz/theme.py similarity index 100% rename from oonidata/src/oonidata/dataviz/theme.py rename to oonipipeline/src/oonipipeline/dataviz/theme.py diff --git a/oonidata/src/oonidata/dataviz/utils.py b/oonipipeline/src/oonipipeline/dataviz/utils.py similarity index 100% rename from oonidata/src/oonidata/dataviz/utils.py rename to oonipipeline/src/oonipipeline/dataviz/utils.py diff --git a/oonidata/src/oonidata/dataviz/viz.py b/oonipipeline/src/oonipipeline/dataviz/viz.py similarity index 100% rename from oonidata/src/oonidata/dataviz/viz.py rename to oonipipeline/src/oonipipeline/dataviz/viz.py diff --git a/oonidata/src/oonidata/dataviz/web.py b/oonipipeline/src/oonipipeline/dataviz/web.py similarity index 98% rename from oonidata/src/oonidata/dataviz/web.py rename to oonipipeline/src/oonipipeline/dataviz/web.py index 88cd66a0..9bc0c159 100644 --- a/oonidata/src/oonidata/dataviz/web.py +++ b/oonipipeline/src/oonipipeline/dataviz/web.py @@ -23,7 +23,7 @@ get_df_dns_analysis_raw, ) from flask import Flask, request, render_template -from oonidata.fingerprintdb import FingerprintDB +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB from oonidata.netinfo import NetinfoDB from oonidata.transforms import measurement_to_observations diff --git a/oonidata/src/oonidata/db/__init__.py b/oonipipeline/src/oonipipeline/db/__init__.py similarity index 100% rename from oonidata/src/oonidata/db/__init__.py rename to oonipipeline/src/oonipipeline/db/__init__.py diff --git a/oonidata/src/oonidata/db/connections.py b/oonipipeline/src/oonipipeline/db/connections.py similarity index 100% rename from oonidata/src/oonidata/db/connections.py rename to oonipipeline/src/oonipipeline/db/connections.py diff --git a/oonidata/src/oonidata/db/create_tables.py b/oonipipeline/src/oonipipeline/db/create_tables.py similarity index 100% rename from oonidata/src/oonidata/db/create_tables.py rename to oonipipeline/src/oonipipeline/db/create_tables.py diff --git a/oonidata/src/oonidata/db/speed_estimate.py b/oonipipeline/src/oonipipeline/db/speed_estimate.py similarity index 100% rename from oonidata/src/oonidata/db/speed_estimate.py rename to oonipipeline/src/oonipipeline/db/speed_estimate.py diff --git a/oonidata/src/oonidata/fingerprintdb.py b/oonipipeline/src/oonipipeline/fingerprintdb.py similarity index 100% rename from oonidata/src/oonidata/fingerprintdb.py rename to oonipipeline/src/oonipipeline/fingerprintdb.py diff --git a/oonidata/src/oonidata/transforms/nettests/__init__.py b/oonipipeline/src/oonipipeline/models/__init__.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/__init__.py rename to oonipipeline/src/oonipipeline/models/__init__.py diff --git a/oonidata/src/oonidata/models/analysis.py b/oonipipeline/src/oonipipeline/models/analysis.py similarity index 100% rename from oonidata/src/oonidata/models/analysis.py rename to oonipipeline/src/oonipipeline/models/analysis.py diff --git a/oonidata/src/oonidata/models/experiment_result.py b/oonipipeline/src/oonipipeline/models/experiment_result.py similarity index 100% rename from oonidata/src/oonidata/models/experiment_result.py rename to oonipipeline/src/oonipipeline/models/experiment_result.py diff --git a/oonidata/src/oonidata/models/observations.py b/oonipipeline/src/oonipipeline/models/observations.py similarity index 100% rename from oonidata/src/oonidata/models/observations.py rename to oonipipeline/src/oonipipeline/models/observations.py diff --git a/oonidata/src/oonidata/workers/__init__.py b/oonipipeline/src/oonipipeline/to-port-workers/__init__.py similarity index 100% rename from oonidata/src/oonidata/workers/__init__.py rename to oonipipeline/src/oonipipeline/to-port-workers/__init__.py diff --git a/oonidata/src/oonidata/workers/common.py b/oonipipeline/src/oonipipeline/to-port-workers/common.py similarity index 100% rename from oonidata/src/oonidata/workers/common.py rename to oonipipeline/src/oonipipeline/to-port-workers/common.py diff --git a/oonidata/src/oonidata/workers/fingerprint_hunter.py b/oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py similarity index 97% rename from oonidata/src/oonidata/workers/fingerprint_hunter.py rename to oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py index 2455a5fa..7d9c5419 100644 --- a/oonidata/src/oonidata/workers/fingerprint_hunter.py +++ b/oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py @@ -15,7 +15,7 @@ import orjson from warcio.archiveiterator import ArchiveIterator -from oonidata.fingerprintdb import FingerprintDB, Fingerprint +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB, Fingerprint log = logging.getLogger("oonidata.processing") diff --git a/oonidata/src/oonidata/workers/response_archiver.py b/oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py similarity index 99% rename from oonidata/src/oonidata/workers/response_archiver.py rename to oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py index 0bf4b93b..f50e5a05 100644 --- a/oonidata/src/oonidata/workers/response_archiver.py +++ b/oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py @@ -23,7 +23,7 @@ from warcio.statusandheaders import StatusAndHeaders from oonidata.analysis.datasources import load_measurement from oonidata.dataclient import date_interval, iter_measurements -from oonidata.fingerprintdb import FingerprintDB, Fingerprint +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB, Fingerprint from oonidata.models.nettests.web_connectivity import WebConnectivity log = logging.getLogger("oonidata.processing") diff --git a/oonidata/src/oonidata/transforms/__init__.py b/oonipipeline/src/oonipipeline/transforms/__init__.py similarity index 100% rename from oonidata/src/oonidata/transforms/__init__.py rename to oonipipeline/src/oonipipeline/transforms/__init__.py diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/__init__.py b/oonipipeline/src/oonipipeline/transforms/nettests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oonidata/src/oonidata/transforms/nettests/browser_web.py b/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/browser_web.py rename to oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py diff --git a/oonidata/src/oonidata/transforms/nettests/dnscheck.py b/oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/dnscheck.py rename to oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py diff --git a/oonidata/src/oonidata/transforms/nettests/facebook_messenger.py b/oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/facebook_messenger.py rename to oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py diff --git a/oonidata/src/oonidata/transforms/nettests/http_header_field_manipulation.py b/oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/http_header_field_manipulation.py rename to oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py diff --git a/oonidata/src/oonidata/transforms/nettests/http_invalid_request_line.py b/oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/http_invalid_request_line.py rename to oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py diff --git a/oonidata/src/oonidata/transforms/nettests/measurement_transformer.py b/oonipipeline/src/oonipipeline/transforms/nettests/measurement_transformer.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/measurement_transformer.py rename to oonipipeline/src/oonipipeline/transforms/nettests/measurement_transformer.py diff --git a/oonidata/src/oonidata/transforms/nettests/signal.py b/oonipipeline/src/oonipipeline/transforms/nettests/signal.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/signal.py rename to oonipipeline/src/oonipipeline/transforms/nettests/signal.py diff --git a/oonidata/src/oonidata/transforms/nettests/stun_reachability.py b/oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/stun_reachability.py rename to oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py diff --git a/oonidata/src/oonidata/transforms/nettests/telegram.py b/oonipipeline/src/oonipipeline/transforms/nettests/telegram.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/telegram.py rename to oonipipeline/src/oonipipeline/transforms/nettests/telegram.py diff --git a/oonidata/src/oonidata/transforms/nettests/tor.py b/oonipipeline/src/oonipipeline/transforms/nettests/tor.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/tor.py rename to oonipipeline/src/oonipipeline/transforms/nettests/tor.py diff --git a/oonidata/src/oonidata/transforms/nettests/urlgetter.py b/oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/urlgetter.py rename to oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py diff --git a/oonidata/src/oonidata/transforms/nettests/web_connectivity.py b/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/web_connectivity.py rename to oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py diff --git a/oonidata/src/oonidata/transforms/nettests/whatsapp.py b/oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py similarity index 100% rename from oonidata/src/oonidata/transforms/nettests/whatsapp.py rename to oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py diff --git a/oonipipeline/src/oonipipeline/workflows/analysis.py b/oonipipeline/src/oonipipeline/workflows/analysis.py index b57c418f..69f68e50 100644 --- a/oonipipeline/src/oonipipeline/workflows/analysis.py +++ b/oonipipeline/src/oonipipeline/workflows/analysis.py @@ -19,7 +19,7 @@ from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer from oonidata.db.connections import ClickhouseConnection -from oonidata.fingerprintdb import FingerprintDB +from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB from oonidata.models.analysis import WebAnalysis from oonidata.models.experiment_result import MeasurementExperimentResult from oonidata.netinfo import NetinfoDB diff --git a/oonipipeline/tests/__init__.py b/oonipipeline/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oonipipeline/tests/conftest.py b/oonipipeline/tests/conftest.py new file mode 100644 index 00000000..02486918 --- /dev/null +++ b/oonipipeline/tests/conftest.py @@ -0,0 +1,107 @@ +import os +from pathlib import Path +from datetime import date +from click.testing import CliRunner + +import pytest + +import orjson +from oonipipeline.db.connections import ClickhouseConnection + +from oonipipeline.fingerprintdb import FingerprintDB +from oonipipeline.netinfo import NetinfoDB + +from oonidata.dataclient import sync_measurements +from oonidata.apiclient import get_measurement_dict_by_uid + +from ._sample_measurements import SAMPLE_MEASUREMENTS + +FIXTURE_PATH = Path(os.path.dirname(os.path.realpath(__file__))) / "data" +DATA_DIR = FIXTURE_PATH / "datadir" + + +@pytest.fixture +def datadir(): + return DATA_DIR + + +@pytest.fixture +def fingerprintdb(datadir): + return FingerprintDB( + datadir=datadir, + download=True, + ) + + +@pytest.fixture +def netinfodb(): + return NetinfoDB(datadir=DATA_DIR, download=True, max_age_seconds=60 * 60 * 24) + + +@pytest.fixture +def raw_measurements(): + output_dir = FIXTURE_PATH / "raw_measurements" + if (output_dir / "signal" / "2022-10-01").exists(): + return output_dir + sync_measurements( + output_dir=output_dir, + probe_cc=["IT"], + test_name=["web_connectivity", "signal"], + start_day=date(2022, 10, 1), + end_day=date(2022, 10, 2), + ) + return output_dir + + +@pytest.fixture +def measurements(): + measurement_dir = FIXTURE_PATH / "measurements" + measurement_dir.mkdir(parents=True, exist_ok=True) + + sampled_measurements = {} + for msmt_uid in SAMPLE_MEASUREMENTS: + sampled_measurements[msmt_uid] = measurement_dir / f"{msmt_uid}.json" + if sampled_measurements[msmt_uid].exists(): + continue + msmt = get_measurement_dict_by_uid(msmt_uid) + with sampled_measurements[msmt_uid].open("wb") as out_file: + out_file.write(orjson.dumps(msmt)) + return sampled_measurements + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +from oonidata.db.create_tables import create_queries + + +def create_db_for_fixture(): + try: + with ClickhouseConnection(conn_url="clickhouse://localhost/") as db: + db.execute("CREATE DATABASE IF NOT EXISTS testing_oonidata") + except: + pytest.skip("no database connection") + + db = ClickhouseConnection(conn_url="clickhouse://localhost/testing_oonidata") + try: + db.execute("SELECT 1") + except: + pytest.skip("no database connection") + for query, _ in create_queries: + db.execute(query) + return db + + +@pytest.fixture +def db_notruncate(): + return create_db_for_fixture() + + +@pytest.fixture +def db(): + db = create_db_for_fixture() + for _, table_name in create_queries: + db.execute(f"TRUNCATE TABLE {table_name};") + return db diff --git a/oonidata/tests/test_analysis.py b/oonipipeline/tests/test_analysis.py similarity index 100% rename from oonidata/tests/test_analysis.py rename to oonipipeline/tests/test_analysis.py diff --git a/oonidata/tests/test_ctrl.py b/oonipipeline/tests/test_ctrl.py similarity index 100% rename from oonidata/tests/test_ctrl.py rename to oonipipeline/tests/test_ctrl.py diff --git a/oonidata/tests/test_db.py b/oonipipeline/tests/test_db.py similarity index 100% rename from oonidata/tests/test_db.py rename to oonipipeline/tests/test_db.py diff --git a/oonidata/tests/test_experiment_results.py b/oonipipeline/tests/test_experiment_results.py similarity index 100% rename from oonidata/tests/test_experiment_results.py rename to oonipipeline/tests/test_experiment_results.py diff --git a/oonidata/tests/test_fingerprints.py b/oonipipeline/tests/test_fingerprints.py similarity index 100% rename from oonidata/tests/test_fingerprints.py rename to oonipipeline/tests/test_fingerprints.py diff --git a/oonidata/tests/test_netinfo.py b/oonipipeline/tests/test_netinfo.py similarity index 100% rename from oonidata/tests/test_netinfo.py rename to oonipipeline/tests/test_netinfo.py diff --git a/oonidata/tests/test_scoring.py b/oonipipeline/tests/test_scoring.py similarity index 100% rename from oonidata/tests/test_scoring.py rename to oonipipeline/tests/test_scoring.py diff --git a/oonidata/tests/test_transforms.py b/oonipipeline/tests/test_transforms.py similarity index 100% rename from oonidata/tests/test_transforms.py rename to oonipipeline/tests/test_transforms.py diff --git a/oonidata/tests/test_workers.py b/oonipipeline/tests/test_workers.py similarity index 100% rename from oonidata/tests/test_workers.py rename to oonipipeline/tests/test_workers.py diff --git a/oonipipeline/tests/to_port_test_full_workflow.py b/oonipipeline/tests/to_port_test_full_workflow.py new file mode 100644 index 00000000..eb9dd5c5 --- /dev/null +++ b/oonipipeline/tests/to_port_test_full_workflow.py @@ -0,0 +1,115 @@ +def _test_full_workflow( + db, cli_runner, fingerprintdb, netinfodb, datadir, tmp_path: Path +): + result = cli_runner.invoke( + cli, + [ + "mkobs", + "--probe-cc", + "BA", + "--start-day", + "2022-10-20", + "--end-day", + "2022-10-21", + "--test-name", + "web_connectivity", + "--create-tables", + "--data-dir", + datadir, + "--clickhouse", + db.clickhouse_url, + # "--archives-dir", + # tmp_path.absolute(), + ], + ) + assert result.exit_code == 0 + # assert len(list(tmp_path.glob("*.warc.gz"))) == 1 + res = db.execute( + "SELECT COUNT(DISTINCT(measurement_uid)) FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" + ) + assert res[0][0] == 200 # type: ignore + res = db.execute( + "SELECT COUNT() FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" + ) + obs_count = res[0][0] # type: ignore + + result = cli_runner.invoke( + cli, + [ + "mkobs", + "--probe-cc", + "BA", + "--start-day", + "2022-10-20", + "--end-day", + "2022-10-21", + "--test-name", + "web_connectivity", + "--create-tables", + "--data-dir", + datadir, + "--clickhouse", + db.clickhouse_url, + ], + ) + assert result.exit_code == 0 + + # Wait for the mutation to finish running + wait_for_mutations(db, "obs_web") + res = db.execute( + "SELECT COUNT() FROM obs_web WHERE bucket_date = '2022-10-20' AND probe_cc = 'BA'" + ) + # By re-running it against the same date, we should still get the same observation count + assert res[0][0] == obs_count # type: ignore + + result = cli_runner.invoke( + cli, + [ + "mkgt", + "--start-day", + "2022-10-20", + "--end-day", + "2022-10-21", + "--data-dir", + datadir, + "--clickhouse", + "clickhouse://localhost/testing_oonidata", + ], + ) + assert result.exit_code == 0 + + # result = cli_runner.invoke( + # cli, + # [ + # "fphunt", + # "--data-dir", + # datadir, + # "--archives-dir", + # tmp_path.absolute(), + # ], + # ) + # assert result.exit_code == 0 + + result = cli_runner.invoke( + cli, + [ + "mkanalysis", + "--probe-cc", + "BA", + "--start-day", + "2022-10-20", + "--end-day", + "2022-10-21", + "--test-name", + "web_connectivity", + "--data-dir", + datadir, + "--clickhouse", + db.clickhouse_url, + ], + ) + assert result.exit_code == 0 + res = db.execute( + "SELECT COUNT(DISTINCT(measurement_uid)) FROM measurement_experiment_result WHERE measurement_uid LIKE '20221020%' AND location_network_cc = 'BA'" + ) + assert res[0][0] == 200 # type: ignore From e43c566fe3210303db4493d354f49ef438cc7ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 18:25:34 +0100 Subject: [PATCH 06/25] Move integration tests into separate directory, write more unit tests --- .gitignore | 8 +- .../{_sample_measurements.py => _fixtures.py} | 23 ++- oonidata/tests/conftest.py | 44 +++- oonidata/tests/data/.gitignore | 4 +- oonidata/tests/{ => integration}/test_cli.py | 0 oonidata/tests/integration/test_dataclient.py | 176 ++++++++++++++++ .../tests/{ => integration}/test_normalize.py | 0 oonidata/tests/test_dataclient.py | 192 +++--------------- 8 files changed, 258 insertions(+), 189 deletions(-) rename oonidata/tests/{_sample_measurements.py => _fixtures.py} (67%) rename oonidata/tests/{ => integration}/test_cli.py (100%) create mode 100644 oonidata/tests/integration/test_dataclient.py rename oonidata/tests/{ => integration}/test_normalize.py (100%) diff --git a/.gitignore b/.gitignore index 5299ac8a..ea5e8da2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ __pycache__ -/.coverage -/coverage.xml +.coverage +coverage.xml + +.ipynb_checkpoints/ + /dist /datadir /output /attic /prof -.ipynb_checkpoints/ diff --git a/oonidata/tests/_sample_measurements.py b/oonidata/tests/_fixtures.py similarity index 67% rename from oonidata/tests/_sample_measurements.py rename to oonidata/tests/_fixtures.py index 45fcdc0a..102dace8 100644 --- a/oonidata/tests/_sample_measurements.py +++ b/oonidata/tests/_fixtures.py @@ -23,14 +23,17 @@ "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2", "20210101190046.780850_US_webconnectivity_3296f126f79ca186", "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6", - "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", - "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" - "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" - "20231222154141.824397_US_browserweb_615428b4802b5297", - "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", - "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", - "20231101164649.235575_RU_tor_ccf7519bf683c022", - "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter - "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected - "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit + # "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", + # "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" + # "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" + # "20231222154141.824397_US_browserweb_615428b4802b5297", + # "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", + # "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", + # "20231101164649.235575_RU_tor_ccf7519bf683c022", + # "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter + # "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected + # "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit ] + +SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"] +SAMPLE_JSONLGZS = ["2024030100_AM_webconnectivity.n1.0.jsonl.gz"] diff --git a/oonidata/tests/conftest.py b/oonidata/tests/conftest.py index edf155a9..196c5c71 100644 --- a/oonidata/tests/conftest.py +++ b/oonidata/tests/conftest.py @@ -10,7 +10,7 @@ from oonidata.dataclient import sync_measurements from oonidata.apiclient import get_measurement_dict_by_uid -from ._sample_measurements import SAMPLE_MEASUREMENTS +from ._fixtures import SAMPLE_MEASUREMENTS, SAMPLE_POSTCANS, SAMPLE_JSONLGZS FIXTURE_PATH = Path(os.path.dirname(os.path.realpath(__file__))) / "data" DATA_DIR = FIXTURE_PATH / "datadir" @@ -36,20 +36,50 @@ def raw_measurements(): return output_dir +def download_from_s3(name, dst_path): + print(name, dst_path) + assert False + + +def make_samples(sample_name, sample_list): + dir = FIXTURE_PATH / sample_name + dir.mkdir(parents=True, exist_ok=True) + + samples = {} + for name in sample_list: + sample_path = dir / name + if not sample_path.exists(): + download_from_s3(name, dir / name) + + samples[name] = dir / name + + return samples + + +@pytest.fixture +def postcans(): + return make_samples("postcans", SAMPLE_POSTCANS) + + +@pytest.fixture +def jsonlgzs(): + return make_samples("jsonlgzs", SAMPLE_JSONLGZS) + + @pytest.fixture -def measurements(should_download=False): +def measurements(): measurement_dir = FIXTURE_PATH / "measurements" measurement_dir.mkdir(parents=True, exist_ok=True) - sampled_measurements = {} + sample_measurements = {} for msmt_uid in SAMPLE_MEASUREMENTS: - sampled_measurements[msmt_uid] = measurement_dir / f"{msmt_uid}.json" - if sampled_measurements[msmt_uid].exists() or not should_download: + sample_measurements[msmt_uid] = measurement_dir / f"{msmt_uid}.json" + if sample_measurements[msmt_uid].exists(): continue msmt = get_measurement_dict_by_uid(msmt_uid) - with sampled_measurements[msmt_uid].open("wb") as out_file: + with sample_measurements[msmt_uid].open("wb") as out_file: out_file.write(orjson.dumps(msmt)) - return sampled_measurements + return sample_measurements @pytest.fixture diff --git a/oonidata/tests/data/.gitignore b/oonidata/tests/data/.gitignore index 3dd48105..74de692e 100644 --- a/oonidata/tests/data/.gitignore +++ b/oonidata/tests/data/.gitignore @@ -1,4 +1,6 @@ /datadir/* /raw_measurements/* /measurements/* -/s3-dump/* \ No newline at end of file +/s3-dump/* +/postcans/* +/jsonlgzs/* diff --git a/oonidata/tests/test_cli.py b/oonidata/tests/integration/test_cli.py similarity index 100% rename from oonidata/tests/test_cli.py rename to oonidata/tests/integration/test_cli.py diff --git a/oonidata/tests/integration/test_dataclient.py b/oonidata/tests/integration/test_dataclient.py new file mode 100644 index 00000000..8fb96f32 --- /dev/null +++ b/oonidata/tests/integration/test_dataclient.py @@ -0,0 +1,176 @@ +from datetime import date, datetime + +from oonidata.dataclient import ( + date_interval, + iter_file_entries, + get_v2_prefixes, + iter_measurements, +) +from oonidata.dataclient import ( + get_file_entries, + get_can_prefixes, + Prefix, + MC_BUCKET_NAME, +) + + +def test_iter_file_entries_new_jsonl(): + fe_list = list( + iter_file_entries( + Prefix( + prefix="jsonl/webconnectivity/IT/20201020/00/", + bucket_name=MC_BUCKET_NAME, + ) + ) + ) + assert len(fe_list) == 41 + for fe in fe_list: + assert fe.testname == "webconnectivity" + assert fe.probe_cc == "IT" + assert fe.size > 0 + assert fe.bucket_name == MC_BUCKET_NAME + assert fe.timestamp == datetime(2020, 10, 20, 0, 0) + assert fe.ext == "jsonl.gz" + + +def test_iter_file_entries_old_format(): + fe_list = list( + iter_file_entries( + Prefix( + prefix="raw/20211020/00/IT/webconnectivity/", bucket_name=MC_BUCKET_NAME + ) + ) + ) + assert len(fe_list) == 6 + for fe in fe_list: + assert fe.testname == "webconnectivity" + assert fe.probe_cc == "IT" + assert fe.size > 0 + assert fe.bucket_name == MC_BUCKET_NAME + assert fe.timestamp == datetime(2021, 10, 20, 0, 0) + + +def test_get_v2_prefixes(): + prefixes = list(get_v2_prefixes(set(), set(), date(2020, 1, 1), date(2020, 1, 2))) + # assert len(prefixes) == 2516 + assert len(prefixes) == 2905 + + +def test_get_file_entries(): + fe_list = get_file_entries( + probe_cc=None, + test_name=None, + start_day=date(2021, 1, 1), + end_day=date(2021, 1, 2), + from_cans=False, + ) + # assert len(fe_list) == 1125 + assert len(fe_list) == 3320 + + +def test_get_file_entries_for_cc(): + from oonidata.dataclient import ProgressStatus + + def progress_callback(p): + assert p.total_prefixes == 10 + assert ( + p.progress_status == ProgressStatus.LISTING_BEGIN + or p.progress_status == ProgressStatus.LISTING + ) + + fe_list = get_file_entries( + probe_cc="IT", + test_name="webconnectivity", + start_day=date(2022, 8, 1), + end_day=date(2022, 8, 11), + from_cans=True, + progress_callback=progress_callback, + ) + # assert len(fe_list) == 1125 + assert len(fe_list) == 454 + + +def test_get_can_prefixes(): + # print(get_can_prefixes(set(), set(), date(2019, 6, 2), date(2020, 10, 21))) + # print(get_can_prefixes(set(), set(), date(2020, 6, 2), date(2020, 10, 21))) + start_day = date(2020, 6, 1) + end_day = date(2020, 6, 11) + prefixes = get_can_prefixes(start_day, end_day) + assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( + prefixes + ), "Duplicate prefixes" + assert len(prefixes) == len( + list(date_interval(start_day, end_day)) + ), "Inconsistent prefix length" + + start_day = date(2020, 10, 12) + end_day = date(2020, 10, 22) + prefixes = get_can_prefixes(start_day, end_day) + + assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( + prefixes + ), "Duplicate prefixes" + assert len(prefixes) == len( + list(date_interval(start_day, end_day)) + ), "Inconsistent prefix length" + + start_day = date(2019, 1, 1) + end_day = date(2020, 10, 22) + prefixes = get_can_prefixes(start_day, end_day) + + assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( + prefixes + ), "Duplicate prefixes" + assert len(prefixes) == len( + list(date_interval(start_day, end_day)) + ), "Inconsistent prefix length" + + +def test_iter_measurements(caplog): + import logging + + caplog.set_level(logging.DEBUG, logger="oonidata.dataclient") + msmt_count_cans = 0 + report_id_cans = [] + msmt_count_jsonl = 0 + report_id_jsonl = [] + for msmt in iter_measurements( + start_day=date(2018, 1, 1), + end_day=date(2018, 1, 2), + probe_cc=["IT"], + test_name=["whatsapp"], + from_cans=True, + progress_callback=lambda x: print(x), + ): + msmt_count_cans += 1 + report_id_cans.append(msmt["report_id"]) + assert msmt["measurement_uid"] is not None + + for msmt in iter_measurements( + start_day=date(2018, 1, 1), + end_day=date(2018, 1, 2), + probe_cc=["IT"], + test_name=["whatsapp"], + from_cans=False, + progress_callback=lambda x: print(x), + ): + report_id_jsonl.append(msmt["report_id"]) + msmt_count_jsonl += 1 + + assert set(report_id_jsonl) == set(report_id_cans) + # TODO: these are disabled due to: https://github.com/ooni/backend/issues/613 + # We ought to probably come up with a workaround in the meantime + # assert msmt_count_jsonl == msmt_count_cans + # assert report_id_jsonl == report_id_cans + + count = 0 + for _ in iter_measurements( + start_day=date(2022, 10, 20), + end_day=date(2022, 10, 21), + probe_cc=["BA"], + test_name=["web_connectivity"], + from_cans=True, + progress_callback=lambda x: print(x), + ): + count += 1 + assert count == 200 \ No newline at end of file diff --git a/oonidata/tests/test_normalize.py b/oonidata/tests/integration/test_normalize.py similarity index 100% rename from oonidata/tests/test_normalize.py rename to oonidata/tests/integration/test_normalize.py diff --git a/oonidata/tests/test_dataclient.py b/oonidata/tests/test_dataclient.py index 8fb96f32..97961dfd 100644 --- a/oonidata/tests/test_dataclient.py +++ b/oonidata/tests/test_dataclient.py @@ -1,176 +1,32 @@ -from datetime import date, datetime +import pytest -from oonidata.dataclient import ( - date_interval, - iter_file_entries, - get_v2_prefixes, - iter_measurements, -) -from oonidata.dataclient import ( - get_file_entries, - get_can_prefixes, - Prefix, - MC_BUCKET_NAME, -) +from copy import deepcopy +from oonidata.dataclient import stream_postcan, stream_jsonl -def test_iter_file_entries_new_jsonl(): - fe_list = list( - iter_file_entries( - Prefix( - prefix="jsonl/webconnectivity/IT/20201020/00/", - bucket_name=MC_BUCKET_NAME, - ) - ) - ) - assert len(fe_list) == 41 - for fe in fe_list: - assert fe.testname == "webconnectivity" - assert fe.probe_cc == "IT" - assert fe.size > 0 - assert fe.bucket_name == MC_BUCKET_NAME - assert fe.timestamp == datetime(2020, 10, 20, 0, 0) - assert fe.ext == "jsonl.gz" +def test_stream_postcan(postcans, jsonlgzs): + json_msmts = [] + postcan_msmts = [] + msmt_count = 0 + with postcans["2024030100_AM_webconnectivity.n1.0.tar.gz"].open("rb") as in_file: + for msmt in stream_postcan(in_file): + assert msmt["measurement_uid"] is not None + postcan_msmts.append(msmt) + msmt_count += 1 -def test_iter_file_entries_old_format(): - fe_list = list( - iter_file_entries( - Prefix( - prefix="raw/20211020/00/IT/webconnectivity/", bucket_name=MC_BUCKET_NAME - ) - ) - ) - assert len(fe_list) == 6 - for fe in fe_list: - assert fe.testname == "webconnectivity" - assert fe.probe_cc == "IT" - assert fe.size > 0 - assert fe.bucket_name == MC_BUCKET_NAME - assert fe.timestamp == datetime(2021, 10, 20, 0, 0) + assert msmt_count == 100 + with jsonlgzs["2024030100_AM_webconnectivity.n1.0.jsonl.gz"].open("rb") as in_file: + for msmt in stream_jsonl(in_file): + json_msmts.append(msmt) + msmt_count += 1 -def test_get_v2_prefixes(): - prefixes = list(get_v2_prefixes(set(), set(), date(2020, 1, 1), date(2020, 1, 2))) - # assert len(prefixes) == 2516 - assert len(prefixes) == 2905 + assert len(json_msmts) == len(postcan_msmts) - -def test_get_file_entries(): - fe_list = get_file_entries( - probe_cc=None, - test_name=None, - start_day=date(2021, 1, 1), - end_day=date(2021, 1, 2), - from_cans=False, - ) - # assert len(fe_list) == 1125 - assert len(fe_list) == 3320 - - -def test_get_file_entries_for_cc(): - from oonidata.dataclient import ProgressStatus - - def progress_callback(p): - assert p.total_prefixes == 10 - assert ( - p.progress_status == ProgressStatus.LISTING_BEGIN - or p.progress_status == ProgressStatus.LISTING - ) - - fe_list = get_file_entries( - probe_cc="IT", - test_name="webconnectivity", - start_day=date(2022, 8, 1), - end_day=date(2022, 8, 11), - from_cans=True, - progress_callback=progress_callback, - ) - # assert len(fe_list) == 1125 - assert len(fe_list) == 454 - - -def test_get_can_prefixes(): - # print(get_can_prefixes(set(), set(), date(2019, 6, 2), date(2020, 10, 21))) - # print(get_can_prefixes(set(), set(), date(2020, 6, 2), date(2020, 10, 21))) - start_day = date(2020, 6, 1) - end_day = date(2020, 6, 11) - prefixes = get_can_prefixes(start_day, end_day) - assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( - prefixes - ), "Duplicate prefixes" - assert len(prefixes) == len( - list(date_interval(start_day, end_day)) - ), "Inconsistent prefix length" - - start_day = date(2020, 10, 12) - end_day = date(2020, 10, 22) - prefixes = get_can_prefixes(start_day, end_day) - - assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( - prefixes - ), "Duplicate prefixes" - assert len(prefixes) == len( - list(date_interval(start_day, end_day)) - ), "Inconsistent prefix length" - - start_day = date(2019, 1, 1) - end_day = date(2020, 10, 22) - prefixes = get_can_prefixes(start_day, end_day) - - assert len(set([p.prefix.split("/")[-1] for p in prefixes])) == len( - prefixes - ), "Duplicate prefixes" - assert len(prefixes) == len( - list(date_interval(start_day, end_day)) - ), "Inconsistent prefix length" - - -def test_iter_measurements(caplog): - import logging - - caplog.set_level(logging.DEBUG, logger="oonidata.dataclient") - msmt_count_cans = 0 - report_id_cans = [] - msmt_count_jsonl = 0 - report_id_jsonl = [] - for msmt in iter_measurements( - start_day=date(2018, 1, 1), - end_day=date(2018, 1, 2), - probe_cc=["IT"], - test_name=["whatsapp"], - from_cans=True, - progress_callback=lambda x: print(x), - ): - msmt_count_cans += 1 - report_id_cans.append(msmt["report_id"]) - assert msmt["measurement_uid"] is not None - - for msmt in iter_measurements( - start_day=date(2018, 1, 1), - end_day=date(2018, 1, 2), - probe_cc=["IT"], - test_name=["whatsapp"], - from_cans=False, - progress_callback=lambda x: print(x), - ): - report_id_jsonl.append(msmt["report_id"]) - msmt_count_jsonl += 1 - - assert set(report_id_jsonl) == set(report_id_cans) - # TODO: these are disabled due to: https://github.com/ooni/backend/issues/613 - # We ought to probably come up with a workaround in the meantime - # assert msmt_count_jsonl == msmt_count_cans - # assert report_id_jsonl == report_id_cans - - count = 0 - for _ in iter_measurements( - start_day=date(2022, 10, 20), - end_day=date(2022, 10, 21), - probe_cc=["BA"], - test_name=["web_connectivity"], - from_cans=True, - progress_callback=lambda x: print(x), - ): - count += 1 - assert count == 200 \ No newline at end of file + for idx, msmt in enumerate(json_msmts): + pc_msmt = deepcopy(postcan_msmts[idx]) + # postcans should be identical to jsonl if not for the report_id + pc_msmt.pop("measurement_uid") + assert pc_msmt["report_id"] == msmt["report_id"] + assert pc_msmt == msmt From e443c178dd4034f2577c15e7b10c1f78879767c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 18:38:43 +0100 Subject: [PATCH 07/25] Fix most import errors with oonipipeline --- .../src/oonipipeline/analysis/control.py | 9 ++-- .../src/oonipipeline/analysis/datasources.py | 4 +- .../src/oonipipeline/analysis/signal.py | 6 +-- .../src/oonipipeline/analysis/web_analysis.py | 20 +++++---- .../analysis/website_experiment_results.py | 6 +-- .../src/oonipipeline/db/create_tables.py | 11 ++--- .../oonipipeline/models/experiment_result.py | 2 +- .../src/oonipipeline/models/observations.py | 3 +- .../src/oonipipeline}/netinfo.py | 0 .../src/oonipipeline/transforms/__init__.py | 42 ------------------- .../oonipipeline/transforms/observations.py | 42 +++++++++++++++++++ .../src/oonipipeline/workflows/analysis.py | 21 +++++----- .../{to-port-workers => workflows}/common.py | 6 +-- .../oonipipeline/workflows/ground_truths.py | 8 ++-- .../oonipipeline/workflows/observations.py | 18 ++++---- 15 files changed, 99 insertions(+), 99 deletions(-) rename {oonidata/src/oonidata => oonipipeline/src/oonipipeline}/netinfo.py (100%) create mode 100644 oonipipeline/src/oonipipeline/transforms/observations.py rename oonipipeline/src/oonipipeline/{to-port-workers => workflows}/common.py (98%) diff --git a/oonipipeline/src/oonipipeline/analysis/control.py b/oonipipeline/src/oonipipeline/analysis/control.py index 9145b59a..5e7a89cd 100644 --- a/oonipipeline/src/oonipipeline/analysis/control.py +++ b/oonipipeline/src/oonipipeline/analysis/control.py @@ -4,12 +4,13 @@ from collections.abc import Iterable from typing import Any, Generator, Optional, Tuple, List, NamedTuple -from oonidata.models.observations import WebControlObservation, WebObservation -from oonidata.netinfo import NetinfoDB -from oonidata.db.connections import ClickhouseConnection +from ..models.observations import WebControlObservation, WebObservation +from ..netinfo import NetinfoDB -log = logging.getLogger("oonidata.processing") +from ..db.connections import ClickhouseConnection + +log = logging.getLogger(__name__) class WebGroundTruth(NamedTuple): diff --git a/oonipipeline/src/oonipipeline/analysis/datasources.py b/oonipipeline/src/oonipipeline/analysis/datasources.py index 7548fa56..c2d137d5 100644 --- a/oonipipeline/src/oonipipeline/analysis/datasources.py +++ b/oonipipeline/src/oonipipeline/analysis/datasources.py @@ -2,8 +2,8 @@ from datetime import date, timedelta from typing import Generator, List, Optional -from oonidata.db.connections import ClickhouseConnection -from oonidata.models.observations import WebObservation +from ..db.connections import ClickhouseConnection +from ..models.observations import WebObservation def iter_web_observations( diff --git a/oonipipeline/src/oonipipeline/analysis/signal.py b/oonipipeline/src/oonipipeline/analysis/signal.py index 1991aca4..d879a676 100644 --- a/oonipipeline/src/oonipipeline/analysis/signal.py +++ b/oonipipeline/src/oonipipeline/analysis/signal.py @@ -1,14 +1,14 @@ from typing import List, Generator -from oonidata.models.experiment_result import ( +from ..models.experiment_result import ( BlockingScope, ExperimentResult, Outcome, fp_to_scope, iter_experiment_results, ) -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB -from oonidata.models.observations import WebObservation +from ..fingerprintdb import FingerprintDB +from ..models.observations import WebObservation def make_signal_experiment_result( diff --git a/oonipipeline/src/oonipipeline/analysis/web_analysis.py b/oonipipeline/src/oonipipeline/analysis/web_analysis.py index 522407ab..d2218668 100644 --- a/oonipipeline/src/oonipipeline/analysis/web_analysis.py +++ b/oonipipeline/src/oonipipeline/analysis/web_analysis.py @@ -1,8 +1,10 @@ +import logging +import ipaddress +import dataclasses + from collections import defaultdict from dataclasses import dataclass -import dataclasses from datetime import datetime, timezone -import ipaddress from typing import ( Generator, Iterable, @@ -10,19 +12,19 @@ List, Dict, ) -from oonidata.db.connections import ClickhouseConnection -from oonidata.analysis.control import ( + +from ..db.connections import ClickhouseConnection +from ..analysis.control import ( WebGroundTruth, BodyDB, ) -from oonidata.models.analysis import WebAnalysis +from ..fingerprintdb import FingerprintDB -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB -from oonidata.models.observations import WebControlObservation, WebObservation +from ..models.analysis import WebAnalysis +from ..models.observations import WebControlObservation, WebObservation -import logging -log = logging.getLogger("oonidata.processing") +log = logging.getLogger(__name__) SYSTEM_RESOLVERS = ["system", "getaddrinfo", "golang_net_resolver", "go", "unknown"] CLOUD_PROVIDERS_ASNS = [ diff --git a/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py b/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py index 42047e68..c7198c25 100644 --- a/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py +++ b/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py @@ -1,10 +1,10 @@ from dataclasses import dataclass from datetime import datetime, timezone import logging -from typing import Dict, Generator, List, NamedTuple, Optional, Tuple +from typing import Dict, Generator, List, Optional, Tuple -from oonidata.models.analysis import WebAnalysis -from oonidata.models.experiment_result import MeasurementExperimentResult +from ..models.analysis import WebAnalysis +from ..models.experiment_result import MeasurementExperimentResult log = logging.getLogger("oonidata.analysis") diff --git a/oonipipeline/src/oonipipeline/db/create_tables.py b/oonipipeline/src/oonipipeline/db/create_tables.py index 8485b6a1..ced34a65 100644 --- a/oonipipeline/src/oonipipeline/db/create_tables.py +++ b/oonipipeline/src/oonipipeline/db/create_tables.py @@ -1,15 +1,16 @@ from datetime import datetime -from enum import Enum from typing import NamedTuple, Optional, Tuple, List, Any, Type, Mapping, Dict from dataclasses import fields -from oonidata.db.connections import ClickhouseConnection -from oonidata.models.experiment_result import ( + +from .connections import ClickhouseConnection + +from ..models.experiment_result import ( ExperimentResult, MeasurementExperimentResult, ) -from oonidata.models.analysis import WebAnalysis -from oonidata.models.observations import ( +from ..models.analysis import WebAnalysis +from ..models.observations import ( ObservationBase, WebControlObservation, WebObservation, diff --git a/oonipipeline/src/oonipipeline/models/experiment_result.py b/oonipipeline/src/oonipipeline/models/experiment_result.py index 939fe7b2..24542446 100644 --- a/oonipipeline/src/oonipipeline/models/experiment_result.py +++ b/oonipipeline/src/oonipipeline/models/experiment_result.py @@ -8,7 +8,7 @@ from tabulate import tabulate from oonidata.datautils import maybe_elipse -from oonidata.models.observations import MeasurementMeta +from ..models.observations import MeasurementMeta log = logging.getLogger("oonidata.events") diff --git a/oonipipeline/src/oonipipeline/models/observations.py b/oonipipeline/src/oonipipeline/models/observations.py index 50c7f3eb..45fd8873 100644 --- a/oonipipeline/src/oonipipeline/models/observations.py +++ b/oonipipeline/src/oonipipeline/models/observations.py @@ -1,8 +1,7 @@ -from dataclasses import dataclass, field import dataclasses +from dataclasses import dataclass, field from datetime import datetime from typing import ( - NamedTuple, Optional, List, Tuple, diff --git a/oonidata/src/oonidata/netinfo.py b/oonipipeline/src/oonipipeline/netinfo.py similarity index 100% rename from oonidata/src/oonidata/netinfo.py rename to oonipipeline/src/oonipipeline/netinfo.py diff --git a/oonipipeline/src/oonipipeline/transforms/__init__.py b/oonipipeline/src/oonipipeline/transforms/__init__.py index 7f9d2aed..e69de29b 100644 --- a/oonipipeline/src/oonipipeline/transforms/__init__.py +++ b/oonipipeline/src/oonipipeline/transforms/__init__.py @@ -1,42 +0,0 @@ -from oonidata.netinfo import NetinfoDB - -from oonidata.transforms.nettests.dnscheck import DNSCheckTransformer -from oonidata.transforms.nettests.http_header_field_manipulation import ( - HTTPHeaderFieldManipulationTransformer, -) -from oonidata.transforms.nettests.signal import SignalTransformer -from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer -from oonidata.transforms.nettests.whatsapp import WhatsappTransformer -from oonidata.transforms.nettests.telegram import TelegramTransformer -from oonidata.transforms.nettests.stun_reachability import StunReachabilityTransformer -from oonidata.transforms.nettests.tor import TorTransformer -from oonidata.transforms.nettests.browser_web import BrowserWebTransformer -from oonidata.transforms.nettests.urlgetter import UrlGetterTransformer -from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer -from oonidata.transforms.nettests.http_invalid_request_line import ( - HTTPInvalidRequestLineTransformer, -) - -NETTEST_TRANSFORMERS = { - "dnscheck": DNSCheckTransformer, - "signal": SignalTransformer, - "facebook_messenger": FacebookMessengerTransformer, - "whatsapp": WhatsappTransformer, - "telegram": TelegramTransformer, - "stunreachability": StunReachabilityTransformer, - "tor": TorTransformer, - "browser_web": BrowserWebTransformer, - "urlgetter": UrlGetterTransformer, - "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, - "http_invalid_request_line": HTTPInvalidRequestLineTransformer, - "web_connectivity": WebConnectivityTransformer, -} - - -def measurement_to_observations(msmt, netinfodb: NetinfoDB): - if msmt.test_name in NETTEST_TRANSFORMERS: - transformer = NETTEST_TRANSFORMERS[msmt.test_name]( - measurement=msmt, netinfodb=netinfodb - ) - return transformer.make_observations(msmt) - return [[]] diff --git a/oonipipeline/src/oonipipeline/transforms/observations.py b/oonipipeline/src/oonipipeline/transforms/observations.py new file mode 100644 index 00000000..58aab210 --- /dev/null +++ b/oonipipeline/src/oonipipeline/transforms/observations.py @@ -0,0 +1,42 @@ +from .nettests.dnscheck import DNSCheckTransformer +from .nettests.http_header_field_manipulation import ( + HTTPHeaderFieldManipulationTransformer, +) +from .nettests.signal import SignalTransformer +from .nettests.facebook_messenger import FacebookMessengerTransformer +from .nettests.whatsapp import WhatsappTransformer +from .nettests.telegram import TelegramTransformer +from .nettests.stun_reachability import StunReachabilityTransformer +from .nettests.tor import TorTransformer +from .nettests.browser_web import BrowserWebTransformer +from .nettests.urlgetter import UrlGetterTransformer +from .nettests.web_connectivity import WebConnectivityTransformer +from .nettests.http_invalid_request_line import ( + HTTPInvalidRequestLineTransformer, +) + +from ..netinfo import NetinfoDB + +NETTEST_TRANSFORMERS = { + "dnscheck": DNSCheckTransformer, + "signal": SignalTransformer, + "facebook_messenger": FacebookMessengerTransformer, + "whatsapp": WhatsappTransformer, + "telegram": TelegramTransformer, + "stunreachability": StunReachabilityTransformer, + "tor": TorTransformer, + "browser_web": BrowserWebTransformer, + "urlgetter": UrlGetterTransformer, + "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, + "http_invalid_request_line": HTTPInvalidRequestLineTransformer, + "web_connectivity": WebConnectivityTransformer, +} + + +def measurement_to_observations(msmt, netinfodb: NetinfoDB): + if msmt.test_name in NETTEST_TRANSFORMERS: + transformer = NETTEST_TRANSFORMERS[msmt.test_name]( + measurement=msmt, netinfodb=netinfodb + ) + return transformer.make_observations(msmt) + return [[]] diff --git a/oonipipeline/src/oonipipeline/workflows/analysis.py b/oonipipeline/src/oonipipeline/workflows/analysis.py index 69f68e50..1256aab6 100644 --- a/oonipipeline/src/oonipipeline/workflows/analysis.py +++ b/oonipipeline/src/oonipipeline/workflows/analysis.py @@ -12,18 +12,19 @@ from dask.distributed import wait as dask_wait from dask.distributed import as_completed -from oonidata.analysis.control import BodyDB, WebGroundTruthDB -from oonidata.analysis.datasources import iter_web_observations -from oonidata.analysis.web_analysis import make_web_analysis -from oonidata.analysis.website_experiment_results import make_website_experiment_results from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer -from oonidata.db.connections import ClickhouseConnection -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB -from oonidata.models.analysis import WebAnalysis -from oonidata.models.experiment_result import MeasurementExperimentResult -from oonidata.netinfo import NetinfoDB -from oonidata.workers.ground_truths import maybe_build_web_ground_truth + +from ..analysis.control import BodyDB, WebGroundTruthDB +from ..analysis.datasources import iter_web_observations +from ..analysis.web_analysis import make_web_analysis +from ..analysis.website_experiment_results import make_website_experiment_results +from ..db.connections import ClickhouseConnection +from ..fingerprintdb import FingerprintDB +from ..models.analysis import WebAnalysis +from ..models.experiment_result import MeasurementExperimentResult +from ..netinfo import NetinfoDB +from .ground_truths import maybe_build_web_ground_truth from .common import ( get_obs_count_by_cc, diff --git a/oonipipeline/src/oonipipeline/to-port-workers/common.py b/oonipipeline/src/oonipipeline/workflows/common.py similarity index 98% rename from oonipipeline/src/oonipipeline/to-port-workers/common.py rename to oonipipeline/src/oonipipeline/workflows/common.py index 76d6671d..25764d3a 100644 --- a/oonipipeline/src/oonipipeline/to-port-workers/common.py +++ b/oonipipeline/src/oonipipeline/workflows/common.py @@ -20,10 +20,8 @@ from oonidata.dataclient import ( MeasurementListProgress, ) -from oonidata.db.connections import ( - ClickhouseConnection, -) -from oonidata.db.create_tables import create_queries +from ..db.connections import ClickhouseConnection +from ..db.create_tables import create_queries log = logging.getLogger("oonidata.processing") diff --git a/oonipipeline/src/oonipipeline/workflows/ground_truths.py b/oonipipeline/src/oonipipeline/workflows/ground_truths.py index 71ff8015..2add1c5d 100644 --- a/oonipipeline/src/oonipipeline/workflows/ground_truths.py +++ b/oonipipeline/src/oonipipeline/workflows/ground_truths.py @@ -10,13 +10,13 @@ from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer -from oonidata.analysis.control import WebGroundTruthDB, iter_web_ground_truths -from oonidata.netinfo import NetinfoDB +from ..analysis.control import WebGroundTruthDB, iter_web_ground_truths +from ..netinfo import NetinfoDB -from oonidata.db.connections import ( +from ..db.connections import ( ClickhouseConnection, ) -from oonidata.workers.common import run_progress_thread +from .common import run_progress_thread log = logging.getLogger("oonidata.processing") diff --git a/oonipipeline/src/oonipipeline/workflows/observations.py b/oonipipeline/src/oonipipeline/workflows/observations.py index 49c00710..931cec53 100644 --- a/oonipipeline/src/oonipipeline/workflows/observations.py +++ b/oonipipeline/src/oonipipeline/workflows/observations.py @@ -17,23 +17,21 @@ with workflow.unsafe.imports_passed_through(): import statsd - from oonidata.analysis.datasources import load_measurement from oonidata.datautils import PerfTimer - from oonidata.models.nettests import SupportedDataformats - - from oonidata.netinfo import NetinfoDB - from oonidata.dataclient import ( date_interval, list_file_entries_batches, stream_measurements, ccs_set, + load_measurement, ) - from oonidata.db.connections import ( - ClickhouseConnection, - ) - from oonidata.transforms import measurement_to_observations - from oonidata.workers.common import ( + from oonidata.models.nettests import SupportedDataformats + + from ..netinfo import NetinfoDB + from ..db.connections import ClickhouseConnection + from ..transforms.observations import measurement_to_observations + + from .common import ( get_prev_range, make_db_rows, maybe_delete_prev_range, From 167c57a3e603fbb4d5fc70570d30663ca7ad277a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 18:40:07 +0100 Subject: [PATCH 08/25] Fix broken import in cli --- oonidata/src/oonidata/cli/command.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/oonidata/src/oonidata/cli/command.py b/oonidata/src/oonidata/cli/command.py index 8e160f71..ab812818 100644 --- a/oonidata/src/oonidata/cli/command.py +++ b/oonidata/src/oonidata/cli/command.py @@ -12,8 +12,6 @@ from oonidata.dataclient import ( sync_measurements, ) -from oonidata.netinfo import NetinfoDB - log = logging.getLogger("oonidata") From 954e9a5816b0d38050c03aaab0a7d4fd74a96288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 18:42:28 +0100 Subject: [PATCH 09/25] Fix imports in web app --- oonipipeline/src/oonipipeline/dataviz/web.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/oonipipeline/src/oonipipeline/dataviz/web.py b/oonipipeline/src/oonipipeline/dataviz/web.py index 9bc0c159..7cb342a7 100644 --- a/oonipipeline/src/oonipipeline/dataviz/web.py +++ b/oonipipeline/src/oonipipeline/dataviz/web.py @@ -1,32 +1,36 @@ from dataclasses import asdict import json from pathlib import Path -from oonidata.analysis.control import ( + +from flask import Flask, request, render_template + +from oonidata.dataclient import load_measurement +from oonidata.apiclient import get_measurement_dict_by_uid + +from ..analysis.control import ( BodyDB, WebGroundTruthDB, iter_ground_truths_from_web_control, ) -from oonidata.analysis.datasources import load_measurement -from oonidata.analysis.web_analysis import make_web_analysis -from oonidata.analysis.website_experiment_results import make_website_experiment_results -from oonidata.apiclient import get_measurement_dict_by_uid -from oonidata.dataviz.viz import ( +from ..analysis.web_analysis import make_web_analysis +from ..analysis.website_experiment_results import make_website_experiment_results + +from .viz import ( plot_blocking_world_map, plot_blocking_of_domain_in_asn, plot_blocking_of_domain_by_asn, ) -from oonidata.dataviz.viz import ( +from .viz import ( get_df_blocking_world_map, get_df_blocking_of_domain_in_asn, get_df_blocking_of_domain_by_asn, get_df_dns_analysis, get_df_dns_analysis_raw, ) -from flask import Flask, request, render_template -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB -from oonidata.netinfo import NetinfoDB +from ..fingerprintdb import FingerprintDB +from ..netinfo import NetinfoDB -from oonidata.transforms import measurement_to_observations +from ..transforms.observations import measurement_to_observations app = Flask(__name__) From fc9f320c29993503525d71f02f21e2035f0a8d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 18:48:56 +0100 Subject: [PATCH 10/25] It's actually better to keep all models in oonidata --- .../src/oonidata}/models/analysis.py | 0 .../src/oonidata}/models/experiment_result.py | 0 .../src/oonidata}/models/observations.py | 0 oonipipeline/src/oonipipeline/models/__init__.py | 0 .../{nettests => }/measurement_transformer.py | 0 .../oonipipeline/transforms/nettests/browser_web.py | 13 +++++-------- .../oonipipeline/transforms/nettests/dnscheck.py | 3 ++- .../transforms/nettests/facebook_messenger.py | 3 ++- .../nettests/http_header_field_manipulation.py | 3 ++- .../nettests/http_invalid_request_line.py | 3 ++- .../src/oonipipeline/transforms/nettests/signal.py | 3 ++- .../transforms/nettests/stun_reachability.py | 9 +++------ .../oonipipeline/transforms/nettests/telegram.py | 3 ++- .../src/oonipipeline/transforms/nettests/tor.py | 3 ++- .../oonipipeline/transforms/nettests/urlgetter.py | 6 +++--- .../transforms/nettests/web_connectivity.py | 6 ++++-- .../oonipipeline/transforms/nettests/whatsapp.py | 5 +++-- oonipipeline/src/oonipipeline/workflows/analysis.py | 5 +++-- 18 files changed, 35 insertions(+), 30 deletions(-) rename {oonipipeline/src/oonipipeline => oonidata/src/oonidata}/models/analysis.py (100%) rename {oonipipeline/src/oonipipeline => oonidata/src/oonidata}/models/experiment_result.py (100%) rename {oonipipeline/src/oonipipeline => oonidata/src/oonidata}/models/observations.py (100%) delete mode 100644 oonipipeline/src/oonipipeline/models/__init__.py rename oonipipeline/src/oonipipeline/transforms/{nettests => }/measurement_transformer.py (100%) diff --git a/oonipipeline/src/oonipipeline/models/analysis.py b/oonidata/src/oonidata/models/analysis.py similarity index 100% rename from oonipipeline/src/oonipipeline/models/analysis.py rename to oonidata/src/oonidata/models/analysis.py diff --git a/oonipipeline/src/oonipipeline/models/experiment_result.py b/oonidata/src/oonidata/models/experiment_result.py similarity index 100% rename from oonipipeline/src/oonipipeline/models/experiment_result.py rename to oonidata/src/oonidata/models/experiment_result.py diff --git a/oonipipeline/src/oonipipeline/models/observations.py b/oonidata/src/oonidata/models/observations.py similarity index 100% rename from oonipipeline/src/oonipipeline/models/observations.py rename to oonidata/src/oonidata/models/observations.py diff --git a/oonipipeline/src/oonipipeline/models/__init__.py b/oonipipeline/src/oonipipeline/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/measurement_transformer.py b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py similarity index 100% rename from oonipipeline/src/oonipipeline/transforms/nettests/measurement_transformer.py rename to oonipipeline/src/oonipipeline/transforms/measurement_transformer.py diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py b/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py index 1ef158cd..62fda2e0 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/browser_web.py @@ -2,18 +2,15 @@ import dataclasses from oonidata.models.nettests import BrowserWeb from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer, make_web_observation + +from ..measurement_transformer import MeasurementTransformer class BrowserWebTransformer(MeasurementTransformer): def make_observations(self, msmt: BrowserWeb) -> Tuple[List[WebObservation]]: - bw_obs = WebObservation( - **dataclasses.asdict(self.measurement_meta) - ) - + bw_obs = WebObservation(**dataclasses.asdict(self.measurement_meta)) + bw_obs.http_failure = msmt.test_keys.result bw_obs.http_runtime = msmt.test_keys.load_time_ms - return ( - ([bw_obs],) - ) + return ([bw_obs],) diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py b/oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py index 4da24723..5b6ccd57 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/dnscheck.py @@ -2,7 +2,8 @@ from oonidata.models.nettests import DNSCheck from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class DNSCheckTransformer(MeasurementTransformer): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py b/oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py index 73d01148..a9760a80 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/facebook_messenger.py @@ -1,7 +1,8 @@ from typing import List, Tuple from oonidata.models.nettests import FacebookMessenger from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class FacebookMessengerTransformer(MeasurementTransformer): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py b/oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py index f447f50a..68834184 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/http_header_field_manipulation.py @@ -4,7 +4,8 @@ from typing import List, Tuple from oonidata.models.nettests import HTTPHeaderFieldManipulation from oonidata.models.observations import HTTPMiddleboxObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class HTTPHeaderFieldManipulationTransformer(MeasurementTransformer): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py b/oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py index b789a2e5..ae217517 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/http_invalid_request_line.py @@ -4,7 +4,8 @@ from oonidata.models.dataformats import maybe_binary_data_to_bytes from oonidata.models.nettests import HTTPInvalidRequestLine from oonidata.models.observations import HTTPMiddleboxObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer def detect_target_index(sent_data): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/signal.py b/oonipipeline/src/oonipipeline/transforms/nettests/signal.py index 40d218ea..9382601e 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/signal.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/signal.py @@ -3,7 +3,8 @@ from oonidata.models.nettests import Signal from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer SIGNAL_ROOT_CA_OLD = """-----BEGIN CERTIFICATE----- diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py b/oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py index 2442581d..230a151f 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/stun_reachability.py @@ -2,15 +2,12 @@ from oonidata.models.nettests import StunReachability from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class StunReachabilityTransformer(MeasurementTransformer): def make_observations(self, msmt: StunReachability) -> Tuple[List[WebObservation]]: dns_observations = self.make_dns_observations(msmt.test_keys.queries) - return ( - self.consume_web_observations( - dns_observations=dns_observations - ), - ) + return (self.consume_web_observations(dns_observations=dns_observations),) diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/telegram.py b/oonipipeline/src/oonipipeline/transforms/nettests/telegram.py index 5d85380d..19afa607 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/telegram.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/telegram.py @@ -2,7 +2,8 @@ from oonidata.models.nettests import Telegram from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class TelegramTransformer(MeasurementTransformer): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/tor.py b/oonipipeline/src/oonipipeline/transforms/nettests/tor.py index aac36895..106a639c 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/tor.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/tor.py @@ -1,7 +1,8 @@ from typing import List, Tuple from oonidata.models.nettests import Tor from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class TorTransformer(MeasurementTransformer): diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py b/oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py index e527eb3a..c64b0f02 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/urlgetter.py @@ -2,7 +2,8 @@ from oonidata.models.nettests import UrlGetter from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class UrlGetterTransformer(MeasurementTransformer): @@ -10,8 +11,7 @@ def make_observations(self, msmt: UrlGetter) -> Tuple[List[WebObservation]]: dns_observations = self.make_dns_observations(msmt.test_keys.queries) tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) tls_observations = self.make_tls_observations( - msmt.test_keys.tls_handshakes, - msmt.test_keys.network_events + msmt.test_keys.tls_handshakes, msmt.test_keys.network_events ) http_observations = self.make_http_observations(msmt.test_keys.requests) diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py b/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py index be8ac276..a634c46b 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/web_connectivity.py @@ -5,8 +5,10 @@ from oonidata.datautils import is_ip_bogon from oonidata.models.nettests import WebConnectivity from oonidata.models.observations import WebControlObservation, WebObservation -from oonidata.netinfo import NetinfoDB -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer + +from ...netinfo import NetinfoDB def make_web_control_observations( diff --git a/oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py b/oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py index bb4773fb..cce6a58d 100644 --- a/oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py +++ b/oonipipeline/src/oonipipeline/transforms/nettests/whatsapp.py @@ -1,7 +1,8 @@ from typing import List, Tuple from oonidata.models.nettests import Whatsapp from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + +from ..measurement_transformer import MeasurementTransformer class WhatsappTransformer(MeasurementTransformer): @@ -9,7 +10,7 @@ def make_observations(self, msmt: Whatsapp) -> Tuple[List[WebObservation]]: dns_observations = self.make_dns_observations(msmt.test_keys.queries) tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) tls_observations = self.make_tls_observations( - msmt.test_keys.tls_handshakes, + msmt.test_keys.tls_handshakes, msmt.test_keys.network_events, ) http_observations = self.make_http_observations(msmt.test_keys.requests) diff --git a/oonipipeline/src/oonipipeline/workflows/analysis.py b/oonipipeline/src/oonipipeline/workflows/analysis.py index 1256aab6..5e6ad582 100644 --- a/oonipipeline/src/oonipipeline/workflows/analysis.py +++ b/oonipipeline/src/oonipipeline/workflows/analysis.py @@ -14,6 +14,8 @@ from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer +from oonidata.models.analysis import WebAnalysis +from oonidata.models.experiment_result import MeasurementExperimentResult from ..analysis.control import BodyDB, WebGroundTruthDB from ..analysis.datasources import iter_web_observations @@ -21,8 +23,7 @@ from ..analysis.website_experiment_results import make_website_experiment_results from ..db.connections import ClickhouseConnection from ..fingerprintdb import FingerprintDB -from ..models.analysis import WebAnalysis -from ..models.experiment_result import MeasurementExperimentResult + from ..netinfo import NetinfoDB from .ground_truths import maybe_build_web_ground_truth From d5c55b947c2ed8e91f33c0832077e31a958b394c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sat, 16 Mar 2024 19:09:14 +0100 Subject: [PATCH 11/25] Attempt at porting analysis over to temporalio --- .../oonipipeline/to-port-workers/__init__.py | 0 .../src/oonipipeline/workflows/analysis.py | 178 +++++++++--------- .../oonipipeline/workflows/ground_truths.py | 1 + .../oonipipeline/workflows/observations.py | 55 +++--- .../to_port}/fingerprint_hunter.py | 2 +- .../to_port}/response_archiver.py | 7 +- 6 files changed, 127 insertions(+), 116 deletions(-) delete mode 100644 oonipipeline/src/oonipipeline/to-port-workers/__init__.py rename oonipipeline/src/oonipipeline/{to-port-workers => workflows/to_port}/fingerprint_hunter.py (97%) rename oonipipeline/src/oonipipeline/{to-port-workers => workflows/to_port}/response_archiver.py (98%) diff --git a/oonipipeline/src/oonipipeline/to-port-workers/__init__.py b/oonipipeline/src/oonipipeline/to-port-workers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/oonipipeline/src/oonipipeline/workflows/analysis.py b/oonipipeline/src/oonipipeline/workflows/analysis.py index 5e6ad582..a971e8e6 100644 --- a/oonipipeline/src/oonipipeline/workflows/analysis.py +++ b/oonipipeline/src/oonipipeline/workflows/analysis.py @@ -1,16 +1,15 @@ +import asyncio import dataclasses import logging import pathlib -from datetime import date, datetime, timezone + +from datetime import date, datetime, timedelta, timezone from typing import Dict, List -import orjson -import statsd +from temporalio import workflow, activity -from dask.distributed import Client as DaskClient -from dask.distributed import progress as dask_progress -from dask.distributed import wait as dask_wait -from dask.distributed import as_completed +import orjson +import statsd from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer @@ -35,6 +34,8 @@ optimize_all_tables, ) +from .observations import ObservationsWorkflowParams, MakeObservationsParams + log = logging.getLogger("oonidata.processing") @@ -56,17 +57,26 @@ def make_ctrl( db_lookup.close() -def make_analysis_in_a_day( - probe_cc: List[str], - test_name: List[str], - clickhouse: str, - data_dir: pathlib.Path, - fast_fail: bool, - day: date, -): +@dataclasses.dataclass +class MakeAnalysisParams: + probe_cc: List[str] + test_name: List[str] + clickhouse: str + data_dir: str + fast_fail: bool + day: date + + +@activity.defn +def make_analysis_in_a_day(params: MakeAnalysisParams) -> dict: t_total = PerfTimer() log.info("Optimizing all tables") - optimize_all_tables(clickhouse) + optimize_all_tables(params.clickhouse) + data_dir = pathlib.Path(params.data_dir) + clickhouse = params.clickhouse + day = params.day + probe_cc = params.probe_cc + test_name = params.test_name statsd_client = statsd.StatsClient("localhost", 8125) fingerprintdb = FingerprintDB(datadir=data_dir, download=False) @@ -188,7 +198,7 @@ def make_analysis_in_a_day( ] ], ) - return idx + return {"count": idx} def make_cc_batches( @@ -246,74 +256,74 @@ def make_cc_batches( return cc_batches -def start_analysis( - probe_cc: List[str], - test_name: List[str], - start_day: date, - end_day: date, - data_dir: pathlib.Path, - clickhouse: str, - parallelism: int, - fast_fail: bool, - rebuild_ground_truths: bool, - log_level: int = logging.INFO, -): - t_total = PerfTimer() - dask_client = DaskClient( - threads_per_worker=2, - n_workers=parallelism, - ) +@workflow.defn +class AnalysisWorkflow: + @workflow.run + async def run(self, params: ObservationsWorkflowParams) -> dict: + # TODO(art): should this be a parameter or is it better to remove it? + rebuild_ground_truths = True + t_total = PerfTimer() - t = PerfTimer() - # TODO: maybe use dask for this too - log.info("building ground truth databases") - for day in date_interval(start_day, end_day): - make_ctrl( - clickhouse=clickhouse, - data_dir=data_dir, - rebuild_ground_truths=rebuild_ground_truths, - day=day, - ) - log.info(f"built ground truth db in {t.pretty}") - - with ClickhouseConnection(clickhouse) as db: - cnt_by_cc = get_obs_count_by_cc( - db, start_day=start_day, end_day=end_day, test_name=test_name - ) - cc_batches = make_cc_batches( - cnt_by_cc=cnt_by_cc, - probe_cc=probe_cc, - parallelism=parallelism, - ) - log.info( - f"starting processing of {len(cc_batches)} batches over {(end_day - start_day).days} days (parallelism = {parallelism})" - ) - log.info(f"({cc_batches} from {start_day} to {end_day}") + t = PerfTimer() + start_day = datetime.strptime(params.start_day, "%Y-%m-%d").date() + end_day = datetime.strptime(params.end_day, "%Y-%m-%d").date() - future_list = [] - for probe_cc in cc_batches: + log.info("building ground truth databases") for day in date_interval(start_day, end_day): - t = dask_client.submit( - make_analysis_in_a_day, - probe_cc, - test_name, - clickhouse, - data_dir, - fast_fail, - day, + make_ctrl( + clickhouse=params.clickhouse, + data_dir=pathlib.Path(params.data_dir), + rebuild_ground_truths=rebuild_ground_truths, + day=day, + ) + log.info(f"built ground truth db in {t.pretty}") + + with ClickhouseConnection(params.clickhouse) as db: + cnt_by_cc = get_obs_count_by_cc( + db, start_day=start_day, end_day=end_day, test_name=params.test_name ) - future_list.append(t) - - log.debug("starting progress monitoring") - dask_progress(future_list) - log.debug("waiting on task_list") - dask_wait(future_list) - total_obs_count = 0 - for _, result in as_completed(future_list, with_results=True): - total_obs_count += result # type: ignore - - log.info(f"produces a total of {total_obs_count} analysis") - obs_per_sec = round(total_obs_count / t_total.s) - log.info(f"finished processing {start_day} - {end_day} speed: {obs_per_sec}obs/s)") - log.info(f"{total_obs_count} msmts in {t_total.pretty}") - dask_client.shutdown() + cc_batches = make_cc_batches( + cnt_by_cc=cnt_by_cc, + probe_cc=params.probe_cc, + parallelism=params.parallelism, + ) + log.info( + f"starting processing of {len(cc_batches)} batches over {(end_day - start_day).days} days (parallelism = {params.parallelism})" + ) + log.info(f"({cc_batches} from {start_day} to {end_day}") + + task_list = [] + async with asyncio.TaskGroup() as tg: + for probe_cc in cc_batches: + for day in date_interval(start_day, end_day): + task = tg.create_task( + workflow.execute_activity( + make_analysis_in_a_day, + MakeAnalysisParams( + probe_cc=params.probe_cc, + test_name=params.test_name, + clickhouse=params.clickhouse, + data_dir=params.data_dir, + fast_fail=params.fast_fail, + day=day, + ), + start_to_close_timeout=timedelta(minutes=30), + ) + ) + task_list.append(task) + + t = PerfTimer() + # size, msmt_count = + total_obs_count = 0 + for task in task_list: + res = task.result() + + total_obs_count += res["count"] + + log.info(f"produces a total of {total_obs_count} analysis") + obs_per_sec = round(total_obs_count / t_total.s) + log.info( + f"finished processing {start_day} - {end_day} speed: {obs_per_sec}obs/s)" + ) + log.info(f"{total_obs_count} msmts in {t_total.pretty}") + return {"total_obs_count": total_obs_count} diff --git a/oonipipeline/src/oonipipeline/workflows/ground_truths.py b/oonipipeline/src/oonipipeline/workflows/ground_truths.py index 2add1c5d..3d679afa 100644 --- a/oonipipeline/src/oonipipeline/workflows/ground_truths.py +++ b/oonipipeline/src/oonipipeline/workflows/ground_truths.py @@ -10,6 +10,7 @@ from oonidata.dataclient import date_interval from oonidata.datautils import PerfTimer + from ..analysis.control import WebGroundTruthDB, iter_web_ground_truths from ..netinfo import NetinfoDB diff --git a/oonipipeline/src/oonipipeline/workflows/observations.py b/oonipipeline/src/oonipipeline/workflows/observations.py index 931cec53..1b6529be 100644 --- a/oonipipeline/src/oonipipeline/workflows/observations.py +++ b/oonipipeline/src/oonipipeline/workflows/observations.py @@ -2,21 +2,20 @@ import pathlib import logging import dataclasses -from datetime import date, datetime, timedelta, timezone +from dataclasses import dataclass +from datetime import datetime, timedelta from typing import ( List, Sequence, Tuple, ) -from temporalio import workflow, activity -from dataclasses import dataclass -with workflow.unsafe.imports_passed_through(): - import clickhouse_driver +from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): import statsd + import clickhouse_driver from oonidata.datautils import PerfTimer from oonidata.dataclient import ( date_interval, @@ -41,29 +40,6 @@ log = logging.getLogger("oonidata.processing") -@dataclass -class ObservationsWorkflowParams: - probe_cc: List[str] - test_name: List[str] - start_day: str - end_day: str - clickhouse: str - data_dir: str - parallelism: int - fast_fail: bool - log_level: int = logging.INFO - - -@dataclass -class MakeObservationsParams: - probe_cc: List[str] - test_name: List[str] - clickhouse: str - data_dir: str - fast_fail: bool - bucket_date: str - - def write_observations_to_db( msmt: SupportedDataformats, netinfodb: NetinfoDB, @@ -145,6 +121,29 @@ def make_observations_for_file_entry_batch( return idx +@dataclass +class ObservationsWorkflowParams: + probe_cc: List[str] + test_name: List[str] + start_day: str + end_day: str + clickhouse: str + data_dir: str + parallelism: int + fast_fail: bool + log_level: int = logging.INFO + + +@dataclass +class MakeObservationsParams: + probe_cc: List[str] + test_name: List[str] + clickhouse: str + data_dir: str + fast_fail: bool + bucket_date: str + + @activity.defn def make_observation_in_day(params: MakeObservationsParams) -> dict: statsd_client = statsd.StatsClient("localhost", 8125) diff --git a/oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py b/oonipipeline/src/oonipipeline/workflows/to_port/fingerprint_hunter.py similarity index 97% rename from oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py rename to oonipipeline/src/oonipipeline/workflows/to_port/fingerprint_hunter.py index 7d9c5419..05019d42 100644 --- a/oonipipeline/src/oonipipeline/to-port-workers/fingerprint_hunter.py +++ b/oonipipeline/src/oonipipeline/workflows/to_port/fingerprint_hunter.py @@ -15,7 +15,7 @@ import orjson from warcio.archiveiterator import ArchiveIterator -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB, Fingerprint +from ..fingerprintdb import FingerprintDB, Fingerprint log = logging.getLogger("oonidata.processing") diff --git a/oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py b/oonipipeline/src/oonipipeline/workflows/to_port/response_archiver.py similarity index 98% rename from oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py rename to oonipipeline/src/oonipipeline/workflows/to_port/response_archiver.py index f50e5a05..f40688e5 100644 --- a/oonipipeline/src/oonipipeline/to-port-workers/response_archiver.py +++ b/oonipipeline/src/oonipipeline/workflows/to_port/response_archiver.py @@ -21,11 +21,12 @@ from warcio.warcwriter import WARCWriter from warcio.statusandheaders import StatusAndHeaders -from oonidata.analysis.datasources import load_measurement -from oonidata.dataclient import date_interval, iter_measurements -from oonipipeline.src.oonipipeline.fingerprintdb import FingerprintDB, Fingerprint + +from oonidata.dataclient import date_interval, iter_measurements, load_measurement from oonidata.models.nettests.web_connectivity import WebConnectivity +from ..fingerprintdb import FingerprintDB, Fingerprint + log = logging.getLogger("oonidata.processing") From 8311221b6ada2f9ce5f00994a873ce7970914475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 09:20:23 -0400 Subject: [PATCH 12/25] Refactor structure of unit and integration tests --- .gitignore | 1 + oonidata/tests/integration/conftest.py | 24 ++++++++++++++++++++ oonidata/tests/unit/__init__.py | 0 oonidata/tests/{ => unit}/_fixtures.py | 0 oonidata/tests/{ => unit}/conftest.py | 6 ----- oonidata/tests/{ => unit}/test_dataclient.py | 0 oonidata/tests/{ => unit}/test_dataformat.py | 0 7 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 oonidata/tests/integration/conftest.py create mode 100644 oonidata/tests/unit/__init__.py rename oonidata/tests/{ => unit}/_fixtures.py (100%) rename oonidata/tests/{ => unit}/conftest.py (95%) rename oonidata/tests/{ => unit}/test_dataclient.py (100%) rename oonidata/tests/{ => unit}/test_dataformat.py (100%) diff --git a/.gitignore b/.gitignore index ea5e8da2..cdc294a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__ .coverage +.coverage.* coverage.xml .ipynb_checkpoints/ diff --git a/oonidata/tests/integration/conftest.py b/oonidata/tests/integration/conftest.py new file mode 100644 index 00000000..f7f989ea --- /dev/null +++ b/oonidata/tests/integration/conftest.py @@ -0,0 +1,24 @@ +import os +from pathlib import Path +from datetime import date +from click.testing import CliRunner + +import pytest + +import orjson + +from oonidata.dataclient import sync_measurements +from oonidata.apiclient import get_measurement_dict_by_uid + +FIXTURE_PATH = Path(os.path.dirname(os.path.realpath(__file__))) / "data" +DATA_DIR = FIXTURE_PATH / "datadir" + + +@pytest.fixture +def datadir(): + return DATA_DIR + + +@pytest.fixture +def cli_runner(): + return CliRunner() diff --git a/oonidata/tests/unit/__init__.py b/oonidata/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oonidata/tests/_fixtures.py b/oonidata/tests/unit/_fixtures.py similarity index 100% rename from oonidata/tests/_fixtures.py rename to oonidata/tests/unit/_fixtures.py diff --git a/oonidata/tests/conftest.py b/oonidata/tests/unit/conftest.py similarity index 95% rename from oonidata/tests/conftest.py rename to oonidata/tests/unit/conftest.py index 196c5c71..bdfdc22c 100644 --- a/oonidata/tests/conftest.py +++ b/oonidata/tests/unit/conftest.py @@ -37,7 +37,6 @@ def raw_measurements(): def download_from_s3(name, dst_path): - print(name, dst_path) assert False @@ -80,8 +79,3 @@ def measurements(): with sample_measurements[msmt_uid].open("wb") as out_file: out_file.write(orjson.dumps(msmt)) return sample_measurements - - -@pytest.fixture -def cli_runner(): - return CliRunner() diff --git a/oonidata/tests/test_dataclient.py b/oonidata/tests/unit/test_dataclient.py similarity index 100% rename from oonidata/tests/test_dataclient.py rename to oonidata/tests/unit/test_dataclient.py diff --git a/oonidata/tests/test_dataformat.py b/oonidata/tests/unit/test_dataformat.py similarity index 100% rename from oonidata/tests/test_dataformat.py rename to oonidata/tests/unit/test_dataformat.py From 395a7426aebb6420b656dd8534b516c77a5afe51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 09:20:48 -0400 Subject: [PATCH 13/25] Set python version to 3.11 --- oonidata/pyproject.toml | 1 + oonidata/src/oonidata/dataclient.py | 4 ++-- .../src/oonidata/legacy/normalize_yamlooni.py | 18 +++++++----------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/oonidata/pyproject.toml b/oonidata/pyproject.toml index f700860c..f2f8bc5e 100644 --- a/oonidata/pyproject.toml +++ b/oonidata/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "memray", "viztracer", ] +python = "3.11" path = ".venv/" [tool.hatch.version] diff --git a/oonidata/src/oonidata/dataclient.py b/oonidata/src/oonidata/dataclient.py index 551677a4..b3fc21fb 100644 --- a/oonidata/src/oonidata/dataclient.py +++ b/oonidata/src/oonidata/dataclient.py @@ -207,7 +207,7 @@ def stream_yamllz4(body: io.BytesIO, s3path: str): bucket_tstamp = s3path.split("/")[-2] rfn = f"{bucket_tstamp}/" + s3path.split("/")[-1] # The normalize function already add the measurement_uid - yield from iter_yaml_msmt_normalized(in_file, bucket_tstamp, rfn) + yield from iter_yaml_msmt_normalized(in_file) def stream_oldcan(body: io.BytesIO, s3path: str) -> Generator[dict, None, None]: @@ -239,7 +239,7 @@ def stream_oldcan(body: io.BytesIO, s3path: str) -> Generator[dict, None, None]: bucket_tstamp = s3path.split("/")[-2] rfn = f"{bucket_tstamp}/" + s3path.split("/")[-1] # The normalize function already add the measurement_uid - yield from iter_yaml_msmt_normalized(in_file, bucket_tstamp, rfn) + yield from iter_yaml_msmt_normalized(in_file) def stream_measurements(bucket_name, s3path, ext): diff --git a/oonidata/src/oonidata/legacy/normalize_yamlooni.py b/oonidata/src/oonidata/legacy/normalize_yamlooni.py index 9fa7c534..f5ee479f 100644 --- a/oonidata/src/oonidata/legacy/normalize_yamlooni.py +++ b/oonidata/src/oonidata/legacy/normalize_yamlooni.py @@ -3,6 +3,7 @@ # from datetime import datetime +import io from itertools import groupby import functools import hashlib @@ -389,7 +390,7 @@ def normalize_process(entry): return entry -def normalize_entry(entry, bucket_date, perma_fname, esha): +def normalize_entry(entry: dict, esha: bytes): """Autoclaving""" hashuuid = esha[:16] # sha1 is 20 bytes @@ -400,11 +401,8 @@ def normalize_entry(entry, bucket_date, perma_fname, esha): test_name = test_name_mappings.get(test_name, test_name.lower()) entry["test_name"] = test_name - entry["bucket_date"] = bucket_date - if not entry.get("id"): entry["id"] = str(uuid.UUID(bytes=hashuuid)) - entry["report_filename"] = perma_fname # Ensure all the keys in the schema are present for key in schema: @@ -429,7 +427,7 @@ def normalize_entry(entry, bucket_date, perma_fname, esha): else: test_start_time = datetime.strptime(tst, "%Y-%m-%d %H:%M:%S") measurement_start_time = datetime.strptime( - entry.get("measurement_start_time"), "%Y-%m-%d %H:%M:%S" + entry["measurement_start_time"], "%Y-%m-%d %H:%M:%S" ) except KeyError: # Failback to using the start_time @@ -492,7 +490,7 @@ class TruncatedReportError(BlobSlicerError): pass -def stream_yaml_blobs(fd): +def stream_yaml_blobs(fd: io.BytesIO): """Detects YAML objects from a stream. Returns an iterator of (offset, blob) """ @@ -561,11 +559,9 @@ def generate_report_id(header): ## Entry points -def iter_yaml_msmt_normalized(data, bucket_tstamp: str, report_fn: str): +def iter_yaml_msmt_normalized(data): """Yields normalized measurements from a YAML bytestream""" - assert bucket_tstamp.startswith("20") - assert len(bucket_tstamp) == 10 - assert len(report_fn.split("/")) == 2, report_fn + # Taken from autoclaving.py stream_yaml_reports blobgen = stream_yaml_blobs(data) @@ -601,7 +597,7 @@ def iter_yaml_msmt_normalized(data, bucket_tstamp: str, report_fn: str): header.pop("test_start_time") entry.update(header) try: - d = normalize_entry(entry, bucket_tstamp, report_fn, esha_d) + d = normalize_entry(entry=entry, esha=esha_d) msmt_uid = trivial_id(raw_entry, d) d["measurement_uid"] = msmt_uid yield d From 905c6162fcdc25ca7bc529067ac1f432e6bdf035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 09:42:45 -0400 Subject: [PATCH 14/25] Fix oonidata tests --- oonidata/src/oonidata/s3client.py | 328 ++++++++++++++++++++++++++++ oonidata/tests/unit/conftest.py | 24 +- oonidata/tests/unit/data/.gitignore | 3 + 3 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 oonidata/src/oonidata/s3client.py create mode 100644 oonidata/tests/unit/data/.gitignore diff --git a/oonidata/src/oonidata/s3client.py b/oonidata/src/oonidata/s3client.py new file mode 100644 index 00000000..95bebfe7 --- /dev/null +++ b/oonidata/src/oonidata/s3client.py @@ -0,0 +1,328 @@ +import gzip +import io +import logging +import pathlib +import shutil +import tarfile +from datetime import date, datetime +from typing import Generator, List, Optional, Set +from urllib.parse import urlparse + +import boto3 +from botocore import UNSIGNED as botoSigUNSIGNED +from botocore.config import Config as botoConfig + +import lz4.frame +import orjson + +from .datautils import trivial_id +from .legacy.normalize_yamlooni import iter_yaml_msmt_normalized + +log = logging.getLogger(__name__) + +LEGACY_BUCKET_NAME = "ooni-data" +NEW_BUCKET_NAME = "ooni-data-eu-fra" + + +def read_to_bytesio(body: io.BytesIO) -> io.BytesIO: + read_body = io.BytesIO() + shutil.copyfileobj(body, read_body) + read_body.seek(0) + del body + return read_body + + +def stream_jsonl(body: io.BytesIO) -> Generator[dict, None, None]: + """ + JSONL is the most simple OONI measurement format. They are basically just a + bunch of report files concatenated together for the same test_name, country, + timestamp and compressed with gzip. + """ + with gzip.GzipFile(fileobj=body) as in_file: + for line in in_file: + if line == "": + continue + yield orjson.loads(line) + + +def stream_postcan(body: io.BytesIO) -> Generator[dict, None, None]: + """ + Postcans are the newer format, where each individual measurement entry has + it's own file inside of a tarball. + + Here is an example of the tar layout: + + $ tar tf 2024030100_AM_webconnectivity.n1.0.tar.gz | head -n 3 + var/lib/ooniapi/measurements/incoming/2024030100_AM_webconnectivity/20240301003627.966169_AM_webconnectivity_76f66893a38a3de6.post + var/lib/ooniapi/measurements/incoming/2024030100_AM_webconnectivity/20240301003629.092464_AM_webconnectivity_48be39a609d1dcb7.post + var/lib/ooniapi/measurements/incoming/2024030100_AM_webconnectivity/20240301003630.694204_AM_webconnectivity_a95c7da2775bf109.post + + You should not expect the prefix of + `var/lib/ooniapi/measurement/incoming/XXX` to remain constant over time, but + rather you should only use the last of the file name to determine the + measurement ID + `20240301003630.694204_AM_webconnectivity_a95c7da2775bf109.post`. + + Each `.post` file is a JSON document, which contains two top level keys: + + { + "format": "json", + "content": { + } + } + + Format is always set to `json` and content as the data for the actual + measurement. + + FIXME + Some older postcans have the .gz extension, but are actually not + compressed, tar needs to be able to re-seek back to the beginning of the + file in the event of it not finding the gzip magic header when operating in + "transparent compression mode". When we we fix that in the source data, we + might be able to avoid this. + """ + with tarfile.open(fileobj=body, mode="r|*") as tar: + for m in tar: + assert m.name.endswith(".post"), f"{m.name} doesn't end with .post" + in_file = tar.extractfile(m) + + assert in_file is not None, "found empty tarfile in {m.name}" + + j = orjson.loads(in_file.read()) + assert j["format"] == "json", "postcan with non json format" + + msmt = j["content"] + # extract msmt_uid from filename e.g: + # ... /20210614004521.999962_JO_signal_68eb19b439326d60.post + msmt_uid = pathlib.PurePath(m.name).with_suffix("").name + msmt["measurement_uid"] = msmt_uid + yield msmt + + +def stream_jsonlz4(body: io.BytesIO): + """ + lz4.frame requires the input stream to be seekable, so we need to load it + in memory + """ + read_body = read_to_bytesio(body) + + with lz4.frame.open(read_body, mode="rb") as in_file: + for line in in_file: + try: + msmt = orjson.loads(line) + except ValueError: + log.error("stream_jsonlz4: unable to parse json measurement") + continue + + msmt_uid = trivial_id(line, msmt) # type: ignore due to bad types in lz4 + msmt["measurement_uid"] = msmt_uid + yield msmt + + +def stream_yamllz4(body: io.BytesIO): + """ + lz4.frame requires the input stream to be seekable, so we need to load it + in memory + """ + read_body = read_to_bytesio(body) + + with lz4.frame.open(read_body) as in_file: + # The normalize function already add the measurement_uid + yield from iter_yaml_msmt_normalized(in_file) + + +def stream_oldcan(body: io.BytesIO) -> Generator[dict, None, None]: + """ + lz4.frame requires the input stream to be seekable, so we need to load it + in memory. + """ + read_body = read_to_bytesio(body) + + with lz4.frame.open(read_body) as lz4_file: + with tarfile.open(fileobj=lz4_file) as tar: # type: ignore due to bad types in lz4 + for m in tar: + in_file = tar.extractfile(m) + assert in_file is not None, "{m.name} is None" + + if m.name.endswith(".json"): + for line in in_file: + msmt = orjson.loads(line) + msmt_uid = trivial_id(line, msmt) + msmt["measurement_uid"] = msmt_uid + yield msmt + + elif m.name.endswith(".yaml"): + # The normalize function already add the measurement_uid + yield from iter_yaml_msmt_normalized(in_file) + + +def create_s3_anonymous_client(): + return boto3.client("s3", config=botoConfig(signature_version=botoSigUNSIGNED)) + + +class OONIMeasurementLister: + def __init__( + self, + *, + probe_cc_filter: Optional[set] = None, + test_name_filter: Optional[set] = None, + ): + self.s3 = create_s3_anonymous_client() + self.probe_cc_filter = probe_cc_filter + self.test_name_filter = test_name_filter + + def get_body(self, bucket_name: str, key: str) -> io.BytesIO: + return self.s3.get_object(Bucket=bucket_name, Key=key)["Body"] + + def apply_filter(self, stream_func, body): + """ + Some of the formats don't support doing filtering by path, so we + do it here as well. + """ + for msmt in stream_func(body): + + if ( + self.probe_cc_filter is not None + and msmt["probe_cc"] not in self.probe_cc_filter + ): + continue + + if ( + self.test_name_filter is not None + and msmt["test_name"] not in self.test_name_filter + ): + continue + + yield msmt + + def measurements(self, s3_url: str): + u = urlparse(s3_url) + bucket_name = u.netloc + assert u.scheme == "s3", "must be s3 URL" + assert bucket_name in ["ooni-data-eu-fra", "ooni-data"] + + body = self.get_body(bucket_name=bucket_name, key=u.path) + s3path = pathlib.PurePath(u.path) + + stream_func = None + if s3path.name.endswith("jsonl.gz"): + stream_func = stream_jsonl + elif s3path.name.endswith("tar.gz"): + stream_func = stream_postcan + elif s3path.name.endswith("tar.lz4"): + stream_func = stream_oldcan + elif s3path.name.endswith("json.lz4"): + stream_func = stream_jsonlz4 + elif s3path.name.endswith("yaml.lz4"): + stream_func = stream_yamllz4 + + assert stream_func is not None, f"invalid format for {s3path.name}" + yield from self.apply_filter(stream_func, body) + + +def get_v2_search_prefixes(s3, testnames: Set[str], ccs: Set[str]) -> List[str]: + """ + get_search_prefixes will return all the prefixes inside of the new jsonl + bucket that match the given testnames and ccs. + If the ccs list is empty we will return prefixes for all countries for + which that particular testname as measurements. + """ + prefixes = [] + paginator = s3.get_paginator("list_objects_v2") + for tn in testnames: + for r in paginator.paginate( + Bucket=NEW_BUCKET_NAME, Prefix=f"jsonl/{tn}/", Delimiter="/" + ): + for f in r.get("CommonPrefixes", []): + prefix = f["Prefix"] + cc = prefix.split("/")[-2] + if ccs and cc not in ccs: + continue + prefixes.append(prefix) + return prefixes + + +def get_v2_prefixes( + ccs: Set[str], testnames: Set[str], start_day: date, end_day: date +) -> List[str]: + legacy_prefixes = [ + Prefix(bucket_name=NEW_BUCKET_NAME, prefix=f"raw/{d:%Y%m%d}") + for d in date_interval(max(date(2020, 10, 20), start_day), end_day) + ] + if not testnames: + testnames = list_all_testnames() + prefixes = [] + if start_day < date(2020, 10, 21): + prefixes = get_v2_search_prefixes(testnames, ccs) + combos = list(itertools.product(prefixes, date_interval(start_day, end_day))) + # This results in a faster listing in cases where we need only a small time + # window or few testnames. For larger windows of time, we are better off + # just listing everything. + if len(combos) < 1_000_000: # XXX we might want to tweak this parameter a bit + prefixes = [ + Prefix(bucket_name=NEW_BUCKET_NAME, prefix=f"{p}{d:%Y%m%d}") + for p, d in combos + ] + + return prefixes + legacy_prefixes + + +def get_can_prefixes(start_day: date, end_day: date) -> List[Prefix]: + """ + Returns the list of search prefixes for cans. In most cases, since we don't + have the country code or test name in the path, all we do is return the + range of dates. + """ + new_cans = [ + Prefix(prefix=f"canned/{d:%Y-%m-%d}", bucket_name=NEW_BUCKET_NAME) + for d in date_interval( + # The new cans are between 2020-06-02 and 2020-10-21 inclusive + max(date(2020, 6, 2), start_day), + min(date(2020, 10, 22), end_day), + ) + ] + old_cans = [ + Prefix(prefix=f"canned/{d:%Y-%m-%d}", bucket_name=LEGACY_BUCKET_NAME) + for d in date_interval( + # The new cans are between 2020-06-02 and 2020-10-21 inclusive + # Note: the cans between 2020-06-02 and 2020-10-21 appears to be duplicated between the new and old cans. + # TODO: check if they are actually identical or not. + max(date(2012, 12, 5), start_day), + min(date(2020, 6, 2), end_day), + ) + ] + return old_cans + new_cans + + +def iter_file_entries(prefix: Prefix) -> Generator[FileEntry, None, None]: + s3_client = create_s3_client() + paginator = s3_client.get_paginator("list_objects_v2") + for r in paginator.paginate(Bucket=prefix.bucket_name, Prefix=prefix.prefix): + for obj_dict in r.get("Contents", []): + try: + if obj_dict["Key"].endswith(".json.gz"): + # We ignore the legacy can index files + continue + + yield FileEntry.from_obj_dict(prefix.bucket_name, obj_dict) + except ValueError as exc: + log.error(exc) + + +def make_measurement_listers( + bucket_start_day: date, + bucket_end_day: date, + probe_cc_filter: Optional[set] = None, + test_name_filter: Optional[set] = None, +): + start_timestamp = datetime.combine(bucket_start_day, datetime.min.time()) + end_timestamp = datetime.combine(bucket_end_day, datetime.min.time()) + + prefix_list = get_v2_prefixes(ccs, testnames, start_day, end_day) + if from_cans == True: + prefix_list = get_can_prefixes(start_day, end_day) + prefix_list + + log.debug(f"using prefix list {prefix_list}") + file_entries = [] + prefix_idx = 0 + total_prefixes = len(prefix_list) diff --git a/oonidata/tests/unit/conftest.py b/oonidata/tests/unit/conftest.py index bdfdc22c..93416731 100644 --- a/oonidata/tests/unit/conftest.py +++ b/oonidata/tests/unit/conftest.py @@ -1,13 +1,14 @@ import os from pathlib import Path -from datetime import date +from datetime import date, datetime +import shutil from click.testing import CliRunner import pytest import orjson -from oonidata.dataclient import sync_measurements +from oonidata.dataclient import sync_measurements, create_s3_client from oonidata.apiclient import get_measurement_dict_by_uid from ._fixtures import SAMPLE_MEASUREMENTS, SAMPLE_POSTCANS, SAMPLE_JSONLGZS @@ -37,7 +38,24 @@ def raw_measurements(): def download_from_s3(name, dst_path): - assert False + s3_client = create_s3_client() + # 2024030100_AM_webconnectivity.n1.0.tar.gz + # s3://ooni-data-eu-fra/raw/20240301/raw/20240301/00/AM/webconnectivity/2024030100_AM_webconnectivity.n1.0.tar.gz + p = name.split("_") + cc = p[1] + testname = p[2].split(".")[0] + dt = datetime.strptime(p[0], "%Y%m%d%H") + day_ts = dt.strftime("%Y%m%d") + hour_ts = dt.strftime("%H") + key = f"raw/{day_ts}/{hour_ts}/{cc}/{testname}/{name}" + + try: + shutil.copyfileobj( + s3_client.get_object(Bucket="ooni-data-eu-fra", Key=key)["Body"], + dst_path.open("wb"), + ) + except: + raise Exception(f"failed to download {name} from {key}") def make_samples(sample_name, sample_list): diff --git a/oonidata/tests/unit/data/.gitignore b/oonidata/tests/unit/data/.gitignore new file mode 100644 index 00000000..b0362314 --- /dev/null +++ b/oonidata/tests/unit/data/.gitignore @@ -0,0 +1,3 @@ +/jsonlgzs +/measurements +/postcans \ No newline at end of file From 0156f1244fba767c80a3a510ba91557b78707d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 10:00:32 -0400 Subject: [PATCH 15/25] Fix tests for oonipipeline Add workflow for running oonidata and oonipipeline tests separately --- .github/workflows/test_oonidata.yml | 48 +++++++++++ .github/workflows/test_oonipipeline.yml | 43 ++++++++++ .github/workflows/tests.yml | 86 ------------------- oonipipeline/pyproject.toml | 1 + .../src/oonipipeline/analysis/control.py | 3 +- .../src/oonipipeline/analysis/datasources.py | 3 +- .../src/oonipipeline/analysis/signal.py | 5 +- .../src/oonipipeline/analysis/web_analysis.py | 5 +- .../analysis/website_experiment_results.py | 4 +- .../src/oonipipeline/db/create_tables.py | 6 +- .../transforms/measurement_transformer.py | 14 +-- oonipipeline/tests/_fixtures.py | 39 +++++++++ oonipipeline/tests/conftest.py | 14 ++- oonipipeline/tests/data/.gitignore | 1 + ...{test_workers.py => fixme_test_workers.py} | 37 +++++--- oonipipeline/tests/test_analysis.py | 20 +++-- oonipipeline/tests/test_ctrl.py | 12 +-- oonipipeline/tests/test_db.py | 2 +- oonipipeline/tests/test_experiment_results.py | 16 ++-- oonipipeline/tests/test_fingerprints.py | 3 +- oonipipeline/tests/test_scoring.py | 10 ++- oonipipeline/tests/test_transforms.py | 15 +++- 22 files changed, 234 insertions(+), 153 deletions(-) create mode 100644 .github/workflows/test_oonidata.yml create mode 100644 .github/workflows/test_oonipipeline.yml delete mode 100644 .github/workflows/tests.yml create mode 100644 oonipipeline/tests/_fixtures.py create mode 100644 oonipipeline/tests/data/.gitignore rename oonipipeline/tests/{test_workers.py => fixme_test_workers.py} (92%) diff --git a/.github/workflows/test_oonidata.yml b/.github/workflows/test_oonidata.yml new file mode 100644 index 00000000..38ae03e3 --- /dev/null +++ b/.github/workflows/test_oonidata.yml @@ -0,0 +1,48 @@ +name: test oonidata +on: push +jobs: + run_tests: + name: ${{ matrix.os }} / ${{ matrix.python-version }} + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [Ubuntu] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install hatch + run: pip install hatch + + - name: Set up datadir cache + uses: actions/cache@v3 + with: + path: oonidata/tests/data/datadir/ + key: oonidata-tests-data-datadir + + - name: Set up measurements cache + uses: actions/cache@v3 + with: + path: oonidata/tests/data/measurements/ + key: oonidata-tests-data-measurements + + - name: Set up raw_measurements cache + uses: actions/cache@v3 + with: + path: oonidata/tests/data/measurements/ + key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} + + - name: Run all tests + run: make test-cov + working-directory: ./oonidata/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: oonidata + working-directory: ./oonidata/ diff --git a/.github/workflows/test_oonipipeline.yml b/.github/workflows/test_oonipipeline.yml new file mode 100644 index 00000000..08966b19 --- /dev/null +++ b/.github/workflows/test_oonipipeline.yml @@ -0,0 +1,43 @@ +name: test oonipipeline +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Set up datadir cache + uses: actions/cache@v3 + with: + path: oonipipeline/tests/data/datadir/ + key: oonipipeline-tests-data-datadir + + - name: Set up measurements cache + uses: actions/cache@v3 + with: + path: oonipipeline/tests/data/measurements/ + key: oonipipeline-tests-data-measurements + + - name: Set up raw_measurements cache + uses: actions/cache@v3 + with: + path: oonipipeline/tests/data/measurements/ + key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} + + - name: Run all tests + run: make test-cov + working-directory: ./oonipipeline/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: oonipipeline + working-directory: ./oonipipeline/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index bf575c18..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Tests -on: - push: - branches: - - main - pull_request: - branches: - - "*" -jobs: - Tests: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [Ubuntu] - python-version: [3.8, 3.9, "3.10"] # python 3.7 temporarily disabled due to lack of altair - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Get full Python version - id: full-python-version - run: | - echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Install poetry - run: | - curl -fsS https://install.python-poetry.org | python - --preview -y - - - name: Add poetry to PATH - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Set up poetry cache - uses: actions/cache@v3 - with: - path: "$HOME/.cache/pypoetry/virtualenvs" - key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }} - - - name: Set up datadir cache - uses: actions/cache@v3 - with: - path: tests/data/datadir/ - key: tests-data-datadir - - - name: Set up measurements cache - uses: actions/cache@v3 - with: - path: tests/data/measurements/ - key: tests-data-measurements-${{ hashFiles('tests/_sample_measurements.py') }} - - - name: Set up raw_measurements cache - uses: actions/cache@v3 - with: - path: tests/data/measurements/ - key: tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} - - - name: Install clickhouse - run: | - sudo apt-get install -y apt-transport-https ca-certificates dirmngr - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 - echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ - /etc/apt/sources.list.d/clickhouse.list - sudo apt-get update - sudo apt-get install -y clickhouse-server clickhouse-client - - - name: start clickhouse - run: sudo service clickhouse-server start - - - name: Install dependencies - run: poetry install - - - name: Run all tests - env: - PYTHONUNBUFFERED: "1" - run: poetry run pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=term tests - - - name: Upload coverage to codecov - uses: codecov/codecov-action@v3 diff --git a/oonipipeline/pyproject.toml b/oonipipeline/pyproject.toml index a69b2f82..fb2f8a98 100644 --- a/oonipipeline/pyproject.toml +++ b/oonipipeline/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "memray", "viztracer", ] +python = "3.11" path = ".venv/" [tool.hatch.version] diff --git a/oonipipeline/src/oonipipeline/analysis/control.py b/oonipipeline/src/oonipipeline/analysis/control.py index 5e7a89cd..1f910381 100644 --- a/oonipipeline/src/oonipipeline/analysis/control.py +++ b/oonipipeline/src/oonipipeline/analysis/control.py @@ -5,7 +5,8 @@ from typing import Any, Generator, Optional, Tuple, List, NamedTuple -from ..models.observations import WebControlObservation, WebObservation +from oonidata.models.observations import WebControlObservation, WebObservation + from ..netinfo import NetinfoDB from ..db.connections import ClickhouseConnection diff --git a/oonipipeline/src/oonipipeline/analysis/datasources.py b/oonipipeline/src/oonipipeline/analysis/datasources.py index c2d137d5..95a81523 100644 --- a/oonipipeline/src/oonipipeline/analysis/datasources.py +++ b/oonipipeline/src/oonipipeline/analysis/datasources.py @@ -2,8 +2,9 @@ from datetime import date, timedelta from typing import Generator, List, Optional +from oonidata.models.observations import WebObservation + from ..db.connections import ClickhouseConnection -from ..models.observations import WebObservation def iter_web_observations( diff --git a/oonipipeline/src/oonipipeline/analysis/signal.py b/oonipipeline/src/oonipipeline/analysis/signal.py index d879a676..95d0eb99 100644 --- a/oonipipeline/src/oonipipeline/analysis/signal.py +++ b/oonipipeline/src/oonipipeline/analysis/signal.py @@ -1,14 +1,15 @@ from typing import List, Generator -from ..models.experiment_result import ( +from oonidata.models.experiment_result import ( BlockingScope, ExperimentResult, Outcome, fp_to_scope, iter_experiment_results, ) +from oonidata.models.observations import WebObservation + from ..fingerprintdb import FingerprintDB -from ..models.observations import WebObservation def make_signal_experiment_result( diff --git a/oonipipeline/src/oonipipeline/analysis/web_analysis.py b/oonipipeline/src/oonipipeline/analysis/web_analysis.py index d2218668..48cf7677 100644 --- a/oonipipeline/src/oonipipeline/analysis/web_analysis.py +++ b/oonipipeline/src/oonipipeline/analysis/web_analysis.py @@ -12,6 +12,8 @@ List, Dict, ) +from oonidata.models.analysis import WebAnalysis +from oonidata.models.observations import WebControlObservation, WebObservation from ..db.connections import ClickhouseConnection from ..analysis.control import ( @@ -20,9 +22,6 @@ ) from ..fingerprintdb import FingerprintDB -from ..models.analysis import WebAnalysis -from ..models.observations import WebControlObservation, WebObservation - log = logging.getLogger(__name__) diff --git a/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py b/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py index c7198c25..de4c4e3d 100644 --- a/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py +++ b/oonipipeline/src/oonipipeline/analysis/website_experiment_results.py @@ -3,8 +3,8 @@ import logging from typing import Dict, Generator, List, Optional, Tuple -from ..models.analysis import WebAnalysis -from ..models.experiment_result import MeasurementExperimentResult +from oonidata.models.analysis import WebAnalysis +from oonidata.models.experiment_result import MeasurementExperimentResult log = logging.getLogger("oonidata.analysis") diff --git a/oonipipeline/src/oonipipeline/db/create_tables.py b/oonipipeline/src/oonipipeline/db/create_tables.py index ced34a65..596f45fc 100644 --- a/oonipipeline/src/oonipipeline/db/create_tables.py +++ b/oonipipeline/src/oonipipeline/db/create_tables.py @@ -5,12 +5,12 @@ from .connections import ClickhouseConnection -from ..models.experiment_result import ( +from oonidata.models.experiment_result import ( ExperimentResult, MeasurementExperimentResult, ) -from ..models.analysis import WebAnalysis -from ..models.observations import ( +from oonidata.models.analysis import WebAnalysis +from oonidata.models.observations import ( ObservationBase, WebControlObservation, WebObservation, diff --git a/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py index a282e9a0..bc5604b1 100644 --- a/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py +++ b/oonipipeline/src/oonipipeline/transforms/measurement_transformer.py @@ -40,7 +40,7 @@ get_certificate_meta, removeprefix, ) -from oonidata.netinfo import NetinfoDB +from ..netinfo import NetinfoDB log = logging.getLogger("oonidata.transforms") @@ -180,9 +180,11 @@ def measurement_to_http_observation( request_body_is_truncated=http_transaction.request.body_is_truncated, request_headers_list=http_transaction.request.headers_list_bytes, request_method=http_transaction.request.method or "", - request_body_length=len(http_transaction.request.body_bytes) - if http_transaction.request.body_bytes - else 0, + request_body_length=( + len(http_transaction.request.body_bytes) + if http_transaction.request.body_bytes + else 0 + ), network=network, alpn=http_transaction.alpn, failure=normalize_failure(http_transaction.failure), @@ -902,7 +904,9 @@ def consume_web_observations( for idx, obs in enumerate(web_obs_list): obs.observation_id = f"{obs.measurement_uid}_{idx}" - obs.created_at = datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None) + obs.created_at = datetime.now(timezone.utc).replace( + microsecond=0, tzinfo=None + ) return web_obs_list diff --git a/oonipipeline/tests/_fixtures.py b/oonipipeline/tests/_fixtures.py new file mode 100644 index 00000000..102dace8 --- /dev/null +++ b/oonipipeline/tests/_fixtures.py @@ -0,0 +1,39 @@ +SAMPLE_MEASUREMENTS = [ + "20220107222458.184469_IL_webconnectivity_d32af5597d7eeccc", # "https://ooni.org/", + "20220607115854.978538_BR_webconnectivity_d47c958eb0986d1b", # "https://ooni.org/", + "20220608132401.787399_AM_webconnectivity_2285fc373f62729e", # "http://hahr.am", + "20220608155654.044764_AM_webconnectivity_ccb727b4812234a5", # "https://aysor.am", + "20220608122138.241075_IR_webconnectivity_c4240e52c7ca025f", # "https://www.youtube.com/", + "20220608121828.356206_RU_webconnectivity_80e3fa60eb2cd026", # "http://proxy.org/", + "20220627131742.081225_GB_webconnectivity_e1e2cf4db492b748", # "https://ooni.org/", + "20220627030703.592775_IR_webconnectivity_80e199b3c572f8d3", # "https://thepiratebay.org/", + "20220627134426.194308_DE_webconnectivity_15675b61ec62e268", # "https://thepiratebay.org/", + "20220627125833.737451_FR_webconnectivity_bca9ad9d3371919a", # "https://thepiratebay.org/", + "20220625234824.235023_HU_webconnectivity_3435a5df0e743d39", # "https://thepiratebay.org/", + "20220924222854.036406_IR_webconnectivity_7aedefe4aaac824c", # "https://doh.dns.apple.com/dns-query?dns=q80BAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB", + "20221020235950.432819_NL_signal_27b05458f186a906", + "20221016235944.266268_GB_signal_1265ff650ee17b44", + "20210926222047.205897_UZ_signal_95fab4a2e669573f", + "20221018174612.488229_IR_signal_f8640b28061bec06", + "20221013000000.517636_US_dnscheck_bfd6d991e70afa0e", + "20221114002335.786418_BR_webconnectivity_6b203219ec4ded0e", + "20230427235943.206438_US_telegram_ac585306869eca7b", + "20211018232506.972850_IN_whatsapp_44970a56806dbfb3", + "20220124235953.650143_ES_facebookmessenger_0e048a26b89a9d70", + "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2", + "20210101190046.780850_US_webconnectivity_3296f126f79ca186", + "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6", + # "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", + # "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" + # "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" + # "20231222154141.824397_US_browserweb_615428b4802b5297", + # "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", + # "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", + # "20231101164649.235575_RU_tor_ccf7519bf683c022", + # "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter + # "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected + # "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit +] + +SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"] +SAMPLE_JSONLGZS = ["2024030100_AM_webconnectivity.n1.0.jsonl.gz"] diff --git a/oonipipeline/tests/conftest.py b/oonipipeline/tests/conftest.py index 02486918..9de6dc67 100644 --- a/oonipipeline/tests/conftest.py +++ b/oonipipeline/tests/conftest.py @@ -6,15 +6,16 @@ import pytest import orjson -from oonipipeline.db.connections import ClickhouseConnection - -from oonipipeline.fingerprintdb import FingerprintDB -from oonipipeline.netinfo import NetinfoDB from oonidata.dataclient import sync_measurements from oonidata.apiclient import get_measurement_dict_by_uid -from ._sample_measurements import SAMPLE_MEASUREMENTS +from oonipipeline.db.connections import ClickhouseConnection +from oonipipeline.db.create_tables import create_queries +from oonipipeline.fingerprintdb import FingerprintDB +from oonipipeline.netinfo import NetinfoDB + +from ._fixtures import SAMPLE_MEASUREMENTS FIXTURE_PATH = Path(os.path.dirname(os.path.realpath(__file__))) / "data" DATA_DIR = FIXTURE_PATH / "datadir" @@ -74,9 +75,6 @@ def cli_runner(): return CliRunner() -from oonidata.db.create_tables import create_queries - - def create_db_for_fixture(): try: with ClickhouseConnection(conn_url="clickhouse://localhost/") as db: diff --git a/oonipipeline/tests/data/.gitignore b/oonipipeline/tests/data/.gitignore new file mode 100644 index 00000000..fca7e39f --- /dev/null +++ b/oonipipeline/tests/data/.gitignore @@ -0,0 +1 @@ +/datadir diff --git a/oonipipeline/tests/test_workers.py b/oonipipeline/tests/fixme_test_workers.py similarity index 92% rename from oonipipeline/tests/test_workers.py rename to oonipipeline/tests/fixme_test_workers.py index 08c9cdb2..17556d91 100644 --- a/oonipipeline/tests/test_workers.py +++ b/oonipipeline/tests/fixme_test_workers.py @@ -4,29 +4,44 @@ import sqlite3 from typing import List, Tuple from unittest.mock import MagicMock +import time -from oonidata.analysis.datasources import load_measurement - -from oonidata.dataclient import stream_jsonl +from oonidata.dataclient import stream_jsonl, load_measurement from oonidata.models.nettests.dnscheck import DNSCheck from oonidata.models.nettests.web_connectivity import WebConnectivity from oonidata.models.nettests.http_invalid_request_line import HTTPInvalidRequestLine from oonidata.models.observations import HTTPMiddleboxObservation -from oonidata.workers.analysis import make_analysis_in_a_day, make_cc_batches, make_ctrl -from oonidata.workers.common import ( + +from oonipipeline.workflows.analysis import ( + make_analysis_in_a_day, + make_cc_batches, + make_ctrl, +) +from oonipipeline.workflows.common import ( get_obs_count_by_cc, get_prev_range, maybe_delete_prev_range, ) -from oonidata.workers.observations import ( +from oonipipeline.workflows.observations import ( make_observations_for_file_entry_batch, write_observations_to_db, ) -from oonidata.workers.response_archiver import ResponseArchiver -from oonidata.workers.fingerprint_hunter import fingerprint_hunter -from oonidata.transforms import measurement_to_observations -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer -from tests.test_cli import wait_for_mutations +from oonipipeline.workflows.response_archiver import ResponseArchiver +from oonipipeline.workflows.fingerprint_hunter import fingerprint_hunter +from oonipipeline.transforms import measurement_to_observations +from oonipipeline.transforms.nettests.measurement_transformer import ( + MeasurementTransformer, +) + + +def wait_for_mutations(db, table_name): + while True: + res = db.execute( + f"SELECT * FROM system.mutations WHERE is_done=0 AND table='{table_name}';" + ) + if len(res) == 0: # type: ignore + break + time.sleep(1) def test_get_prev_range(db): diff --git a/oonipipeline/tests/test_analysis.py b/oonipipeline/tests/test_analysis.py index 9cea29f7..044411e8 100644 --- a/oonipipeline/tests/test_analysis.py +++ b/oonipipeline/tests/test_analysis.py @@ -5,21 +5,23 @@ from unittest.mock import MagicMock import pytest -from oonidata.analysis.datasources import load_measurement -from oonidata.analysis.web_analysis import make_web_analysis + +from oonidata.dataclient import load_measurement +from oonidata.models.nettests.signal import Signal +from oonidata.models.nettests.web_connectivity import WebConnectivity +from oonidata.models.observations import WebObservation, print_nice, print_nice_vertical from oonidata.datautils import validate_cert_chain -from oonidata.analysis.control import ( + +from oonipipeline.analysis.web_analysis import make_web_analysis +from oonipipeline.analysis.control import ( BodyDB, WebGroundTruth, iter_ground_truths_from_web_control, WebGroundTruthDB, ) -from oonidata.analysis.signal import make_signal_experiment_result -from oonidata.models.nettests.signal import Signal -from oonidata.models.nettests.web_connectivity import WebConnectivity -from oonidata.models.observations import WebObservation, print_nice, print_nice_vertical -from oonidata.transforms.nettests.signal import SIGNAL_PEM_STORE -from oonidata.transforms import measurement_to_observations +from oonipipeline.analysis.signal import make_signal_experiment_result +from oonipipeline.transforms.nettests.signal import SIGNAL_PEM_STORE +from oonipipeline.transforms.observations import measurement_to_observations def test_signal(fingerprintdb, netinfodb, measurements): diff --git a/oonipipeline/tests/test_ctrl.py b/oonipipeline/tests/test_ctrl.py index c5d86b81..1a7f384a 100644 --- a/oonipipeline/tests/test_ctrl.py +++ b/oonipipeline/tests/test_ctrl.py @@ -1,14 +1,14 @@ from datetime import date, datetime -import pytest -from oonidata.analysis.datasources import iter_web_observations -from oonidata.db.connections import ClickhouseConnection -from oonidata.analysis.control import ( +from oonidata.models.observations import WebObservation, print_nice_vertical + +from oonipipeline.analysis.datasources import iter_web_observations +from oonipipeline.db.connections import ClickhouseConnection +from oonipipeline.analysis.control import ( WebGroundTruthDB, iter_web_ground_truths, ) -from oonidata.models.observations import WebObservation, print_nice_vertical -from oonidata.workers.observations import make_observations_for_file_entry_batch +from oonipipeline.workflows.observations import make_observations_for_file_entry_batch def test_web_ground_truth_from_clickhouse(db, datadir, netinfodb, tmp_path): diff --git a/oonipipeline/tests/test_db.py b/oonipipeline/tests/test_db.py index 7c7ce936..a1b025a5 100644 --- a/oonipipeline/tests/test_db.py +++ b/oonipipeline/tests/test_db.py @@ -2,7 +2,7 @@ from clickhouse_driver import Client -from oonidata.db.connections import ClickhouseConnection +from oonipipeline.db.connections import ClickhouseConnection def test_flush_rows(db): diff --git a/oonipipeline/tests/test_experiment_results.py b/oonipipeline/tests/test_experiment_results.py index 5e399f76..ffa1a551 100644 --- a/oonipipeline/tests/test_experiment_results.py +++ b/oonipipeline/tests/test_experiment_results.py @@ -1,14 +1,18 @@ from pprint import pprint -from oonidata.analysis.control import ( + +from oonidata.models.observations import print_nice, print_nice_vertical +from oonidata.dataclient import load_measurement + +from oonipipeline.analysis.control import ( BodyDB, WebGroundTruthDB, iter_ground_truths_from_web_control, ) -from oonidata.analysis.datasources import load_measurement -from oonidata.analysis.web_analysis import make_web_analysis -from oonidata.analysis.website_experiment_results import make_website_experiment_results -from oonidata.models.observations import print_nice, print_nice_vertical -from oonidata.transforms import measurement_to_observations +from oonipipeline.analysis.web_analysis import make_web_analysis +from oonipipeline.analysis.website_experiment_results import ( + make_website_experiment_results, +) +from oonipipeline.transforms.observations import measurement_to_observations # Check this for wc 0.5 overwriting tls analsysis diff --git a/oonipipeline/tests/test_fingerprints.py b/oonipipeline/tests/test_fingerprints.py index d1ef89e9..2809e0c5 100644 --- a/oonipipeline/tests/test_fingerprints.py +++ b/oonipipeline/tests/test_fingerprints.py @@ -1,6 +1,7 @@ -from oonidata.analysis.datasources import load_measurement from oonidata.models.nettests.web_connectivity import WebConnectivity +from oonidata.dataclient import load_measurement + def test_fingerprintdb(fingerprintdb, measurements): dns_blocked = load_measurement( diff --git a/oonipipeline/tests/test_scoring.py b/oonipipeline/tests/test_scoring.py index a55691cb..790fd2c8 100644 --- a/oonipipeline/tests/test_scoring.py +++ b/oonipipeline/tests/test_scoring.py @@ -1,13 +1,15 @@ from unittest.mock import MagicMock import pytest -from oonidata.analysis.control import ( + +from oonidata.models.experiment_result import print_nice_er +from oonidata.dataclient import load_measurement + +from oonipipeline.analysis.control import ( WebGroundTruthDB, iter_ground_truths_from_web_control, ) -from oonidata.analysis.datasources import load_measurement -from oonidata.models.experiment_result import print_nice_er -from oonidata.transforms import measurement_to_observations +from oonipipeline.transforms.observations import measurement_to_observations def test_tcp_scoring(measurements, netinfodb, fingerprintdb): diff --git a/oonipipeline/tests/test_transforms.py b/oonipipeline/tests/test_transforms.py index 2a2a5651..62d98f4e 100644 --- a/oonipipeline/tests/test_transforms.py +++ b/oonipipeline/tests/test_transforms.py @@ -1,5 +1,7 @@ from typing import List -from oonidata.analysis.datasources import load_measurement + + +from oonidata.dataclient import load_measurement from oonidata.models.nettests.dnscheck import DNSCheck from oonidata.models.nettests.telegram import Telegram from oonidata.models.nettests.signal import Signal @@ -10,8 +12,11 @@ from oonidata.models.nettests.urlgetter import UrlGetter from oonidata.models.nettests.browser_web import BrowserWeb from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer -from oonidata.transforms import measurement_to_observations + +from oonipipeline.transforms.measurement_transformer import ( + MeasurementTransformer, +) +from oonipipeline.transforms.observations import measurement_to_observations def test_wc_v5_observations(netinfodb, measurements): @@ -183,7 +188,9 @@ def test_telegram_obs(netinfodb, measurements): def test_stunreachability_obs(netinfodb, measurements): msmt = load_measurement( - msmt_path=measurements["20221224235924.922622_BR_stunreachability_905c61a34356a9b2"] + msmt_path=measurements[ + "20221224235924.922622_BR_stunreachability_905c61a34356a9b2" + ] ) assert isinstance(msmt, StunReachability) web_obs: List[WebObservation] = measurement_to_observations( From b6c947ad1932b77931b3ec9b04b2b67296bd1c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 10:01:26 -0400 Subject: [PATCH 16/25] In pipeline tests we also need clickhouse --- .github/workflows/test_oonipipeline.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test_oonipipeline.yml b/.github/workflows/test_oonipipeline.yml index 08966b19..bb90741a 100644 --- a/.github/workflows/test_oonipipeline.yml +++ b/.github/workflows/test_oonipipeline.yml @@ -32,6 +32,15 @@ jobs: path: oonipipeline/tests/data/measurements/ key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} + - name: Install clickhouse + run: | + sudo apt-get install -y apt-transport-https ca-certificates dirmngr + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ + /etc/apt/sources.list.d/clickhouse.list + sudo apt-get update + sudo apt-get install -y clickhouse-server clickhouse-client + - name: Run all tests run: make test-cov working-directory: ./oonipipeline/ From c40dba2a140ec84c8e09f1b49f182fdd7d3c992e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 10:06:38 -0400 Subject: [PATCH 17/25] Fix raw measurement cache path --- .github/workflows/test_oonidata.yml | 2 +- .github/workflows/test_oonipipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_oonidata.yml b/.github/workflows/test_oonidata.yml index 38ae03e3..b7b7133e 100644 --- a/.github/workflows/test_oonidata.yml +++ b/.github/workflows/test_oonidata.yml @@ -34,7 +34,7 @@ jobs: - name: Set up raw_measurements cache uses: actions/cache@v3 with: - path: oonidata/tests/data/measurements/ + path: oonidata/tests/data/raw_measurements/ key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} - name: Run all tests diff --git a/.github/workflows/test_oonipipeline.yml b/.github/workflows/test_oonipipeline.yml index bb90741a..4ed8ee93 100644 --- a/.github/workflows/test_oonipipeline.yml +++ b/.github/workflows/test_oonipipeline.yml @@ -29,7 +29,7 @@ jobs: - name: Set up raw_measurements cache uses: actions/cache@v3 with: - path: oonipipeline/tests/data/measurements/ + path: oonipipeline/tests/data/raw_measurements/ key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} - name: Install clickhouse From 57b57bcefe490a8ae0158bed6a779efa4664d1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 10:09:45 -0400 Subject: [PATCH 18/25] We don't actually have a makefile yet --- .github/workflows/test_oonidata.yml | 2 +- .github/workflows/test_oonipipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_oonidata.yml b/.github/workflows/test_oonidata.yml index b7b7133e..cc62264c 100644 --- a/.github/workflows/test_oonidata.yml +++ b/.github/workflows/test_oonidata.yml @@ -38,7 +38,7 @@ jobs: key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }} - name: Run all tests - run: make test-cov + run: hatch run cov working-directory: ./oonidata/ - name: Upload coverage to codecov diff --git a/.github/workflows/test_oonipipeline.yml b/.github/workflows/test_oonipipeline.yml index 4ed8ee93..62879ea2 100644 --- a/.github/workflows/test_oonipipeline.yml +++ b/.github/workflows/test_oonipipeline.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y clickhouse-server clickhouse-client - name: Run all tests - run: make test-cov + run: hatch run cov working-directory: ./oonipipeline/ - name: Upload coverage to codecov From ab5f34e16135d6deb3eec559c541068a8f09a3bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Sun, 17 Mar 2024 10:11:02 -0400 Subject: [PATCH 19/25] Add note about why oonipipeline has only 1 python version in the test matrix --- .github/workflows/test_oonipipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_oonipipeline.yml b/.github/workflows/test_oonipipeline.yml index 62879ea2..371d334a 100644 --- a/.github/workflows/test_oonipipeline.yml +++ b/.github/workflows/test_oonipipeline.yml @@ -6,6 +6,9 @@ jobs: steps: - uses: actions/checkout@v3 + # Contrary to oonidata, which is a tool for end users, oonipipeline + # targets a single python version to minimize the likelyhood of breakage + # in production. - name: Set up Python 3.11 uses: actions/setup-python@v4 with: From be685b0c91864fcc9373485333975500d2e278d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 13:57:02 +0200 Subject: [PATCH 20/25] Drop python version from oonidata pyproject.yml --- oonidata/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/oonidata/pyproject.toml b/oonidata/pyproject.toml index f2f8bc5e..f700860c 100644 --- a/oonidata/pyproject.toml +++ b/oonidata/pyproject.toml @@ -55,7 +55,6 @@ dependencies = [ "memray", "viztracer", ] -python = "3.11" path = ".venv/" [tool.hatch.version] From 3ca3e0dc12cc516603b054548d43e1f32a81c9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 13:59:43 +0200 Subject: [PATCH 21/25] Update .gitignores --- .gitignore | 1 - oonipipeline/tests/data/.gitignore | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cdc294a7..999050a9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ coverage.xml .ipynb_checkpoints/ /dist -/datadir /output /attic /prof diff --git a/oonipipeline/tests/data/.gitignore b/oonipipeline/tests/data/.gitignore index fca7e39f..213f74a9 100644 --- a/oonipipeline/tests/data/.gitignore +++ b/oonipipeline/tests/data/.gitignore @@ -1 +1,2 @@ /datadir +/measurements From 92fa8b54b61d6791acbe546c4849f1749a78993b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 14:02:59 +0200 Subject: [PATCH 22/25] Uncomment fixtures --- oonidata/tests/unit/_fixtures.py | 20 ++++++++++---------- oonipipeline/tests/_fixtures.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/oonidata/tests/unit/_fixtures.py b/oonidata/tests/unit/_fixtures.py index 102dace8..47960b32 100644 --- a/oonidata/tests/unit/_fixtures.py +++ b/oonidata/tests/unit/_fixtures.py @@ -23,16 +23,16 @@ "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2", "20210101190046.780850_US_webconnectivity_3296f126f79ca186", "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6", - # "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", - # "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" - # "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" - # "20231222154141.824397_US_browserweb_615428b4802b5297", - # "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", - # "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", - # "20231101164649.235575_RU_tor_ccf7519bf683c022", - # "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter - # "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected - # "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit + "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", + "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" + "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" + "20231222154141.824397_US_browserweb_615428b4802b5297", + "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", + "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", + "20231101164649.235575_RU_tor_ccf7519bf683c022", + "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter + "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected + "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit ] SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"] diff --git a/oonipipeline/tests/_fixtures.py b/oonipipeline/tests/_fixtures.py index 102dace8..47960b32 100644 --- a/oonipipeline/tests/_fixtures.py +++ b/oonipipeline/tests/_fixtures.py @@ -23,16 +23,16 @@ "20210101181154.037019_CH_webconnectivity_68ce38aa9e3182c2", "20210101190046.780850_US_webconnectivity_3296f126f79ca186", "20231031032643.267235_GR_dnscheck_abcbfc460b9424b6", - # "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", - # "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" - # "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" - # "20231222154141.824397_US_browserweb_615428b4802b5297", - # "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", - # "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", - # "20231101164649.235575_RU_tor_ccf7519bf683c022", - # "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter - # "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected - # "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit + "20221224235924.922622_BR_stunreachability_905c61a34356a9b2", + "20211224011542.635260_IR_urlgetter_38d73cdfee442409", # "dnslookup://example.org" + "20221224180301.892770_VE_urlgetter_0a02e27d0c651b8f", # "https://dolartoday.com/" + "20231222154141.824397_US_browserweb_615428b4802b5297", + "20231101164541.763506_NP_httpinvalidrequestline_0cf676868fa36cc4", + "20231101164544.534107_BR_httpheaderfieldmanipulation_4caa0b0556f0b141", + "20231101164649.235575_RU_tor_ccf7519bf683c022", + "20221101055235.141387_RU_webconnectivity_046ce024dd76b564", # ru_blocks_twitter + "20230907000740.785053_BR_httpinvalidrequestline_bdfe6d70dcbda5e9", # middlebox detected + "20221110235922.335062_IR_webconnectivity_e4114ee32b8dbf74", # Iran blocking reddit ] SAMPLE_POSTCANS = ["2024030100_AM_webconnectivity.n1.0.tar.gz"] From 814b78a2bb18f94fad4c86b768183373f334b54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 14:06:52 +0200 Subject: [PATCH 23/25] Drop un-used deps from oonidata --- oonidata/pyproject.toml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/oonidata/pyproject.toml b/oonidata/pyproject.toml index f700860c..8aa36a38 100644 --- a/oonidata/pyproject.toml +++ b/oonidata/pyproject.toml @@ -15,26 +15,13 @@ dependencies = [ "lz4 ~= 4.0", "requests ~= 2.27", "cryptography ~= 38.0.3", - "clickhouse-driver ~= 0.2", "click ~= 8.0.0", "lxml ~= 4.9", - "maxminddb ~= 2.2", "orjson ~= 3.8", "mashumaro ~= 3.0", "pyOpenSSL ~= 22.1", - "fastapi ~= 0.108.0", "tabulate ~= 0.9.0", - "warcio ~= 1.7.4", - "msgpack ~= 1.0.4", - "statsd ~= 4.0.1", "click-loglevel ~= 0.5.0", - "dask ~= 2023.3.2", - "altair ~= 4.2.0", - "numpy ~= 1.23.5", - "pandas ~= 2.0.0", - "flask ~= 2.2.0", - "jupyterlab ~= 4.0.7", - "temporalio ~= 1.5.1", ] [tool.hatch.build.targets.sdist] From 1edda8e19d8c8a2ae72aaf882fa049dd2ffb5c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 14:11:07 +0200 Subject: [PATCH 24/25] Drop EOL 3.7 python --- .github/workflows/test_oonidata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_oonidata.yml b/.github/workflows/test_oonidata.yml index cc62264c..d7b90cf6 100644 --- a/.github/workflows/test_oonidata.yml +++ b/.github/workflows/test_oonidata.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: os: [Ubuntu] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 From dd710140e9056669aa5e8b318f5d88fbcd3f0a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 12 Apr 2024 17:29:11 +0200 Subject: [PATCH 25/25] Improvements to the tests --- oonipipeline/pyproject.toml | 1 + oonipipeline/tests/conftest.py | 38 +++++++++++++++---- oonipipeline/tests/docker-compose.yml | 6 +++ oonipipeline/tests/test_analysis.py | 4 +- ...est_full_workflow.py => test_worfklows.py} | 3 ++ 5 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 oonipipeline/tests/docker-compose.yml rename oonipipeline/tests/{to_port_test_full_workflow.py => test_worfklows.py} (99%) diff --git a/oonipipeline/pyproject.toml b/oonipipeline/pyproject.toml index fb2f8a98..c5ee396d 100644 --- a/oonipipeline/pyproject.toml +++ b/oonipipeline/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "pytest-profiling", "memray", "viztracer", + "pytest-docker", ] python = "3.11" path = ".venv/" diff --git a/oonipipeline/tests/conftest.py b/oonipipeline/tests/conftest.py index 9de6dc67..030b8bb3 100644 --- a/oonipipeline/tests/conftest.py +++ b/oonipipeline/tests/conftest.py @@ -7,6 +7,8 @@ import orjson +from clickhouse_driver import Client as ClickhouseClient + from oonidata.dataclient import sync_measurements from oonidata.apiclient import get_measurement_dict_by_uid @@ -21,6 +23,26 @@ DATA_DIR = FIXTURE_PATH / "datadir" +def is_clickhouse_running(url): + try: + with ClickhouseClient.from_url(url) as client: + client.execute("SELECT 1") + return True + except Exception: + return False + + +@pytest.fixture(scope="session") +def clickhouse_server(docker_ip, docker_services): + """Ensure that HTTP service is up and responsive.""" + port = docker_services.port_for("clickhouse", 9000) + url = "clickhouse://{}:{}/default".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) + ) + yield url + + @pytest.fixture def datadir(): return DATA_DIR @@ -75,14 +97,14 @@ def cli_runner(): return CliRunner() -def create_db_for_fixture(): +def create_db_for_fixture(conn_url): try: - with ClickhouseConnection(conn_url="clickhouse://localhost/") as db: + with ClickhouseConnection(conn_url=conn_url) as db: db.execute("CREATE DATABASE IF NOT EXISTS testing_oonidata") except: pytest.skip("no database connection") - db = ClickhouseConnection(conn_url="clickhouse://localhost/testing_oonidata") + db = ClickhouseConnection(conn_url=conn_url.replace("default", "testing_oonidata")) try: db.execute("SELECT 1") except: @@ -93,13 +115,13 @@ def create_db_for_fixture(): @pytest.fixture -def db_notruncate(): - return create_db_for_fixture() +def db_notruncate(clickhouse_server): + yield create_db_for_fixture(clickhouse_server) @pytest.fixture -def db(): - db = create_db_for_fixture() +def db(clickhouse_server): + db = create_db_for_fixture(clickhouse_server) for _, table_name in create_queries: db.execute(f"TRUNCATE TABLE {table_name};") - return db + yield db diff --git a/oonipipeline/tests/docker-compose.yml b/oonipipeline/tests/docker-compose.yml new file mode 100644 index 00000000..7546ca5b --- /dev/null +++ b/oonipipeline/tests/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2' +services: + clickhouse: + image: "clickhouse/clickhouse-server" + ports: + - "9000:9000" diff --git a/oonipipeline/tests/test_analysis.py b/oonipipeline/tests/test_analysis.py index 044411e8..242da280 100644 --- a/oonipipeline/tests/test_analysis.py +++ b/oonipipeline/tests/test_analysis.py @@ -184,9 +184,7 @@ def test_website_dns_blocking_event(fingerprintdb, netinfodb, measurements): def make_experiment_result_from_wc_ctrl(msmt_path, fingerprintdb, netinfodb): msmt = load_measurement(msmt_path=msmt_path) assert isinstance(msmt, WebConnectivity) - web_observations, web_control_observations = measurement_to_observations( - msmt, netinfodb=netinfodb - ) + _, web_control_observations = measurement_to_observations(msmt, netinfodb=netinfodb) assert msmt.test_keys.control assert isinstance(msmt.input, str) diff --git a/oonipipeline/tests/to_port_test_full_workflow.py b/oonipipeline/tests/test_worfklows.py similarity index 99% rename from oonipipeline/tests/to_port_test_full_workflow.py rename to oonipipeline/tests/test_worfklows.py index eb9dd5c5..5a11a9d6 100644 --- a/oonipipeline/tests/to_port_test_full_workflow.py +++ b/oonipipeline/tests/test_worfklows.py @@ -1,3 +1,6 @@ +from pathlib import Path + + def _test_full_workflow( db, cli_runner, fingerprintdb, netinfodb, datadir, tmp_path: Path ):