From a3c616f74012e5ee0cf1166e108ccf6df2570f66 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 8 Jan 2021 14:09:47 -0600 Subject: [PATCH 01/30] visa update cron job --- fence/job/visa_update_cronjob.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 fence/job/visa_update_cronjob.py diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py new file mode 100644 index 000000000..9213d9639 --- /dev/null +++ b/fence/job/visa_update_cronjob.py @@ -0,0 +1,12 @@ +from fence.models import GA4GHVisaV1 +from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient + +# Base class that collects only specific type of refresh token + + +class Visa_Update(object): + def __init__( + self, + visa_type=None + ): + self.visa_type = visa_type \ No newline at end of file From 75911349e40ab7c2aebe6fbaf7bd8dd746588a16 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 8 Jan 2021 16:01:35 -0600 Subject: [PATCH 02/30] initialize --- fence/job/visa_update_cronjob.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 9213d9639..2ed70394b 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -1,12 +1,22 @@ +import asyncio + +from fence.config import config from fence.models import GA4GHVisaV1 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient # Base class that collects only specific type of refresh token -class Visa_Update(object): +class Visa_Token_Update(object): def __init__( self, - visa_type=None + visa_type=None, + concurrency=None, # number of concurrent users going through the visa update flow + thread_pool_size=None, # number of Docker container CPU used for jwt verification + buffer_size=None, # max size of asyncio queue ): - self.visa_type = visa_type \ No newline at end of file + self.visa_type = visa_type or "ras" + self.concurrency = concurrency or 3 + self.thread_pool_size = thread_pool_size or 2 + self.buffer_size = buffer_size or 10 + From b9aed53186397b952b9749d161f8540b0816e7b8 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 11 Jan 2021 11:42:46 -0600 Subject: [PATCH 03/30] notes --- fence/job/visa_update_cronjob.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 2ed70394b..8793febfa 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -8,6 +8,7 @@ class Visa_Token_Update(object): + def __init__( self, visa_type=None, @@ -20,3 +21,10 @@ def __init__( self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 + def update_tokens(self): + """ + Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use + looking at the type field in the ga4gh table. + + """ + client = RASClient() \ No newline at end of file From 39e0423a050f8f2e1ade2c195f14c2d4a3cd78e9 Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 13 Jan 2021 09:07:26 -0600 Subject: [PATCH 04/30] draft flow --- fence/job/visa_update_cronjob.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 8793febfa..b4ecd389a 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -20,11 +20,36 @@ def __init__( self.concurrency = concurrency or 3 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 + self.n_workers = self.thread_pool_size + self.concurrency - def update_tokens(self): + async def update_tokens(self): """ Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use looking at the type field in the ga4gh table. + """ + + queue = asyncio.Queue(maxsize=self.buffer_size) + + # Iterate through all users in upstream_refresh_token table (or should it be the ga4gh_visa_v1 table since we want to maximize access) + # No case where there might be a refresh token but no visa so should iterate through the visa table + + # + + + # Check what type of visa they have + + # Depending on the type of visa change their OauthClient + + + + async def producer(self, queue): + """ + TODO: Rename this + Producer: Produces users and puts them in a queue for processing + + """ + async def consumer(self, queue): """ - client = RASClient() \ No newline at end of file + TODO: Rename this + Consumer: Create workers that does the visa update flow From 5ed4941d8ee0722c97e763eff9a31c74989355f6 Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 14 Jan 2021 10:27:48 -0600 Subject: [PATCH 05/30] more notes --- fence/job/visa_update_cronjob.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index b4ecd389a..5a782d1d3 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -27,18 +27,9 @@ async def update_tokens(self): Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use looking at the type field in the ga4gh table. """ - - queue = asyncio.Queue(maxsize=self.buffer_size) - - # Iterate through all users in upstream_refresh_token table (or should it be the ga4gh_visa_v1 table since we want to maximize access) - # No case where there might be a refresh token but no visa so should iterate through the visa table - # - - - # Check what type of visa they have + queue = asyncio.Queue(maxsize=self.buffer_size) - # Depending on the type of visa change their OauthClient @@ -48,6 +39,9 @@ async def producer(self, queue): Producer: Produces users and puts them in a queue for processing """ + # while there are users: + # fill queue with users until queue is full + # could use a counter to query through each user in db and use this counter for Prometheus (???) async def consumer(self, queue): """ From 159d309b86c39493abd30b46809fafc5b52e2e1a Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 19 Jan 2021 09:29:47 -0600 Subject: [PATCH 06/30] testing --- fence/job/visa_update_cronjob.py | 23 +- poetry.lock | 1399 ++++++++++++++++-------------- tests/ras/test_ras.py | 90 ++ 3 files changed, 853 insertions(+), 659 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 5a782d1d3..9e0d06106 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -1,9 +1,18 @@ import asyncio +from cdislogging import get_logger +from userdatamodel.driver import SQLAlchemyDriver + from fence.config import config -from fence.models import GA4GHVisaV1 +from fence.models import ( + GA4GHVisaV1, + User, + UpstreamRefreshToken, + query_for_user, +) from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient + # Base class that collects only specific type of refresh token @@ -22,7 +31,7 @@ def __init__( self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency - async def update_tokens(self): + def update_tokens(self, db_session): """ Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use looking at the type field in the ga4gh table. @@ -30,6 +39,14 @@ async def update_tokens(self): queue = asyncio.Queue(maxsize=self.buffer_size) + all_visas = db_session.query(GA4GHVisaV1).all() + print("--------------------------------------------") + for visa in all_visas: + username = visa.user.username + print(username) + u = query_for_user(db_session, username) + print(u.ga4gh_visas_v1) + @@ -47,3 +64,5 @@ async def consumer(self, queue): """ TODO: Rename this Consumer: Create workers that does the visa update flow + """ + # update visa stuff here \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 11f33208b..575bbce0b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,108 +1,108 @@ [[package]] -name = "addict" -version = "2.3.0" -description = "Addict is a dictionary whose items can be set using both attribute and item syntax." category = "main" +description = "Addict is a dictionary whose items can be set using both attribute and item syntax." +name = "addict" optional = false python-versions = "*" +version = "2.4.0" [[package]] -name = "aniso8601" -version = "8.0.0" -description = "A library for parsing ISO 8601 strings." category = "main" +description = "A library for parsing ISO 8601 strings." +name = "aniso8601" optional = false python-versions = "*" +version = "8.1.0" [[package]] -name = "argparse" -version = "1.4.0" -description = "Python command-line parsing library" category = "main" +description = "Python command-line parsing library" +name = "argparse" optional = false python-versions = "*" +version = "1.4.0" [[package]] -name = "atomicwrites" -version = "1.4.0" -description = "Atomic file writes." category = "dev" +description = "Atomic file writes." +name = "atomicwrites" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.4.0" [[package]] -name = "attrs" -version = "20.2.0" -description = "Classes Without Boilerplate" category = "dev" +description = "Classes Without Boilerplate" +name = "attrs" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.3.0" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] +tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] [[package]] -name = "authlib" -version = "0.11" -description = "The ultimate Python library in building OAuth and OpenID Connect servers." category = "main" +description = "The ultimate Python library in building OAuth and OpenID Connect servers." +name = "authlib" optional = false python-versions = "*" +version = "0.11" [package.dependencies] cryptography = "*" requests = "*" [[package]] -name = "authutils" -version = "4.0.0" -description = "Gen3 auth utility functions" category = "main" +description = "Gen3 auth utility functions" +name = "authutils" optional = false python-versions = "*" +version = "4.0.0" [package.dependencies] -addict = ">=2.1,<3.0" Authlib = ">=0.11,<1.0" +Flask = ">=0.10.1" +PyJWT = ">=1.5,<2.0" +Werkzeug = ">=0.12,<1.0" +addict = ">=2.1,<3.0" cached-property = ">=1.4,<2.0" cdiserrors = ">=0.1,<1.0" cdislogging = "*" -Flask = ">=0.10.1" -PyJWT = ">=1.5,<2.0" python-keystoneclient = ">=1.8,<2.0" requests = ">=2.6,<3.0" -Werkzeug = ">=0.12,<1.0" xmltodict = ">=0.9,<1.0" [[package]] -name = "babel" -version = "2.8.0" -description = "Internationalization utilities" category = "main" +description = "Internationalization utilities" +name = "babel" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.9.0" [package.dependencies] pytz = ">=2015.7" [[package]] -name = "backoff" -version = "1.10.0" -description = "Function decoration for backoff and retry" category = "main" +description = "Function decoration for backoff and retry" +name = "backoff" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "1.10.0" [[package]] -name = "bcrypt" -version = "3.2.0" -description = "Modern password hashing for your software and your servers" category = "main" +description = "Modern password hashing for your software and your servers" +name = "bcrypt" optional = false python-versions = ">=3.6" +version = "3.2.0" [package.dependencies] cffi = ">=1.1" @@ -113,20 +113,20 @@ tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)"] typecheck = ["mypy"] [[package]] -name = "boto" -version = "2.49.0" -description = "Amazon Web Services Library" category = "main" +description = "Amazon Web Services Library" +name = "boto" optional = false python-versions = "*" +version = "2.49.0" [[package]] -name = "boto3" -version = "1.9.253" -description = "The AWS SDK for Python" category = "main" +description = "The AWS SDK for Python" +name = "boto3" optional = false python-versions = "*" +version = "1.9.253" [package.dependencies] botocore = ">=1.12.253,<1.13.0" @@ -134,130 +134,134 @@ jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.2.0,<0.3.0" [[package]] -name = "botocore" -version = "1.12.253" -description = "Low-level, data-driven core of boto 3." category = "main" +description = "Low-level, data-driven core of boto 3." +name = "botocore" optional = false python-versions = "*" +version = "1.12.253" [package.dependencies] docutils = ">=0.10,<0.16" jmespath = ">=0.7.1,<1.0.0" -python-dateutil = {version = ">=2.1,<3.0.0", markers = "python_version >= \"2.7\""} -urllib3 = {version = ">=1.20,<1.26", markers = "python_version >= \"3.4\""} + +[package.dependencies.python-dateutil] +python = ">=2.7" +version = ">=2.1,<3.0.0" + +[package.dependencies.urllib3] +python = ">=3.4" +version = ">=1.20,<1.26" [[package]] -name = "cached-property" -version = "1.5.2" -description = "A decorator for caching properties in classes." category = "main" +description = "A decorator for caching properties in classes." +name = "cached-property" optional = false python-versions = "*" +version = "1.5.2" [[package]] -name = "cachetools" -version = "4.1.1" -description = "Extensible memoizing collections and decorators" category = "main" +description = "Extensible memoizing collections and decorators" +name = "cachetools" optional = false python-versions = "~=3.5" +version = "4.2.0" [[package]] -name = "cdiserrors" -version = "0.1.2" -description = "The auth system for the gdcapi." category = "main" +description = "The auth system for the gdcapi." +name = "cdiserrors" optional = false python-versions = "*" +version = "0.1.2" [package.dependencies] -cdislogging = ">=0.0.2" Flask = "*" Werkzeug = "*" +cdislogging = ">=0.0.2" [[package]] -name = "cdislogging" -version = "1.0.0" -description = "Standardized logging tool and format for cdis applications" category = "main" +description = "Standardized logging tool and format for cdis applications" +name = "cdislogging" optional = false python-versions = "*" +version = "1.0.0" [[package]] -name = "cdispyutils" -version = "1.0.4" -description = "General utilities for Gen3 development" category = "main" +description = "General utilities for Gen3 development" +name = "cdispyutils" optional = false python-versions = "*" +version = "1.0.4" [package.dependencies] -cdiserrors = ">=0.1,<1.0" -cryptography = "2.8" Flask = "*" PyJWT = ">=1.5,<2.0" +cdiserrors = ">=0.1,<1.0" +cryptography = "2.8" requests = ">=2.5,<3.0" [package.extras] profiling = ["Werkzeug (>=0.9,<1.0)", "matplotlib (>=2.2,<3.0)", "numpy (>=1.15,<2.0)"] [[package]] -name = "cdisutilstest" -version = "0.2.4" -description = "Collection of test data and tools" category = "dev" +description = "Collection of test data and tools" +name = "cdisutilstest" optional = false python-versions = "*" +version = "0.2.4" [package.source] +reference = "bdfdeb05e45407e839fd954ce6d195d847cd8024" type = "git" url = "https://github.com/uc-cdis/cdisutils-test" -reference = "1.0.0" -resolved_reference = "bdfdeb05e45407e839fd954ce6d195d847cd8024" - [[package]] -name = "certifi" -version = "2020.6.20" -description = "Python package for providing Mozilla's CA Bundle." category = "main" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" optional = false python-versions = "*" +version = "2020.12.5" [[package]] -name = "cffi" -version = "1.14.3" -description = "Foreign Function Interface for Python calling C code." category = "main" +description = "Foreign Function Interface for Python calling C code." +name = "cffi" optional = false python-versions = "*" +version = "1.14.4" [package.dependencies] pycparser = "*" [[package]] -name = "chardet" -version = "3.0.4" -description = "Universal encoding detector for Python 2 and 3" category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" optional = false -python-versions = "*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "4.0.0" [[package]] -name = "click" -version = "7.1.2" -description = "Composable command line interface toolkit" category = "main" +description = "Composable command line interface toolkit" +name = "click" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" [[package]] -name = "codacy-coverage" -version = "1.3.11" -description = "Codacy coverage reporter for Python" category = "dev" +description = "Codacy coverage reporter for Python" +name = "codacy-coverage" optional = false python-versions = "*" +version = "1.3.11" [package.dependencies] requests = ">=2.9.1" @@ -267,31 +271,32 @@ dev = ["check-manifest"] test = ["coverage", "nosetests"] [[package]] -name = "colorama" -version = "0.4.4" -description = "Cross-platform colored terminal text." category = "dev" +description = "Cross-platform colored terminal text." +marker = "sys_platform == \"win32\"" +name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.4" [[package]] -name = "coverage" -version = "5.3" -description = "Code coverage measurement for Python" category = "dev" +description = "Code coverage measurement for Python" +name = "coverage" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "5.3.1" [package.extras] toml = ["toml"] [[package]] -name = "coveralls" -version = "2.1.2" -description = "Show coverage stats online via coveralls.io" category = "dev" +description = "Show coverage stats online via coveralls.io" +name = "coveralls" optional = false python-versions = ">= 3.5" +version = "2.2.0" [package.dependencies] coverage = ">=4.1,<6.0" @@ -302,12 +307,12 @@ requests = ">=1.0.0" yaml = ["PyYAML (>=3.10)"] [[package]] -name = "cryptography" -version = "2.8" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +name = "cryptography" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "2.8" [package.dependencies] cffi = ">=1.8,<1.11.3 || >1.11.3" @@ -321,12 +326,12 @@ pep8test = ["flake8", "flake8-import-order", "pep8-naming"] test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] [[package]] -name = "debtcollector" -version = "2.2.0" -description = "A collection of Python deprecation patterns and strategies that help you collect your technical debt in a non-destructive manner." category = "main" +description = "A collection of Python deprecation patterns and strategies that help you collect your technical debt in a non-destructive manner." +name = "debtcollector" optional = false python-versions = ">=3.6" +version = "2.2.0" [package.dependencies] pbr = ">=2.0.0,<2.1.0 || >2.1.0" @@ -334,51 +339,51 @@ six = ">=1.10.0" wrapt = ">=1.7.0" [[package]] -name = "decorator" -version = "4.4.2" -description = "Decorators for Humans" category = "main" +description = "Decorators for Humans" +name = "decorator" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "4.4.2" [[package]] -name = "dnspython" -version = "2.0.0" -description = "DNS toolkit" category = "main" +description = "DNS toolkit" +name = "dnspython" optional = false python-versions = ">=3.6" +version = "2.1.0" [package.extras] +curio = ["curio (>=1.2)", "sniffio (>=1.1)"] dnssec = ["cryptography (>=2.6)"] doh = ["requests", "requests-toolbelt"] idna = ["idna (>=2.1)"] -curio = ["curio (>=1.2)", "sniffio (>=1.1)"] trio = ["trio (>=0.14.0)", "sniffio (>=1.1)"] [[package]] -name = "docopt" -version = "0.6.2" -description = "Pythonic argument parser, that will make you smile" category = "dev" +description = "Pythonic argument parser, that will make you smile" +name = "docopt" optional = false python-versions = "*" +version = "0.6.2" [[package]] -name = "docutils" -version = "0.15.2" -description = "Docutils -- Python Documentation Utilities" category = "main" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.15.2" [[package]] -name = "ecdsa" -version = "0.16.0" -description = "ECDSA cryptographic signature library (pure python)" category = "main" +description = "ECDSA cryptographic signature library (pure python)" +name = "ecdsa" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.16.1" [package.dependencies] six = ">=1.9.0" @@ -388,30 +393,30 @@ gmpy = ["gmpy"] gmpy2 = ["gmpy2"] [[package]] -name = "email-validator" -version = "1.1.1" -description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." category = "main" +description = "A robust email syntax and deliverability validation library for Python 2.x/3.x." +name = "email-validator" optional = false python-versions = "*" +version = "1.1.2" [package.dependencies] dnspython = ">=1.15.0" idna = ">=2.0.0" [[package]] -name = "flask" -version = "1.1.2" -description = "A simple framework for building complex web applications." category = "main" +description = "A simple framework for building complex web applications." +name = "flask" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "1.1.2" [package.dependencies] -click = ">=5.1" -itsdangerous = ">=0.24" Jinja2 = ">=2.10.1" Werkzeug = ">=0.15" +click = ">=5.1" +itsdangerous = ">=0.24" [package.extras] dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] @@ -419,28 +424,28 @@ docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx- dotenv = ["python-dotenv"] [[package]] -name = "flask-cors" -version = "3.0.9" -description = "A Flask extension adding a decorator for CORS support" category = "main" +description = "A Flask extension adding a decorator for CORS support" +name = "flask-cors" optional = false python-versions = "*" +version = "3.0.10" [package.dependencies] Flask = ">=0.9" Six = "*" [[package]] -name = "flask-restful" -version = "0.3.8" -description = "Simple framework for creating REST APIs" category = "main" +description = "Simple framework for creating REST APIs" +name = "flask-restful" optional = false python-versions = "*" +version = "0.3.8" [package.dependencies] -aniso8601 = ">=0.82" Flask = ">=0.8" +aniso8601 = ">=0.82" pytz = "*" six = ">=1.3.0" @@ -448,36 +453,36 @@ six = ">=1.3.0" docs = ["sphinx"] [[package]] -name = "flask-sqlalchemy-session" -version = "1.1" -description = "SQL Alchemy session scoped on Flask requests." category = "main" +description = "SQL Alchemy session scoped on Flask requests." +name = "flask-sqlalchemy-session" optional = false python-versions = "*" +version = "1.1" [package.dependencies] Flask = ">=0.9" -sqlalchemy = ">=0.9" Werkzeug = ">=0.6.1" +sqlalchemy = ">=0.9" [package.extras] docs = ["Sphinx (>=1.2.3)", "alabaster (>=0.6.3)"] [[package]] -name = "future" -version = "0.18.2" -description = "Clean single-source support for Python 3 and 2" category = "main" +description = "Clean single-source support for Python 3 and 2" +name = "future" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.18.2" [[package]] -name = "gen3authz" -version = "0.4.0" -description = "Gen3 authz client" category = "main" +description = "Gen3 authz client" +name = "gen3authz" optional = false python-versions = "*" +version = "0.4.0" [package.dependencies] backoff = ">=1.6,<2.0" @@ -485,12 +490,12 @@ cdiserrors = ">=0.1,<1.0" requests = ">=2.18,<3.0" [[package]] -name = "gen3cirrus" -version = "1.3.0" -description = "" category = "main" +description = "" +name = "gen3cirrus" optional = false python-versions = "*" +version = "1.3.0" [package.dependencies] backoff = ">=1.6,<2.0" @@ -502,39 +507,39 @@ google-cloud-storage = ">=1.10,<2.0" oauth2client = ">=2.0.0,<4.0dev" [[package]] -name = "gen3config" -version = "0.1.7" -description = "Gen3 Configuration Library" category = "main" +description = "Gen3 Configuration Library" +name = "gen3config" optional = false python-versions = "*" +version = "0.1.7" [package.dependencies] +PyYAML = ">=5.1,<6.0" cdiserrors = ">=0.1,<1.0" cdislogging = ">=1.0,<2.0" -PyYAML = ">=5.1,<6.0" six = ">=1.0,<2.0" [[package]] -name = "gen3users" -version = "0.6.0" -description = "Utils for Gen3 commons user management" category = "main" +description = "Utils for Gen3 commons user management" +name = "gen3users" optional = false python-versions = "*" +version = "0.6.0" [package.dependencies] +PyYAML = ">=5.1,<6.0" cdislogging = ">=1.0.0,<1.1.0" click = "*" -PyYAML = ">=5.1,<6.0" [[package]] -name = "google-api-core" -version = "1.23.0" -description = "Google API client core library" category = "main" +description = "Google API client core library" +name = "google-api-core" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +version = "1.25.0" [package.dependencies] google-auth = ">=1.21.1,<2.0dev" @@ -542,6 +547,7 @@ googleapis-common-protos = ">=1.6.0,<2.0dev" protobuf = ">=3.12.0" pytz = "*" requests = ">=2.18.0,<3.0.0dev" +setuptools = ">=40.3.0" six = ">=1.13.0" [package.extras] @@ -550,12 +556,12 @@ grpcgcp = ["grpcio-gcp (>=0.2.2)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2)"] [[package]] -name = "google-api-python-client" -version = "1.11.0" -description = "Google API Client Library for Python" category = "main" +description = "Google API Client Library for Python" +name = "google-api-python-client" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.11.0" [package.dependencies] google-api-core = ">=1.18.0,<2dev" @@ -566,29 +572,33 @@ six = ">=1.6.1,<2dev" uritemplate = ">=3.0.0,<4dev" [[package]] -name = "google-auth" -version = "1.22.1" -description = "Google Authentication Library" category = "main" +description = "Google Authentication Library" +name = "google-auth" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +version = "1.24.0" [package.dependencies] cachetools = ">=2.0.0,<5.0" pyasn1-modules = ">=0.2.1" -rsa = {version = ">=3.1.4,<5", markers = "python_version >= \"3.5\""} +setuptools = ">=40.3.0" six = ">=1.9.0" +[package.dependencies.rsa] +python = ">=3.6" +version = ">=3.1.4,<5" + [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] [[package]] -name = "google-auth-httplib2" -version = "0.0.4" -description = "Google Authentication Library: httplib2 transport" category = "main" +description = "Google Authentication Library: httplib2 transport" +name = "google-auth-httplib2" optional = false python-versions = "*" +version = "0.0.4" [package.dependencies] google-auth = "*" @@ -596,40 +606,42 @@ httplib2 = ">=0.9.1" six = "*" [[package]] -name = "google-cloud-core" -version = "1.4.3" -description = "Google Cloud API client core library" category = "main" +description = "Google Cloud API client core library" +name = "google-cloud-core" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +version = "1.5.0" [package.dependencies] -google-api-core = ">=1.19.0,<2.0.0dev" +google-api-core = ">=1.21.0,<2.0.0dev" +six = ">=1.12.0" [package.extras] grpc = ["grpcio (>=1.8.2,<2.0dev)"] [[package]] -name = "google-cloud-storage" -version = "1.32.0" -description = "Google Cloud Storage API client library" category = "main" +description = "Google Cloud Storage API client library" +name = "google-cloud-storage" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +version = "1.35.0" [package.dependencies] google-auth = ">=1.11.0,<2.0dev" google-cloud-core = ">=1.4.1,<2.0dev" -google-resumable-media = ">=1.0.0,<2.0dev" +google-resumable-media = ">=1.2.0,<2.0dev" requests = ">=2.18.0,<3.0.0dev" [[package]] -name = "google-crc32c" -version = "1.0.0" -description = "A python wrapper of the C library 'Google CRC32C'" category = "main" +description = "A python wrapper of the C library 'Google CRC32C'" +marker = "python_version >= \"3.5\"" +name = "google-crc32c" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" +version = "1.1.1" [package.dependencies] cffi = ">=1.0.0" @@ -638,28 +650,31 @@ cffi = ">=1.0.0" testing = ["pytest"] [[package]] -name = "google-resumable-media" -version = "1.1.0" -description = "Utilities for Google Media Downloads and Resumable Uploads" category = "main" +description = "Utilities for Google Media Downloads and Resumable Uploads" +name = "google-resumable-media" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +version = "1.2.0" [package.dependencies] -google-crc32c = {version = ">=1.0,<2.0dev", markers = "python_version >= \"3.5\""} six = "*" +[package.dependencies.google-crc32c] +python = ">=3.5" +version = ">=1.0,<2.0dev" + [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] -name = "googleapis-common-protos" -version = "1.52.0" -description = "Common protobufs used in Google APIs" category = "main" +description = "Common protobufs used in Google APIs" +name = "googleapis-common-protos" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.52.0" [package.dependencies] protobuf = ">=3.6.0" @@ -668,73 +683,82 @@ protobuf = ">=3.6.0" grpc = ["grpcio (>=1.0.0)"] [[package]] -name = "httplib2" -version = "0.18.1" -description = "A comprehensive HTTP client library." category = "main" +description = "A comprehensive HTTP client library." +name = "httplib2" optional = false python-versions = "*" +version = "0.18.1" [[package]] -name = "idna" -version = "2.10" -description = "Internationalized Domain Names in Applications (IDNA)" category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.10" [[package]] -name = "importlib-metadata" -version = "2.0.0" -description = "Read metadata from Python packages" category = "main" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" +version = "3.4.0" [package.dependencies] zipp = ">=0.5" +[package.dependencies.typing-extensions] +python = "<3.8" +version = ">=3.6.4" + [package.extras] -docs = ["sphinx", "rst.linker"] -testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] -name = "importlib-resources" -version = "3.0.0" -description = "Read resources from Python packages" category = "main" +description = "Read resources from Python packages" +marker = "python_version < \"3.7\"" +name = "importlib-resources" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" +version = "5.0.0" [package.dependencies] -zipp = {version = ">=0.4", markers = "python_version < \"3.8\""} +[package.dependencies.zipp] +python = "<3.8" +version = ">=0.4" [package.extras] -docs = ["sphinx", "rst.linker", "jaraco.packaging"] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "pytest-black (>=0.3.7)", "pytest-mypy"] [[package]] -name = "iso8601" -version = "0.1.13" -description = "Simple module to parse ISO 8601 dates" category = "main" +description = "Simple module to parse ISO 8601 dates" +name = "iso8601" optional = false python-versions = "*" +version = "0.1.13" [[package]] -name = "itsdangerous" -version = "1.1.0" -description = "Various helpers to pass data to untrusted environments and back." category = "main" +description = "Various helpers to pass data to untrusted environments and back." +name = "itsdangerous" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.1.0" [[package]] -name = "jinja2" -version = "2.11.2" -description = "A very fast and expressive template engine." category = "main" +description = "A very fast and expressive template engine." +name = "jinja2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.11.2" [package.dependencies] MarkupSafe = ">=0.23" @@ -743,42 +767,44 @@ MarkupSafe = ">=0.23" i18n = ["Babel (>=0.8)"] [[package]] -name = "jmespath" -version = "0.9.2" -description = "JSON Matching Expressions" category = "main" +description = "JSON Matching Expressions" +name = "jmespath" optional = false python-versions = "*" +version = "0.9.2" [[package]] -name = "markdown" -version = "3.3.2" -description = "Python implementation of Markdown." category = "main" +description = "Python implementation of Markdown." +name = "markdown" optional = false python-versions = ">=3.6" +version = "3.3.3" [package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" [package.extras] testing = ["coverage", "pyyaml"] [[package]] -name = "markupsafe" -version = "1.1.1" -description = "Safely add untrusted strings to HTML/XML markup." category = "main" +description = "Safely add untrusted strings to HTML/XML markup." +name = "markupsafe" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.1.1" [[package]] -name = "mock" -version = "2.0.0" -description = "Rolling backport of unittest.mock for all Pythons" category = "dev" +description = "Rolling backport of unittest.mock for all Pythons" +name = "mock" optional = false python-versions = "*" +version = "2.0.0" [package.dependencies] pbr = ">=0.11" @@ -789,33 +815,34 @@ docs = ["sphinx", "Pygments (<2)", "jinja2 (<2.7)", "sphinx (<1.3)"] test = ["unittest2 (>=1.1.0)"] [[package]] -name = "more-itertools" -version = "8.5.0" -description = "More routines for operating on iterables, beyond itertools" category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" optional = false python-versions = ">=3.5" +version = "8.6.0" [[package]] -name = "moto" -version = "1.3.15" -description = "A library that allows your python tests to easily mock out the boto library" category = "dev" +description = "A library that allows your python tests to easily mock out the boto library" +name = "moto" optional = false python-versions = "*" +version = "1.3.15" [package.dependencies] +Jinja2 = ">=2.10.1" +MarkupSafe = "<2.0" boto = ">=2.36.0" boto3 = ">=1.9.201" botocore = ">=1.12.201" -Jinja2 = ">=2.10.1" -MarkupSafe = "<2.0" mock = "*" more-itertools = "*" python-dateutil = ">=2.1,<3.0.0" pytz = "*" requests = ">=2.5" responses = ">=0.9.0" +setuptools = "*" six = ">1.9" werkzeug = "*" xmltodict = "*" @@ -823,11 +850,11 @@ zipp = "*" [package.extras] acm = ["cryptography (>=2.3.0)"] -all = ["cryptography (>=2.3.0)", "PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (<0.15)", "docker (>=2.5.1)", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,<0.96 || >0.96)", "idna (>=2.5,<3)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0,<4.0)", "sshpubkeys (>=3.1.0)"] +all = ["cryptography (>=2.3.0)", "PyYAML (>=5.1)", "python-jose (>=3.1.0,<4.0.0)", "ecdsa (<0.15)", "docker (>=2.5.1)", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,<0.96 || >0.96)", "idna (>=2.5,<3)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0,<4.0)", "sshpubkeys (>=3.1.0)"] awslambda = ["docker (>=2.5.1)"] batch = ["docker (>=2.5.1)"] cloudformation = ["PyYAML (>=5.1)", "cfn-lint (>=0.4.0)"] -cognitoidp = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (<0.15)"] +cognitoidp = ["python-jose (>=3.1.0,<4.0.0)", "ecdsa (<0.15)"] ec2 = ["cryptography (>=2.3.0)", "sshpubkeys (>=3.1.0,<4.0)", "sshpubkeys (>=3.1.0)"] iam = ["cryptography (>=2.3.0)"] iotdata = ["jsondiff (>=1.1.2)"] @@ -835,39 +862,41 @@ server = ["flask"] xray = ["aws-xray-sdk (>=0.93,<0.96 || >0.96)"] [[package]] -name = "msgpack" -version = "1.0.0" -description = "MessagePack (de)serializer." category = "main" +description = "MessagePack (de)serializer." +name = "msgpack" optional = false python-versions = "*" +version = "1.0.2" [[package]] -name = "netaddr" -version = "0.8.0" -description = "A network address manipulation library for Python" category = "main" +description = "A network address manipulation library for Python" +name = "netaddr" optional = false python-versions = "*" +version = "0.8.0" [package.dependencies] -importlib-resources = {version = "*", markers = "python_version < \"3.7\""} +[package.dependencies.importlib-resources] +python = "<3.7" +version = "*" [[package]] -name = "netifaces" -version = "0.10.9" -description = "Portable network interface information." category = "main" +description = "Portable network interface information." +name = "netifaces" optional = false python-versions = "*" +version = "0.10.9" [[package]] -name = "oauth2client" -version = "3.0.0" -description = "OAuth 2.0 client library" category = "main" +description = "OAuth 2.0 client library" +name = "oauth2client" optional = false python-versions = "*" +version = "3.0.0" [package.dependencies] httplib2 = ">=0.9.1" @@ -877,42 +906,49 @@ rsa = ">=3.1.4" six = ">=1.6.1" [[package]] -name = "oslo.config" -version = "8.3.2" -description = "Oslo Configuration API" category = "main" +description = "Oslo Configuration API" +name = "oslo.config" optional = false python-versions = ">=3.6" +version = "8.4.0" [package.dependencies] +PyYAML = ">=3.12" debtcollector = ">=1.2.0" -importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} netaddr = ">=0.7.18" "oslo.i18n" = ">=3.15.3" -PyYAML = ">=3.12" requests = ">=2.18.0" rfc3986 = ">=1.2.0" stevedore = ">=1.20.0" +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=1.7.0" + +[package.extras] +rst_generator = ["rst2txt (>=1.1.0)", "sphinx (>=1.8.0,<2.1.0 || >2.1.0)"] +test = ["bandit (>=1.6.0,<1.7.0)", "coverage (>=4.0,<4.4 || >4.4)", "fixtures (>=3.0.0)", "hacking (>=3.0.1,<3.1.0)", "mypy (>=0.720)", "oslo.log (>=3.36.0)", "oslotest (>=3.2.0)", "pre-commit (>=2.6.0)", "requests-mock (>=1.5.0)", "stestr (>=2.1.0)", "testscenarios (>=0.4)", "testtools (>=2.2.0)"] + [[package]] -name = "oslo.i18n" -version = "5.0.1" -description = "Oslo i18n library" category = "main" +description = "Oslo i18n library" +name = "oslo.i18n" optional = false python-versions = ">=3.6" +version = "5.0.1" [package.dependencies] pbr = ">=2.0.0,<2.1.0 || >2.1.0" six = ">=1.10.0" [[package]] -name = "oslo.serialization" -version = "4.0.1" -description = "Oslo Serialization library" category = "main" +description = "Oslo Serialization library" +name = "oslo.serialization" optional = false python-versions = ">=3.6" +version = "4.0.1" [package.dependencies] msgpack = ">=0.5.2" @@ -921,12 +957,12 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" pytz = ">=2013.6" [[package]] -name = "oslo.utils" -version = "4.6.0" -description = "Oslo Utility library" category = "main" +description = "Oslo Utility library" +name = "oslo.utils" optional = false python-versions = ">=3.6" +version = "4.7.0" [package.dependencies] debtcollector = ">=1.2.0" @@ -941,24 +977,23 @@ pytz = ">=2013.6" six = ">=1.10.0" [[package]] -name = "packaging" -version = "20.4" -description = "Core utilities for Python packages" category = "main" +description = "Core utilities for Python packages" +name = "packaging" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.8" [package.dependencies] pyparsing = ">=2.0.2" -six = "*" [[package]] -name = "paramiko" -version = "2.7.2" -description = "SSH2 protocol library" category = "main" +description = "SSH2 protocol library" +name = "paramiko" optional = false python-versions = "*" +version = "2.7.2" [package.dependencies] bcrypt = ">=3.1.3" @@ -972,104 +1007,106 @@ gssapi = ["pyasn1 (>=0.1.7)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] invoke = ["invoke (>=1.3)"] [[package]] -name = "pbr" -version = "2.0.0" -description = "Python Build Reasonableness" category = "main" +description = "Python Build Reasonableness" +name = "pbr" optional = false python-versions = "*" +version = "2.0.0" [[package]] -name = "pluggy" -version = "0.13.1" -description = "plugin and hook calling mechanisms for python" category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" [package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" [package.extras] dev = ["pre-commit", "tox"] [[package]] -name = "prettytable" -version = "0.7.2" -description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format." category = "main" +description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format." +name = "prettytable" optional = false python-versions = "*" +version = "0.7.2" [[package]] -name = "protobuf" -version = "3.13.0" -description = "Protocol Buffers" category = "main" +description = "Protocol Buffers" +name = "protobuf" optional = false python-versions = "*" +version = "3.14.0" [package.dependencies] six = ">=1.9" [[package]] -name = "psycopg2" -version = "2.8.6" -description = "psycopg2 - Python-PostgreSQL Database Adapter" category = "main" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +name = "psycopg2" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "2.8.6" [[package]] -name = "py" -version = "1.9.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "main" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.10.0" [[package]] -name = "pyasn1" -version = "0.4.8" -description = "ASN.1 types and codecs" category = "main" +description = "ASN.1 types and codecs" +name = "pyasn1" optional = false python-versions = "*" +version = "0.4.8" [[package]] -name = "pyasn1-modules" -version = "0.2.8" -description = "A collection of ASN.1-based protocols modules." category = "main" +description = "A collection of ASN.1-based protocols modules." +name = "pyasn1-modules" optional = false python-versions = "*" +version = "0.2.8" [package.dependencies] pyasn1 = ">=0.4.6,<0.5.0" [[package]] -name = "pycparser" -version = "2.20" -description = "C parser in Python" category = "main" +description = "C parser in Python" +name = "pycparser" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.20" [[package]] -name = "pycryptodome" -version = "3.9.8" -description = "Cryptographic library for Python" category = "main" +description = "Cryptographic library for Python" +name = "pycryptodome" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.9.9" [[package]] -name = "pyjwt" -version = "1.7.1" -description = "JSON Web Token implementation in Python" category = "main" +description = "JSON Web Token implementation in Python" +name = "pyjwt" optional = false python-versions = "*" +version = "1.7.1" [package.extras] crypto = ["cryptography (>=1.4)"] @@ -1077,12 +1114,12 @@ flake8 = ["flake8", "flake8-import-order", "pep8-naming"] test = ["pytest (>=4.0.1,<5.0.0)", "pytest-cov (>=2.6.0,<3.0.0)", "pytest-runner (>=4.2,<5.0.0)"] [[package]] -name = "pynacl" -version = "1.4.0" -description = "Python binding to the Networking and Cryptography (NaCl) library" category = "main" +description = "Python binding to the Networking and Cryptography (NaCl) library" +name = "pynacl" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.4.0" [package.dependencies] cffi = ">=1.4.1" @@ -1093,37 +1130,38 @@ docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)", "hypothesis (>=3.27.0)"] [[package]] -name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" category = "main" +description = "Python parsing module" +name = "pyparsing" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.7" [[package]] -name = "pytest" -version = "3.10.1" -description = "pytest: simple powerful testing with Python" category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.10.1" [package.dependencies] atomicwrites = ">=1.0" attrs = ">=17.4.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} +colorama = "*" more-itertools = ">=4.0.0" pluggy = ">=0.7" py = ">=1.5.0" +setuptools = "*" six = ">=1.10.0" [[package]] -name = "pytest-cov" -version = "2.9.0" -description = "Pytest plugin for measuring coverage." category = "dev" +description = "Pytest plugin for measuring coverage." +name = "pytest-cov" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.9.0" [package.dependencies] coverage = ">=4.4" @@ -1133,39 +1171,39 @@ pytest = ">=3.6" testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] -name = "pytest-flask" -version = "0.11.0" -description = "A set of py.test fixtures to test Flask applications." category = "dev" +description = "A set of py.test fixtures to test Flask applications." +name = "pytest-flask" optional = false python-versions = "*" +version = "0.11.0" [package.dependencies] Flask = "*" -pytest = "*" Werkzeug = ">=0.7" +pytest = "*" [package.extras] docs = ["sphinx", "sphinx-rtd-theme"] [[package]] -name = "python-dateutil" -version = "2.8.1" -description = "Extensions to the standard Python datetime module" category = "main" +description = "Extensions to the standard Python datetime module" +name = "python-dateutil" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -name = "python-jose" -version = "2.0.2" -description = "JOSE implementation in Python" category = "main" +description = "JOSE implementation in Python" +name = "python-jose" optional = false python-versions = "*" +version = "2.0.2" [package.dependencies] ecdsa = "<1.0" @@ -1178,16 +1216,17 @@ cryptography = ["cryptography"] pycrypto = ["pycrypto (>=2.6.0,<2.7.0)"] [[package]] -name = "python-keystoneclient" -version = "1.8.1" -description = "Client Library for OpenStack Identity" category = "main" +description = "Client Library for OpenStack Identity" +name = "python-keystoneclient" optional = false python-versions = "*" +version = "1.8.1" [package.dependencies] -argparse = "*" Babel = ">=1.3" +PrettyTable = ">=0.7,<0.8" +argparse = "*" debtcollector = ">=0.3.0" iso8601 = ">=0.1.9" netaddr = ">=0.7.12,<0.7.16 || >0.7.16" @@ -1196,52 +1235,51 @@ netaddr = ">=0.7.12,<0.7.16 || >0.7.16" "oslo.serialization" = ">=1.4.0" "oslo.utils" = ">=2.0.0" pbr = ">=1.6" -PrettyTable = ">=0.7,<0.8" requests = ">=2.5.2,<2.8.0 || >2.8.0" six = ">=1.9.0" stevedore = ">=1.5.0" [[package]] -name = "pytz" -version = "2020.1" -description = "World timezone definitions, modern and historical" category = "main" +description = "World timezone definitions, modern and historical" +name = "pytz" optional = false python-versions = "*" +version = "2020.5" [[package]] -name = "pyyaml" -version = "5.3.1" -description = "YAML parser and emitter for Python" category = "main" +description = "YAML parser and emitter for Python" +name = "pyyaml" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "5.3.1" [[package]] -name = "requests" -version = "2.24.0" -description = "Python HTTP for Humans." category = "main" +description = "Python HTTP for Humans." +name = "requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.25.1" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<4" +chardet = ">=3.0.2,<5" idna = ">=2.5,<3" -urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" +urllib3 = ">=1.21.1,<1.27" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [[package]] -name = "responses" -version = "0.12.0" -description = "A utility library for mocking out the `requests` Python library." category = "dev" +description = "A utility library for mocking out the `requests` Python library." +name = "responses" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.12.1" [package.dependencies] requests = ">=2.0" @@ -1252,65 +1290,65 @@ urllib3 = ">=1.25.10" tests = ["coverage (>=3.7.1,<6.0.0)", "pytest-cov", "pytest-localserver", "flake8", "pytest (>=4.6,<5.0)", "pytest (>=4.6)"] [[package]] -name = "retry" -version = "0.9.2" -description = "Easy to use retry decorator." category = "main" +description = "Easy to use retry decorator." +name = "retry" optional = false python-versions = "*" +version = "0.9.2" [package.dependencies] decorator = ">=3.4.2" py = ">=1.4.26,<2.0.0" [[package]] -name = "rfc3986" -version = "1.4.0" -description = "Validating URI References per RFC 3986" category = "main" +description = "Validating URI References per RFC 3986" +name = "rfc3986" optional = false python-versions = "*" +version = "1.4.0" [package.extras] idna2008 = ["idna"] [[package]] -name = "rsa" -version = "4.6" -description = "Pure-Python RSA implementation" category = "main" +description = "Pure-Python RSA implementation" +name = "rsa" optional = false python-versions = ">=3.5, <4" +version = "4.7" [package.dependencies] pyasn1 = ">=0.1.3" [[package]] -name = "s3transfer" -version = "0.2.1" -description = "An Amazon S3 Transfer Manager" category = "main" +description = "An Amazon S3 Transfer Manager" +name = "s3transfer" optional = false python-versions = "*" +version = "0.2.1" [package.dependencies] botocore = ">=1.12.36,<2.0.0" [[package]] -name = "six" -version = "1.15.0" -description = "Python 2 and 3 compatibility utilities" category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "1.15.0" [[package]] -name = "sqlalchemy" -version = "1.3.20" -description = "Database Abstraction Library" category = "main" +description = "Database Abstraction Library" +name = "sqlalchemy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.3.22" [package.extras] mssql = ["pyodbc"] @@ -1325,24 +1363,27 @@ postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql"] [[package]] -name = "stevedore" -version = "3.2.2" -description = "Manage dynamic plugins for Python applications" category = "main" +description = "Manage dynamic plugins for Python applications" +name = "stevedore" optional = false python-versions = ">=3.6" +version = "3.3.0" [package.dependencies] -importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} pbr = ">=2.0.0,<2.1.0 || >2.1.0" +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=1.7.0" + [[package]] -name = "storageclient" -version = "0.1.0" -description = "" category = "main" +description = "Python client to interact with ceph/cleversafe backend" +name = "storageclient" optional = false python-versions = "*" +version = "0.1.0" [package.dependencies] boto = ">=2.36.0,<3.0.0" @@ -1355,26 +1396,33 @@ requests = ">=2.5.2,<3.0.0" s3transfer = ">=0.2.0,<0.3.0" [package.source] +reference = "885c59d83fc075527af44968c353d1ac93bec37e" type = "git" url = "https://github.com/uc-cdis/storage-client" -reference = "1.0.1" -resolved_reference = "885c59d83fc075527af44968c353d1ac93bec37e" +[[package]] +category = "main" +description = "Backported and Experimental Type Hints for Python 3.5+" +marker = "python_version < \"3.8\"" +name = "typing-extensions" +optional = false +python-versions = "*" +version = "3.7.4.3" [[package]] -name = "uritemplate" -version = "3.0.1" -description = "URI templates" category = "main" +description = "URI templates" +name = "uritemplate" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.0.1" [[package]] -name = "urllib3" -version = "1.25.11" -description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "1.25.11" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -1382,24 +1430,24 @@ secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "cer socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -name = "userdatamodel" -version = "2.3.3" -description = "" category = "main" +description = "" +name = "userdatamodel" optional = false python-versions = "*" +version = "2.3.3" [package.dependencies] cdislogging = "*" sqlalchemy = ">=1.3.3,<1.4.0" [[package]] -name = "werkzeug" -version = "0.16.1" -description = "The comprehensive WSGI web application library." category = "main" +description = "The comprehensive WSGI web application library." +name = "werkzeug" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.16.1" [package.extras] dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] @@ -1407,46 +1455,45 @@ termcolor = ["termcolor"] watchdog = ["watchdog"] [[package]] -name = "wrapt" -version = "1.12.1" -description = "Module for decorators, wrappers and monkey patching." category = "main" +description = "Module for decorators, wrappers and monkey patching." +name = "wrapt" optional = false python-versions = "*" +version = "1.12.1" [[package]] -name = "xmltodict" -version = "0.12.0" -description = "Makes working with XML feel like you are working with JSON" category = "main" +description = "Makes working with XML feel like you are working with JSON" +name = "xmltodict" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.12.0" [[package]] -name = "zipp" -version = "3.3.1" -description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" +description = "Backport of pathlib-compatible object wrapper for zip files" +name = "zipp" optional = false python-versions = ">=3.6" +version = "3.4.0" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [metadata] -lock-version = "1.1" -python-versions = "^3.6" content-hash = "e9492e3d39fd0b71c6d90ffce984fc8b27d54a462b2ad707a01b775662eb035a" +python-versions = "^3.6" [metadata.files] addict = [ - {file = "addict-2.3.0-py3-none-any.whl", hash = "sha256:69904f5650015d93043522b27316649b6dc877ab1eabb9ea9eb7b11386882edc"}, - {file = "addict-2.3.0.tar.gz", hash = "sha256:e8085624d18a9eecd7ec8a427f378f1e5036dfafba34c33fa29f0be8cd3a80b2"}, + {file = "addict-2.4.0-py3-none-any.whl", hash = "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc"}, + {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] aniso8601 = [ - {file = "aniso8601-8.0.0-py2.py3-none-any.whl", hash = "sha256:c033f63d028b9a58e3ab0c2c7d0532ab4bfa7452bfc788fbfe3ddabd327b181a"}, - {file = "aniso8601-8.0.0.tar.gz", hash = "sha256:529dcb1f5f26ee0df6c0a1ee84b7b27197c3c50fc3a6321d66c544689237d072"}, + {file = "aniso8601-8.1.0-py2.py3-none-any.whl", hash = "sha256:51047d4fb51d7b8afd522b70f2d21a1b2487cbb7f7bd84ea852e9aa7808e7704"}, + {file = "aniso8601-8.1.0.tar.gz", hash = "sha256:246bf8d3611527030889e6df970878969d3a2f760ba3eb694fa1fb10e6ce53f9"}, ] argparse = [ {file = "argparse-1.4.0-py2.py3-none-any.whl", hash = "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"}, @@ -1457,8 +1504,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, - {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] authlib = [ {file = "Authlib-0.11-py2.py3-none-any.whl", hash = "sha256:3a226f231e962a16dd5f6fcf0c113235805ba206e294717a64fa8e04ae3ad9c4"}, @@ -1468,8 +1515,8 @@ authutils = [ {file = "authutils-4.0.0.tar.gz", hash = "sha256:d67cd5025d8584a23a3ba583719e6116807f13450b611d1d82b112cdb13a1631"}, ] babel = [ - {file = "Babel-2.8.0-py2.py3-none-any.whl", hash = "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"}, - {file = "Babel-2.8.0.tar.gz", hash = "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38"}, + {file = "Babel-2.9.0-py2.py3-none-any.whl", hash = "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5"}, + {file = "Babel-2.9.0.tar.gz", hash = "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05"}, ] backoff = [ {file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"}, @@ -1501,8 +1548,8 @@ cached-property = [ {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, ] cachetools = [ - {file = "cachetools-4.1.1-py3-none-any.whl", hash = "sha256:513d4ff98dd27f85743a8dc0e92f55ddb1b49e060c2d5961512855cda2c01a98"}, - {file = "cachetools-4.1.1.tar.gz", hash = "sha256:bbaa39c3dede00175df2dc2b03d0cf18dd2d32a7de7beb68072d13043c9edb20"}, + {file = "cachetools-4.2.0-py3-none-any.whl", hash = "sha256:c6b07a6ded8c78bf36730b3dc452dfff7d95f2a12a2fed856b1a0cb13ca78c61"}, + {file = "cachetools-4.2.0.tar.gz", hash = "sha256:3796e1de094f0eaca982441c92ce96c68c89cced4cd97721ab297ea4b16db90e"}, ] cdiserrors = [ {file = "cdiserrors-0.1.2.tar.gz", hash = "sha256:d6a91162aa76c7af37b79a8bba1bc5ebd06beb331ac522d36a74c4b300f6f333"}, @@ -1515,50 +1562,50 @@ cdispyutils = [ ] cdisutilstest = [] certifi = [ - {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, - {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, + {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, + {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, ] cffi = [ - {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"}, - {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"}, - {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"}, - {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"}, - {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"}, - {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"}, - {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"}, - {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"}, - {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"}, - {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, - {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, - {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, - {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, - {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, - {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, - {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"}, - {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"}, - {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"}, - {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"}, + {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, + {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, + {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, + {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, + {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, + {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, + {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, + {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, + {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, + {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, + {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, + {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, + {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, + {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, + {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, + {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, + {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, + {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, + {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, ] chardet = [ - {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, - {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, + {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, + {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, @@ -1573,44 +1620,59 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-5.3-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270"}, - {file = "coverage-5.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4"}, - {file = "coverage-5.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9"}, - {file = "coverage-5.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729"}, - {file = "coverage-5.3-cp27-cp27m-win32.whl", hash = "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d"}, - {file = "coverage-5.3-cp27-cp27m-win_amd64.whl", hash = "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418"}, - {file = "coverage-5.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9"}, - {file = "coverage-5.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5"}, - {file = "coverage-5.3-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822"}, - {file = "coverage-5.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097"}, - {file = "coverage-5.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9"}, - {file = "coverage-5.3-cp35-cp35m-win32.whl", hash = "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636"}, - {file = "coverage-5.3-cp35-cp35m-win_amd64.whl", hash = "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f"}, - {file = "coverage-5.3-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237"}, - {file = "coverage-5.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54"}, - {file = "coverage-5.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7"}, - {file = "coverage-5.3-cp36-cp36m-win32.whl", hash = "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a"}, - {file = "coverage-5.3-cp36-cp36m-win_amd64.whl", hash = "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d"}, - {file = "coverage-5.3-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8"}, - {file = "coverage-5.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f"}, - {file = "coverage-5.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c"}, - {file = "coverage-5.3-cp37-cp37m-win32.whl", hash = "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751"}, - {file = "coverage-5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709"}, - {file = "coverage-5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516"}, - {file = "coverage-5.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f"}, - {file = "coverage-5.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259"}, - {file = "coverage-5.3-cp38-cp38-win32.whl", hash = "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82"}, - {file = "coverage-5.3-cp38-cp38-win_amd64.whl", hash = "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221"}, - {file = "coverage-5.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978"}, - {file = "coverage-5.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21"}, - {file = "coverage-5.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24"}, - {file = "coverage-5.3-cp39-cp39-win32.whl", hash = "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7"}, - {file = "coverage-5.3-cp39-cp39-win_amd64.whl", hash = "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7"}, - {file = "coverage-5.3.tar.gz", hash = "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0"}, + {file = "coverage-5.3.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d"}, + {file = "coverage-5.3.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7"}, + {file = "coverage-5.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528"}, + {file = "coverage-5.3.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044"}, + {file = "coverage-5.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b"}, + {file = "coverage-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297"}, + {file = "coverage-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb"}, + {file = "coverage-5.3.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899"}, + {file = "coverage-5.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36"}, + {file = "coverage-5.3.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500"}, + {file = "coverage-5.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7"}, + {file = "coverage-5.3.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f"}, + {file = "coverage-5.3.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b"}, + {file = "coverage-5.3.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec"}, + {file = "coverage-5.3.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714"}, + {file = "coverage-5.3.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b"}, + {file = "coverage-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7"}, + {file = "coverage-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72"}, + {file = "coverage-5.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b"}, + {file = "coverage-5.3.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4"}, + {file = "coverage-5.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105"}, + {file = "coverage-5.3.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448"}, + {file = "coverage-5.3.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277"}, + {file = "coverage-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f"}, + {file = "coverage-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c"}, + {file = "coverage-5.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd"}, + {file = "coverage-5.3.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4"}, + {file = "coverage-5.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff"}, + {file = "coverage-5.3.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8"}, + {file = "coverage-5.3.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e"}, + {file = "coverage-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2"}, + {file = "coverage-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879"}, + {file = "coverage-5.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b"}, + {file = "coverage-5.3.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497"}, + {file = "coverage-5.3.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059"}, + {file = "coverage-5.3.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631"}, + {file = "coverage-5.3.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830"}, + {file = "coverage-5.3.1-cp38-cp38-win32.whl", hash = "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae"}, + {file = "coverage-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606"}, + {file = "coverage-5.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f"}, + {file = "coverage-5.3.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1"}, + {file = "coverage-5.3.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8"}, + {file = "coverage-5.3.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4"}, + {file = "coverage-5.3.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d"}, + {file = "coverage-5.3.1-cp39-cp39-win32.whl", hash = "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98"}, + {file = "coverage-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1"}, + {file = "coverage-5.3.1-pp36-none-any.whl", hash = "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3"}, + {file = "coverage-5.3.1-pp37-none-any.whl", hash = "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c"}, + {file = "coverage-5.3.1.tar.gz", hash = "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b"}, ] coveralls = [ - {file = "coveralls-2.1.2-py2.py3-none-any.whl", hash = "sha256:b3b60c17b03a0dee61952a91aed6f131e0b2ac8bd5da909389c53137811409e1"}, - {file = "coveralls-2.1.2.tar.gz", hash = "sha256:4430b862baabb3cf090d36d84d331966615e4288d8a8c5957e0fd456d0dd8bd6"}, + {file = "coveralls-2.2.0-py2.py3-none-any.whl", hash = "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc"}, + {file = "coveralls-2.2.0.tar.gz", hash = "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617"}, ] cryptography = [ {file = "cryptography-2.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"}, @@ -1644,8 +1706,8 @@ decorator = [ {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"}, ] dnspython = [ - {file = "dnspython-2.0.0-py3-none-any.whl", hash = "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"}, - {file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"}, + {file = "dnspython-2.1.0-py3-none-any.whl", hash = "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216"}, + {file = "dnspython-2.1.0.zip", hash = "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"}, ] docopt = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, @@ -1656,20 +1718,20 @@ docutils = [ {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, ] ecdsa = [ - {file = "ecdsa-0.16.0-py2.py3-none-any.whl", hash = "sha256:ca359c971594dceebf334f3d623dae43163ab161c7d09f28cae70a86df26eb7a"}, - {file = "ecdsa-0.16.0.tar.gz", hash = "sha256:494c6a853e9ed2e9be33d160b41d47afc50a6629b993d2b9c5ad7bb226add892"}, + {file = "ecdsa-0.16.1-py2.py3-none-any.whl", hash = "sha256:881fa5e12bb992972d3d1b3d4dfbe149ab76a89f13da02daa5ea1ec7dea6e747"}, + {file = "ecdsa-0.16.1.tar.gz", hash = "sha256:cfc046a2ddd425adbd1a78b3c46f0d1325c657811c0f45ecc3a0a6236c1e50ff"}, ] email-validator = [ - {file = "email_validator-1.1.1-py2.py3-none-any.whl", hash = "sha256:5f246ae8d81ce3000eade06595b7bb55a4cf350d559e890182a1466a21f25067"}, - {file = "email_validator-1.1.1.tar.gz", hash = "sha256:63094045c3e802c3d3d575b18b004a531c36243ca8d1cec785ff6bfcb04185bb"}, + {file = "email-validator-1.1.2.tar.gz", hash = "sha256:1a13bd6050d1db4475f13e444e169b6fe872434922d38968c67cea9568cce2f0"}, + {file = "email_validator-1.1.2-py2.py3-none-any.whl", hash = "sha256:094b1d1c60d790649989d38d34f69e1ef07792366277a2cf88684d03495d018f"}, ] flask = [ {file = "Flask-1.1.2-py2.py3-none-any.whl", hash = "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557"}, {file = "Flask-1.1.2.tar.gz", hash = "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060"}, ] flask-cors = [ - {file = "Flask-Cors-3.0.9.tar.gz", hash = "sha256:6bcfc100288c5d1bcb1dbb854babd59beee622ffd321e444b05f24d6d58466b8"}, - {file = "Flask_Cors-3.0.9-py2.py3-none-any.whl", hash = "sha256:cee4480aaee421ed029eaa788f4049e3e26d15b5affb6a880dade6bafad38324"}, + {file = "Flask-Cors-3.0.10.tar.gz", hash = "sha256:b60839393f3b84a0f3746f6cdca56c1ad7426aa738b70d6c61375857823181de"}, + {file = "Flask_Cors-3.0.10-py2.py3-none-any.whl", hash = "sha256:74efc975af1194fc7891ff5cd85b0f7478be4f7f59fe158102e91abb72bb4438"}, ] flask-restful = [ {file = "Flask-RESTful-0.3.8.tar.gz", hash = "sha256:5ea9a5991abf2cb69b4aac19793faac6c032300505b325687d7c305ffaa76915"}, @@ -1694,52 +1756,55 @@ gen3users = [ {file = "gen3users-0.6.0.tar.gz", hash = "sha256:3b9b56798a7d8b34712389dbbab93c00b0f92524f890513f899c31630ea986da"}, ] google-api-core = [ - {file = "google-api-core-1.23.0.tar.gz", hash = "sha256:1bb3c485c38eacded8d685b1759968f6cf47dd9432922d34edb90359eaa391e2"}, - {file = "google_api_core-1.23.0-py2.py3-none-any.whl", hash = "sha256:94d8c707d358d8d9e8b0045c42be20efb58433d308bd92cf748511c7825569c8"}, + {file = "google-api-core-1.25.0.tar.gz", hash = "sha256:d967beae8d8acdb88fb2f6f769e2ee0ee813042576a08891bded3b8e234150ae"}, + {file = "google_api_core-1.25.0-py2.py3-none-any.whl", hash = "sha256:4656345cba9627ab1290eab51300a6397cc50370d99366133df1ae64b744e1eb"}, ] google-api-python-client = [ {file = "google-api-python-client-1.11.0.tar.gz", hash = "sha256:caf4015800ef1a18d06d117f47f0219c0c0641f21978f6b1bb5ede7912fab97b"}, {file = "google_api_python_client-1.11.0-py2.py3-none-any.whl", hash = "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6"}, ] google-auth = [ - {file = "google-auth-1.22.1.tar.gz", hash = "sha256:9c0f71789438d703f77b94aad4ea545afaec9a65f10e6cc1bc8b89ce242244bb"}, - {file = "google_auth-1.22.1-py2.py3-none-any.whl", hash = "sha256:712dd7d140a9a1ea218e5688c7fcb04af71b431a29ec9ce433e384c60e387b98"}, + {file = "google-auth-1.24.0.tar.gz", hash = "sha256:0b0e026b412a0ad096e753907559e4bdb180d9ba9f68dd9036164db4fdc4ad2e"}, + {file = "google_auth-1.24.0-py2.py3-none-any.whl", hash = "sha256:ce752cc51c31f479dbf9928435ef4b07514b20261b021c7383bee4bda646acb8"}, ] google-auth-httplib2 = [ {file = "google-auth-httplib2-0.0.4.tar.gz", hash = "sha256:8d092cc60fb16517b12057ec0bba9185a96e3b7169d86ae12eae98e645b7bc39"}, {file = "google_auth_httplib2-0.0.4-py2.py3-none-any.whl", hash = "sha256:aeaff501738b289717fac1980db9711d77908a6c227f60e4aa1923410b43e2ee"}, ] google-cloud-core = [ - {file = "google-cloud-core-1.4.3.tar.gz", hash = "sha256:21afb70c1b0bce8eeb8abb5dca63c5fd37fc8aea18f4b6d60e803bd3d27e6b80"}, - {file = "google_cloud_core-1.4.3-py2.py3-none-any.whl", hash = "sha256:75abff9056977809937127418323faa3917f32df68490704d39a4f0d492ebc2b"}, + {file = "google-cloud-core-1.5.0.tar.gz", hash = "sha256:1277a015f8eeb014c48f2ec094ed5368358318f1146cf49e8de389962dc19106"}, + {file = "google_cloud_core-1.5.0-py2.py3-none-any.whl", hash = "sha256:99a8a15f406f53f2b11bda1f45f952a9cdfbdbba8abf40c75651019d800879f5"}, ] google-cloud-storage = [ - {file = "google-cloud-storage-1.32.0.tar.gz", hash = "sha256:da12b7bd79bbe978a7945a44b600604fbc10ece2935d31f243e751f99135e34f"}, - {file = "google_cloud_storage-1.32.0-py2.py3-none-any.whl", hash = "sha256:063bd12b5ceb4045e8681dc5cce8c3ceeb1203f7c5c3e59f5c9b75bb79a5f59b"}, + {file = "google-cloud-storage-1.35.0.tar.gz", hash = "sha256:555c0db2f88f3419f123bf9c621d7fd92f7c9e4f8b11f08eda57facacba16a9e"}, + {file = "google_cloud_storage-1.35.0-py2.py3-none-any.whl", hash = "sha256:7b48b74683dafec5da315c7b0861ab168c208e04c763aa5db7ea8d7ecd8b6c5d"}, ] google-crc32c = [ - {file = "google-crc32c-1.0.0.tar.gz", hash = "sha256:9439b960b6ecd847557675d130fc3626d762bf535da595c20a6949a705fb3eae"}, - {file = "google_crc32c-1.0.0-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:7b5ccdc7697ca54351d2965d4241f907d53f26f5288710bed505f8c3776ed235"}, - {file = "google_crc32c-1.0.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f3b859200c3bc73925b1719ed8b1f6d8d73b6620b42dbc121c4df58423045e34"}, - {file = "google_crc32c-1.0.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:1a613f43534c9a345cc86fc6531bda477e2473cb876b6e26aee22b8060917069"}, - {file = "google_crc32c-1.0.0-cp35-cp35m-win32.whl", hash = "sha256:b7ee33659231c8205bb05559781ac61a325f31b06b917b3e997bea5c2c49ff4d"}, - {file = "google_crc32c-1.0.0-cp35-cp35m-win_amd64.whl", hash = "sha256:176cef33c9ad2a56977efd084646b378e50ab14b43a7c0a16e956bc3e3ec130a"}, - {file = "google_crc32c-1.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b6fad0842a02abd270f8b660db082d37d197ab80aa4db6a2ddbfcf472eade9e7"}, - {file = "google_crc32c-1.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:00b34d4c9ac565b2be553f81f58e5861e51d43af2043ed7cbfe1853ee2f54671"}, - {file = "google_crc32c-1.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:438d6c314a52d50a9523460024e655a3d27774adde47d72eebccc89dc9eec992"}, - {file = "google_crc32c-1.0.0-cp36-cp36m-win32.whl", hash = "sha256:6fd5d861421c37786b9c1a87dc7b0d8349a426151a461d5724b76c5a07f6ae9b"}, - {file = "google_crc32c-1.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cda3a6829e8b5bf6058615e53387430d004590c9b0ad808e53fea5bec35bbe44"}, - {file = "google_crc32c-1.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7f44c5259f6b2f8b2b6f668dbaa954693a10e97811345c193e46b933c2dd5165"}, - {file = "google_crc32c-1.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:337566ce49d7ea7493f95bd6bc89ab08640caa91b6105cea0be57ed026980e74"}, - {file = "google_crc32c-1.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f54c90058e3f56e55fa0f699c6f4ceaaa825ea7f17ef2adbf07b2b06b27455e7"}, - {file = "google_crc32c-1.0.0-cp37-cp37m-win32.whl", hash = "sha256:ec4d91c9236b0576d9d2b23c7eb85c6a6372b88afe2d0c64681cf11629586f74"}, - {file = "google_crc32c-1.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:17223ac9135eab28e874ff1e221810190d109a1abd482451d0776dc388be14de"}, - {file = "google_crc32c-1.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cf373207380e54c42da6c88baf1f7a31c2d9f29b87c9c922d5147d219eed55aa"}, - {file = "google_crc32c-1.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:41fb6c22cd72ae3db4d98d28dbb768d53397c8fc3cb8ab945fd434e842e622d4"}, + {file = "google-crc32c-1.1.1.tar.gz", hash = "sha256:d585f5d962812354a6113f962d06d67c4177993d8ed1b99e3fbb9753b1b98d65"}, + {file = "google_crc32c-1.1.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703d3f2fe4426f403ac8eb1229bbf458780df795bd4fdb04b0b73400dd58c108"}, + {file = "google_crc32c-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:75a43a19a6df9366d4ce6c171ab46ea60a0995eedf3ab988c111796e02e8e5ce"}, + {file = "google_crc32c-1.1.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:38d8808446fd38c7084a7b21bfc0aa22cd97eda320f3a391f524390de84377f8"}, + {file = "google_crc32c-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:4c5f7c9a2d500ca4b6142708832439410fb2b1422b5cebbad7f455bc0c40753b"}, + {file = "google_crc32c-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:3060d2e0d415a15de5189c2f6f521723c69f45633f8b4a62db8402c456b583bc"}, + {file = "google_crc32c-1.1.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:2c4d71f5212f039de575c3ec17ee5b2c14f293ec9f95ce7d054194aa55499405"}, + {file = "google_crc32c-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:46654a852cea70d0a5ccbfb078bdadca43c01448e96f301e5677d3bd4a0a808d"}, + {file = "google_crc32c-1.1.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:be277a1861ffef89cf68f34a1acb63173811df9f7e4d3f136a6ff2f578d1f54d"}, + {file = "google_crc32c-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:df5bccf9f38e72c77c607f27781f3e0c5cf77a1938020829872eb167f295c744"}, + {file = "google_crc32c-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2071754849915d070d6eb04a6522e05a1ecf2a58e55230c50367e183d6dc825b"}, + {file = "google_crc32c-1.1.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8875822807ab288b8034ba29a2d9b927036e31455b20e4831473e4f3d8b3f01c"}, + {file = "google_crc32c-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:783278dbe80c79637bfff011a2a61e02d268fdb38766be5bf1c56c43076c881a"}, + {file = "google_crc32c-1.1.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5a92ce8dd5dd27d3386dfb2a53fd272814f03e2f41b6e941061345fc36954f54"}, + {file = "google_crc32c-1.1.1-cp38-cp38-win32.whl", hash = "sha256:e10342adc665c0d22d76ea0d044cf588a96dfbea56b564a82d9ec5ae0634c848"}, + {file = "google_crc32c-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:c981f2fc2dc50a400cca9c0536ffe9080f8c713908633617007530ddf302eec8"}, + {file = "google_crc32c-1.1.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:94e65d56383f234b1f39dffc3a5647769918e4f13d49db11df7b176035781a39"}, + {file = "google_crc32c-1.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1aed1db85b5862b969e93e9fef80e9c61d989447d52ea236f9f76af76dd84a51"}, + {file = "google_crc32c-1.1.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:a9d784f37637b992447a80d4312a10f948400e4cd7a626f22d23131238cda722"}, + {file = "google_crc32c-1.1.1-cp39-cp39-win32.whl", hash = "sha256:15f6d38bfc860463169b142da327a2e875266d4c0db83a3ec8e6366364ac9dd7"}, + {file = "google_crc32c-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:c83720353c57a5a800b51d3118e0297b00ede1413a9b49a0f75e61c96ee91e98"}, ] google-resumable-media = [ - {file = "google-resumable-media-1.1.0.tar.gz", hash = "sha256:dcdab13e95bc534d268f87d5293e482cce5bc86dfce6ca0f2e2e89cbb73ef38c"}, - {file = "google_resumable_media-1.1.0-py2.py3-none-any.whl", hash = "sha256:ecabcd90d74a6e00331af0c87f500f238c44870d9626e01b385dda3af7b99269"}, + {file = "google-resumable-media-1.2.0.tar.gz", hash = "sha256:ee98b1921e5bda94867a08c864e55b4763d63887664f49ee1c231988f56b9d43"}, + {file = "google_resumable_media-1.2.0-py2.py3-none-any.whl", hash = "sha256:dbe670cd7f02f3586705fd5a108c8ab8552fa36a1cad8afbc5a54e982cf34f0c"}, ] googleapis-common-protos = [ {file = "googleapis-common-protos-1.52.0.tar.gz", hash = "sha256:560716c807117394da12cecb0a54da5a451b5cf9866f1d37e9a5e2329a665351"}, @@ -1754,12 +1819,12 @@ idna = [ {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] importlib-metadata = [ - {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, - {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, + {file = "importlib_metadata-3.4.0-py3-none-any.whl", hash = "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771"}, + {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, ] importlib-resources = [ - {file = "importlib_resources-3.0.0-py2.py3-none-any.whl", hash = "sha256:d028f66b66c0d5732dae86ba4276999855e162a749c92620a38c1d779ed138a7"}, - {file = "importlib_resources-3.0.0.tar.gz", hash = "sha256:19f745a6eca188b490b1428c8d1d4a0d2368759f32370ea8fb89cad2ab1106c3"}, + {file = "importlib_resources-5.0.0-py3-none-any.whl", hash = "sha256:ea17df80a0ff04b5dbd3d96dbeab1842acfd1c6c902eaeb8c8858abf2720161e"}, + {file = "importlib_resources-5.0.0.tar.gz", hash = "sha256:4743f090ed8946e713745ec0e660249ef9fb0b9843eacc5b5ff931d2fd5aa67f"}, ] iso8601 = [ {file = "iso8601-0.1.13-py2.py3-none-any.whl", hash = "sha256:694be0743e9f1535ea873bfc7bd6fb62380c62b75822761859428073a17fd39c"}, @@ -1779,8 +1844,8 @@ jmespath = [ {file = "jmespath-0.9.2.tar.gz", hash = "sha256:54c441e2e08b23f12d7fa7d8e6761768c47c969e6aed10eead57505ba760aee9"}, ] markdown = [ - {file = "Markdown-3.3.2-py3-none-any.whl", hash = "sha256:77b7bff443b1f97b4814fa438c181fd5882e31edb01b422856b3feca91475f3e"}, - {file = "Markdown-3.3.2.tar.gz", hash = "sha256:4b71fbd2db30c1dfb400f22b25f41cb823fc1db0aa8b7b67d120644f92cc1011"}, + {file = "Markdown-3.3.3-py3-none-any.whl", hash = "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328"}, + {file = "Markdown-3.3.3.tar.gz", hash = "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1822,32 +1887,42 @@ mock = [ {file = "mock-2.0.0.tar.gz", hash = "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba"}, ] more-itertools = [ - {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"}, - {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"}, + {file = "more-itertools-8.6.0.tar.gz", hash = "sha256:b3a9005928e5bed54076e6e549c792b306fddfe72b2d1d22dd63d42d5d3899cf"}, + {file = "more_itertools-8.6.0-py3-none-any.whl", hash = "sha256:8e1a2a43b2f2727425f2b5839587ae37093f19153dc26c0927d1048ff6557330"}, ] moto = [ {file = "moto-1.3.15-py2.py3-none-any.whl", hash = "sha256:3be7e1f406ef7e9c222dbcbfd8cefa2cb1062200e26deae49b5df446e17be3df"}, {file = "moto-1.3.15.tar.gz", hash = "sha256:fd98f7b219084ba8aadad263849c4dbe8be73979e035d8dc5c86e11a86f11b7f"}, ] msgpack = [ - {file = "msgpack-1.0.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08"}, - {file = "msgpack-1.0.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be"}, - {file = "msgpack-1.0.0-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a"}, - {file = "msgpack-1.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf"}, - {file = "msgpack-1.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8"}, - {file = "msgpack-1.0.0-cp36-cp36m-win32.whl", hash = "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1"}, - {file = "msgpack-1.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2"}, - {file = "msgpack-1.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97"}, - {file = "msgpack-1.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e"}, - {file = "msgpack-1.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140"}, - {file = "msgpack-1.0.0-cp37-cp37m-win32.whl", hash = "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272"}, - {file = "msgpack-1.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322"}, - {file = "msgpack-1.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab"}, - {file = "msgpack-1.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84"}, - {file = "msgpack-1.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e"}, - {file = "msgpack-1.0.0-cp38-cp38-win32.whl", hash = "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408"}, - {file = "msgpack-1.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d"}, - {file = "msgpack-1.0.0.tar.gz", hash = "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0"}, + {file = "msgpack-1.0.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9"}, + {file = "msgpack-1.0.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192"}, + {file = "msgpack-1.0.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841"}, + {file = "msgpack-1.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"}, + {file = "msgpack-1.0.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326"}, + {file = "msgpack-1.0.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439"}, + {file = "msgpack-1.0.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f"}, + {file = "msgpack-1.0.2-cp36-cp36m-win32.whl", hash = "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2"}, + {file = "msgpack-1.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc"}, + {file = "msgpack-1.0.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54"}, + {file = "msgpack-1.0.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87"}, + {file = "msgpack-1.0.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c"}, + {file = "msgpack-1.0.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1"}, + {file = "msgpack-1.0.2-cp37-cp37m-win32.whl", hash = "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a"}, + {file = "msgpack-1.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b"}, + {file = "msgpack-1.0.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06"}, + {file = "msgpack-1.0.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c"}, + {file = "msgpack-1.0.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e"}, + {file = "msgpack-1.0.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83"}, + {file = "msgpack-1.0.2-cp38-cp38-win32.whl", hash = "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9"}, + {file = "msgpack-1.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009"}, + {file = "msgpack-1.0.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694"}, + {file = "msgpack-1.0.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759"}, + {file = "msgpack-1.0.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887"}, + {file = "msgpack-1.0.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e"}, + {file = "msgpack-1.0.2-cp39-cp39-win32.whl", hash = "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33"}, + {file = "msgpack-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f"}, + {file = "msgpack-1.0.2.tar.gz", hash = "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984"}, ] netaddr = [ {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, @@ -1881,8 +1956,8 @@ oauth2client = [ {file = "oauth2client-3.0.0.tar.gz", hash = "sha256:5b5b056ec6f2304e7920b632885bd157fa71d1a7f3ddd00a43b1541a8d1a2460"}, ] "oslo.config" = [ - {file = "oslo.config-8.3.2-py3-none-any.whl", hash = "sha256:490bc3dc05ad69f77215dc650b1aa0a1bd5e5fb0415e925b6045552b828213aa"}, - {file = "oslo.config-8.3.2.tar.gz", hash = "sha256:c214ab7b30946b7341248edc7ea000944163b460d395af99fd182598b82d5833"}, + {file = "oslo.config-8.4.0-py3-none-any.whl", hash = "sha256:4532e94c2da67cdeeebfb82c517073e66ec5d944bbb34cb824a4c0541b3206fb"}, + {file = "oslo.config-8.4.0.tar.gz", hash = "sha256:537d62c581de2227aebfc4c71ee6fb455b853be920e0b30b65ebbe8d8128725c"}, ] "oslo.i18n" = [ {file = "oslo.i18n-5.0.1-py3-none-any.whl", hash = "sha256:99a6453b9b7a9d1603ba6c32e6ab8c738af95f6573215682a33c8028340bdccd"}, @@ -1893,12 +1968,12 @@ oauth2client = [ {file = "oslo.serialization-4.0.1.tar.gz", hash = "sha256:f84d3dca7ffbb86394e273094c674532b6144223eca8990a38836ba99728d53e"}, ] "oslo.utils" = [ - {file = "oslo.utils-4.6.0-py3-none-any.whl", hash = "sha256:12f5b6fecd77422ad83e5822a8289d96b35a3c5a0c9e8892687badefb1516891"}, - {file = "oslo.utils-4.6.0.tar.gz", hash = "sha256:460065e6d8bff195df2b543ed0f27589be3bef233b6df98f1b9157d200102fd7"}, + {file = "oslo.utils-4.7.0-py3-none-any.whl", hash = "sha256:51c1127c518e44beb4b565eb86f6f33e0a441d9a3e98eaff9f58a20933687b5d"}, + {file = "oslo.utils-4.7.0.tar.gz", hash = "sha256:b95b02c60dd4fac00d46525f73d27dc4e196c19281215aea84e8d389466b1b90"}, ] packaging = [ - {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, - {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, + {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, + {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -1918,24 +1993,24 @@ prettytable = [ {file = "prettytable-0.7.2.zip", hash = "sha256:a53da3b43d7a5c229b5e3ca2892ef982c46b7923b51e98f0db49956531211c4f"}, ] protobuf = [ - {file = "protobuf-3.13.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9c2e63c1743cba12737169c447374fab3dfeb18111a460a8c1a000e35836b18c"}, - {file = "protobuf-3.13.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1e834076dfef9e585815757a2c7e4560c7ccc5962b9d09f831214c693a91b463"}, - {file = "protobuf-3.13.0-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:df3932e1834a64b46ebc262e951cd82c3cf0fa936a154f0a42231140d8237060"}, - {file = "protobuf-3.13.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8c35bcbed1c0d29b127c886790e9d37e845ffc2725cc1db4bd06d70f4e8359f4"}, - {file = "protobuf-3.13.0-cp35-cp35m-win32.whl", hash = "sha256:339c3a003e3c797bc84499fa32e0aac83c768e67b3de4a5d7a5a9aa3b0da634c"}, - {file = "protobuf-3.13.0-cp35-cp35m-win_amd64.whl", hash = "sha256:361acd76f0ad38c6e38f14d08775514fbd241316cce08deb2ce914c7dfa1184a"}, - {file = "protobuf-3.13.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9edfdc679a3669988ec55a989ff62449f670dfa7018df6ad7f04e8dbacb10630"}, - {file = "protobuf-3.13.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5db9d3e12b6ede5e601b8d8684a7f9d90581882925c96acf8495957b4f1b204b"}, - {file = "protobuf-3.13.0-cp36-cp36m-win32.whl", hash = "sha256:c8abd7605185836f6f11f97b21200f8a864f9cb078a193fe3c9e235711d3ff1e"}, - {file = "protobuf-3.13.0-cp36-cp36m-win_amd64.whl", hash = "sha256:4d1174c9ed303070ad59553f435846a2f877598f59f9afc1b89757bdf846f2a7"}, - {file = "protobuf-3.13.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bba42f439bf45c0f600c3c5993666fcb88e8441d011fad80a11df6f324eef33"}, - {file = "protobuf-3.13.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:c0c5ab9c4b1eac0a9b838f1e46038c3175a95b0f2d944385884af72876bd6bc7"}, - {file = "protobuf-3.13.0-cp37-cp37m-win32.whl", hash = "sha256:f68eb9d03c7d84bd01c790948320b768de8559761897763731294e3bc316decb"}, - {file = "protobuf-3.13.0-cp37-cp37m-win_amd64.whl", hash = "sha256:91c2d897da84c62816e2f473ece60ebfeab024a16c1751aaf31100127ccd93ec"}, - {file = "protobuf-3.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3dee442884a18c16d023e52e32dd34a8930a889e511af493f6dc7d4d9bf12e4f"}, - {file = "protobuf-3.13.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e7662437ca1e0c51b93cadb988f9b353fa6b8013c0385d63a70c8a77d84da5f9"}, - {file = "protobuf-3.13.0-py2.py3-none-any.whl", hash = "sha256:d69697acac76d9f250ab745b46c725edf3e98ac24763990b24d58c16c642947a"}, - {file = "protobuf-3.13.0.tar.gz", hash = "sha256:6a82e0c8bb2bf58f606040cc5814e07715b2094caeba281e2e7d0b0e2e397db5"}, + {file = "protobuf-3.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:629b03fd3caae7f815b0c66b41273f6b1900a579e2ccb41ef4493a4f5fb84f3a"}, + {file = "protobuf-3.14.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:5b7a637212cc9b2bcf85dd828b1178d19efdf74dbfe1ddf8cd1b8e01fdaaa7f5"}, + {file = "protobuf-3.14.0-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:43b554b9e73a07ba84ed6cf25db0ff88b1e06be610b37656e292e3cbb5437472"}, + {file = "protobuf-3.14.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:5e9806a43232a1fa0c9cf5da8dc06f6910d53e4390be1fa06f06454d888a9142"}, + {file = "protobuf-3.14.0-cp35-cp35m-win32.whl", hash = "sha256:1c51fda1bbc9634246e7be6016d860be01747354ed7015ebe38acf4452f470d2"}, + {file = "protobuf-3.14.0-cp35-cp35m-win_amd64.whl", hash = "sha256:4b74301b30513b1a7494d3055d95c714b560fbb630d8fb9956b6f27992c9f980"}, + {file = "protobuf-3.14.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:86a75477addde4918e9a1904e5c6af8d7b691f2a3f65587d73b16100fbe4c3b2"}, + {file = "protobuf-3.14.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ecc33531a213eee22ad60e0e2aaea6c8ba0021f0cce35dbf0ab03dee6e2a23a1"}, + {file = "protobuf-3.14.0-cp36-cp36m-win32.whl", hash = "sha256:72230ed56f026dd664c21d73c5db73ebba50d924d7ba6b7c0d81a121e390406e"}, + {file = "protobuf-3.14.0-cp36-cp36m-win_amd64.whl", hash = "sha256:0fc96785262042e4863b3f3b5c429d4636f10d90061e1840fce1baaf59b1a836"}, + {file = "protobuf-3.14.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4e75105c9dfe13719b7293f75bd53033108f4ba03d44e71db0ec2a0e8401eafd"}, + {file = "protobuf-3.14.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2a7e2fe101a7ace75e9327b9c946d247749e564a267b0515cf41dfe450b69bac"}, + {file = "protobuf-3.14.0-cp37-cp37m-win32.whl", hash = "sha256:b0d5d35faeb07e22a1ddf8dce620860c8fe145426c02d1a0ae2688c6e8ede36d"}, + {file = "protobuf-3.14.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8971c421dbd7aad930c9bd2694122f332350b6ccb5202a8b7b06f3f1a5c41ed5"}, + {file = "protobuf-3.14.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9616f0b65a30851e62f1713336c931fcd32c057202b7ff2cfbfca0fc7d5e3043"}, + {file = "protobuf-3.14.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:22bcd2e284b3b1d969c12e84dc9b9a71701ec82d8ce975fdda19712e1cfd4e00"}, + {file = "protobuf-3.14.0-py2.py3-none-any.whl", hash = "sha256:0e247612fadda953047f53301a7b0407cb0c3cb4ae25a6fde661597a04039b3c"}, + {file = "protobuf-3.14.0.tar.gz", hash = "sha256:1d63eb389347293d8915fb47bee0951c7b5dab522a4a60118b9a18f33e21f8ce"}, ] psycopg2 = [ {file = "psycopg2-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:068115e13c70dc5982dfc00c5d70437fe37c014c808acce119b5448361c03725"}, @@ -1953,8 +2028,8 @@ psycopg2 = [ {file = "psycopg2-2.8.6.tar.gz", hash = "sha256:fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, @@ -1991,36 +2066,41 @@ pycparser = [ {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, ] pycryptodome = [ - {file = "pycryptodome-3.9.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:50348edd283afdccddc0938cdc674484533912ba8a99a27c7bfebb75030aa856"}, - {file = "pycryptodome-3.9.8-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:80d57177a0b7c14d4594c62bbb47fe2f6309ad3b0a34348a291d570925c97a82"}, - {file = "pycryptodome-3.9.8-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:fbe65d5cfe04ff2f7684160d50f5118bdefb01e3af4718eeb618bfed40f19d94"}, - {file = "pycryptodome-3.9.8-cp27-cp27m-win32.whl", hash = "sha256:bcd5b8416e73e4b0d48afba3704d8c826414764dafaed7a1a93c442188d90ccc"}, - {file = "pycryptodome-3.9.8-cp27-cp27m-win_amd64.whl", hash = "sha256:360955eece2cd0fa694a708d10303c6abd7b39614fa2547b6bd245da76198beb"}, - {file = "pycryptodome-3.9.8-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:1e655746f539421d923fd48df8f6f40b3443d80b75532501c0085b64afed9df5"}, - {file = "pycryptodome-3.9.8-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:709b9f144d23e290b9863121d1ace14a72e01f66ea9c903fbdc690520dfdfcf0"}, - {file = "pycryptodome-3.9.8-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:6276478ada411aca97c0d5104916354b3d740d368407912722bd4d11aa9ee4c2"}, - {file = "pycryptodome-3.9.8-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:54bdedd28476dea8a3cd86cb67c0df1f0e3d71cae8022354b0f879c41a3d27b2"}, - {file = "pycryptodome-3.9.8-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f521178e5a991ffd04182ed08f552daca1affcb826aeda0e1945cd989a9d4345"}, - {file = "pycryptodome-3.9.8-cp35-cp35m-win32.whl", hash = "sha256:a207231a52426de3ff20f5608f0687261a3329d97a036c51f7d4c606a6f30c23"}, - {file = "pycryptodome-3.9.8-cp35-cp35m-win_amd64.whl", hash = "sha256:2b998dc45ef5f4e5cf5248a6edfcd8d8e9fb5e35df8e4259b13a1b10eda7b16b"}, - {file = "pycryptodome-3.9.8-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:03d5cca8618620f45fd40f827423f82b86b3a202c8d44108601b0f5f56b04299"}, - {file = "pycryptodome-3.9.8-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:f78a68c2c820e4731e510a2df3eef0322f24fde1781ced970bf497b6c7d92982"}, - {file = "pycryptodome-3.9.8-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:132a56abba24e2e06a479d8e5db7a48271a73a215f605017bbd476d31f8e71c1"}, - {file = "pycryptodome-3.9.8-cp36-cp36m-win32.whl", hash = "sha256:67dcad1b8b201308586a8ca2ffe89df1e4f731d5a4cdd0610cc4ea790351c739"}, - {file = "pycryptodome-3.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:b56638d58a3a4be13229c6a815cd448f9e3ce40c00880a5398471b42ee86f50e"}, - {file = "pycryptodome-3.9.8-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:bec2bcdf7c9ce7f04d718e51887f3b05dc5c1cfaf5d2c2e9065ecddd1b2f6c9a"}, - {file = "pycryptodome-3.9.8-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:abc2e126c9490e58a36a0f83516479e781d83adfb134576a5cbe5c6af2a3e93c"}, - {file = "pycryptodome-3.9.8-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ef39c98d9b8c0736d91937d193653e47c3b19ddf4fc3bccdc5e09aaa4b0c5d21"}, - {file = "pycryptodome-3.9.8-cp37-cp37m-win32.whl", hash = "sha256:4350a42028240c344ee855f032c7d4ad6ff4f813bfbe7121547b7dc579ecc876"}, - {file = "pycryptodome-3.9.8-cp37-cp37m-win_amd64.whl", hash = "sha256:c8bf40cf6e281a4378e25846924327e728a887e8bf0ee83b2604a0f4b61692e8"}, - {file = "pycryptodome-3.9.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d8074c8448cfd0705dfa71ca333277fce9786d0b9cac75d120545de6253f996a"}, - {file = "pycryptodome-3.9.8-cp38-cp38-manylinux1_i686.whl", hash = "sha256:8063a712fba642f78d3c506b0896846601b6de7f5c3d534e388ad0cc07f5a149"}, - {file = "pycryptodome-3.9.8-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:dd302b6ae3965afeb5ef1b0d92486f986c0e65183cd7835973f0b593800590e6"}, - {file = "pycryptodome-3.9.8-cp38-cp38-win32.whl", hash = "sha256:02e51e1d5828d58f154896ddfd003e2e7584869c275e5acbe290443575370fba"}, - {file = "pycryptodome-3.9.8-cp38-cp38-win_amd64.whl", hash = "sha256:55eb61aca2c883db770999f50d091ff7c14016f2769ad7bca3d9b75d1d7c1b68"}, - {file = "pycryptodome-3.9.8-cp39-cp39-manylinux1_i686.whl", hash = "sha256:39ef9fb52d6ec7728fce1f1693cb99d60ce302aeebd59bcedea70ca3203fda60"}, - {file = "pycryptodome-3.9.8-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:de6e1cd75677423ff64712c337521e62e3a7a4fc84caabbd93207752e831a85a"}, - {file = "pycryptodome-3.9.8.tar.gz", hash = "sha256:0e24171cf01021bc5dc17d6a9d4f33a048f09d62cc3f62541e95ef104588bda4"}, + {file = "pycryptodome-3.9.9-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:5598dc6c9dbfe882904e54584322893eff185b98960bbe2cdaaa20e8a437b6e5"}, + {file = "pycryptodome-3.9.9-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1cfdb92dca388e27e732caa72a1cc624520fe93752a665c3b6cd8f1a91b34916"}, + {file = "pycryptodome-3.9.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5f19e6ef750f677d924d9c7141f54bade3cd56695bbfd8a9ef15d0378557dfe4"}, + {file = "pycryptodome-3.9.9-cp27-cp27m-win32.whl", hash = "sha256:a3d8a9efa213be8232c59cdc6b65600276508e375e0a119d710826248fd18d37"}, + {file = "pycryptodome-3.9.9-cp27-cp27m-win_amd64.whl", hash = "sha256:50826b49fbca348a61529693b0031cdb782c39060fb9dca5ac5dff858159dc5a"}, + {file = "pycryptodome-3.9.9-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:19cb674df6c74a14b8b408aa30ba8a89bd1c01e23505100fb45f930fbf0ed0d9"}, + {file = "pycryptodome-3.9.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:28f75e58d02019a7edc7d4135203d2501dfc47256d175c72c9798f9a129a49a7"}, + {file = "pycryptodome-3.9.9-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:6d3baaf82681cfb1a842f1c8f77beac791ceedd99af911e4f5fabec32bae2259"}, + {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:946399d15eccebafc8ce0257fc4caffe383c75e6b0633509bd011e357368306c"}, + {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:eb01f9997e4d6a8ec8a1ad1f676ba5a362781ff64e8189fe2985258ba9cb9706"}, + {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:411745c6dce4eff918906eebcde78771d44795d747e194462abb120d2e537cd9"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:8f9f84059039b672a5a705b3c5aa21747867bacc30a72e28bf0d147cc8ef85ed"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:7798e73225a699651888489fbb1dbc565e03a509942a8ce6194bbe6fb582a41f"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:46e96aeb8a9ca8b1edf9b1fd0af4bf6afcf3f1ca7fa35529f5d60b98f3e4e959"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:843e5f10ecdf9d307032b8b91afe9da1d6ed5bb89d0bbec5c8dcb4ba44008e11"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-win32.whl", hash = "sha256:b68794fba45bdb367eeb71249c26d23e61167510a1d0c3d6cf0f2f14636e62ee"}, + {file = "pycryptodome-3.9.9-cp36-cp36m-win_amd64.whl", hash = "sha256:60febcf5baf70c566d9d9351c47fbd8321da9a4edf2eff45c4c31c86164ca794"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:4ed27951b0a17afd287299e2206a339b5b6d12de9321e1a1575261ef9c4a851b"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:9000877383e2189dafd1b2fc68c6c726eca9a3cfb6d68148fbb72ccf651959b6"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:faa682c404c218e8788c3126c9a4b8fbcc54dc245b5b6e8ea5b46f3b63bd0c84"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:62c488a21c253dadc9f731a32f0ac61e4e436d81a1ea6f7d1d9146ed4d20d6bd"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-win32.whl", hash = "sha256:834b790bbb6bd18956f625af4004d9c15eed12d5186d8e57851454ae76d52215"}, + {file = "pycryptodome-3.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:70d807d11d508433daf96244ec1c64e55039e8a35931fc5ea9eee94dbe3cb6b5"}, + {file = "pycryptodome-3.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:27397aee992af69d07502126561d851ba3845aa808f0e55c71ad0efa264dd7d4"}, + {file = "pycryptodome-3.9.9-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d7ec2bd8f57c559dd24e71891c51c25266a8deb66fc5f02cc97c7fb593d1780a"}, + {file = "pycryptodome-3.9.9-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e15bde67ccb7d4417f627dd16ffe2f5a4c2941ce5278444e884cb26d73ecbc61"}, + {file = "pycryptodome-3.9.9-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5c3c4865730dfb0263f822b966d6d58429d8b1e560d1ddae37685fd9e7c63161"}, + {file = "pycryptodome-3.9.9-cp38-cp38-win32.whl", hash = "sha256:76b1a34d74bb2c91bce460cdc74d1347592045627a955e9a252554481c17c52f"}, + {file = "pycryptodome-3.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:6e4227849e4231a3f5b35ea5bdedf9a82b3883500e5624f00a19156e9a9ef861"}, + {file = "pycryptodome-3.9.9-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2a68df525b387201a43b27b879ce8c08948a430e883a756d6c9e3acdaa7d7bd8"}, + {file = "pycryptodome-3.9.9-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:a4599c0ca0fc027c780c1c45ed996d5bef03e571470b7b1c7171ec1e1a90914c"}, + {file = "pycryptodome-3.9.9-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b4e6b269a8ddaede774e5c3adbef6bf452ee144e6db8a716d23694953348cd86"}, + {file = "pycryptodome-3.9.9-cp39-cp39-win32.whl", hash = "sha256:a199e9ca46fc6e999e5f47fce342af4b56c7de85fae893c69ab6aa17531fb1e1"}, + {file = "pycryptodome-3.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:6e89bb3826e6f84501e8e3b205c22595d0c5492c2f271cbb9ee1c48eb1866645"}, + {file = "pycryptodome-3.9.9.tar.gz", hash = "sha256:910e202a557e1131b1c1b3f17a63914d57aac55cf9fb9b51644962841c3995c4"}, ] pyjwt = [ {file = "PyJWT-1.7.1-py2.py3-none-any.whl", hash = "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e"}, @@ -2073,8 +2153,8 @@ python-keystoneclient = [ {file = "python_keystoneclient-1.8.1-py2.py3-none-any.whl", hash = "sha256:6835a58e0a533947fca317fff487f040af5fbc1d7b78c75d9f812c09b0307740"}, ] pytz = [ - {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"}, - {file = "pytz-2020.1.tar.gz", hash = "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"}, + {file = "pytz-2020.5-py2.py3-none-any.whl", hash = "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4"}, + {file = "pytz-2020.5.tar.gz", hash = "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"}, ] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, @@ -2090,12 +2170,12 @@ pyyaml = [ {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] requests = [ - {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, - {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, + {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, + {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] responses = [ - {file = "responses-0.12.0-py2.py3-none-any.whl", hash = "sha256:0de50fbf600adf5ef9f0821b85cc537acca98d66bc7776755924476775c1989c"}, - {file = "responses-0.12.0.tar.gz", hash = "sha256:e80d5276011a4b79ecb62c5f82ba07aa23fb31ecbc95ee7cad6de250a3c97444"}, + {file = "responses-0.12.1-py2.py3-none-any.whl", hash = "sha256:ef265bd3200bdef5ec17912fc64a23570ba23597fd54ca75c18650fa1699213d"}, + {file = "responses-0.12.1.tar.gz", hash = "sha256:2e5764325c6b624e42b428688f2111fea166af46623cb0127c05f6afb14d3457"}, ] retry = [ {file = "retry-0.9.2-py2.py3-none-any.whl", hash = "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606"}, @@ -2106,8 +2186,8 @@ rfc3986 = [ {file = "rfc3986-1.4.0.tar.gz", hash = "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d"}, ] rsa = [ - {file = "rsa-4.6-py3-none-any.whl", hash = "sha256:6166864e23d6b5195a5cfed6cd9fed0fe774e226d8f854fcb23b7bbef0350233"}, - {file = "rsa-4.6.tar.gz", hash = "sha256:109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa"}, + {file = "rsa-4.7-py3-none-any.whl", hash = "sha256:a8774e55b59fd9fc893b0d05e9bfc6f47081f46ff5b46f39ccf24631b7be356b"}, + {file = "rsa-4.7.tar.gz", hash = "sha256:69805d6b69f56eb05b62daea3a7dbd7aa44324ad1306445e05da8060232d00f4"}, ] s3transfer = [ {file = "s3transfer-0.2.1-py2.py3-none-any.whl", hash = "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba"}, @@ -2118,50 +2198,55 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.3.20-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27m-win32.whl", hash = "sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27m-win_amd64.whl", hash = "sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327"}, - {file = "SQLAlchemy-1.3.20-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-win32.whl", hash = "sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6"}, - {file = "SQLAlchemy-1.3.20-cp35-cp35m-win_amd64.whl", hash = "sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-win32.whl", hash = "sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b"}, - {file = "SQLAlchemy-1.3.20-cp36-cp36m-win_amd64.whl", hash = "sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-win32.whl", hash = "sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61"}, - {file = "SQLAlchemy-1.3.20-cp37-cp37m-win_amd64.whl", hash = "sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-win32.whl", hash = "sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf"}, - {file = "SQLAlchemy-1.3.20-cp38-cp38-win_amd64.whl", hash = "sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-win32.whl", hash = "sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d"}, - {file = "SQLAlchemy-1.3.20-cp39-cp39-win_amd64.whl", hash = "sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1"}, - {file = "SQLAlchemy-1.3.20.tar.gz", hash = "sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:61628715931f4962e0cdb2a7c87ff39eea320d2aa96bd471a3c293d146f90394"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:81d8d099a49f83111cce55ec03cc87eef45eec0d90f9842b4fc674f860b857b0"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:d055ff750fcab69ca4e57b656d9c6ad33682e9b8d564f2fbe667ab95c63591b0"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-win32.whl", hash = "sha256:9bf572e4f5aa23f88dd902f10bb103cb5979022a38eec684bfa6d61851173fec"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-win_amd64.whl", hash = "sha256:7d4b8de6bb0bc736161cb0bbd95366b11b3eb24dd6b814a143d8375e75af9990"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4a84c7c7658dd22a33dab2e2aa2d17c18cb004a42388246f2e87cb4085ef2811"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:f1e88b30da8163215eab643962ae9d9252e47b4ea53404f2c4f10f24e70ddc62"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:f115150cc4361dd46153302a640c7fa1804ac207f9cc356228248e351a8b4676"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6aaa13ee40c4552d5f3a59f543f0db6e31712cc4009ec7385407be4627259d41"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3ab5b44a07b8c562c6dcb7433c6a6c6e03266d19d64f87b3333eda34e3b9936b"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:426ece890153ccc52cc5151a1a0ed540a5a7825414139bb4c95a868d8da54a52"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-win32.whl", hash = "sha256:bd4b1af45fd322dcd1fb2a9195b4f93f570d1a5902a842e3e6051385fac88f9c"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-win_amd64.whl", hash = "sha256:62285607a5264d1f91590abd874d6a498e229d5840669bd7d9f654cfaa599bd0"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:314f5042c0b047438e19401d5f29757a511cfc2f0c40d28047ca0e4c95eabb5b"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:62fb881ba51dbacba9af9b779211cf9acff3442d4f2993142015b22b3cd1f92a"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:bde677047305fe76c7ee3e4492b545e0018918e44141cc154fe39e124e433991"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:0c6406a78a714a540d980a680b86654feadb81c8d0eecb59f3d6c554a4c69f19"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-win32.whl", hash = "sha256:95bde07d19c146d608bccb9b16e144ec8f139bcfe7fd72331858698a71c9b4f5"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-win_amd64.whl", hash = "sha256:888d5b4b5aeed0d3449de93ea80173653e939e916cc95fe8527079e50235c1d2"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:d53f59744b01f1440a1b0973ed2c3a7de204135c593299ee997828aad5191693"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:70121f0ae48b25ef3e56e477b88cd0b0af0e1f3a53b5554071aa6a93ef378a03"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:54da615e5b92c339e339fe8536cce99fe823b6ed505d4ea344852aefa1c205fb"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:68428818cf80c60dc04aa0f38da20ad39b28aba4d4d199f949e7d6e04444ea86"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-win32.whl", hash = "sha256:17610d573e698bf395afbbff946544fbce7c5f4ee77b5bcb1f821b36345fae7a"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-win_amd64.whl", hash = "sha256:216ba5b4299c95ed179b58f298bda885a476b16288ab7243e89f29f6aeced7e0"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:0c72b90988be749e04eff0342dcc98c18a14461eb4b2ad59d611b57b31120f90"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:491fe48adc07d13e020a8b07ef82eefc227003a046809c121bea81d3dbf1832d"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:f8191fef303025879e6c3548ecd8a95aafc0728c764ab72ec51a0bdf0c91a341"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:108580808803c7732f34798eb4a329d45b04c562ed83ee90f09f6a184a42b766"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-win32.whl", hash = "sha256:bab5a1e15b9466a25c96cda19139f3beb3e669794373b9ce28c4cf158c6e841d"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-win_amd64.whl", hash = "sha256:318b5b727e00662e5fc4b4cd2bf58a5116d7c1b4dd56ffaa7d68f43458a8d1ed"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:1418f5e71d6081aa1095a1d6b567a562d2761996710bdce9b6e6ba20a03d0864"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5a7f224cdb7233182cec2a45d4c633951268d6a9bcedac37abbf79dd07012aea"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:715b34578cc740b743361f7c3e5f584b04b0f1344f45afc4e87fbac4802eb0a0"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2ff132a379838b1abf83c065be54cef32b47c987aedd06b82fc76476c85225eb"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-win32.whl", hash = "sha256:c389d7cc2b821853fb018c85457da3e7941db64f4387720a329bc7ff06a27963"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-win_amd64.whl", hash = "sha256:04f995fcbf54e46cddeb4f75ce9dfc17075d6ae04ac23b2bacb44b3bc6f6bf11"}, + {file = "SQLAlchemy-1.3.22.tar.gz", hash = "sha256:758fc8c4d6c0336e617f9f6919f9daea3ab6bb9b07005eda9a1a682e24a6cacc"}, ] stevedore = [ - {file = "stevedore-3.2.2-py3-none-any.whl", hash = "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62"}, - {file = "stevedore-3.2.2.tar.gz", hash = "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0"}, + {file = "stevedore-3.3.0-py3-none-any.whl", hash = "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a"}, + {file = "stevedore-3.3.0.tar.gz", hash = "sha256:3a5bbd0652bf552748871eaa73a4a8dc2899786bc497a2aa1fcb4dcdb0debeee"}, ] storageclient = [] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] uritemplate = [ {file = "uritemplate-3.0.1-py2.py3-none-any.whl", hash = "sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f"}, {file = "uritemplate-3.0.1.tar.gz", hash = "sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae"}, @@ -2185,6 +2270,6 @@ xmltodict = [ {file = "xmltodict-0.12.0.tar.gz", hash = "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21"}, ] zipp = [ - {file = "zipp-3.3.1-py3-none-any.whl", hash = "sha256:16522f69653f0d67be90e8baa4a46d66389145b734345d68a257da53df670903"}, - {file = "zipp-3.3.1.tar.gz", hash = "sha256:c1532a8030c32fd52ff6a288d855fe7adef5823ba1d26a29a68fd6314aa72baa"}, + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, ] diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 6272658a0..724754ad7 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -7,6 +7,7 @@ from fence.models import User, UpstreamRefreshToken, GA4GHVisaV1 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient from fence.config import config +from fence.job.visa_update_cronjob import Visa_Token_Update import tests.utils logger = get_logger(__name__, log_level="debug") @@ -331,3 +332,92 @@ def test_update_visa_token_with_invalid_visa( for query_visa in query_visas: assert query_visa.ga4gh_visa assert query_visa.ga4gh_visa == encoded_visa + + +@mock.patch("fence.resources.openid.ras_oauth2.RASOauth2Client.get_userinfo") +@mock.patch("fence.resources.openid.ras_oauth2.RASOauth2Client.get_access_token") +@mock.patch( + "fence.resources.openid.ras_oauth2.RASOauth2Client.get_value_from_discovery_doc" +) +def test_cronjob( + mock_discovery, + mock_get_token, + mock_userinfo, + config, + db_session, + rsa_private_key, + kid, + kid_2, +): + """ + Test to check visa table is updated when getting new visa + """ + + mock_discovery.return_value = "https://ras/token_endpoint" + new_token = "refresh12345abcdefg" + token_response = { + "access_token": "abcdef12345", + "id_token": "id12345abcdef", + "refresh_token": new_token, + } + mock_get_token.return_value = token_response + + userinfo_response = { + "sub": "abcd-asdj-sajpiasj12iojd-asnoin", + "name": "", + "preferred_username": "someuser@era.com", + "UID": "", + "UserID": "admin_user", + "email": "", + } + + test_user = add_test_user(db_session) + add_visa_manually(db_session, test_user, rsa_private_key, kid) + add_refresh_token(db_session, test_user) + + visa_query = db_session.query(GA4GHVisaV1).filter_by(user=test_user).first() + initial_visa = visa_query.ga4gh_visa + assert initial_visa + + oidc = config.get("OPENID_CONNECT", {}) + ras_client = RASClient( + oidc["ras"], + HTTP_PROXY=config.get("HTTP_PROXY"), + logger=logger, + ) + + new_visa = { + "iss": "https://stsstg.nih.gov", + "sub": "abcde12345aspdij", + "iat": int(time.time()), + "exp": int(time.time()) + 1000, + "scope": "openid ga4gh_passport_v1 email profile", + "jti": "jtiajoidasndokmasdl", + "txn": "sapidjspa.asipidja", + "name": "", + "ga4gh_visa_v1": { + "type": "https://ras/visa/v1", + "asserted": int(time.time()), + "value": "https://nig/passport/dbgap", + "source": "https://ncbi/gap", + }, + } + + headers = {"kid": kid_2} + + encoded_visa = jwt.encode( + new_visa, key=rsa_private_key, headers=headers, algorithm="RS256" + ).decode("utf-8") + + userinfo_response["ga4gh_passport_v1"] = [encoded_visa] + mock_userinfo.return_value = userinfo_response + + ras_client.update_user_visas(test_user) + + query_visa = db_session.query(GA4GHVisaV1).first() + assert query_visa.ga4gh_visa + assert query_visa.ga4gh_visa == encoded_visa + + cronjob = Visa_Token_Update() + + cronjob.update_tokens(db_session) From 10a3170f73a0a42b450ee301cb3f06d1988893ec Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 22 Jan 2021 10:16:00 -0600 Subject: [PATCH 07/30] add window method --- fence/job/visa_update_cronjob.py | 87 +++++++++++++++++++++++--------- poetry.lock | 6 +-- tests/ras/test_ras.py | 19 ++++--- 3 files changed, 76 insertions(+), 36 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 9e0d06106..931c75d53 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -2,6 +2,7 @@ from cdislogging import get_logger from userdatamodel.driver import SQLAlchemyDriver +import random from fence.config import config from fence.models import ( @@ -13,56 +14,92 @@ from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient -# Base class that collects only specific type of refresh token +logger = get_logger(__name__, log_level="debug") class Visa_Token_Update(object): - def __init__( self, visa_type=None, - concurrency=None, # number of concurrent users going through the visa update flow - thread_pool_size=None, # number of Docker container CPU used for jwt verification - buffer_size=None, # max size of asyncio queue + concurrency=None, # number of concurrent users going through the visa update flow + thread_pool_size=None, # number of Docker container CPU used for jwt verification + buffer_size=None, # max size of asyncio queue ): self.visa_type = visa_type or "ras" self.concurrency = concurrency or 3 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency - - def update_tokens(self, db_session): + + async def update_tokens(self, db_session): """ - Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use - looking at the type field in the ga4gh table. + Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use + looking at the type field in the ga4gh table. """ - queue = asyncio.Queue(maxsize=self.buffer_size) + # producers = [asyncio.create_task(self.producer(i, db_session, queue)) for i in range(1)] + producer = [ + asyncio.create_task(self.producer(db_session, queue, window_idx=0)) + for _ in range(1) + ] + workers = [asyncio.create_task(self.worker(j, queue)) for j in range(5)] - all_visas = db_session.query(GA4GHVisaV1).all() - print("--------------------------------------------") - for visa in all_visas: - username = visa.user.username - print(username) - u = query_for_user(db_session, username) - print(u.ga4gh_visas_v1) + await asyncio.gather(*producer) + + await queue.join() + for w in workers: + w.cancel() - + async def window(self, db_session, queue, window_idx): + window_size = 8 + start, stop = window_size * window_idx, window_size * (window_idx + 1) + visas = db_session.query(GA4GHVisaV1).slice(start, stop).all() - async def producer(self, queue): + return visas + + async def producer(self, db_session, queue, window_idx): """ TODO: Rename this Producer: Produces users and puts them in a queue for processing """ - # while there are users: - # fill queue with users until queue is full - # could use a counter to query through each user in db and use this counter for Prometheus (???) + window_size = 8 + while True: + visas = await self.window(db_session, queue, window_idx) + if visas == None: + break + for visa in visas: + print("Producer producing visa for user {}".format(visa.user.username)) + await queue.put(visa) + if len(visas) < window_size: + break + window_idx += 1 - async def consumer(self, queue): + async def worker(self, name, queue): """ TODO: Rename this - Consumer: Create workers that does the visa update flow + worker: Create workers that does the visa update flow + """ + # update visa stuff here + while True: + visa = await queue.get() + client = self._pick_client(visa) + username = visa.user.username + user = visa.user + # print("worker {} working on user {}".format(name, username)) + client.update_user_visas(user) + await asyncio.sleep(random.random()) + queue.task_done() + + def _pick_client(self, visa): + """ + Pick oidc client according to the visa provider """ - # update visa stuff here \ No newline at end of file + if visa.type == "https://ras/visa/v1": + oidc = config.get("OPENID_CONNECT", {}) + return RASClient( + oidc["ras"], + HTTP_PROXY=config.get("HTTP_PROXY"), + logger=logger, + ) diff --git a/poetry.lock b/poetry.lock index 575bbce0b..8fce0d51a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -725,7 +725,7 @@ marker = "python_version < \"3.7\"" name = "importlib-resources" optional = false python-versions = ">=3.6" -version = "5.0.0" +version = "5.1.0" [package.dependencies] [package.dependencies.zipp] @@ -1823,8 +1823,8 @@ importlib-metadata = [ {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, ] importlib-resources = [ - {file = "importlib_resources-5.0.0-py3-none-any.whl", hash = "sha256:ea17df80a0ff04b5dbd3d96dbeab1842acfd1c6c902eaeb8c8858abf2720161e"}, - {file = "importlib_resources-5.0.0.tar.gz", hash = "sha256:4743f090ed8946e713745ec0e660249ef9fb0b9843eacc5b5ff931d2fd5aa67f"}, + {file = "importlib_resources-5.1.0-py3-none-any.whl", hash = "sha256:885b8eae589179f661c909d699a546cf10d83692553e34dca1bf5eb06f7f6217"}, + {file = "importlib_resources-5.1.0.tar.gz", hash = "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380"}, ] iso8601 = [ {file = "iso8601-0.1.13-py2.py3-none-any.whl", hash = "sha256:694be0743e9f1535ea873bfc7bd6fb62380c62b75822761859428073a17fd39c"}, diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 724754ad7..792dfbbe8 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -1,3 +1,4 @@ +import asyncio import time import mock import jwt @@ -13,8 +14,8 @@ logger = get_logger(__name__, log_level="debug") -def add_test_user(db_session): - test_user = User(username="admin_user", id="5678", is_admin=True) +def add_test_user(db_session, username="admin_user", id="5678", is_admin=True): + test_user = User(username=username, id=id, is_admin=is_admin) db_session.add(test_user) db_session.commit() return test_user @@ -353,6 +354,8 @@ def test_cronjob( Test to check visa table is updated when getting new visa """ + n_users = 50 + mock_discovery.return_value = "https://ras/token_endpoint" new_token = "refresh12345abcdefg" token_response = { @@ -371,9 +374,11 @@ def test_cronjob( "email": "", } - test_user = add_test_user(db_session) - add_visa_manually(db_session, test_user, rsa_private_key, kid) - add_refresh_token(db_session, test_user) + for i in range(n_users): + username = "admin_user_{}".format(i) + test_user = add_test_user(db_session, username, i) + add_visa_manually(db_session, test_user, rsa_private_key, kid) + add_refresh_token(db_session, test_user) visa_query = db_session.query(GA4GHVisaV1).filter_by(user=test_user).first() initial_visa = visa_query.ga4gh_visa @@ -415,9 +420,7 @@ def test_cronjob( ras_client.update_user_visas(test_user) query_visa = db_session.query(GA4GHVisaV1).first() - assert query_visa.ga4gh_visa - assert query_visa.ga4gh_visa == encoded_visa cronjob = Visa_Token_Update() - cronjob.update_tokens(db_session) + asyncio.run(cronjob.update_tokens(db_session)) From cfc4c0967334460ccebbb9227bad0e9219fda1b2 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 22 Jan 2021 16:34:56 -0600 Subject: [PATCH 08/30] updated as designed --- fence/job/visa_update_cronjob.py | 92 +++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 31 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 931c75d53..62f0c4e1f 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -20,13 +20,11 @@ class Visa_Token_Update(object): def __init__( self, - visa_type=None, concurrency=None, # number of concurrent users going through the visa update flow thread_pool_size=None, # number of Docker container CPU used for jwt verification buffer_size=None, # max size of asyncio queue ): - self.visa_type = visa_type or "ras" - self.concurrency = concurrency or 3 + self.concurrency = concurrency or 2 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency @@ -37,60 +35,88 @@ async def update_tokens(self, db_session): looking at the type field in the ga4gh table. """ queue = asyncio.Queue(maxsize=self.buffer_size) - # producers = [asyncio.create_task(self.producer(i, db_session, queue)) for i in range(1)] - producer = [ + semaphore = asyncio.Queue(maxsize=self.n_workers) + producers = [ asyncio.create_task(self.producer(db_session, queue, window_idx=0)) for _ in range(1) ] - workers = [asyncio.create_task(self.worker(j, queue)) for j in range(5)] + workers = [ + asyncio.create_task(self.worker(j, queue, semaphore)) + for j in range(self.n_workers) + ] + updaters = [ + asyncio.create_task(self.updater(i, semaphore)) + for i in range(self.concurrency) + ] - await asyncio.gather(*producer) + await asyncio.gather(*producers) + await queue.join() # blocks until everything in queue is complete - await queue.join() + await asyncio.gather(*workers) + await semaphore.join() # blocks until everything in semaphore is complete for w in workers: w.cancel() + for u in updaters: + u.cancel() async def window(self, db_session, queue, window_idx): + """ + window function to get chunks of data from the table + """ window_size = 8 start, stop = window_size * window_idx, window_size * (window_idx + 1) - visas = db_session.query(GA4GHVisaV1).slice(start, stop).all() - - return visas + users = db_session.query(User).slice(start, stop).all() + return users async def producer(self, db_session, queue, window_idx): """ - TODO: Rename this - Producer: Produces users and puts them in a queue for processing - + Produces users from db and puts them in a queue for processing """ window_size = 8 while True: - visas = await self.window(db_session, queue, window_idx) - if visas == None: + users = await self.window(db_session, queue, window_idx) + + if users == None: break - for visa in visas: - print("Producer producing visa for user {}".format(visa.user.username)) - await queue.put(visa) - if len(visas) < window_size: + for user in users: + # print("Producer producing user for user {}".format(user.username)) + await queue.put(user) + if len(users) < window_size: break window_idx += 1 - async def worker(self, name, queue): + async def worker(self, name, queue, semaphore): """ - TODO: Rename this - worker: Create workers that does the visa update flow + Create tasks to pass tot updater to update visas AND pass updated visas to _verify_jwt_token for verification """ - # update visa stuff here while True: - visa = await queue.get() - client = self._pick_client(visa) - username = visa.user.username - user = visa.user - # print("worker {} working on user {}".format(name, username)) - client.update_user_visas(user) - await asyncio.sleep(random.random()) + user = await queue.get() + await semaphore.put(user) + # print("Adding {} to semaphore".format(user.username)) queue.task_done() + if queue.empty(): + break + + async def updater(self, name, semaphore): + """ + Update visas in the semaphore + """ + while True: + user = await semaphore.get() + if user.ga4gh_visas_v1: + for visa in user.ga4gh_visas_v1: + client = self._pick_client(visa) + print( + "Updater {} updating visa for user {}".format( + name, user.username + ) + ) + client.update_user_visas(user) + await asyncio.sleep(random.random()) + else: + print("User {} doesnt have visa. Skipping . . ".format(user.username)) + semaphore.task_done() def _pick_client(self, visa): """ @@ -103,3 +129,7 @@ def _pick_client(self, visa): HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger, ) + + def _verify_jwt_token(self, visa): + # TODO: Once local jwt verification is ready use thread_pool_size to determine how many users we want to verify the token for + pass From 854db6d8bdab8029655826b26f371ff9ed0a51e3 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 11:29:48 -0600 Subject: [PATCH 09/30] fence-create+logging --- bin/fence_create.py | 4 +++ fence/job/visa_update_cronjob.py | 45 +++++++++++++++++++++++++------- fence/scripting/fence_create.py | 15 +++++++++++ tests/ras/test_ras.py | 24 +++++------------ 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/bin/fence_create.py b/bin/fence_create.py index f191f8c92..90782203f 100755 --- a/bin/fence_create.py +++ b/bin/fence_create.py @@ -33,6 +33,7 @@ force_update_google_link, migrate_database, google_list_authz_groups, + update_user_visas, ) from fence.settings import CONFIG_SEARCH_FOLDERS @@ -339,6 +340,7 @@ def parse_arguments(): "Fence is providing access to. Includes Fence Project.auth_id and Google Bucket " "Access Group", ) + subparsers.add_parser("visa-update", help="Update visas and refresh tokens for users with valid visas and refresh tokens") return parser.parse_args() @@ -543,6 +545,8 @@ def main(): ) elif args.action == "migrate": migrate_database(DB) + elif args.action == "visa-update": + update_user_visas(DB) if __name__ == "__main__": diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 62f0c4e1f..53a4c7347 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -1,8 +1,9 @@ import asyncio +import datetime +import time from cdislogging import get_logger from userdatamodel.driver import SQLAlchemyDriver -import random from fence.config import config from fence.models import ( @@ -20,22 +21,41 @@ class Visa_Token_Update(object): def __init__( self, + window_size=None, # concurrency=None, # number of concurrent users going through the visa update flow thread_pool_size=None, # number of Docker container CPU used for jwt verification buffer_size=None, # max size of asyncio queue + logger=logger, ): + """ + args: + window_size: size of chunk of users we want to take from each iteration + concurrency: number of concurrent users going through the visa update flow + thread_pool_size: number of Docker container CPU used for jwt verifcation + buffer_size: max size of queue + """ + self.window_size = window_size or 8 self.concurrency = concurrency or 2 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency + self.logger = logger async def update_tokens(self, db_session): """ Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use looking at the type field in the ga4gh table. """ + start_time = time.time() + self.logger.info("Initializing Visa Update Cronjob . . .") + self.logger.info("Total concurrency size: {}".format(self.concurrency)) + self.logger.info("Total thread pool size: {}".format(self.thread_pool_size)) + self.logger.info("Total buffer size: {}".format(self.buffer_size)) + self.logger.info("Total numbner of workers: {}".format(self.n_workers)) + queue = asyncio.Queue(maxsize=self.buffer_size) semaphore = asyncio.Queue(maxsize=self.n_workers) + producers = [ asyncio.create_task(self.producer(db_session, queue, window_idx=0)) for _ in range(1) @@ -50,21 +70,28 @@ async def update_tokens(self, db_session): ] await asyncio.gather(*producers) - await queue.join() # blocks until everything in queue is complete + self.logger.info("Producers done producing") + await queue.join() await asyncio.gather(*workers) - await semaphore.join() # blocks until everything in semaphore is complete + await semaphore.join() # blocks until everything in semaphore is complete for w in workers: w.cancel() for u in updaters: u.cancel() + self.logger.info( + "Visa cron job completed in {}".format( + datetime.timedelta(seconds=time.time() - start_time) + ) + ) + async def window(self, db_session, queue, window_idx): """ window function to get chunks of data from the table """ - window_size = 8 + window_size = self.window_size start, stop = window_size * window_idx, window_size * (window_idx + 1) users = db_session.query(User).slice(start, stop).all() return users @@ -80,7 +107,7 @@ async def producer(self, db_session, queue, window_idx): if users == None: break for user in users: - # print("Producer producing user for user {}".format(user.username)) + self.logger.info("Producer producing user {}".format(user.username)) await queue.put(user) if len(users) < window_size: break @@ -93,7 +120,6 @@ async def worker(self, name, queue, semaphore): while True: user = await queue.get() await semaphore.put(user) - # print("Adding {} to semaphore".format(user.username)) queue.task_done() if queue.empty(): break @@ -107,15 +133,16 @@ async def updater(self, name, semaphore): if user.ga4gh_visas_v1: for visa in user.ga4gh_visas_v1: client = self._pick_client(visa) - print( + self.logger.info( "Updater {} updating visa for user {}".format( name, user.username ) ) client.update_user_visas(user) - await asyncio.sleep(random.random()) else: - print("User {} doesnt have visa. Skipping . . ".format(user.username)) + self.logger.info( + "User {} doesnt have visa. Skipping . . .".format(user.username) + ) semaphore.task_done() def _pick_client(self, visa): diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 0e7579c7e..d6846bd53 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -4,6 +4,7 @@ from yaml import safe_load import json import pprint +import asyncio from cirrus import GoogleCloudManager from cirrus.google_cloud.errors import GoogleAuthError @@ -34,6 +35,7 @@ generate_signed_refresh_token, issued_and_expiration_times, ) +from fence.job.visa_update_cronjob import Visa_Token_Update from fence.models import ( Client, GoogleServiceAccount, @@ -1494,3 +1496,16 @@ def google_list_authz_groups(db): print(", ".join(item[:-1])) return google_authz + + +async def update_user_visas(db): + """ + Update visas and refresh tokens for users with valid visas and refresh tokens + + db (string): database instance + """ + driver = SQLAlchemyDriver(db) + job = Visa_Token_Update() + + with driver.session as db_session: + asyncio.run(job.update_tokens(db_session)) diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 792dfbbe8..495c068c6 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -351,7 +351,7 @@ def test_cronjob( kid_2, ): """ - Test to check visa table is updated when getting new visa + Test to check visa table is updated when updating visas using cronjob """ n_users = 50 @@ -380,17 +380,6 @@ def test_cronjob( add_visa_manually(db_session, test_user, rsa_private_key, kid) add_refresh_token(db_session, test_user) - visa_query = db_session.query(GA4GHVisaV1).filter_by(user=test_user).first() - initial_visa = visa_query.ga4gh_visa - assert initial_visa - - oidc = config.get("OPENID_CONNECT", {}) - ras_client = RASClient( - oidc["ras"], - HTTP_PROXY=config.get("HTTP_PROXY"), - logger=logger, - ) - new_visa = { "iss": "https://stsstg.nih.gov", "sub": "abcde12345aspdij", @@ -417,10 +406,11 @@ def test_cronjob( userinfo_response["ga4gh_passport_v1"] = [encoded_visa] mock_userinfo.return_value = userinfo_response - ras_client.update_user_visas(test_user) - - query_visa = db_session.query(GA4GHVisaV1).first() + # test "fence-create visa-update" + job = Visa_Token_Update() + asyncio.run(job.update_tokens(db_session)) - cronjob = Visa_Token_Update() + query_visas = db_session.query(GA4GHVisaV1).all() - asyncio.run(cronjob.update_tokens(db_session)) + for visa in query_visas: + assert visa.ga4gh_visa == encoded_visa From 3aef8c70996a3e48e7093c92b512869ee71e616a Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 11:40:50 -0600 Subject: [PATCH 10/30] fix asyncio --- fence/scripting/fence_create.py | 3 +- poetry.lock | 94 ++++++++++++++++++++------------- tests/ras/test_ras.py | 4 +- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index d6846bd53..238f8439e 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1508,4 +1508,5 @@ async def update_user_visas(db): job = Visa_Token_Update() with driver.session as db_session: - asyncio.run(job.update_tokens(db_session)) + loop = asyncio.get_event_loop() + loop.run_until_complete(job.update_tokens(db_session)) diff --git a/poetry.lock b/poetry.lock index 8fce0d51a..6a5731608 100644 --- a/poetry.lock +++ b/poetry.lock @@ -167,7 +167,7 @@ description = "Extensible memoizing collections and decorators" name = "cachetools" optional = false python-versions = "~=3.5" -version = "4.2.0" +version = "4.2.1" [[package]] category = "main" @@ -641,7 +641,7 @@ marker = "python_version >= \"3.5\"" name = "google-crc32c" optional = false python-versions = ">=3.6" -version = "1.1.1" +version = "1.1.2" [package.dependencies] cffi = ">=1.0.0" @@ -1252,8 +1252,8 @@ category = "main" description = "YAML parser and emitter for Python" name = "pyyaml" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "5.3.1" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +version = "5.4.1" [[package]] category = "main" @@ -1548,8 +1548,8 @@ cached-property = [ {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, ] cachetools = [ - {file = "cachetools-4.2.0-py3-none-any.whl", hash = "sha256:c6b07a6ded8c78bf36730b3dc452dfff7d95f2a12a2fed856b1a0cb13ca78c61"}, - {file = "cachetools-4.2.0.tar.gz", hash = "sha256:3796e1de094f0eaca982441c92ce96c68c89cced4cd97721ab297ea4b16db90e"}, + {file = "cachetools-4.2.1-py3-none-any.whl", hash = "sha256:1d9d5f567be80f7c07d765e21b814326d78c61eb0c3a637dffc0e5d1796cb2e2"}, + {file = "cachetools-4.2.1.tar.gz", hash = "sha256:f469e29e7aa4cff64d8de4aad95ce76de8ea1125a16c68e0d93f65c3c3dc92e9"}, ] cdiserrors = [ {file = "cdiserrors-0.1.2.tar.gz", hash = "sha256:d6a91162aa76c7af37b79a8bba1bc5ebd06beb331ac522d36a74c4b300f6f333"}, @@ -1780,27 +1780,35 @@ google-cloud-storage = [ {file = "google_cloud_storage-1.35.0-py2.py3-none-any.whl", hash = "sha256:7b48b74683dafec5da315c7b0861ab168c208e04c763aa5db7ea8d7ecd8b6c5d"}, ] google-crc32c = [ - {file = "google-crc32c-1.1.1.tar.gz", hash = "sha256:d585f5d962812354a6113f962d06d67c4177993d8ed1b99e3fbb9753b1b98d65"}, - {file = "google_crc32c-1.1.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703d3f2fe4426f403ac8eb1229bbf458780df795bd4fdb04b0b73400dd58c108"}, - {file = "google_crc32c-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:75a43a19a6df9366d4ce6c171ab46ea60a0995eedf3ab988c111796e02e8e5ce"}, - {file = "google_crc32c-1.1.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:38d8808446fd38c7084a7b21bfc0aa22cd97eda320f3a391f524390de84377f8"}, - {file = "google_crc32c-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:4c5f7c9a2d500ca4b6142708832439410fb2b1422b5cebbad7f455bc0c40753b"}, - {file = "google_crc32c-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:3060d2e0d415a15de5189c2f6f521723c69f45633f8b4a62db8402c456b583bc"}, - {file = "google_crc32c-1.1.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:2c4d71f5212f039de575c3ec17ee5b2c14f293ec9f95ce7d054194aa55499405"}, - {file = "google_crc32c-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:46654a852cea70d0a5ccbfb078bdadca43c01448e96f301e5677d3bd4a0a808d"}, - {file = "google_crc32c-1.1.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:be277a1861ffef89cf68f34a1acb63173811df9f7e4d3f136a6ff2f578d1f54d"}, - {file = "google_crc32c-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:df5bccf9f38e72c77c607f27781f3e0c5cf77a1938020829872eb167f295c744"}, - {file = "google_crc32c-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2071754849915d070d6eb04a6522e05a1ecf2a58e55230c50367e183d6dc825b"}, - {file = "google_crc32c-1.1.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8875822807ab288b8034ba29a2d9b927036e31455b20e4831473e4f3d8b3f01c"}, - {file = "google_crc32c-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:783278dbe80c79637bfff011a2a61e02d268fdb38766be5bf1c56c43076c881a"}, - {file = "google_crc32c-1.1.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5a92ce8dd5dd27d3386dfb2a53fd272814f03e2f41b6e941061345fc36954f54"}, - {file = "google_crc32c-1.1.1-cp38-cp38-win32.whl", hash = "sha256:e10342adc665c0d22d76ea0d044cf588a96dfbea56b564a82d9ec5ae0634c848"}, - {file = "google_crc32c-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:c981f2fc2dc50a400cca9c0536ffe9080f8c713908633617007530ddf302eec8"}, - {file = "google_crc32c-1.1.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:94e65d56383f234b1f39dffc3a5647769918e4f13d49db11df7b176035781a39"}, - {file = "google_crc32c-1.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1aed1db85b5862b969e93e9fef80e9c61d989447d52ea236f9f76af76dd84a51"}, - {file = "google_crc32c-1.1.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:a9d784f37637b992447a80d4312a10f948400e4cd7a626f22d23131238cda722"}, - {file = "google_crc32c-1.1.1-cp39-cp39-win32.whl", hash = "sha256:15f6d38bfc860463169b142da327a2e875266d4c0db83a3ec8e6366364ac9dd7"}, - {file = "google_crc32c-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:c83720353c57a5a800b51d3118e0297b00ede1413a9b49a0f75e61c96ee91e98"}, + {file = "google-crc32c-1.1.2.tar.gz", hash = "sha256:dff5bd1236737f66950999d25de7a78144548ebac7788d30ada8c1b6ead60b27"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:8ed8f6dc4f55850cba2eb22b78902ad37f397ee02692d3b8e00842e9af757321"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:110157fb19ab5db15603debfaf5fcfbac9627576787d9caf8618ff96821a7a1f"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:80abca603187093ea089cd1215c3779040dda55d3cdabc0cd5ea0e10df7bff99"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6789db0b12aab12a0f04de22ed8412dfa5f6abd5a342ea19f15355064e1cc387"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:ea170341a4a9078a067b431044cd56c73553425833a7c2bb81734777a230ad4b"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:a64e0e8ed6076a8d867fc4622ad821c55eba8dff1b48b18f56b7c2392e22ab9d"}, + {file = "google_crc32c-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9372211acbcc207f63ffaffea1d05f3244a21311e4710721ffff3e8b7a0d24d0"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:0ae3cf54e0d4d83c8af1afe96fc0970fbf32f1b29275f3bfd44ce25c4b622a2b"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:34a97937f164147aefa53c3277364fd3bfa7fd244cbebbd5a976fa8325fb496b"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:91ad96ee2958311d0bb75ffe5c25c87fb521ef547c09e04a8bb6143e75fb1367"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5ea1055fe470334ced844270e7c808b04fe31e3e6394675daa77f6789ca9eff"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:e6458c41236d37cb982120b070ebcc115687c852bee24cdd18792da2640cf44d"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:e5af77656e8d367701f40f80a91c985ca43319f322f0a36ba9f93909d0bc4cb2"}, + {file = "google_crc32c-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ae7b9e7e2ca1b06c3a68b6ef223947a52c30ffae329b1a2be3402756073f2732"}, + {file = "google_crc32c-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:7c5138ed2e815189ba524756e027ac5833365e86115b1c2e6d9e833974a58d82"}, + {file = "google_crc32c-1.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a6c8a712ffae56c805ca732b735af02860b246bed2c1acb38ea954a8b2dc4581"}, + {file = "google_crc32c-1.1.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:49838ede42592154f9fcd21d07c7a43a67b00a36e252f82ae72542fde09dc51f"}, + {file = "google_crc32c-1.1.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:ef2ed6d0ac4de4ac602903e203eccd25ec8e37f1446fe1a3d2953a658035e0a5"}, + {file = "google_crc32c-1.1.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:51f4aa06125bf0641f65fb83268853545dbeb36b98ccfec69ef57dcb6b73b176"}, + {file = "google_crc32c-1.1.2-cp38-cp38-win32.whl", hash = "sha256:1dc6904c0d958f43102c85d70792cca210d3d051ddbeecd0eff10abcd981fdfa"}, + {file = "google_crc32c-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:298a9a922d35b123a73be80233d0f19c6ea01f008743561a8937f9dd83fb586b"}, + {file = "google_crc32c-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ab2b31395fbeeae6d15c98bd7f8b9fb76a18f18f87adc11b1f6dbe8f90d8382f"}, + {file = "google_crc32c-1.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d4a0d4fb938c2c3c0076445c9bd1215a3bd3df557b88d8b05ec2889ca0c92f8d"}, + {file = "google_crc32c-1.1.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:d0630670d27785d7e610e72752dc8087436d00d2c7115e149c0a754babb56d3e"}, + {file = "google_crc32c-1.1.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:364eb36e8d9d34542c17b0c410035b0557edd4300a92ed736b237afaa0fd6dae"}, + {file = "google_crc32c-1.1.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:0dd9b61d0c63043b013349c9ec8a83ec2b05c96410c5bc257da5d0de743fc171"}, + {file = "google_crc32c-1.1.2-cp39-cp39-win32.whl", hash = "sha256:92ed6062792b989e84621e07a5f3d37da9cc3153b77d23a582921f14863af31d"}, + {file = "google_crc32c-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:78cf5b1bd30f3a6033b41aa4ce8c796870bc4645a15d3ef47a4b05d31b0a6dc1"}, ] google-resumable-media = [ {file = "google-resumable-media-1.2.0.tar.gz", hash = "sha256:ee98b1921e5bda94867a08c864e55b4763d63887664f49ee1c231988f56b9d43"}, @@ -2157,17 +2165,27 @@ pytz = [ {file = "pytz-2020.5.tar.gz", hash = "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"}, ] pyyaml = [ - {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, - {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, - {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, - {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, - {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, - {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, - {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, - {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, - {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, - {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, - {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 495c068c6..0dfec9cf0 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -408,7 +408,9 @@ def test_cronjob( # test "fence-create visa-update" job = Visa_Token_Update() - asyncio.run(job.update_tokens(db_session)) + loop = asyncio.get_event_loop() + loop.run_until_complete(job.update_tokens(db_session)) + # asyncio.run(job.update_tokens(db_session)) query_visas = db_session.query(GA4GHVisaV1).all() From 14e64fbb22ed560246eb08f86f835eeb844a8911 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 12:00:59 -0600 Subject: [PATCH 11/30] fix asyncio for python<3.6 --- fence/job/visa_update_cronjob.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 53a4c7347..c3b02bf64 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -55,17 +55,18 @@ async def update_tokens(self, db_session): queue = asyncio.Queue(maxsize=self.buffer_size) semaphore = asyncio.Queue(maxsize=self.n_workers) + loop = asyncio.get_event_loop() producers = [ - asyncio.create_task(self.producer(db_session, queue, window_idx=0)) + loop.create_task(self.producer(db_session, queue, window_idx=0)) for _ in range(1) ] workers = [ - asyncio.create_task(self.worker(j, queue, semaphore)) + loop.create_task(self.worker(j, queue, semaphore)) for j in range(self.n_workers) ] updaters = [ - asyncio.create_task(self.updater(i, semaphore)) + loop.create_task(self.updater(i, semaphore)) for i in range(self.concurrency) ] From 7c581ebedc88ce4bdc9b67bd4e0ac934eb7c460d Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 15:08:57 -0600 Subject: [PATCH 12/30] db session fixes --- bin/fence_create.py | 5 ++++- fence/job/visa_update_cronjob.py | 6 +++--- fence/resources/openid/idp_oauth2.py | 10 +++++----- fence/resources/openid/ras_oauth2.py | 8 ++++---- fence/scripting/fence_create.py | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/bin/fence_create.py b/bin/fence_create.py index 90782203f..63922766c 100755 --- a/bin/fence_create.py +++ b/bin/fence_create.py @@ -340,7 +340,10 @@ def parse_arguments(): "Fence is providing access to. Includes Fence Project.auth_id and Google Bucket " "Access Group", ) - subparsers.add_parser("visa-update", help="Update visas and refresh tokens for users with valid visas and refresh tokens") + subparsers.add_parser( + "visa-update", + help="Update visas and refresh tokens for users with valid visas and refresh tokens", + ) return parser.parse_args() diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index c3b02bf64..65d13213c 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -66,7 +66,7 @@ async def update_tokens(self, db_session): for j in range(self.n_workers) ] updaters = [ - loop.create_task(self.updater(i, semaphore)) + loop.create_task(self.updater(i, semaphore, db_session)) for i in range(self.concurrency) ] @@ -125,7 +125,7 @@ async def worker(self, name, queue, semaphore): if queue.empty(): break - async def updater(self, name, semaphore): + async def updater(self, name, semaphore, db_session): """ Update visas in the semaphore """ @@ -139,7 +139,7 @@ async def updater(self, name, semaphore): name, user.username ) ) - client.update_user_visas(user) + client.update_user_visas(user, db_session) else: self.logger.info( "User {} doesnt have visa. Skipping . . .".format(user.username) diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index 08a4f11f3..8c41e0571 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -133,7 +133,7 @@ def get_user_id(self, code): """ raise NotImplementedError() - def get_access_token(self, user, token_endpoint): + def get_access_token(self, user, token_endpoint, db_session): """ Get access_token using a refresh_token and store it in upstream_refresh_token table. @@ -158,11 +158,11 @@ def get_access_token(self, user, token_endpoint): ) new_refresh_token = token_response["refresh_token"] - self.store_refresh_token(user, refresh_token=new_refresh_token, expires=expires) + self.store_refresh_token(user, refresh_token=new_refresh_token, expires=expires, db_session=db_session) return token_response - def store_refresh_token(self, user, refresh_token, expires): + def store_refresh_token(self, user, refresh_token, expires, db_session): """ Store refresh token in db. """ @@ -172,6 +172,6 @@ def store_refresh_token(self, user, refresh_token, expires): refresh_token=refresh_token, expires=expires, ) - current_db_session = current_session.object_session(upstream_refresh_token) + current_db_session = db_session.object_session(upstream_refresh_token) current_db_session.add(upstream_refresh_token) - current_session.commit() + db_session.commit() diff --git a/fence/resources/openid/ras_oauth2.py b/fence/resources/openid/ras_oauth2.py index ef4dd057a..3f3bd7a85 100644 --- a/fence/resources/openid/ras_oauth2.py +++ b/fence/resources/openid/ras_oauth2.py @@ -103,7 +103,7 @@ def get_user_id(self, code): return {"username": username} @backoff.on_exception(backoff.expo, Exception, **DEFAULT_BACKOFF_SETTINGS) - def update_user_visas(self, user): + def update_user_visas(self, user, db_session): """ Updates user's RAS refresh token and uses the new access token to retrieve new visas from RAS's /userinfo endpoint and update the db with the new visa. @@ -111,7 +111,7 @@ def update_user_visas(self, user): - delete user's visas from db if we're not able to get a new visa """ user.ga4gh_visas_v1 = [] - current_session.commit() + db_session.commit() try: token_endpoint = self.get_value_from_discovery_doc("token_endpoint", "") @@ -139,7 +139,7 @@ def update_user_visas(self, user): ga4gh_visa=encoded_visa, ) - current_db_session = current_session.object_session(visa) + current_db_session = db_session.object_session(visa) current_db_session.add(visa) except Exception as e: @@ -147,4 +147,4 @@ def update_user_visas(self, user): f"Could not process visa '{encoded_visa}' - skipping this visa" ) self.logger.exception("{}: {}".format(err_msg, e), exc_info=True) - current_session.commit() + db_session.commit() diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 238f8439e..3ae756942 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1498,7 +1498,7 @@ def google_list_authz_groups(db): return google_authz -async def update_user_visas(db): +def update_user_visas(db): """ Update visas and refresh tokens for users with valid visas and refresh tokens From c17309ed39be9a2a8ebee847845eefec5d451869 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 15:15:53 -0600 Subject: [PATCH 13/30] small fixes --- fence/resources/openid/idp_oauth2.py | 4 ++-- fence/resources/openid/ras_oauth2.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index 8c41e0571..7bb12fc48 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -133,7 +133,7 @@ def get_user_id(self, code): """ raise NotImplementedError() - def get_access_token(self, user, token_endpoint, db_session): + def get_access_token(self, user, token_endpoint, db_session=None): """ Get access_token using a refresh_token and store it in upstream_refresh_token table. @@ -162,7 +162,7 @@ def get_access_token(self, user, token_endpoint, db_session): return token_response - def store_refresh_token(self, user, refresh_token, expires, db_session): + def store_refresh_token(self, user, refresh_token, expires, db_session=current_session): """ Store refresh token in db. """ diff --git a/fence/resources/openid/ras_oauth2.py b/fence/resources/openid/ras_oauth2.py index 3f3bd7a85..95f930a69 100644 --- a/fence/resources/openid/ras_oauth2.py +++ b/fence/resources/openid/ras_oauth2.py @@ -118,7 +118,7 @@ def update_user_visas(self, user, db_session): userinfo_endpoint = self.get_value_from_discovery_doc( "userinfo_endpoint", "" ) - token = self.get_access_token(user, token_endpoint) + token = self.get_access_token(user, token_endpoint, db_session) userinfo = self.get_userinfo(token, userinfo_endpoint) encoded_visas = userinfo.get("ga4gh_passport_v1", []) except Exception as e: From f080e186331527821a38f00464d179624e7c2af5 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 15:17:48 -0600 Subject: [PATCH 14/30] fixes --- fence/resources/openid/ras_oauth2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/resources/openid/ras_oauth2.py b/fence/resources/openid/ras_oauth2.py index 95f930a69..a3bb35200 100644 --- a/fence/resources/openid/ras_oauth2.py +++ b/fence/resources/openid/ras_oauth2.py @@ -103,7 +103,7 @@ def get_user_id(self, code): return {"username": username} @backoff.on_exception(backoff.expo, Exception, **DEFAULT_BACKOFF_SETTINGS) - def update_user_visas(self, user, db_session): + def update_user_visas(self, user, db_session=current_session): """ Updates user's RAS refresh token and uses the new access token to retrieve new visas from RAS's /userinfo endpoint and update the db with the new visa. From eb8c64ad296c800e86ce52e644575e5b5392e636 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 25 Jan 2021 16:35:27 -0600 Subject: [PATCH 15/30] try --- fence/job/visa_update_cronjob.py | 30 +++++++++++++++------------- fence/resources/openid/idp_oauth2.py | 2 ++ tests/ras/test_ras.py | 6 +++++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 65d13213c..5e849f806 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -130,27 +130,29 @@ async def updater(self, name, semaphore, db_session): Update visas in the semaphore """ while True: - user = await semaphore.get() - if user.ga4gh_visas_v1: - for visa in user.ga4gh_visas_v1: - client = self._pick_client(visa) - self.logger.info( - "Updater {} updating visa for user {}".format( - name, user.username + try: + user = await semaphore.get() + if user.ga4gh_visas_v1: + for visa in user.ga4gh_visas_v1: + client = self._pick_client(visa) + self.logger.info( + "Updater {} updating visa for user {}".format( + name, user.username + ) ) + client.update_user_visas(user, db_session) + else: + self.logger.info( + "User {} doesnt have visa. Skipping . . .".format(user.username) ) - client.update_user_visas(user, db_session) - else: - self.logger.info( - "User {} doesnt have visa. Skipping . . .".format(user.username) - ) - semaphore.task_done() + finally: + semaphore.task_done() def _pick_client(self, visa): """ Pick oidc client according to the visa provider """ - if visa.type == "https://ras/visa/v1": + if "ras" in visa.type: oidc = config.get("OPENID_CONNECT", {}) return RASClient( oidc["ras"], diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index 7bb12fc48..b0efedf27 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -155,6 +155,8 @@ def get_access_token(self, user, token_endpoint, db_session=None): url=token_endpoint, proxies=self.get_proxies(), refresh_token=refresh_token, + grant_type="refresh_token", + client_id=self.settings["client_id"], ) new_refresh_token = token_response["refresh_token"] diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 0dfec9cf0..b34e57beb 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -354,7 +354,8 @@ def test_cronjob( Test to check visa table is updated when updating visas using cronjob """ - n_users = 50 + n_users = 20 + n_users_no_visa = 20 mock_discovery.return_value = "https://ras/token_endpoint" new_token = "refresh12345abcdefg" @@ -379,6 +380,9 @@ def test_cronjob( test_user = add_test_user(db_session, username, i) add_visa_manually(db_session, test_user, rsa_private_key, kid) add_refresh_token(db_session, test_user) + for j in range(n_users_no_visa): + username = "no_visa_{}".format(j) + test_user = add_test_user(db_session, username, j+n_users) new_visa = { "iss": "https://stsstg.nih.gov", From ab7b5c795491373135d79d876c2a0d26a96ab027 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 26 Jan 2021 09:42:55 -0600 Subject: [PATCH 16/30] remove grant type --- fence/resources/openid/idp_oauth2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index b0efedf27..233d83baa 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -155,7 +155,6 @@ def get_access_token(self, user, token_endpoint, db_session=None): url=token_endpoint, proxies=self.get_proxies(), refresh_token=refresh_token, - grant_type="refresh_token", client_id=self.settings["client_id"], ) new_refresh_token = token_response["refresh_token"] From b92406cee7ec59b5dac89cf36ea09abc7d63f721 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 26 Jan 2021 11:30:18 -0600 Subject: [PATCH 17/30] remove try --- fence/job/visa_update_cronjob.py | 33 +++++++++++++--------------- fence/resources/openid/idp_oauth2.py | 1 - 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 5e849f806..cb5d65c2d 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -118,35 +118,32 @@ async def worker(self, name, queue, semaphore): """ Create tasks to pass tot updater to update visas AND pass updated visas to _verify_jwt_token for verification """ - while True: + while queue.empty(): user = await queue.get() await semaphore.put(user) queue.task_done() - if queue.empty(): - break async def updater(self, name, semaphore, db_session): """ Update visas in the semaphore """ while True: - try: - user = await semaphore.get() - if user.ga4gh_visas_v1: - for visa in user.ga4gh_visas_v1: - client = self._pick_client(visa) - self.logger.info( - "Updater {} updating visa for user {}".format( - name, user.username - ) - ) - client.update_user_visas(user, db_session) - else: + user = await semaphore.get() + if user.ga4gh_visas_v1: + for visa in user.ga4gh_visas_v1: + client = self._pick_client(visa) self.logger.info( - "User {} doesnt have visa. Skipping . . .".format(user.username) + "Updater {} updating visa for user {}".format( + name, user.username + ) ) - finally: - semaphore.task_done() + client.update_user_visas(user, db_session) + else: + self.logger.info( + "User {} doesnt have visa. Skipping . . .".format(user.username) + ) + + semaphore.task_done() def _pick_client(self, visa): """ diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index 233d83baa..7bb12fc48 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -155,7 +155,6 @@ def get_access_token(self, user, token_endpoint, db_session=None): url=token_endpoint, proxies=self.get_proxies(), refresh_token=refresh_token, - client_id=self.settings["client_id"], ) new_refresh_token = token_response["refresh_token"] From 955e3d064af45ba13a2f380445517f1ba7833a0e Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 26 Jan 2021 11:53:10 -0600 Subject: [PATCH 18/30] add not --- fence/job/visa_update_cronjob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index cb5d65c2d..4dbf89b4e 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -118,7 +118,7 @@ async def worker(self, name, queue, semaphore): """ Create tasks to pass tot updater to update visas AND pass updated visas to _verify_jwt_token for verification """ - while queue.empty(): + while not queue.empty(): user = await queue.get() await semaphore.put(user) queue.task_done() From 18ca3dc5db38838cab49db42e27e2eed5b169c49 Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 27 Jan 2021 11:20:41 -0600 Subject: [PATCH 19/30] add variables --- bin/fence_create.py | 30 +++++++- fence/job/visa_update_cronjob.py | 1 + fence/resources/openid/idp_oauth2.py | 11 ++- fence/scripting/fence_create.py | 12 ++- poetry.lock | 106 +++++++++++++-------------- tests/ras/test_ras.py | 4 +- 6 files changed, 102 insertions(+), 62 deletions(-) diff --git a/bin/fence_create.py b/bin/fence_create.py index 63922766c..1cf25a6d1 100755 --- a/bin/fence_create.py +++ b/bin/fence_create.py @@ -340,10 +340,26 @@ def parse_arguments(): "Fence is providing access to. Includes Fence Project.auth_id and Google Bucket " "Access Group", ) - subparsers.add_parser( - "visa-update", + update_visas = subparsers.add_parser( + "update-visas", help="Update visas and refresh tokens for users with valid visas and refresh tokens", ) + update_visas.add_argument( + "--window-size", + required=False, + help="size of chunk of users we want to take from each query to db", + ) + update_visas.add_argument( + "--concurrency", + required=False, + help="number of concurrent users going through the visa update flow", + ) + update_visas.add_argument( + "--thread-pool-size", + required=False, + help="number of Docker container CPU used for jwt verifcation", + ) + update_visas.add_argument("--buffer-size", required=False, help="max size of queue") return parser.parse_args() @@ -548,8 +564,14 @@ def main(): ) elif args.action == "migrate": migrate_database(DB) - elif args.action == "visa-update": - update_user_visas(DB) + elif args.action == "update-visas": + update_user_visas( + DB, + window_size=args.window_size, + concurrency=args.concurrency, + thread_pool_size=args.thread_pool_size, + buffer_size=args.buffer_size, + ) if __name__ == "__main__": diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 4dbf89b4e..b1e3f5c3e 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -158,5 +158,6 @@ def _pick_client(self, visa): ) def _verify_jwt_token(self, visa): + # NOT IMPLEMENTED # TODO: Once local jwt verification is ready use thread_pool_size to determine how many users we want to verify the token for pass diff --git a/fence/resources/openid/idp_oauth2.py b/fence/resources/openid/idp_oauth2.py index 7bb12fc48..763c7d09c 100644 --- a/fence/resources/openid/idp_oauth2.py +++ b/fence/resources/openid/idp_oauth2.py @@ -158,11 +158,18 @@ def get_access_token(self, user, token_endpoint, db_session=None): ) new_refresh_token = token_response["refresh_token"] - self.store_refresh_token(user, refresh_token=new_refresh_token, expires=expires, db_session=db_session) + self.store_refresh_token( + user, + refresh_token=new_refresh_token, + expires=expires, + db_session=db_session, + ) return token_response - def store_refresh_token(self, user, refresh_token, expires, db_session=current_session): + def store_refresh_token( + self, user, refresh_token, expires, db_session=current_session + ): """ Store refresh token in db. """ diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 3ae756942..6fe587a6d 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1498,14 +1498,24 @@ def google_list_authz_groups(db): return google_authz -def update_user_visas(db): +def update_user_visas( + db, window_size=None, concurrency=None, thread_pool_size=None, buffer_size=None +): """ Update visas and refresh tokens for users with valid visas and refresh tokens db (string): database instance + window_size (int): size of chunk of users we want to take from each iteration + concurrency (int): number of concurrent users going through the visa update flow + thread_pool_size (int): number of Docker container CPU used for jwt verifcation + buffer_size (int): max size of queue """ driver = SQLAlchemyDriver(db) job = Visa_Token_Update() + job.window_size = window_size + job.concurrency = concurrency + job.thread_pool_size = thread_pool_size + job.buffer_size = buffer_size with driver.session as db_session: loop = asyncio.get_event_loop() diff --git a/poetry.lock b/poetry.lock index 6a5731608..1a2abd7fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -285,7 +285,7 @@ description = "Code coverage measurement for Python" name = "coverage" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.3.1" +version = "5.4" [package.extras] toml = ["toml"] @@ -539,7 +539,7 @@ description = "Google API client core library" name = "google-api-core" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.25.0" +version = "1.25.1" [package.dependencies] google-auth = ">=1.21.1,<2.0dev" @@ -1620,55 +1620,55 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-5.3.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d"}, - {file = "coverage-5.3.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7"}, - {file = "coverage-5.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528"}, - {file = "coverage-5.3.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044"}, - {file = "coverage-5.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b"}, - {file = "coverage-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297"}, - {file = "coverage-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb"}, - {file = "coverage-5.3.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899"}, - {file = "coverage-5.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36"}, - {file = "coverage-5.3.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500"}, - {file = "coverage-5.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7"}, - {file = "coverage-5.3.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f"}, - {file = "coverage-5.3.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b"}, - {file = "coverage-5.3.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec"}, - {file = "coverage-5.3.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714"}, - {file = "coverage-5.3.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b"}, - {file = "coverage-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7"}, - {file = "coverage-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72"}, - {file = "coverage-5.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b"}, - {file = "coverage-5.3.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4"}, - {file = "coverage-5.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105"}, - {file = "coverage-5.3.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448"}, - {file = "coverage-5.3.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277"}, - {file = "coverage-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f"}, - {file = "coverage-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c"}, - {file = "coverage-5.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd"}, - {file = "coverage-5.3.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4"}, - {file = "coverage-5.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff"}, - {file = "coverage-5.3.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8"}, - {file = "coverage-5.3.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e"}, - {file = "coverage-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2"}, - {file = "coverage-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879"}, - {file = "coverage-5.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b"}, - {file = "coverage-5.3.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497"}, - {file = "coverage-5.3.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059"}, - {file = "coverage-5.3.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631"}, - {file = "coverage-5.3.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830"}, - {file = "coverage-5.3.1-cp38-cp38-win32.whl", hash = "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae"}, - {file = "coverage-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606"}, - {file = "coverage-5.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f"}, - {file = "coverage-5.3.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1"}, - {file = "coverage-5.3.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8"}, - {file = "coverage-5.3.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4"}, - {file = "coverage-5.3.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d"}, - {file = "coverage-5.3.1-cp39-cp39-win32.whl", hash = "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98"}, - {file = "coverage-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1"}, - {file = "coverage-5.3.1-pp36-none-any.whl", hash = "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3"}, - {file = "coverage-5.3.1-pp37-none-any.whl", hash = "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c"}, - {file = "coverage-5.3.1.tar.gz", hash = "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b"}, + {file = "coverage-5.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:6d9c88b787638a451f41f97446a1c9fd416e669b4d9717ae4615bd29de1ac135"}, + {file = "coverage-5.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:66a5aae8233d766a877c5ef293ec5ab9520929c2578fd2069308a98b7374ea8c"}, + {file = "coverage-5.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9754a5c265f991317de2bac0c70a746efc2b695cf4d49f5d2cddeac36544fb44"}, + {file = "coverage-5.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:fbb17c0d0822684b7d6c09915677a32319f16ff1115df5ec05bdcaaee40b35f3"}, + {file = "coverage-5.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b7f7421841f8db443855d2854e25914a79a1ff48ae92f70d0a5c2f8907ab98c9"}, + {file = "coverage-5.4-cp27-cp27m-win32.whl", hash = "sha256:4a780807e80479f281d47ee4af2eb2df3e4ccf4723484f77da0bb49d027e40a1"}, + {file = "coverage-5.4-cp27-cp27m-win_amd64.whl", hash = "sha256:87c4b38288f71acd2106f5d94f575bc2136ea2887fdb5dfe18003c881fa6b370"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6809ebcbf6c1049002b9ac09c127ae43929042ec1f1dbd8bb1615f7cd9f70a0"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ba7ca81b6d60a9f7a0b4b4e175dcc38e8fef4992673d9d6e6879fd6de00dd9b8"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:89fc12c6371bf963809abc46cced4a01ca4f99cba17be5e7d416ed7ef1245d19"}, + {file = "coverage-5.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4a8eb7785bd23565b542b01fb39115a975fefb4a82f23d407503eee2c0106247"}, + {file = "coverage-5.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:7e40d3f8eb472c1509b12ac2a7e24158ec352fc8567b77ab02c0db053927e339"}, + {file = "coverage-5.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1ccae21a076d3d5f471700f6d30eb486da1626c380b23c70ae32ab823e453337"}, + {file = "coverage-5.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:755c56beeacac6a24c8e1074f89f34f4373abce8b662470d3aa719ae304931f3"}, + {file = "coverage-5.4-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:322549b880b2d746a7672bf6ff9ed3f895e9c9f108b714e7360292aa5c5d7cf4"}, + {file = "coverage-5.4-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:60a3307a84ec60578accd35d7f0c71a3a971430ed7eca6567399d2b50ef37b8c"}, + {file = "coverage-5.4-cp35-cp35m-win32.whl", hash = "sha256:1375bb8b88cb050a2d4e0da901001347a44302aeadb8ceb4b6e5aa373b8ea68f"}, + {file = "coverage-5.4-cp35-cp35m-win_amd64.whl", hash = "sha256:16baa799ec09cc0dcb43a10680573269d407c159325972dd7114ee7649e56c66"}, + {file = "coverage-5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2f2cf7a42d4b7654c9a67b9d091ec24374f7c58794858bff632a2039cb15984d"}, + {file = "coverage-5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b62046592b44263fa7570f1117d372ae3f310222af1fc1407416f037fb3af21b"}, + {file = "coverage-5.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:812eaf4939ef2284d29653bcfee9665f11f013724f07258928f849a2306ea9f9"}, + {file = "coverage-5.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:859f0add98707b182b4867359e12bde806b82483fb12a9ae868a77880fc3b7af"}, + {file = "coverage-5.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:04b14e45d6a8e159c9767ae57ecb34563ad93440fc1b26516a89ceb5b33c1ad5"}, + {file = "coverage-5.4-cp36-cp36m-win32.whl", hash = "sha256:ebfa374067af240d079ef97b8064478f3bf71038b78b017eb6ec93ede1b6bcec"}, + {file = "coverage-5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:84df004223fd0550d0ea7a37882e5c889f3c6d45535c639ce9802293b39cd5c9"}, + {file = "coverage-5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1b811662ecf72eb2d08872731636aee6559cae21862c36f74703be727b45df90"}, + {file = "coverage-5.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6b588b5cf51dc0fd1c9e19f622457cc74b7d26fe295432e434525f1c0fae02bc"}, + {file = "coverage-5.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3fe50f1cac369b02d34ad904dfe0771acc483f82a1b54c5e93632916ba847b37"}, + {file = "coverage-5.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:32ab83016c24c5cf3db2943286b85b0a172dae08c58d0f53875235219b676409"}, + {file = "coverage-5.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:68fb816a5dd901c6aff352ce49e2a0ffadacdf9b6fae282a69e7a16a02dad5fb"}, + {file = "coverage-5.4-cp37-cp37m-win32.whl", hash = "sha256:a636160680c6e526b84f85d304e2f0bb4e94f8284dd765a1911de9a40450b10a"}, + {file = "coverage-5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:bb32ca14b4d04e172c541c69eec5f385f9a075b38fb22d765d8b0ce3af3a0c22"}, + {file = "coverage-5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4d7165a4e8f41eca6b990c12ee7f44fef3932fac48ca32cecb3a1b2223c21f"}, + {file = "coverage-5.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a565f48c4aae72d1d3d3f8e8fb7218f5609c964e9c6f68604608e5958b9c60c3"}, + {file = "coverage-5.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fff1f3a586246110f34dc762098b5afd2de88de507559e63553d7da643053786"}, + {file = "coverage-5.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a839e25f07e428a87d17d857d9935dd743130e77ff46524abb992b962eb2076c"}, + {file = "coverage-5.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:6625e52b6f346a283c3d563d1fd8bae8956daafc64bb5bbd2b8f8a07608e3994"}, + {file = "coverage-5.4-cp38-cp38-win32.whl", hash = "sha256:5bee3970617b3d74759b2d2df2f6a327d372f9732f9ccbf03fa591b5f7581e39"}, + {file = "coverage-5.4-cp38-cp38-win_amd64.whl", hash = "sha256:03ed2a641e412e42cc35c244508cf186015c217f0e4d496bf6d7078ebe837ae7"}, + {file = "coverage-5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:14a9f1887591684fb59fdba8feef7123a0da2424b0652e1b58dd5b9a7bb1188c"}, + {file = "coverage-5.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9564ac7eb1652c3701ac691ca72934dd3009997c81266807aef924012df2f4b3"}, + {file = "coverage-5.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:0f48fc7dc82ee14aeaedb986e175a429d24129b7eada1b7e94a864e4f0644dde"}, + {file = "coverage-5.4-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:107d327071061fd4f4a2587d14c389a27e4e5c93c7cba5f1f59987181903902f"}, + {file = "coverage-5.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0cdde51bfcf6b6bd862ee9be324521ec619b20590787d1655d005c3fb175005f"}, + {file = "coverage-5.4-cp39-cp39-win32.whl", hash = "sha256:c67734cff78383a1f23ceba3b3239c7deefc62ac2b05fa6a47bcd565771e5880"}, + {file = "coverage-5.4-cp39-cp39-win_amd64.whl", hash = "sha256:c669b440ce46ae3abe9b2d44a913b5fd86bb19eb14a8701e88e3918902ecd345"}, + {file = "coverage-5.4-pp36-none-any.whl", hash = "sha256:c0ff1c1b4d13e2240821ef23c1efb1f009207cb3f56e16986f713c2b0e7cd37f"}, + {file = "coverage-5.4-pp37-none-any.whl", hash = "sha256:cd601187476c6bed26a0398353212684c427e10a903aeafa6da40c63309d438b"}, + {file = "coverage-5.4.tar.gz", hash = "sha256:6d2e262e5e8da6fa56e774fb8e2643417351427604c2b177f8e8c5f75fc928ca"}, ] coveralls = [ {file = "coveralls-2.2.0-py2.py3-none-any.whl", hash = "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc"}, @@ -1756,8 +1756,8 @@ gen3users = [ {file = "gen3users-0.6.0.tar.gz", hash = "sha256:3b9b56798a7d8b34712389dbbab93c00b0f92524f890513f899c31630ea986da"}, ] google-api-core = [ - {file = "google-api-core-1.25.0.tar.gz", hash = "sha256:d967beae8d8acdb88fb2f6f769e2ee0ee813042576a08891bded3b8e234150ae"}, - {file = "google_api_core-1.25.0-py2.py3-none-any.whl", hash = "sha256:4656345cba9627ab1290eab51300a6397cc50370d99366133df1ae64b744e1eb"}, + {file = "google-api-core-1.25.1.tar.gz", hash = "sha256:0e152ec37b8481d1be1258d95844a5a7031cd3d83d7c7046d9e9b2d807042440"}, + {file = "google_api_core-1.25.1-py2.py3-none-any.whl", hash = "sha256:292dd636ed381098d24b7093ccb826b2278a12d886a3fc982084069aa24a8fbb"}, ] google-api-python-client = [ {file = "google-api-python-client-1.11.0.tar.gz", hash = "sha256:caf4015800ef1a18d06d117f47f0219c0c0641f21978f6b1bb5ede7912fab97b"}, diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index b34e57beb..20376ce7f 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -382,7 +382,7 @@ def test_cronjob( add_refresh_token(db_session, test_user) for j in range(n_users_no_visa): username = "no_visa_{}".format(j) - test_user = add_test_user(db_session, username, j+n_users) + test_user = add_test_user(db_session, username, j + n_users) new_visa = { "iss": "https://stsstg.nih.gov", @@ -410,7 +410,7 @@ def test_cronjob( userinfo_response["ga4gh_passport_v1"] = [encoded_visa] mock_userinfo.return_value = userinfo_response - # test "fence-create visa-update" + # test "fence-create update-visa" job = Visa_Token_Update() loop = asyncio.get_event_loop() loop.run_until_complete(job.update_tokens(db_session)) From c986f651d7771e59a4482ddb4dca9afcdc1bc42b Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 27 Jan 2021 13:32:04 -0600 Subject: [PATCH 20/30] fix --- fence/job/visa_update_cronjob.py | 3 +-- fence/scripting/fence_create.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index b1e3f5c3e..bf4fb3718 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -92,8 +92,7 @@ async def window(self, db_session, queue, window_idx): """ window function to get chunks of data from the table """ - window_size = self.window_size - start, stop = window_size * window_idx, window_size * (window_idx + 1) + start, stop = self.window_size * window_idx, self.window_size * (window_idx + 1) users = db_session.query(User).slice(start, stop).all() return users diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 6fe587a6d..5191f0e1c 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1511,12 +1511,12 @@ def update_user_visas( buffer_size (int): max size of queue """ driver = SQLAlchemyDriver(db) - job = Visa_Token_Update() - job.window_size = window_size - job.concurrency = concurrency - job.thread_pool_size = thread_pool_size - job.buffer_size = buffer_size - + job = Visa_Token_Update( + window_size=window_size, + concurrency=concurrency, + thread_pool_size=thread_pool_size, + buffer_size=buffer_size, + ) with driver.session as db_session: loop = asyncio.get_event_loop() loop.run_until_complete(job.update_tokens(db_session)) From 478628bc5d796ff095fcdcb34b11f5b7cfedc6db Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 27 Jan 2021 14:07:45 -0600 Subject: [PATCH 21/30] int --- fence/scripting/fence_create.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 5191f0e1c..cf11f97e8 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1512,10 +1512,10 @@ def update_user_visas( """ driver = SQLAlchemyDriver(db) job = Visa_Token_Update( - window_size=window_size, - concurrency=concurrency, - thread_pool_size=thread_pool_size, - buffer_size=buffer_size, + window_size=int(window_size) if not window_size else None, + concurrency=int(concurrency) if not concurrency else None, + thread_pool_size=int(thread_pool_size) if not thread_pool_size else Non, + buffer_size=int(buffer_size) if not buffer_size else None, ) with driver.session as db_session: loop = asyncio.get_event_loop() From f51238e6d4abfaec283cae9b2c4f5e0b8d8eaa0a Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 27 Jan 2021 14:08:04 -0600 Subject: [PATCH 22/30] fix --- fence/scripting/fence_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index cf11f97e8..62bee2fdf 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1514,7 +1514,7 @@ def update_user_visas( job = Visa_Token_Update( window_size=int(window_size) if not window_size else None, concurrency=int(concurrency) if not concurrency else None, - thread_pool_size=int(thread_pool_size) if not thread_pool_size else Non, + thread_pool_size=int(thread_pool_size) if not thread_pool_size else None, buffer_size=int(buffer_size) if not buffer_size else None, ) with driver.session as db_session: From 9be8bddfbed22f827c27327e8a2cf0c702f4589c Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 27 Jan 2021 14:36:48 -0600 Subject: [PATCH 23/30] change small things --- fence/job/visa_update_cronjob.py | 4 ++-- fence/scripting/fence_create.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index bf4fb3718..c3a73f6e1 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -34,8 +34,8 @@ def __init__( thread_pool_size: number of Docker container CPU used for jwt verifcation buffer_size: max size of queue """ - self.window_size = window_size or 8 - self.concurrency = concurrency or 2 + self.window_size = window_size or 10 + self.concurrency = concurrency or 5 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 62bee2fdf..df0ce5b5b 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1512,10 +1512,10 @@ def update_user_visas( """ driver = SQLAlchemyDriver(db) job = Visa_Token_Update( - window_size=int(window_size) if not window_size else None, - concurrency=int(concurrency) if not concurrency else None, - thread_pool_size=int(thread_pool_size) if not thread_pool_size else None, - buffer_size=int(buffer_size) if not buffer_size else None, + window_size=int(window_size) if window_size else None, + concurrency=int(concurrency) if concurrency else None, + thread_pool_size=int(thread_pool_size) if thread_pool_size else None, + buffer_size=int(buffer_size) if buffer_size else None, ) with driver.session as db_session: loop = asyncio.get_event_loop() From fb29b0ec0f9a28c1d8fdcec14809db18f8727a82 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 8 Feb 2021 11:44:39 -0600 Subject: [PATCH 24/30] fix things --- fence/job/visa_update_cronjob.py | 84 +++++++++-------- poetry.lock | 153 ++++++++++++++++--------------- tests/ras/test_ras.py | 10 +- 3 files changed, 129 insertions(+), 118 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index c3a73f6e1..90fcb1c70 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -21,52 +21,67 @@ class Visa_Token_Update(object): def __init__( self, - window_size=None, # - concurrency=None, # number of concurrent users going through the visa update flow - thread_pool_size=None, # number of Docker container CPU used for jwt verification - buffer_size=None, # max size of asyncio queue + chunk_size=None, + concurrency=None, + thread_pool_size=None, + buffer_size=None, logger=logger, ): """ args: - window_size: size of chunk of users we want to take from each iteration + chunk_size: size of chunk of users we want to take from each iteration concurrency: number of concurrent users going through the visa update flow thread_pool_size: number of Docker container CPU used for jwt verifcation buffer_size: max size of queue """ - self.window_size = window_size or 10 + self.chunk_size = chunk_size or 10 self.concurrency = concurrency or 5 self.thread_pool_size = thread_pool_size or 2 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency self.logger = logger + # Initialize visa clients: + oidc = config.get("OPENID_CONNECT", {}) + if "ras" not in oidc: + self.logger.error("RAS client not configured") + else: + self.ras_client = RASClient( + oidc["ras"], + HTTP_PROXY=config.get("HTTP_PROXY"), + logger=logger, + ) + async def update_tokens(self, db_session): """ - Have dictionary or something to decide which client to use. Can go through the whole list and decide which client to use - looking at the type field in the ga4gh table. + Initialize a producer-consumer workflow. + + Producer: Collects users from db and feeds it to the workers + Worker: Takes in the users from the Producer and passes it to the Updater to update the tokens and passes those updated tokens for JWT validation + Updater: Updates refresh_tokens and visas + """ start_time = time.time() self.logger.info("Initializing Visa Update Cronjob . . .") self.logger.info("Total concurrency size: {}".format(self.concurrency)) self.logger.info("Total thread pool size: {}".format(self.thread_pool_size)) self.logger.info("Total buffer size: {}".format(self.buffer_size)) - self.logger.info("Total numbner of workers: {}".format(self.n_workers)) + self.logger.info("Total number of workers: {}".format(self.n_workers)) queue = asyncio.Queue(maxsize=self.buffer_size) - semaphore = asyncio.Queue(maxsize=self.n_workers) + updater_queue = asyncio.Queue(maxsize=self.n_workers) loop = asyncio.get_event_loop() producers = [ - loop.create_task(self.producer(db_session, queue, window_idx=0)) + loop.create_task(self.producer(db_session, queue, chunk_idx=0)) for _ in range(1) ] workers = [ - loop.create_task(self.worker(j, queue, semaphore)) + loop.create_task(self.worker(j, queue, updater_queue)) for j in range(self.n_workers) ] updaters = [ - loop.create_task(self.updater(i, semaphore, db_session)) + loop.create_task(self.updater(i, updater_queue, db_session)) for i in range(self.concurrency) ] @@ -75,59 +90,57 @@ async def update_tokens(self, db_session): await queue.join() await asyncio.gather(*workers) - await semaphore.join() # blocks until everything in semaphore is complete + await updater_queue.join() # blocks until everything in updater_queue is complete - for w in workers: - w.cancel() for u in updaters: u.cancel() - + self.logger.info( "Visa cron job completed in {}".format( datetime.timedelta(seconds=time.time() - start_time) ) ) - async def window(self, db_session, queue, window_idx): + async def get_user_from_db(self, db_session, queue, chunk_idx): """ - window function to get chunks of data from the table + Window function to get chunks of data from the table """ - start, stop = self.window_size * window_idx, self.window_size * (window_idx + 1) + start, stop = self.chunk_size * chunk_idx, self.chunk_size * (chunk_idx + 1) users = db_session.query(User).slice(start, stop).all() return users - async def producer(self, db_session, queue, window_idx): + async def producer(self, db_session, queue, chunk_idx): """ Produces users from db and puts them in a queue for processing """ - window_size = 8 + chunk_size = self.chunk_size while True: - users = await self.window(db_session, queue, window_idx) + users = await self.get_user_from_db(db_session, queue, chunk_idx) if users == None: break for user in users: self.logger.info("Producer producing user {}".format(user.username)) await queue.put(user) - if len(users) < window_size: + if len(users) < chunk_size: break - window_idx += 1 + chunk_idx += 1 - async def worker(self, name, queue, semaphore): + async def worker(self, name, queue, updater_queue): """ - Create tasks to pass tot updater to update visas AND pass updated visas to _verify_jwt_token for verification + Create tasks to pass to updater to update visas AND pass updated visas to _verify_jwt_token for verification """ while not queue.empty(): user = await queue.get() - await semaphore.put(user) + await updater_queue.put(user) queue.task_done() - async def updater(self, name, semaphore, db_session): + async def updater(self, name, updater_queue, db_session): """ - Update visas in the semaphore + Update visas in the updater_queue """ while True: - user = await semaphore.get() + user = await updater_queue.get() if user.ga4gh_visas_v1: for visa in user.ga4gh_visas_v1: client = self._pick_client(visa) @@ -142,19 +155,14 @@ async def updater(self, name, semaphore, db_session): "User {} doesnt have visa. Skipping . . .".format(user.username) ) - semaphore.task_done() + updater_queue.task_done() def _pick_client(self, visa): """ Pick oidc client according to the visa provider """ if "ras" in visa.type: - oidc = config.get("OPENID_CONNECT", {}) - return RASClient( - oidc["ras"], - HTTP_PROXY=config.get("HTTP_PROXY"), - logger=logger, - ) + return self.ras_client def _verify_jwt_token(self, visa): # NOT IMPLEMENTED diff --git a/poetry.lock b/poetry.lock index 1a2abd7fa..8d0ee19c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,7 +12,7 @@ description = "A library for parsing ISO 8601 strings." name = "aniso8601" optional = false python-versions = "*" -version = "8.1.0" +version = "8.1.1" [[package]] category = "main" @@ -577,7 +577,7 @@ description = "Google Authentication Library" name = "google-auth" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.24.0" +version = "1.25.0" [package.dependencies] cachetools = ">=2.0.0,<5.0" @@ -611,10 +611,11 @@ description = "Google Cloud API client core library" name = "google-cloud-core" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.5.0" +version = "1.6.0" [package.dependencies] google-api-core = ">=1.21.0,<2.0.0dev" +google-auth = ">=1.24.0,<2.0dev" six = ">=1.12.0" [package.extras] @@ -626,7 +627,7 @@ description = "Google Cloud Storage API client library" name = "google-cloud-storage" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.35.0" +version = "1.35.1" [package.dependencies] google-auth = ">=1.11.0,<2.0dev" @@ -688,7 +689,10 @@ description = "A comprehensive HTTP client library." name = "httplib2" optional = false python-versions = "*" -version = "0.18.1" +version = "0.19.0" + +[package.dependencies] +pyparsing = ">=2.4.2,<3" [[package]] category = "main" @@ -742,7 +746,7 @@ description = "Simple module to parse ISO 8601 dates" name = "iso8601" optional = false python-versions = "*" -version = "0.1.13" +version = "0.1.14" [[package]] category = "main" @@ -758,7 +762,7 @@ description = "A very fast and expressive template engine." name = "jinja2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.11.2" +version = "2.11.3" [package.dependencies] MarkupSafe = ">=0.23" @@ -820,7 +824,7 @@ description = "More routines for operating on iterables, beyond itertools" name = "more-itertools" optional = false python-versions = ">=3.5" -version = "8.6.0" +version = "8.7.0" [[package]] category = "dev" @@ -948,7 +952,7 @@ description = "Oslo Serialization library" name = "oslo.serialization" optional = false python-versions = ">=3.6" -version = "4.0.1" +version = "4.1.0" [package.dependencies] msgpack = ">=0.5.2" @@ -982,7 +986,7 @@ description = "Core utilities for Python packages" name = "packaging" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "20.8" +version = "20.9" [package.dependencies] pyparsing = ">=2.0.2" @@ -1245,7 +1249,7 @@ description = "World timezone definitions, modern and historical" name = "pytz" optional = false python-versions = "*" -version = "2020.5" +version = "2021.1" [[package]] category = "main" @@ -1348,7 +1352,7 @@ description = "Database Abstraction Library" name = "sqlalchemy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.3.22" +version = "1.3.23" [package.extras] mssql = ["pyodbc"] @@ -1357,10 +1361,10 @@ mssql_pyodbc = ["pyodbc"] mysql = ["mysqlclient"] oracle = ["cx-oracle"] postgresql = ["psycopg2"] -postgresql_pg8000 = ["pg8000"] +postgresql_pg8000 = ["pg8000 (<1.16.6)"] postgresql_psycopg2binary = ["psycopg2-binary"] postgresql_psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql"] +pymysql = ["pymysql (<1)", "pymysql"] [[package]] category = "main" @@ -1492,8 +1496,8 @@ addict = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] aniso8601 = [ - {file = "aniso8601-8.1.0-py2.py3-none-any.whl", hash = "sha256:51047d4fb51d7b8afd522b70f2d21a1b2487cbb7f7bd84ea852e9aa7808e7704"}, - {file = "aniso8601-8.1.0.tar.gz", hash = "sha256:246bf8d3611527030889e6df970878969d3a2f760ba3eb694fa1fb10e6ce53f9"}, + {file = "aniso8601-8.1.1-py2.py3-none-any.whl", hash = "sha256:f59914762c5049ffd956cad037aa82fe0cabf8baf51900e2af24026761090b0b"}, + {file = "aniso8601-8.1.1.tar.gz", hash = "sha256:be08b19c19ca527af722f2d4ba4dc569db292ec96f7de963746df4bb0bff9250"}, ] argparse = [ {file = "argparse-1.4.0-py2.py3-none-any.whl", hash = "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"}, @@ -1764,20 +1768,20 @@ google-api-python-client = [ {file = "google_api_python_client-1.11.0-py2.py3-none-any.whl", hash = "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6"}, ] google-auth = [ - {file = "google-auth-1.24.0.tar.gz", hash = "sha256:0b0e026b412a0ad096e753907559e4bdb180d9ba9f68dd9036164db4fdc4ad2e"}, - {file = "google_auth-1.24.0-py2.py3-none-any.whl", hash = "sha256:ce752cc51c31f479dbf9928435ef4b07514b20261b021c7383bee4bda646acb8"}, + {file = "google-auth-1.25.0.tar.gz", hash = "sha256:514e39f4190ca972200ba33876da5a8857c5665f2b4ccc36c8b8ee21228aae80"}, + {file = "google_auth-1.25.0-py2.py3-none-any.whl", hash = "sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d"}, ] google-auth-httplib2 = [ {file = "google-auth-httplib2-0.0.4.tar.gz", hash = "sha256:8d092cc60fb16517b12057ec0bba9185a96e3b7169d86ae12eae98e645b7bc39"}, {file = "google_auth_httplib2-0.0.4-py2.py3-none-any.whl", hash = "sha256:aeaff501738b289717fac1980db9711d77908a6c227f60e4aa1923410b43e2ee"}, ] google-cloud-core = [ - {file = "google-cloud-core-1.5.0.tar.gz", hash = "sha256:1277a015f8eeb014c48f2ec094ed5368358318f1146cf49e8de389962dc19106"}, - {file = "google_cloud_core-1.5.0-py2.py3-none-any.whl", hash = "sha256:99a8a15f406f53f2b11bda1f45f952a9cdfbdbba8abf40c75651019d800879f5"}, + {file = "google-cloud-core-1.6.0.tar.gz", hash = "sha256:c6abb18527545379fc82efc4de75ce9a3772ccad2fc645adace593ba097cbb02"}, + {file = "google_cloud_core-1.6.0-py2.py3-none-any.whl", hash = "sha256:40d9c2da2d03549b5ac3dcccf289d4f15e6d1210044c6381ce45c92913e62904"}, ] google-cloud-storage = [ - {file = "google-cloud-storage-1.35.0.tar.gz", hash = "sha256:555c0db2f88f3419f123bf9c621d7fd92f7c9e4f8b11f08eda57facacba16a9e"}, - {file = "google_cloud_storage-1.35.0-py2.py3-none-any.whl", hash = "sha256:7b48b74683dafec5da315c7b0861ab168c208e04c763aa5db7ea8d7ecd8b6c5d"}, + {file = "google-cloud-storage-1.35.1.tar.gz", hash = "sha256:dc076b6af6da991252416639cb93831f8e50c8328d5ac3fb8e03e40cd8de2290"}, + {file = "google_cloud_storage-1.35.1-py2.py3-none-any.whl", hash = "sha256:69fa8feda06768c44dd635c2cd8d59267362a0c8323afc65cf24efc4d88a0fcc"}, ] google-crc32c = [ {file = "google-crc32c-1.1.2.tar.gz", hash = "sha256:dff5bd1236737f66950999d25de7a78144548ebac7788d30ada8c1b6ead60b27"}, @@ -1819,8 +1823,8 @@ googleapis-common-protos = [ {file = "googleapis_common_protos-1.52.0-py2.py3-none-any.whl", hash = "sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24"}, ] httplib2 = [ - {file = "httplib2-0.18.1-py3-none-any.whl", hash = "sha256:ca2914b015b6247791c4866782fa6042f495b94401a0f0bd3e1d6e0ba2236782"}, - {file = "httplib2-0.18.1.tar.gz", hash = "sha256:8af66c1c52c7ffe1aa5dc4bcd7c769885254b0756e6e69f953c7f0ab49a70ba3"}, + {file = "httplib2-0.19.0-py3-none-any.whl", hash = "sha256:749c32603f9bf16c1277f59531d502e8f1c2ca19901ae653b49c4ed698f0820e"}, + {file = "httplib2-0.19.0.tar.gz", hash = "sha256:e0d428dad43c72dbce7d163b7753ffc7a39c097e6788ef10f4198db69b92f08e"}, ] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, @@ -1835,17 +1839,16 @@ importlib-resources = [ {file = "importlib_resources-5.1.0.tar.gz", hash = "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380"}, ] iso8601 = [ - {file = "iso8601-0.1.13-py2.py3-none-any.whl", hash = "sha256:694be0743e9f1535ea873bfc7bd6fb62380c62b75822761859428073a17fd39c"}, - {file = "iso8601-0.1.13-py3-none-any.whl", hash = "sha256:6f02f01dd13320a7f280e58516dc8d1950dfaf77527cc365a398cd9de4d3c692"}, - {file = "iso8601-0.1.13.tar.gz", hash = "sha256:f7dec22af52025d4526be94cc1303c7d8f5379b746a3f54a8c8446384392eeb1"}, + {file = "iso8601-0.1.14-py2.py3-none-any.whl", hash = "sha256:e7e1122f064d626e17d47cd5106bed2c620cb38fe464999e0ddae2b6d2de6004"}, + {file = "iso8601-0.1.14.tar.gz", hash = "sha256:8aafd56fa0290496c5edbb13c311f78fa3a241f0853540da09d9363eae3ebd79"}, ] itsdangerous = [ {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, ] jinja2 = [ - {file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"}, - {file = "Jinja2-2.11.2.tar.gz", hash = "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0"}, + {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"}, + {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"}, ] jmespath = [ {file = "jmespath-0.9.2-py2.py3-none-any.whl", hash = "sha256:3f03b90ac8e0f3ba472e8ebff083e460c89501d8d41979771535efe9a343177e"}, @@ -1895,8 +1898,8 @@ mock = [ {file = "mock-2.0.0.tar.gz", hash = "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba"}, ] more-itertools = [ - {file = "more-itertools-8.6.0.tar.gz", hash = "sha256:b3a9005928e5bed54076e6e549c792b306fddfe72b2d1d22dd63d42d5d3899cf"}, - {file = "more_itertools-8.6.0-py3-none-any.whl", hash = "sha256:8e1a2a43b2f2727425f2b5839587ae37093f19153dc26c0927d1048ff6557330"}, + {file = "more-itertools-8.7.0.tar.gz", hash = "sha256:c5d6da9ca3ff65220c3bfd2a8db06d698f05d4d2b9be57e1deb2be5a45019713"}, + {file = "more_itertools-8.7.0-py3-none-any.whl", hash = "sha256:5652a9ac72209ed7df8d9c15daf4e1aa0e3d2ccd3c87f8265a0673cd9cbc9ced"}, ] moto = [ {file = "moto-1.3.15-py2.py3-none-any.whl", hash = "sha256:3be7e1f406ef7e9c222dbcbfd8cefa2cb1062200e26deae49b5df446e17be3df"}, @@ -1972,16 +1975,16 @@ oauth2client = [ {file = "oslo.i18n-5.0.1.tar.gz", hash = "sha256:3484b71e30f75c437523302d1151c291caf4098928269ceec65ce535456e035b"}, ] "oslo.serialization" = [ - {file = "oslo.serialization-4.0.1-py3-none-any.whl", hash = "sha256:6ed86d8378a845a24363d036bc06aef9a5ddb5b4bbfd3478d3daedee0f2bb984"}, - {file = "oslo.serialization-4.0.1.tar.gz", hash = "sha256:f84d3dca7ffbb86394e273094c674532b6144223eca8990a38836ba99728d53e"}, + {file = "oslo.serialization-4.1.0-py3-none-any.whl", hash = "sha256:a0acf0ff7ca88b3ee6514713571f614b5c20870005ed0eb90408fa7f9f3edb60"}, + {file = "oslo.serialization-4.1.0.tar.gz", hash = "sha256:cecc7794df806c85cb70dbd6c2b3af19bc68047ad29e3c6442be90a0a4de5379"}, ] "oslo.utils" = [ {file = "oslo.utils-4.7.0-py3-none-any.whl", hash = "sha256:51c1127c518e44beb4b565eb86f6f33e0a441d9a3e98eaff9f58a20933687b5d"}, {file = "oslo.utils-4.7.0.tar.gz", hash = "sha256:b95b02c60dd4fac00d46525f73d27dc4e196c19281215aea84e8d389466b1b90"}, ] packaging = [ - {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, - {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, + {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, + {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -2161,8 +2164,8 @@ python-keystoneclient = [ {file = "python_keystoneclient-1.8.1-py2.py3-none-any.whl", hash = "sha256:6835a58e0a533947fca317fff487f040af5fbc1d7b78c75d9f812c09b0307740"}, ] pytz = [ - {file = "pytz-2020.5-py2.py3-none-any.whl", hash = "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4"}, - {file = "pytz-2020.5.tar.gz", hash = "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"}, + {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, + {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, ] pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, @@ -2216,44 +2219,44 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.3.22-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:61628715931f4962e0cdb2a7c87ff39eea320d2aa96bd471a3c293d146f90394"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:81d8d099a49f83111cce55ec03cc87eef45eec0d90f9842b4fc674f860b857b0"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:d055ff750fcab69ca4e57b656d9c6ad33682e9b8d564f2fbe667ab95c63591b0"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27m-win32.whl", hash = "sha256:9bf572e4f5aa23f88dd902f10bb103cb5979022a38eec684bfa6d61851173fec"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27m-win_amd64.whl", hash = "sha256:7d4b8de6bb0bc736161cb0bbd95366b11b3eb24dd6b814a143d8375e75af9990"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4a84c7c7658dd22a33dab2e2aa2d17c18cb004a42388246f2e87cb4085ef2811"}, - {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:f1e88b30da8163215eab643962ae9d9252e47b4ea53404f2c4f10f24e70ddc62"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:f115150cc4361dd46153302a640c7fa1804ac207f9cc356228248e351a8b4676"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6aaa13ee40c4552d5f3a59f543f0db6e31712cc4009ec7385407be4627259d41"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3ab5b44a07b8c562c6dcb7433c6a6c6e03266d19d64f87b3333eda34e3b9936b"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:426ece890153ccc52cc5151a1a0ed540a5a7825414139bb4c95a868d8da54a52"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-win32.whl", hash = "sha256:bd4b1af45fd322dcd1fb2a9195b4f93f570d1a5902a842e3e6051385fac88f9c"}, - {file = "SQLAlchemy-1.3.22-cp35-cp35m-win_amd64.whl", hash = "sha256:62285607a5264d1f91590abd874d6a498e229d5840669bd7d9f654cfaa599bd0"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:314f5042c0b047438e19401d5f29757a511cfc2f0c40d28047ca0e4c95eabb5b"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:62fb881ba51dbacba9af9b779211cf9acff3442d4f2993142015b22b3cd1f92a"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:bde677047305fe76c7ee3e4492b545e0018918e44141cc154fe39e124e433991"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:0c6406a78a714a540d980a680b86654feadb81c8d0eecb59f3d6c554a4c69f19"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-win32.whl", hash = "sha256:95bde07d19c146d608bccb9b16e144ec8f139bcfe7fd72331858698a71c9b4f5"}, - {file = "SQLAlchemy-1.3.22-cp36-cp36m-win_amd64.whl", hash = "sha256:888d5b4b5aeed0d3449de93ea80173653e939e916cc95fe8527079e50235c1d2"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:d53f59744b01f1440a1b0973ed2c3a7de204135c593299ee997828aad5191693"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:70121f0ae48b25ef3e56e477b88cd0b0af0e1f3a53b5554071aa6a93ef378a03"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:54da615e5b92c339e339fe8536cce99fe823b6ed505d4ea344852aefa1c205fb"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:68428818cf80c60dc04aa0f38da20ad39b28aba4d4d199f949e7d6e04444ea86"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-win32.whl", hash = "sha256:17610d573e698bf395afbbff946544fbce7c5f4ee77b5bcb1f821b36345fae7a"}, - {file = "SQLAlchemy-1.3.22-cp37-cp37m-win_amd64.whl", hash = "sha256:216ba5b4299c95ed179b58f298bda885a476b16288ab7243e89f29f6aeced7e0"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:0c72b90988be749e04eff0342dcc98c18a14461eb4b2ad59d611b57b31120f90"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:491fe48adc07d13e020a8b07ef82eefc227003a046809c121bea81d3dbf1832d"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:f8191fef303025879e6c3548ecd8a95aafc0728c764ab72ec51a0bdf0c91a341"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:108580808803c7732f34798eb4a329d45b04c562ed83ee90f09f6a184a42b766"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-win32.whl", hash = "sha256:bab5a1e15b9466a25c96cda19139f3beb3e669794373b9ce28c4cf158c6e841d"}, - {file = "SQLAlchemy-1.3.22-cp38-cp38-win_amd64.whl", hash = "sha256:318b5b727e00662e5fc4b4cd2bf58a5116d7c1b4dd56ffaa7d68f43458a8d1ed"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:1418f5e71d6081aa1095a1d6b567a562d2761996710bdce9b6e6ba20a03d0864"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5a7f224cdb7233182cec2a45d4c633951268d6a9bcedac37abbf79dd07012aea"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:715b34578cc740b743361f7c3e5f584b04b0f1344f45afc4e87fbac4802eb0a0"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2ff132a379838b1abf83c065be54cef32b47c987aedd06b82fc76476c85225eb"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-win32.whl", hash = "sha256:c389d7cc2b821853fb018c85457da3e7941db64f4387720a329bc7ff06a27963"}, - {file = "SQLAlchemy-1.3.22-cp39-cp39-win_amd64.whl", hash = "sha256:04f995fcbf54e46cddeb4f75ce9dfc17075d6ae04ac23b2bacb44b3bc6f6bf11"}, - {file = "SQLAlchemy-1.3.22.tar.gz", hash = "sha256:758fc8c4d6c0336e617f9f6919f9daea3ab6bb9b07005eda9a1a682e24a6cacc"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:fd3b96f8c705af8e938eaa99cbd8fd1450f632d38cad55e7367c33b263bf98ec"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:29cccc9606750fe10c5d0e8bd847f17a97f3850b8682aef1f56f5d5e1a5a64b1"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:927ce09e49bff3104459e1451ce82983b0a3062437a07d883a4c66f0b344c9b5"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-win32.whl", hash = "sha256:b4b0e44d586cd64b65b507fa116a3814a1a53d55dce4836d7c1a6eb2823ff8d1"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27m-win_amd64.whl", hash = "sha256:6b8b8c80c7f384f06825612dd078e4a31f0185e8f1f6b8c19e188ff246334205"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9e9c25522933e569e8b53ccc644dc993cab87e922fb7e142894653880fdd419d"}, + {file = "SQLAlchemy-1.3.23-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:a0e306e9bb76fd93b29ae3a5155298e4c1b504c7cbc620c09c20858d32d16234"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:6c9e6cc9237de5660bcddea63f332428bb83c8e2015c26777281f7ffbd2efb84"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:94f667d86be82dd4cb17d08de0c3622e77ca865320e0b95eae6153faa7b4ecaf"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:751934967f5336a3e26fc5993ccad1e4fee982029f9317eb6153bc0bc3d2d2da"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:63677d0c08524af4c5893c18dbe42141de7178001360b3de0b86217502ed3601"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-win32.whl", hash = "sha256:ddfb511e76d016c3a160910642d57f4587dc542ce5ee823b0d415134790eeeb9"}, + {file = "SQLAlchemy-1.3.23-cp35-cp35m-win_amd64.whl", hash = "sha256:040bdfc1d76a9074717a3f43455685f781c581f94472b010cd6c4754754e1862"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:d1a85dfc5dee741bf49cb9b6b6b8d2725a268e4992507cf151cba26b17d97c37"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:639940bbe1108ac667dcffc79925db2966826c270112e9159439ab6bb14f8d80"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:e8a1750b44ad6422ace82bf3466638f1aa0862dbb9689690d5f2f48cce3476c8"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e5bb3463df697279e5459a7316ad5a60b04b0107f9392e88674d0ece70e9cf70"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-win32.whl", hash = "sha256:e273367f4076bd7b9a8dc2e771978ef2bfd6b82526e80775a7db52bff8ca01dd"}, + {file = "SQLAlchemy-1.3.23-cp36-cp36m-win_amd64.whl", hash = "sha256:ac2244e64485c3778f012951fdc869969a736cd61375fde6096d08850d8be729"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:23927c3981d1ec6b4ea71eb99d28424b874d9c696a21e5fbd9fa322718be3708"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d90010304abb4102123d10cbad2cdf2c25a9f2e66a50974199b24b468509bad5"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:a8bfc1e1afe523e94974132d7230b82ca7fa2511aedde1f537ec54db0399541a"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:269990b3ab53cb035d662dcde51df0943c1417bdab707dc4a7e4114a710504b4"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-win32.whl", hash = "sha256:fdd2ed7395df8ac2dbb10cefc44737b66c6a5cd7755c92524733d7a443e5b7e2"}, + {file = "SQLAlchemy-1.3.23-cp37-cp37m-win_amd64.whl", hash = "sha256:6a939a868fdaa4b504e8b9d4a61f21aac11e3fecc8a8214455e144939e3d2aea"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:24f9569e82a009a09ce2d263559acb3466eba2617203170e4a0af91e75b4f075"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2578dbdbe4dbb0e5126fb37ffcd9793a25dcad769a95f171a2161030bea850ff"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:1fe5d8d39118c2b018c215c37b73fd6893c3e1d4895be745ca8ff6eb83333ed3"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:c7dc052432cd5d060d7437e217dd33c97025287f99a69a50e2dc1478dd610d64"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-win32.whl", hash = "sha256:ecce8c021894a77d89808222b1ff9687ad84db54d18e4bd0500ca766737faaf6"}, + {file = "SQLAlchemy-1.3.23-cp38-cp38-win_amd64.whl", hash = "sha256:37b83bf81b4b85dda273aaaed5f35ea20ad80606f672d94d2218afc565fb0173"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:8be835aac18ec85351385e17b8665bd4d63083a7160a017bef3d640e8e65cadb"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6ec1044908414013ebfe363450c22f14698803ce97fbb47e53284d55c5165848"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:eab063a70cca4a587c28824e18be41d8ecc4457f8f15b2933584c6c6cccd30f0"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:baeb451ee23e264de3f577fee5283c73d9bbaa8cb921d0305c0bbf700094b65b"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-win32.whl", hash = "sha256:94208867f34e60f54a33a37f1c117251be91a47e3bfdb9ab8a7847f20886ad06"}, + {file = "SQLAlchemy-1.3.23-cp39-cp39-win_amd64.whl", hash = "sha256:f4d972139d5000105fcda9539a76452039434013570d6059993120dc2a65e447"}, + {file = "SQLAlchemy-1.3.23.tar.gz", hash = "sha256:6fca33672578666f657c131552c4ef8979c1606e494f78cd5199742dfb26918b"}, ] stevedore = [ {file = "stevedore-3.3.0-py3-none-any.whl", hash = "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a"}, diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 20376ce7f..476d38fdb 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -7,7 +7,6 @@ from fence.models import User, UpstreamRefreshToken, GA4GHVisaV1 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient -from fence.config import config from fence.job.visa_update_cronjob import Visa_Token_Update import tests.utils @@ -340,7 +339,7 @@ def test_update_visa_token_with_invalid_visa( @mock.patch( "fence.resources.openid.ras_oauth2.RASOauth2Client.get_value_from_discovery_doc" ) -def test_cronjob( +def test_visa_update_cronjob( mock_discovery, mock_get_token, mock_userinfo, @@ -355,7 +354,7 @@ def test_cronjob( """ n_users = 20 - n_users_no_visa = 20 + n_users_no_visa = 15 mock_discovery.return_value = "https://ras/token_endpoint" new_token = "refresh12345abcdefg" @@ -376,7 +375,7 @@ def test_cronjob( } for i in range(n_users): - username = "admin_user_{}".format(i) + username = "user_{}".format(i) test_user = add_test_user(db_session, username, i) add_visa_manually(db_session, test_user, rsa_private_key, kid) add_refresh_token(db_session, test_user) @@ -414,9 +413,10 @@ def test_cronjob( job = Visa_Token_Update() loop = asyncio.get_event_loop() loop.run_until_complete(job.update_tokens(db_session)) - # asyncio.run(job.update_tokens(db_session)) query_visas = db_session.query(GA4GHVisaV1).all() + assert len(query_visas) == n_users + for visa in query_visas: assert visa.ga4gh_visa == encoded_visa From 476a3a7b54d92c6055433d74d6c8a169a0603743 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 8 Feb 2021 13:56:17 -0600 Subject: [PATCH 25/30] add help --- bin/fence_create.py | 16 +++++++++------- fence/job/visa_update_cronjob.py | 18 +++++++++--------- fence/scripting/fence_create.py | 6 +++--- tests/ras/test_ras.py | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/bin/fence_create.py b/bin/fence_create.py index 1cf25a6d1..438d756ce 100755 --- a/bin/fence_create.py +++ b/bin/fence_create.py @@ -342,24 +342,26 @@ def parse_arguments(): ) update_visas = subparsers.add_parser( "update-visas", - help="Update visas and refresh tokens for users with valid visas and refresh tokens", + help="Update visas and refresh tokens for users with valid visas and refresh tokens.", ) update_visas.add_argument( - "--window-size", + "--chunk-size", required=False, - help="size of chunk of users we want to take from each query to db", + help="size of chunk of users we want to take from each query to db. Default value: 10", ) update_visas.add_argument( "--concurrency", required=False, - help="number of concurrent users going through the visa update flow", + help="number of concurrent users going through the visa update flow. Default value: 5", ) update_visas.add_argument( "--thread-pool-size", required=False, - help="number of Docker container CPU used for jwt verifcation", + help="number of Docker container CPU used for jwt verifcation. Default value: 3", + ) + update_visas.add_argument( + "--buffer-size", required=False, help="max size of queue. Default value: 10" ) - update_visas.add_argument("--buffer-size", required=False, help="max size of queue") return parser.parse_args() @@ -567,7 +569,7 @@ def main(): elif args.action == "update-visas": update_user_visas( DB, - window_size=args.window_size, + chunk_size=args.chunk_size, concurrency=args.concurrency, thread_pool_size=args.thread_pool_size, buffer_size=args.buffer_size, diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 90fcb1c70..be749e26f 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -36,7 +36,7 @@ def __init__( """ self.chunk_size = chunk_size or 10 self.concurrency = concurrency or 5 - self.thread_pool_size = thread_pool_size or 2 + self.thread_pool_size = thread_pool_size or 3 self.buffer_size = buffer_size or 10 self.n_workers = self.thread_pool_size + self.concurrency self.logger = logger @@ -47,10 +47,10 @@ def __init__( self.logger.error("RAS client not configured") else: self.ras_client = RASClient( - oidc["ras"], - HTTP_PROXY=config.get("HTTP_PROXY"), - logger=logger, - ) + oidc["ras"], + HTTP_PROXY=config.get("HTTP_PROXY"), + logger=logger, + ) async def update_tokens(self, db_session): """ @@ -58,7 +58,7 @@ async def update_tokens(self, db_session): Producer: Collects users from db and feeds it to the workers Worker: Takes in the users from the Producer and passes it to the Updater to update the tokens and passes those updated tokens for JWT validation - Updater: Updates refresh_tokens and visas + Updater: Updates refresh_tokens and visas by calling the update_user_visas from the correct client """ start_time = time.time() @@ -94,14 +94,14 @@ async def update_tokens(self, db_session): for u in updaters: u.cancel() - + self.logger.info( "Visa cron job completed in {}".format( datetime.timedelta(seconds=time.time() - start_time) ) ) - async def get_user_from_db(self, db_session, queue, chunk_idx): + async def get_user_from_db(self, db_session, chunk_idx): """ Window function to get chunks of data from the table """ @@ -115,7 +115,7 @@ async def producer(self, db_session, queue, chunk_idx): """ chunk_size = self.chunk_size while True: - users = await self.get_user_from_db(db_session, queue, chunk_idx) + users = await self.get_user_from_db(db_session, chunk_idx) if users == None: break diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index df0ce5b5b..92d13f570 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -1499,20 +1499,20 @@ def google_list_authz_groups(db): def update_user_visas( - db, window_size=None, concurrency=None, thread_pool_size=None, buffer_size=None + db, chunk_size=None, concurrency=None, thread_pool_size=None, buffer_size=None ): """ Update visas and refresh tokens for users with valid visas and refresh tokens db (string): database instance - window_size (int): size of chunk of users we want to take from each iteration + chunk_size (int): size of chunk of users we want to take from each iteration concurrency (int): number of concurrent users going through the visa update flow thread_pool_size (int): number of Docker container CPU used for jwt verifcation buffer_size (int): max size of queue """ driver = SQLAlchemyDriver(db) job = Visa_Token_Update( - window_size=int(window_size) if window_size else None, + chunk_size=int(chunk_size) if chunk_size else None, concurrency=int(concurrency) if concurrency else None, thread_pool_size=int(thread_pool_size) if thread_pool_size else None, buffer_size=int(buffer_size) if buffer_size else None, diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 476d38fdb..0afcb9ecd 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -5,6 +5,7 @@ from cdislogging import get_logger +from fence.config import config from fence.models import User, UpstreamRefreshToken, GA4GHVisaV1 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient from fence.job.visa_update_cronjob import Visa_Token_Update @@ -343,7 +344,6 @@ def test_visa_update_cronjob( mock_discovery, mock_get_token, mock_userinfo, - config, db_session, rsa_private_key, kid, From 6b7141524165bf2e3c395d70a742c18705b21395 Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 10 Feb 2021 13:05:27 -0600 Subject: [PATCH 26/30] lock update --- poetry.lock | 82 +++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8d0ee19c3..bf4fc47db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -539,11 +539,12 @@ description = "Google API client core library" name = "google-api-core" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.25.1" +version = "1.26.0" [package.dependencies] google-auth = ">=1.21.1,<2.0dev" googleapis-common-protos = ">=1.6.0,<2.0dev" +packaging = ">=14.3" protobuf = ">=3.12.0" pytz = "*" requests = ">=2.18.0,<3.0.0dev" @@ -577,7 +578,7 @@ description = "Google Authentication Library" name = "google-auth" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.25.0" +version = "1.26.0" [package.dependencies] cachetools = ">=2.0.0,<5.0" @@ -1101,8 +1102,8 @@ category = "main" description = "Cryptographic library for Python" name = "pycryptodome" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "3.9.9" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "3.10.1" [[package]] category = "main" @@ -1760,16 +1761,16 @@ gen3users = [ {file = "gen3users-0.6.0.tar.gz", hash = "sha256:3b9b56798a7d8b34712389dbbab93c00b0f92524f890513f899c31630ea986da"}, ] google-api-core = [ - {file = "google-api-core-1.25.1.tar.gz", hash = "sha256:0e152ec37b8481d1be1258d95844a5a7031cd3d83d7c7046d9e9b2d807042440"}, - {file = "google_api_core-1.25.1-py2.py3-none-any.whl", hash = "sha256:292dd636ed381098d24b7093ccb826b2278a12d886a3fc982084069aa24a8fbb"}, + {file = "google-api-core-1.26.0.tar.gz", hash = "sha256:4230ec764d48ca934fe69b85cc217e31e844e176f68df93e252acd55350e730b"}, + {file = "google_api_core-1.26.0-py2.py3-none-any.whl", hash = "sha256:002e44c533299aecd9dd265d200f9eacd9957cddd2c72e2cd1cb5cea127e972d"}, ] google-api-python-client = [ {file = "google-api-python-client-1.11.0.tar.gz", hash = "sha256:caf4015800ef1a18d06d117f47f0219c0c0641f21978f6b1bb5ede7912fab97b"}, {file = "google_api_python_client-1.11.0-py2.py3-none-any.whl", hash = "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6"}, ] google-auth = [ - {file = "google-auth-1.25.0.tar.gz", hash = "sha256:514e39f4190ca972200ba33876da5a8857c5665f2b4ccc36c8b8ee21228aae80"}, - {file = "google_auth-1.25.0-py2.py3-none-any.whl", hash = "sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d"}, + {file = "google-auth-1.26.0.tar.gz", hash = "sha256:26cf3e839be936bbd2f65465cbdecd41590a603a51891a9fd9077716ddc3f726"}, + {file = "google_auth-1.26.0-py2.py3-none-any.whl", hash = "sha256:4c95ef8d2a9afb11015b387a2d2d1d86d8c9170f0b89d07c0f773b852dd695de"}, ] google-auth-httplib2 = [ {file = "google-auth-httplib2-0.0.4.tar.gz", hash = "sha256:8d092cc60fb16517b12057ec0bba9185a96e3b7169d86ae12eae98e645b7bc39"}, @@ -2077,41 +2078,36 @@ pycparser = [ {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, ] pycryptodome = [ - {file = "pycryptodome-3.9.9-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:5598dc6c9dbfe882904e54584322893eff185b98960bbe2cdaaa20e8a437b6e5"}, - {file = "pycryptodome-3.9.9-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1cfdb92dca388e27e732caa72a1cc624520fe93752a665c3b6cd8f1a91b34916"}, - {file = "pycryptodome-3.9.9-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5f19e6ef750f677d924d9c7141f54bade3cd56695bbfd8a9ef15d0378557dfe4"}, - {file = "pycryptodome-3.9.9-cp27-cp27m-win32.whl", hash = "sha256:a3d8a9efa213be8232c59cdc6b65600276508e375e0a119d710826248fd18d37"}, - {file = "pycryptodome-3.9.9-cp27-cp27m-win_amd64.whl", hash = "sha256:50826b49fbca348a61529693b0031cdb782c39060fb9dca5ac5dff858159dc5a"}, - {file = "pycryptodome-3.9.9-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:19cb674df6c74a14b8b408aa30ba8a89bd1c01e23505100fb45f930fbf0ed0d9"}, - {file = "pycryptodome-3.9.9-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:28f75e58d02019a7edc7d4135203d2501dfc47256d175c72c9798f9a129a49a7"}, - {file = "pycryptodome-3.9.9-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:6d3baaf82681cfb1a842f1c8f77beac791ceedd99af911e4f5fabec32bae2259"}, - {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:946399d15eccebafc8ce0257fc4caffe383c75e6b0633509bd011e357368306c"}, - {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:eb01f9997e4d6a8ec8a1ad1f676ba5a362781ff64e8189fe2985258ba9cb9706"}, - {file = "pycryptodome-3.9.9-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:411745c6dce4eff918906eebcde78771d44795d747e194462abb120d2e537cd9"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:8f9f84059039b672a5a705b3c5aa21747867bacc30a72e28bf0d147cc8ef85ed"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:7798e73225a699651888489fbb1dbc565e03a509942a8ce6194bbe6fb582a41f"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:46e96aeb8a9ca8b1edf9b1fd0af4bf6afcf3f1ca7fa35529f5d60b98f3e4e959"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:843e5f10ecdf9d307032b8b91afe9da1d6ed5bb89d0bbec5c8dcb4ba44008e11"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-win32.whl", hash = "sha256:b68794fba45bdb367eeb71249c26d23e61167510a1d0c3d6cf0f2f14636e62ee"}, - {file = "pycryptodome-3.9.9-cp36-cp36m-win_amd64.whl", hash = "sha256:60febcf5baf70c566d9d9351c47fbd8321da9a4edf2eff45c4c31c86164ca794"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:4ed27951b0a17afd287299e2206a339b5b6d12de9321e1a1575261ef9c4a851b"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:9000877383e2189dafd1b2fc68c6c726eca9a3cfb6d68148fbb72ccf651959b6"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:faa682c404c218e8788c3126c9a4b8fbcc54dc245b5b6e8ea5b46f3b63bd0c84"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:62c488a21c253dadc9f731a32f0ac61e4e436d81a1ea6f7d1d9146ed4d20d6bd"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-win32.whl", hash = "sha256:834b790bbb6bd18956f625af4004d9c15eed12d5186d8e57851454ae76d52215"}, - {file = "pycryptodome-3.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:70d807d11d508433daf96244ec1c64e55039e8a35931fc5ea9eee94dbe3cb6b5"}, - {file = "pycryptodome-3.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:27397aee992af69d07502126561d851ba3845aa808f0e55c71ad0efa264dd7d4"}, - {file = "pycryptodome-3.9.9-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d7ec2bd8f57c559dd24e71891c51c25266a8deb66fc5f02cc97c7fb593d1780a"}, - {file = "pycryptodome-3.9.9-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e15bde67ccb7d4417f627dd16ffe2f5a4c2941ce5278444e884cb26d73ecbc61"}, - {file = "pycryptodome-3.9.9-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5c3c4865730dfb0263f822b966d6d58429d8b1e560d1ddae37685fd9e7c63161"}, - {file = "pycryptodome-3.9.9-cp38-cp38-win32.whl", hash = "sha256:76b1a34d74bb2c91bce460cdc74d1347592045627a955e9a252554481c17c52f"}, - {file = "pycryptodome-3.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:6e4227849e4231a3f5b35ea5bdedf9a82b3883500e5624f00a19156e9a9ef861"}, - {file = "pycryptodome-3.9.9-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2a68df525b387201a43b27b879ce8c08948a430e883a756d6c9e3acdaa7d7bd8"}, - {file = "pycryptodome-3.9.9-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:a4599c0ca0fc027c780c1c45ed996d5bef03e571470b7b1c7171ec1e1a90914c"}, - {file = "pycryptodome-3.9.9-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b4e6b269a8ddaede774e5c3adbef6bf452ee144e6db8a716d23694953348cd86"}, - {file = "pycryptodome-3.9.9-cp39-cp39-win32.whl", hash = "sha256:a199e9ca46fc6e999e5f47fce342af4b56c7de85fae893c69ab6aa17531fb1e1"}, - {file = "pycryptodome-3.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:6e89bb3826e6f84501e8e3b205c22595d0c5492c2f271cbb9ee1c48eb1866645"}, - {file = "pycryptodome-3.9.9.tar.gz", hash = "sha256:910e202a557e1131b1c1b3f17a63914d57aac55cf9fb9b51644962841c3995c4"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:3f840c49d38986f6e17dbc0673d37947c88bc9d2d9dba1c01b979b36f8447db1"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:2dea65df54349cdfa43d6b2e8edb83f5f8d6861e5cf7b1fbc3e34c5694c85e27"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e61e363d9a5d7916f3a4ce984a929514c0df3daf3b1b2eb5e6edbb131ee771cf"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:2603c98ae04aac675fefcf71a6c87dc4bb74a75e9071ae3923bbc91a59f08d35"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-win32.whl", hash = "sha256:38661348ecb71476037f1e1f553159b80d256c00f6c0b00502acac891f7116d9"}, + {file = "pycryptodome-3.10.1-cp27-cp27m-win_amd64.whl", hash = "sha256:1723ebee5561628ce96748501cdaa7afaa67329d753933296321f0be55358dce"}, + {file = "pycryptodome-3.10.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:77997519d8eb8a4adcd9a47b9cec18f9b323e296986528186c0e9a7a15d6a07e"}, + {file = "pycryptodome-3.10.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:99b2f3fc51d308286071d0953f92055504a6ffe829a832a9fc7a04318a7683dd"}, + {file = "pycryptodome-3.10.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e0a4d5933a88a2c98bbe19c0c722f5483dc628d7a38338ac2cb64a7dbd34064b"}, + {file = "pycryptodome-3.10.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d3d6958d53ad307df5e8469cc44474a75393a434addf20ecd451f38a72fe29b8"}, + {file = "pycryptodome-3.10.1-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:a8eb8b6ea09ec1c2535bf39914377bc8abcab2c7d30fa9225eb4fe412024e427"}, + {file = "pycryptodome-3.10.1-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:31c1df17b3dc5f39600a4057d7db53ac372f492c955b9b75dd439f5d8b460129"}, + {file = "pycryptodome-3.10.1-cp35-abi3-manylinux1_i686.whl", hash = "sha256:a3105a0eb63eacf98c2ecb0eb4aa03f77f40fbac2bdde22020bb8a536b226bb8"}, + {file = "pycryptodome-3.10.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a92d5c414e8ee1249e850789052608f582416e82422502dc0ac8c577808a9067"}, + {file = "pycryptodome-3.10.1-cp35-abi3-manylinux2010_i686.whl", hash = "sha256:60386d1d4cfaad299803b45a5bc2089696eaf6cdd56f9fc17479a6f89595cfc8"}, + {file = "pycryptodome-3.10.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:501ab36aae360e31d0ec370cf5ce8ace6cb4112060d099b993bc02b36ac83fb6"}, + {file = "pycryptodome-3.10.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:fc7489a50323a0df02378bc2fff86eb69d94cc5639914346c736be981c6a02e7"}, + {file = "pycryptodome-3.10.1-cp35-abi3-win32.whl", hash = "sha256:9b6f711b25e01931f1c61ce0115245a23cdc8b80bf8539ac0363bdcf27d649b6"}, + {file = "pycryptodome-3.10.1-cp35-abi3-win_amd64.whl", hash = "sha256:7fd519b89585abf57bf47d90166903ec7b43af4fe23c92273ea09e6336af5c07"}, + {file = "pycryptodome-3.10.1-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:09c1555a3fa450e7eaca41ea11cd00afe7c91fef52353488e65663777d8524e0"}, + {file = "pycryptodome-3.10.1-pp27-pypy_73-manylinux1_x86_64.whl", hash = "sha256:758949ca62690b1540dfb24ad773c6da9cd0e425189e83e39c038bbd52b8e438"}, + {file = "pycryptodome-3.10.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:e3bf558c6aeb49afa9f0c06cee7fb5947ee5a1ff3bd794b653d39926b49077fa"}, + {file = "pycryptodome-3.10.1-pp27-pypy_73-win32.whl", hash = "sha256:f977cdf725b20f6b8229b0c87acb98c7717e742ef9f46b113985303ae12a99da"}, + {file = "pycryptodome-3.10.1-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6d2df5223b12437e644ce0a3be7809471ffa71de44ccd28b02180401982594a6"}, + {file = "pycryptodome-3.10.1-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:98213ac2b18dc1969a47bc65a79a8fca02a414249d0c8635abb081c7f38c91b6"}, + {file = "pycryptodome-3.10.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:12222a5edc9ca4a29de15fbd5339099c4c26c56e13c2ceddf0b920794f26165d"}, + {file = "pycryptodome-3.10.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:6bbf7fee7b7948b29d7e71fcacf48bac0c57fb41332007061a933f2d996f9713"}, + {file = "pycryptodome-3.10.1.tar.gz", hash = "sha256:3e2e3a06580c5f190df843cdb90ea28d61099cf4924334d5297a995de68e4673"}, ] pyjwt = [ {file = "PyJWT-1.7.1-py2.py3-none-any.whl", hash = "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e"}, From 7e444ec7e926ebcb4728dc028a7fd0fec65d8ac6 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 12 Feb 2021 13:40:40 -0600 Subject: [PATCH 27/30] small fixes --- fence/job/visa_update_cronjob.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index be749e26f..f54eeb0ea 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -45,6 +45,7 @@ def __init__( oidc = config.get("OPENID_CONNECT", {}) if "ras" not in oidc: self.logger.error("RAS client not configured") + self.ras_client = None else: self.ras_client = RASClient( oidc["ras"], @@ -72,10 +73,7 @@ async def update_tokens(self, db_session): updater_queue = asyncio.Queue(maxsize=self.n_workers) loop = asyncio.get_event_loop() - producers = [ - loop.create_task(self.producer(db_session, queue, chunk_idx=0)) - for _ in range(1) - ] + producers = loop.create_task(self.producer(db_session, queue, chunk_idx=0)) workers = [ loop.create_task(self.worker(j, queue, updater_queue)) for j in range(self.n_workers) @@ -85,7 +83,7 @@ async def update_tokens(self, db_session): for i in range(self.concurrency) ] - await asyncio.gather(*producers) + await asyncio.gather(producers) self.logger.info("Producers done producing") await queue.join() @@ -133,6 +131,7 @@ async def worker(self, name, queue, updater_queue): while not queue.empty(): user = await queue.get() await updater_queue.put(user) + self._verify_jwt_token(user.ga4gh_visas_v1) queue.task_done() async def updater(self, name, updater_queue, db_session): @@ -161,8 +160,12 @@ def _pick_client(self, visa): """ Pick oidc client according to the visa provider """ + client = None if "ras" in visa.type: - return self.ras_client + client = self.ras_client + if not client: + raise Exception("Visa Client not set up or not avaialable") + return client def _verify_jwt_token(self, visa): # NOT IMPLEMENTED From e57c3c046789d17555b1f9df1d85f27ebd6f3b2c Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 15 Feb 2021 17:12:48 -0600 Subject: [PATCH 28/30] add to config --- fence/config-default.yaml | 3 +++ fence/job/visa_update_cronjob.py | 8 ++++++-- tests/ras/test_ras.py | 8 ++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 808fa9e1e..e08d02dce 100644 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -770,3 +770,6 @@ SYNAPSE_AUTHZ_TTL: 86400 RAS_REFRESH_EXPIRATION: 1296000 # Number of projects that can be registered to a Google Service Accont SERVICE_ACCOUNT_LIMIT: 6 +USERSYNC: + visa_types: + ras: [https://ras.nih.gov/visas/v1, https://ras.nih.gov/visas/v1.1] \ No newline at end of file diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index f54eeb0ea..d6aad2a9e 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -41,6 +41,8 @@ def __init__( self.n_workers = self.thread_pool_size + self.concurrency self.logger = logger + self.visa_types = config.get("USERSYNC").get("visa_types") + # Initialize visa clients: oidc = config.get("OPENID_CONNECT", {}) if "ras" not in oidc: @@ -161,10 +163,12 @@ def _pick_client(self, visa): Pick oidc client according to the visa provider """ client = None - if "ras" in visa.type: + if visa.type in self.visa_types["ras"]: client = self.ras_client + else: + raise Exception("Visa type {} not configured in fence-config".format(visa.type)) if not client: - raise Exception("Visa Client not set up or not avaialable") + raise Exception("Visa Client not set up or not avaialable for type {}".format(visa.type)) return client def _verify_jwt_token(self, visa): diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 0afcb9ecd..122a92158 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -35,7 +35,7 @@ def add_visa_manually(db_session, user, rsa_private_key, kid): "txn": "sapidjspa.asipidja", "name": "", "ga4gh_visa_v1": { - "type": "https://ras/visa/v1", + "type": "https://ras.nih.gov/visas/v1", "asserted": int(time.time()), "value": "https://nig/passport/dbgap", "source": "https://ncbi/gap", @@ -162,7 +162,7 @@ def test_update_visa_token( "txn": "sapidjspa.asipidja", "name": "", "ga4gh_visa_v1": { - "type": "https://ras/visa/v1", + "type": "https://ras.nih.gov/visas/v1", "asserted": int(time.time()), "value": "https://nig/passport/dbgap", "source": "https://ncbi/gap", @@ -310,7 +310,7 @@ def test_update_visa_token_with_invalid_visa( "txn": "sapidjspa.asipidja", "name": "", "ga4gh_visa_v1": { - "type": "https://ras/visa/v1", + "type": "https://ras.nih.gov/visas/v1", "asserted": int(time.time()), "value": "https://nig/passport/dbgap", "source": "https://ncbi/gap", @@ -393,7 +393,7 @@ def test_visa_update_cronjob( "txn": "sapidjspa.asipidja", "name": "", "ga4gh_visa_v1": { - "type": "https://ras/visa/v1", + "type": "https://ras.nih.gov/visas/v1", "asserted": int(time.time()), "value": "https://nig/passport/dbgap", "source": "https://ncbi/gap", From 8b57e768b372f3524a5538754c884e622990ba9e Mon Sep 17 00:00:00 2001 From: Binam Bajracharya <44302895+BinamB@users.noreply.github.com> Date: Tue, 16 Feb 2021 08:56:59 -0600 Subject: [PATCH 29/30] Update fence/job/visa_update_cronjob.py Co-authored-by: Pauline Ribeyre --- fence/job/visa_update_cronjob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index d6aad2a9e..2368c2d1f 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -41,7 +41,7 @@ def __init__( self.n_workers = self.thread_pool_size + self.concurrency self.logger = logger - self.visa_types = config.get("USERSYNC").get("visa_types") + self.visa_types = config.get("USERSYNC", {}).get("visa_types", {}) # Initialize visa clients: oidc = config.get("OPENID_CONNECT", {}) From fe41c2e647005f76c350df5d5068a87e0685ef17 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 16 Feb 2021 09:25:14 -0600 Subject: [PATCH 30/30] black --- fence/job/visa_update_cronjob.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 2368c2d1f..21bdc4a4b 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -166,9 +166,13 @@ def _pick_client(self, visa): if visa.type in self.visa_types["ras"]: client = self.ras_client else: - raise Exception("Visa type {} not configured in fence-config".format(visa.type)) + raise Exception( + "Visa type {} not configured in fence-config".format(visa.type) + ) if not client: - raise Exception("Visa Client not set up or not avaialable for type {}".format(visa.type)) + raise Exception( + "Visa Client not set up or not avaialable for type {}".format(visa.type) + ) return client def _verify_jwt_token(self, visa):