diff --git a/bin/fence_create.py b/bin/fence_create.py index 438d756ce..1c32f1541 100755 --- a/bin/fence_create.py +++ b/bin/fence_create.py @@ -405,6 +405,9 @@ def main(): STORAGE_CREDENTIALS = os.environ.get("STORAGE_CREDENTIALS") or config.get( "STORAGE_CREDENTIALS" ) + usersync = config.get("USERSYNC", {}) + sync_from_visas = usersync.get("sync_from_visas", False) + fallback_to_dbgap_sftp = usersync.get("fallback_to_dbgap_sftp", False) arborist = None if args.arborist: @@ -467,6 +470,8 @@ def main(): sync_from_local_yaml_file=args.yaml, folder=args.folder, arborist=arborist, + sync_from_visas=sync_from_visas, + fallback_to_dbgap_sftp=fallback_to_dbgap_sftp, ) elif args.action == "dbgap-download-access-files": download_dbgap_files( diff --git a/fence/blueprints/login/ras.py b/fence/blueprints/login/ras.py index e0d647a68..8d450c02e 100644 --- a/fence/blueprints/login/ras.py +++ b/fence/blueprints/login/ras.py @@ -2,7 +2,7 @@ import jwt from flask_sqlalchemy_session import current_session -from fence.models import GA4GHVisaV1, IdentityProvider, User +from fence.models import GA4GHVisaV1, IdentityProvider from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback @@ -57,7 +57,6 @@ def post_login(self, user, token_result): expires=int(decoded_visa["exp"]), ga4gh_visa=encoded_visa, ) - current_session.add(visa) current_session.commit() diff --git a/fence/config-default.yaml b/fence/config-default.yaml index e08d02dce..06638dce6 100644 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -460,7 +460,6 @@ dbGaP: # 'studyX': ['/orgA/', '/orgB/'] # 'studyX.c2': ['/orgB/', '/orgC/'] # 'studyZ': ['/orgD/'] - # Regex to match an assession number that has consent information in forms like: # phs00301123.c999 # phs000123.v3.p1.c3 @@ -770,6 +769,10 @@ SYNAPSE_AUTHZ_TTL: 86400 RAS_REFRESH_EXPIRATION: 1296000 # Number of projects that can be registered to a Google Service Accont SERVICE_ACCOUNT_LIMIT: 6 +# Settings for usersync with visas USERSYNC: + sync_from_visas: false + # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed + fallback_to_dbgap_sftp: false visa_types: - ras: [https://ras.nih.gov/visas/v1, https://ras.nih.gov/visas/v1.1] \ No newline at end of file + ras: [https://ras.nih.gov/visas/v1, https://ras.nih.gov/visas/v1.1] diff --git a/fence/job/visa_update_cronjob.py b/fence/job/visa_update_cronjob.py index 21bdc4a4b..ae875d64c 100644 --- a/fence/job/visa_update_cronjob.py +++ b/fence/job/visa_update_cronjob.py @@ -171,7 +171,7 @@ def _pick_client(self, visa): ) if not client: raise Exception( - "Visa Client not set up or not avaialable for type {}".format(visa.type) + "Visa Client not set up or not available for type {}".format(visa.type) ) return client diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index d8c81da81..23b4d5f90 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -210,6 +210,8 @@ def init_syncer( sync_from_local_yaml_file=None, arborist=None, folder=None, + sync_from_visas=False, + fallback_to_dbgap_sftp=False, ): """ sync ACL files from dbGap to auth db and storage backends @@ -268,6 +270,8 @@ def init_syncer( sync_from_local_yaml_file=sync_from_local_yaml_file, arborist=arborist, folder=folder, + sync_from_visas=sync_from_visas, + fallback_to_dbgap_sftp=fallback_to_dbgap_sftp, ) @@ -309,6 +313,8 @@ def sync_users( sync_from_local_yaml_file=None, arborist=None, folder=None, + sync_from_visas=False, + fallback_to_dbgap_sftp=False, ): syncer = init_syncer( dbGaP, @@ -320,10 +326,15 @@ def sync_users( sync_from_local_yaml_file, arborist, folder, + sync_from_visas, + fallback_to_dbgap_sftp, ) if not syncer: exit(1) - syncer.sync() + if sync_from_visas: + syncer.sync_visas() + else: + syncer.sync() def create_sample_data(DB, yaml_file_path): diff --git a/fence/sync/passport_sync/base_sync.py b/fence/sync/passport_sync/base_sync.py new file mode 100644 index 000000000..0a216a5e0 --- /dev/null +++ b/fence/sync/passport_sync/base_sync.py @@ -0,0 +1,14 @@ +class DefaultVisa(object): + """ + Base class for representation of information in a GA4GH passport describing user, project, and ABAC + information for access control + """ + + def __init__( + self, + logger=None, + ): + self.logger = logger + + def _parse_single_visa(self, user, visa): + pass diff --git a/fence/sync/passport_sync/ras_sync.py b/fence/sync/passport_sync/ras_sync.py new file mode 100644 index 000000000..9ab5fd5fe --- /dev/null +++ b/fence/sync/passport_sync/ras_sync.py @@ -0,0 +1,54 @@ +import jwt +import time + +from fence.sync.passport_sync.base_sync import DefaultVisa + + +class RASVisa(DefaultVisa): + """ + Class representing RAS visas + """ + + def _init__(self, logger): + super(RASVisa, self).__init__( + logger=logger, + ) + + def _parse_single_visa( + self, user, encoded_visa, expires, parse_consent_code, db_session + ): + decoded_visa = {} + try: + decoded_visa = jwt.decode(encoded_visa, verify=False) + except Exception as e: + self.logger.warning("Couldn't decode visa {}".format(e)) + # Remove visas if its invalid or expired + user.ga4gh_visas_v1 = [] + db_session.commit() + finally: + ras_dbgap_permissions = decoded_visa.get("ras_dbgap_permissions", []) + project = {} + info = {} + info["tags"] = {} + + if time.time() < expires: + for permission in ras_dbgap_permissions: + phsid = permission.get("phs_id", "") + version = permission.get("version", "") + participant_set = permission.get("participant_set", "") + consent_group = permission.get("consent_group", "") + full_phsid = phsid + if parse_consent_code and consent_group: + full_phsid += "." + consent_group + privileges = {"read-storage", "read"} + project[full_phsid] = privileges + info["tags"] = {"dbgap_role": permission.get("role", "")} + else: + # Remove visas if its invalid or expired + user.ga4gh_visas_v1 = [] + db_session.commit() + + info["email"] = user.email or "" + info["display_name"] = user.display_name or "" + info["phone_number"] = user.phone_number or "" + return project, info diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 33897e1f8..0f4e46dcd 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1,10 +1,12 @@ import glob +import jwt import os import re import subprocess as sp import yaml import copy from contextlib import contextmanager +from collections import defaultdict from csv import DictReader from io import StringIO from stat import S_ISDIR @@ -32,6 +34,7 @@ from fence.resources.storage import StorageManager from fence.resources.google.access_utils import bulk_update_google_groups from fence.sync import utils +from fence.sync.passport_sync.ras_sync import RASVisa def _format_policy_id(path, privilege): @@ -287,6 +290,8 @@ def __init__( sync_from_local_yaml_file=None, arborist=None, folder=None, + sync_from_visas=False, + fallback_to_dbgap_sftp=False, ): """ Syncs ACL files from dbGap to auth database and storage backends @@ -301,6 +306,8 @@ def __init__( ArboristClient instance if the syncer should also create resources in arborist folder: a local folder where dbgap telemetry files will sync to + sync_from_visas: use visa for sync instead of dbgap + fallback_to_dbgap_sftp: fallback to telemetry files when visa sync fails """ self.sync_from_local_csv_dir = sync_from_local_csv_dir self.sync_from_local_yaml_file = sync_from_local_yaml_file @@ -319,6 +326,12 @@ def __init__( ) self.arborist_client = arborist self.folder = folder + self.sync_from_visas = sync_from_visas + self.fallback_to_dbgap_sftp = fallback_to_dbgap_sftp + + self.auth_source = defaultdict(set) + # auth_source used for logging. username : [source1, source2] + self.visa_types = config.get("USERSYNC", {}).get("visa_types", {}) if storage_credentials: self.storage_manager = StorageManager( @@ -469,7 +482,6 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): self.logger.info( f"using study to common exchange area mapping: {study_common_exchange_areas}" ) - for filepath, privileges in file_dict.items(): self.logger.info("Reading file {}".format(filepath)) if os.stat(filepath).st_size == 0: @@ -626,15 +638,25 @@ def sync_two_user_info_dict(user_info1, user_info2): """ user_info2.update(user_info1) - @staticmethod - def sync_two_phsids_dict(phsids1, phsids2): + def sync_two_phsids_dict( + self, + phsids1, + phsids2, + source1=None, + source2=None, + phsids2_overrides_phsids1=True, + ): """ - Merge pshid1 into phsids2. phsids2 ends up containing the merged dict - (see explanation below). + Merge pshid1 into phsids2. If `phsids2_overrides_phsids1`, values in + phsids1 are overriden by values in phsids2. phsids2 ends up containing + the merged dict (see explanation below). + `source1` and `source2`: for logging. Args: phsids1, phsids2: nested dicts mapping phsids to sets of permissions + source1, source2: source of authz information (eg. dbgap, user_yaml, visas) + Example: { username: { @@ -663,14 +685,23 @@ def sync_two_phsids_dict(phsids1, phsids2): For the other cases, just simple addition """ + for user, projects1 in phsids1.items(): if not phsids2.get(user): + if source1: + self.auth_source[user].add(source1) phsids2[user] = projects1 - else: + elif phsids2_overrides_phsids1: + if source1: + self.auth_source[user].add(source1) + if source2: + self.auth_source[user].add(source2) for phsid1, privilege1 in projects1.items(): if phsid1 not in phsids2[user]: phsids2[user][phsid1] = set() phsids2[user][phsid1].update(privilege1) + elif source2: + self.auth_source[user].add(source2) def sync_to_db_and_storage_backend(self, user_project, user_info, sess): """ @@ -734,7 +765,6 @@ def sync_to_db_and_storage_backend(self, user_project, user_info, sess): self._revoke_from_storage( to_delete, sess, google_bulk_mapping=google_bulk_mapping ) - self._revoke_from_db(sess, to_delete) self._grant_from_storage( @@ -869,7 +899,6 @@ def _grant_from_db(self, sess, to_add, user_info, user_project, auth_provider_li auth_provider = auth_provider_list[0] if "dbgap_role" not in user_info[username]["tags"]: auth_provider = auth_provider_list[1] - user_access = AccessPrivilege( user=u, project=self._projects[project_auth_id], @@ -1234,8 +1263,12 @@ def _sync(self, sess): # the access info is combined - if the user.yaml access is # ["read"] and the CSV file access is ["read-storage"], the # resulting access is ["read", "read-storage"]. - self.sync_two_phsids_dict(user_projects_csv, user_projects) - self.sync_two_phsids_dict(user_yaml.projects, user_projects) + self.sync_two_phsids_dict( + user_projects_csv, user_projects, source1="local_csv", source2="dbgap" + ) + self.sync_two_phsids_dict( + user_yaml.projects, user_projects, source1="user_yaml", source2="dbgap" + ) # Note: if there are multiple dbgap sftp servers configured # this parameter is always from the config for the first dbgap sftp server @@ -1284,6 +1317,10 @@ def _sync(self, sess): ) exit(1) + # Logging authz source + for u, s in self.auth_source.items(): + self.logger.info("Access for user {} from {}".format(u, s)) + def _grant_all_consents_to_c999_users( self, user_projects, user_yaml_project_to_resources ): @@ -1503,7 +1540,6 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): arborist_user_projects = {} try: arborist_users = self.arborist_client.get_users().json["users"] - # construct user information, NOTE the lowering of the username. when adding/ # removing access, the case in the Fence db is used. For combining access, it is # case-insensitive, so we lower @@ -1596,25 +1632,28 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): for policy in user_yaml.policies.get(username, []): self.arborist_client.grant_user_policy(username, policy) - for client_name, client_details in user_yaml.clients.items(): - client_policies = client_details.get("policies", []) - client = session.query(Client).filter_by(name=client_name).first() - # update existing clients, do not create new ones - if not client: - self.logger.warning( - "client to update (`{}`) does not exist in fence: skipping".format( - client_name + if user_yaml: + for client_name, client_details in user_yaml.clients.items(): + client_policies = client_details.get("policies", []) + client = session.query(Client).filter_by(name=client_name).first() + # update existing clients, do not create new ones + if not client: + self.logger.warning( + "client to update (`{}`) does not exist in fence: skipping".format( + client_name + ) ) - ) - continue - try: - self.arborist_client.update_client(client.client_id, client_policies) - except ArboristError as e: - self.logger.info( - "not granting policies {} to client `{}`; {}".format( - client_policies, client_name, str(e) + continue + try: + self.arborist_client.update_client( + client.client_id, client_policies + ) + except ArboristError as e: + self.logger.info( + "not granting policies {} to client `{}`; {}".format( + client_policies, client_name, str(e) + ) ) - ) return True @@ -1693,3 +1732,312 @@ def _is_arborist_healthy(self): ) return False return True + + def _pick_sync_type(self, visa): + """ + Pick type of visa to parse according to the visa provider + """ + sync_client = None + if visa.type in self.visa_types["ras"]: + sync_client = self.ras_sync_client + else: + raise Exception( + "Visa type {} not recognized. Configure in fence-config".format( + visa.type + ) + ) + if not sync_client: + raise Exception("Sync client for {} not configured".format(visa.type)) + + return sync_client + + def parse_user_visas(self, db_session): + """ + Retrieve all visas from fence db and parse to python dict + + Return: + Tuple[[dict, dict]]: + (user_project, user_info) where user_project is a mapping from + usernames to project permissions and user_info is a mapping + from usernames to user details, such as email + + Example: + + ( + { + username: { + 'project1': {'read-storage','write-storage'}, + 'project2': {'read-storage'}, + } + }, + { + username: { + 'email': 'email@mail.com', + 'display_name': 'display name', + 'phone_number': '123-456-789', + 'tags': {'dbgap_role': 'PI'} + } + }, + ) + + """ + user_projects = dict() + user_info = dict() + + users = db_session.query(User).all() + + for user in users: + projects = {} + info = {} + if user.ga4gh_visas_v1: + for visa in user.ga4gh_visas_v1: + project = {} + visa_type = self._pick_sync_type(visa) + encoded_visa = visa.ga4gh_visa + project, info = visa_type._parse_single_visa( + user, + encoded_visa, + visa.expires, + self.parse_consent_code, + db_session, + ) + projects = {**projects, **project} + if projects: + self.auth_source[user.username].add("visas") + user_projects[user.username] = projects + user_info[user.username] = info + + return (user_projects, user_info) + + def _sync_visas(self, sess): + + self.logger.info("Running usersync with Visas") + self.logger.info( + "Fallback to telemetry files: {}".format(self.fallback_to_dbgap_sftp) + ) + + self.ras_sync_client = RASVisa(logger=self.logger) + + dbgap_config = self.dbGaP[0] + user_projects, user_info = self.parse_user_visas(sess) + enable_common_exchange_area_access = dbgap_config.get( + "enable_common_exchange_area_access", False + ) + study_common_exchange_areas = dbgap_config.get( + "study_common_exchange_areas", {} + ) + + try: + user_yaml = UserYAML.from_file( + self.sync_from_local_yaml_file, encrypted=False, logger=self.logger + ) + except (EnvironmentError, AssertionError) as e: + self.logger.error(str(e)) + self.logger.error("aborting early") + return + + # parse projects + user_projects = self.parse_projects(user_projects) + user_yaml.projects = self.parse_projects(user_yaml.projects) + + if self.fallback_to_dbgap_sftp: + # Collect user_info and user_projects from telemetry + user_projects_telemetry = {} + user_info_telemetry = {} + if self.is_sync_from_dbgap_server: + self.logger.debug( + "Pulling telemetry files from {} dbgap sftp servers".format( + len(self.dbGaP) + ) + ) + ( + user_projects_telemetry, + user_info_telemetry, + ) = self._merge_multiple_dbgap_sftp(self.dbGaP, sess) + local_csv_file_list = [] + if self.sync_from_local_csv_dir: + local_csv_file_list = glob.glob( + os.path.join(self.sync_from_local_csv_dir, "*") + ) + + # if syncing from local csv dir dbgap configurations + # come from the first dbgap instance in the fence config file + user_projects_csv, user_info_csv = self._get_user_permissions_from_csv_list( + local_csv_file_list, + encrypted=False, + session=sess, + dbgap_config=self.dbGaP[0], + ) + user_projects_csv = self.parse_projects(user_projects_csv) + user_projects_telemetry = self.parse_projects(user_projects_telemetry) + + # merge all user info dicts into "user_info". + # the user info (such as email) in the user.yaml files + # overrides the user info from the CSV files. + self.sync_two_user_info_dict(user_info_csv, user_info_telemetry) + + # merge all access info dicts into "user_projects". + # the access info is combined - if the user.yaml access is + # ["read"] and the CSV file access is ["read-storage"], the + # resulting access is ["read", "read-storage"]. + self.sync_two_phsids_dict( + user_projects_csv, + user_projects_telemetry, + source1="local_csv", + source2="dbgap", + ) + + # sync phsids so that this adds projects if visas were invalid or adds users that dont have visas. + # `phsids2_overrides_phsids1=True` because We want visa to be the source of truth when its available and not merge any telemetry file info into this. + # We only want visa to be used when visa is not valid or available + self.sync_two_phsids_dict( + user_projects_telemetry, + user_projects, + source1="dbgap", + source2="visa", + phsids2_overrides_phsids1=False, + ) + self.sync_two_user_info_dict(user_info_telemetry, user_info) + + if self.parse_consent_code and enable_common_exchange_area_access: + self.logger.info( + f"using study to common exchange area mapping: {study_common_exchange_areas}" + ) + + # merge all user info dicts into "user_info". + # the user info (such as email) in the user.yaml files + # overrides the user info from the CSV files. + self.sync_two_user_info_dict(user_yaml.user_info, user_info) + + # merge all access info dicts into "user_projects". + # the access info is combined - if the user.yaml access is + # ["read"] and the CSV file access is ["read-storage"], the + # resulting access is ["read", "read-storage"]. + self.sync_two_phsids_dict( + user_yaml.projects, user_projects, source1="user_yaml", source2="visa" + ) + + for username in user_projects.keys(): + for project in user_projects[username].keys(): + phsid = project.split(".") + dbgap_project = phsid[0] + privileges = user_projects[username][project] + if len(phsid) > 1 and self.parse_consent_code: + consent_code = phsid[-1] + + # c999 indicates full access to all consents and access + # to a study-specific exchange area + # access to at least one study-specific exchange area implies access + # to the parent study's common exchange area + # + # NOTE: Handling giving access to all consents is done at + # a later time, when we have full information about possible + # consents + self.logger.debug( + f"got consent code {consent_code} from dbGaP project " + f"{dbgap_project}" + ) + if ( + consent_code == "c999" + and enable_common_exchange_area_access + and dbgap_project in study_common_exchange_areas + ): + self.logger.info( + "found study with consent c999 and Fence " + "is configured to parse exchange area data. Giving user " + f"{username} {privileges} privileges in project: " + f"{study_common_exchange_areas[dbgap_project]}." + ) + self._add_dbgap_project_for_user( + study_common_exchange_areas[dbgap_project], + privileges, + username, + sess, + user_projects, + dbgap_config, + ) + + dbgap_project += "." + consent_code + + if dbgap_project not in self.project_mapping: + self._add_dbgap_project_for_user( + dbgap_project, + privileges, + username, + sess, + user_projects, + dbgap_config, + ) + + for element_dict in self.project_mapping.get(dbgap_project, []): + try: + phsid_privileges = {element_dict["auth_id"]: set(privileges)} + + # need to add dbgap project to arborist + if self.arborist_client: + self._add_dbgap_study_to_arborist( + element_dict["auth_id"], dbgap_config + ) + + if username not in user_projects: + user_projects[username] = {} + user_projects[username].update(phsid_privileges) + + except ValueError as e: + self.logger.info(e) + + # Note: if there are multiple dbgap sftp servers configured + # this parameter is always from the config for the first dbgap sftp server + # not any additional ones + if self.parse_consent_code: + self._grant_all_consents_to_c999_users( + user_projects, user_yaml.project_to_resource + ) + # update fence db + if user_projects: + self.logger.info("Sync to db and storage backend") + self.sync_to_db_and_storage_backend(user_projects, user_info, sess) + else: + self.logger.info("No users for syncing") + + # update the Arborist DB (resources, roles, policies, groups) + if user_yaml.authz: + if not self.arborist_client: + raise EnvironmentError( + "yaml file contains authz section but sync is not configured with" + " arborist client--did you run sync with --arborist arg?" + ) + self.logger.info("Synchronizing arborist...") + success = self._update_arborist(sess, user_yaml) + if success: + self.logger.info("Finished synchronizing arborist") + else: + self.logger.error("Could not synchronize successfully") + exit(1) + else: + self.logger.info("No `authz` section; skipping arborist sync") + + # update arborist db (user access) + if self.arborist_client: + self.logger.info("Synchronizing arborist with authorization info...") + success = self._update_authz_in_arborist(sess, user_projects, user_yaml) + if success: + self.logger.info( + "Finished synchronizing authorization info to arborist" + ) + else: + self.logger.error( + "Could not synchronize authorization info successfully to arborist" + ) + exit(1) + # Logging authz source + for u, s in self.auth_source.items(): + self.logger.info("Access for user {} from {}".format(u, s)) + + def sync_visas(self): + if self.session: + self._sync_visas(self.session) + else: + with self.driver.session as s: + self._sync_visas(s) + # if returns with some failure use telemetry file diff --git a/poetry.lock b/poetry.lock index 0c0a48ad1..7ed2b977c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -218,7 +218,7 @@ description = "Foreign Function Interface for Python calling C code." name = "cffi" optional = false python-versions = "*" -version = "1.14.4" +version = "1.14.5" [package.dependencies] pycparser = "*" @@ -566,7 +566,7 @@ description = "Google Authentication Library" name = "google-auth" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.26.0" +version = "1.26.1" [package.dependencies] cachetools = ">=2.0.0,<5.0" @@ -616,7 +616,7 @@ description = "Google Cloud Storage API client library" name = "google-cloud-storage" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" -version = "1.35.1" +version = "1.36.0" [package.dependencies] google-auth = ">=1.11.0,<2.0dev" @@ -1214,7 +1214,7 @@ description = "Pure-Python RSA implementation" name = "rsa" optional = false python-versions = ">=3.5, <4" -version = "4.7" +version = "4.7.1" [package.dependencies] pyasn1 = ">=0.1.3" @@ -1446,42 +1446,43 @@ certifi = [ {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, ] cffi = [ - {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, - {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, - {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, - {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, - {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, - {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, - {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, - {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, - {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, - {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, - {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, - {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, - {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, - {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, - {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, - {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, - {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, - {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, - {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, + {file = "cffi-1.14.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991"}, + {file = "cffi-1.14.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1"}, + {file = "cffi-1.14.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa"}, + {file = "cffi-1.14.5-cp27-cp27m-win32.whl", hash = "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3"}, + {file = "cffi-1.14.5-cp27-cp27m-win_amd64.whl", hash = "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5"}, + {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482"}, + {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6"}, + {file = "cffi-1.14.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045"}, + {file = "cffi-1.14.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa"}, + {file = "cffi-1.14.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406"}, + {file = "cffi-1.14.5-cp35-cp35m-win32.whl", hash = "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369"}, + {file = "cffi-1.14.5-cp35-cp35m-win_amd64.whl", hash = "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315"}, + {file = "cffi-1.14.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"}, + {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"}, + {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"}, + {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"}, + {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"}, + {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"}, + {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"}, + {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"}, + {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"}, + {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"}, + {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"}, + {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"}, + {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"}, ] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, @@ -1644,8 +1645,8 @@ google-api-python-client = [ {file = "google_api_python_client-1.11.0-py2.py3-none-any.whl", hash = "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6"}, ] google-auth = [ - {file = "google-auth-1.26.0.tar.gz", hash = "sha256:26cf3e839be936bbd2f65465cbdecd41590a603a51891a9fd9077716ddc3f726"}, - {file = "google_auth-1.26.0-py2.py3-none-any.whl", hash = "sha256:4c95ef8d2a9afb11015b387a2d2d1d86d8c9170f0b89d07c0f773b852dd695de"}, + {file = "google-auth-1.26.1.tar.gz", hash = "sha256:1b461d079b5650efe492a7814e95c536ffa9e7a96e39a6d16189c1604f18554f"}, + {file = "google_auth-1.26.1-py2.py3-none-any.whl", hash = "sha256:8ce6862cf4e9252de10045f05fa80393fde831da9c2b45c39288edeee3cde7f2"}, ] google-auth-httplib2 = [ {file = "google-auth-httplib2-0.0.4.tar.gz", hash = "sha256:8d092cc60fb16517b12057ec0bba9185a96e3b7169d86ae12eae98e645b7bc39"}, @@ -1656,8 +1657,8 @@ google-cloud-core = [ {file = "google_cloud_core-1.6.0-py2.py3-none-any.whl", hash = "sha256:40d9c2da2d03549b5ac3dcccf289d4f15e6d1210044c6381ce45c92913e62904"}, ] google-cloud-storage = [ - {file = "google-cloud-storage-1.35.1.tar.gz", hash = "sha256:dc076b6af6da991252416639cb93831f8e50c8328d5ac3fb8e03e40cd8de2290"}, - {file = "google_cloud_storage-1.35.1-py2.py3-none-any.whl", hash = "sha256:69fa8feda06768c44dd635c2cd8d59267362a0c8323afc65cf24efc4d88a0fcc"}, + {file = "google-cloud-storage-1.36.0.tar.gz", hash = "sha256:9517e26cadb4e52fcd11e83cd1b4051e72351a2e518078102ad2f1b9f4d704bc"}, + {file = "google_cloud_storage-1.36.0-py2.py3-none-any.whl", hash = "sha256:0880af3c8706a52785ee2c6dc6856e33aa8bf951083b145a889137344a7201ad"}, ] google-crc32c = [ {file = "google-crc32c-1.1.2.tar.gz", hash = "sha256:dff5bd1236737f66950999d25de7a78144548ebac7788d30ada8c1b6ead60b27"}, @@ -2016,8 +2017,8 @@ rfc3986 = [ {file = "rfc3986-1.4.0.tar.gz", hash = "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d"}, ] rsa = [ - {file = "rsa-4.7-py3-none-any.whl", hash = "sha256:a8774e55b59fd9fc893b0d05e9bfc6f47081f46ff5b46f39ccf24631b7be356b"}, - {file = "rsa-4.7.tar.gz", hash = "sha256:69805d6b69f56eb05b62daea3a7dbd7aa44324ad1306445e05da8060232d00f4"}, + {file = "rsa-4.7.1-py3-none-any.whl", hash = "sha256:74ba16e7ef58920b80b5c54c1c1066d391a2c1e812c466773f74c634eb12253b"}, + {file = "rsa-4.7.1.tar.gz", hash = "sha256:9d74d1ff850745c9802cd6b53382bfeec7f6dbe4e26ee2759241ed1e7b0ecf5d"}, ] s3transfer = [ {file = "s3transfer-0.2.1-py2.py3-none-any.whl", hash = "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba"}, diff --git a/tests/dbgap_sync/conftest.py b/tests/dbgap_sync/conftest.py index 00bd05005..3a3e254da 100644 --- a/tests/dbgap_sync/conftest.py +++ b/tests/dbgap_sync/conftest.py @@ -1,4 +1,6 @@ import os +import time +import jwt from unittest.mock import MagicMock, patch from yaml import safe_load as yaml_load @@ -13,7 +15,7 @@ from fence.sync.sync_users import UserSyncer from fence.resources import userdatamodel as udm -from fence.models import AccessPrivilege, AuthorizationProvider, User +from fence.models import AccessPrivilege, AuthorizationProvider, User, GA4GHVisaV1 from gen3authz.client.arborist.client import ArboristClient @@ -69,7 +71,7 @@ def storage_client(): @pytest.fixture -def syncer(db_session, request): +def syncer(db_session, request, rsa_private_key, kid): if request.param == "google": backend = "google" else: @@ -93,6 +95,16 @@ def syncer(db_session, request): "email": "deleted_user@gmail.com", }, {"username": "TESTUSERD", "is_admin": True, "email": "userD@gmail.com"}, + { + "username": "expired_visa_user", + "is_admin": False, + "email": "expired@expired.com", + }, + { + "username": "invalid_visa_user", + "is_admin": False, + "email": "invalid@invalid.com", + }, ] projects = [ @@ -184,8 +196,112 @@ def mocked_get(path, **kwargs): ) for user in users: - db_session.add(User(**user)) + user = User(**user) + db_session.add(user) + add_visa_manually(db_session, user, rsa_private_key, kid) db_session.commit() return syncer_obj + + +def add_visa_manually(db_session, user, rsa_private_key, kid): + + headers = {"kid": kid} + + decoded_visa = { + "iss": "https://stsstg.nih.gov", + "sub": "abcde12345aspdij", + "iat": int(time.time()), + "exp": int(time.time()) + 1000, + "scope": "openid ga4gh_passport_v1 email profile", + "jti": "jtiajoidasndokmasdl", + "txn": "sapidjspa.asipidja", + "name": "", + "ga4gh_visa_v1": { + "type": "https://ras.nih.gov/visas/v1", + "asserted": int(time.time()), + "value": "https://nig/passport/dbgap", + "source": "https://ncbi/gap", + }, + "ras_dbgap_permissions": [ + { + "consent_name": "Health/Medical/Biomedical", + "phs_id": "phs000991", + "version": "v1", + "participant_set": "p1", + "consent_group": "c1", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + { + "consent_name": "General Research Use (IRB, PUB)", + "phs_id": "phs000961", + "version": "v1", + "participant_set": "p1", + "consent_group": "c1", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + { + "consent_name": "Disease-Specific (Cardiovascular Disease)", + "phs_id": "phs000279", + "version": "v2", + "participant_set": "p1", + "consent_group": "c1", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + { + "consent_name": "Health/Medical/Biomedical (IRB)", + "phs_id": "phs000286", + "version": "v6", + "participant_set": "p2", + "consent_group": "c3", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + { + "consent_name": "Disease-Specific (Focused Disease Only, IRB, NPU)", + "phs_id": "phs000289", + "version": "v6", + "participant_set": "p2", + "consent_group": "c2", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + { + "consent_name": "Disease-Specific (Autism Spectrum Disorder)", + "phs_id": "phs000298", + "version": "v4", + "participant_set": "p3", + "consent_group": "c1", + "role": "designated user", + "expiration": "2020-11-14 00:00:00", + }, + ], + } + + encoded_visa = jwt.encode( + decoded_visa, key=rsa_private_key, headers=headers, algorithm="RS256" + ).decode("utf-8") + + expires = int(decoded_visa["exp"]) + if user.username == "expired_visa_user": + expires -= 100000 + if user.username == "invalid_visa_user": + encoded_visa = encoded_visa[: len(encoded_visa) // 2] + if user.username == "TESTUSERD": + encoded_visa = encoded_visa[: len(encoded_visa) // 2] + + visa = GA4GHVisaV1( + user=user, + source=decoded_visa["ga4gh_visa_v1"]["source"], + type=decoded_visa["ga4gh_visa_v1"]["type"], + asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]), + expires=expires, + ga4gh_visa=encoded_visa, + ) + + db_session.add(visa) + db_session.commit() diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index d4f4a7555..f5ef7300d 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -76,7 +76,7 @@ def test_sync( syncer.sync() users = db_session.query(models.User).all() - assert len(users) == 11 + assert len(users) == 13 if parse_consent_code_config: user = models.query_for_user(session=db_session, username="USERC") @@ -611,3 +611,117 @@ def mock_merge(dbgap_servers, sess): # this function will be called once for each sftp server # the test config file has 2 dbgap sftp servers assert syncer._process_dbgap_files.call_count == 2 + + +@pytest.mark.parametrize("syncer", ["cleversafe", "google"], indirect=True) +@pytest.mark.parametrize("parse_consent_code_config", [False, True]) +@pytest.mark.parametrize("fallback_to_dbgap_sftp", [False, True]) +def test_user_sync_with_visas( + syncer, + db_session, + storage_client, + parse_consent_code_config, + fallback_to_dbgap_sftp, + monkeypatch, +): + # patch the sync to use the parameterized config value + monkeypatch.setitem( + syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config + ) + monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) + monkeypatch.setattr(syncer, "fallback_to_dbgap_sftp", fallback_to_dbgap_sftp) + monkeypatch.setattr(syncer, "sync_from_visas", True) + + syncer.sync_visas() + + users = db_session.query(models.User).all() + + user = models.query_for_user( + session=db_session, username="TESTUSERB" + ) # contains only visa information + + backup_user = models.query_for_user( + session=db_session, username="TESTUSERD" + ) # Contains invalid visa and also in telemetry file + + expired_user = models.query_for_user( + session=db_session, + username="expired_visa_user", + ) + invalid_user = models.query_for_user( + session=db_session, username="invalid_visa_user" + ) + + assert len(invalid_user.project_access) == 0 + assert len(expired_user.project_access) == 0 + + assert len(invalid_user.ga4gh_visas_v1) == 0 + assert len(expired_user.ga4gh_visas_v1) == 0 + + if fallback_to_dbgap_sftp: + assert len(users) == 13 + + if parse_consent_code_config: + assert equal_project_access( + user.project_access, + { + "phs000991.c1": ["read", "read-storage"], + "phs000961.c1": ["read", "read-storage"], + "phs000279.c1": ["read", "read-storage"], + "phs000286.c3": ["read", "read-storage"], + "phs000289.c2": ["read", "read-storage"], + "phs000298.c1": ["read", "read-storage"], + }, + ) + assert equal_project_access( + backup_user.project_access, + { + "phs000179.c1": ["read", "read-storage"], + }, + ) + else: + assert equal_project_access( + user.project_access, + { + "phs000991": ["read", "read-storage"], + "phs000961": ["read", "read-storage"], + "phs000279": ["read", "read-storage"], + "phs000286": ["read", "read-storage"], + "phs000289": ["read", "read-storage"], + "phs000298": ["read", "read-storage"], + }, + ) + assert equal_project_access( + backup_user.project_access, + { + "phs000179": ["read", "read-storage"], + }, + ) + + else: + assert len(users) == 11 + assert len(backup_user.project_access) == 0 + if parse_consent_code_config: + assert equal_project_access( + user.project_access, + { + "phs000991.c1": ["read", "read-storage"], + "phs000961.c1": ["read", "read-storage"], + "phs000279.c1": ["read", "read-storage"], + "phs000286.c3": ["read", "read-storage"], + "phs000289.c2": ["read", "read-storage"], + "phs000298.c1": ["read", "read-storage"], + }, + ) + else: + assert equal_project_access( + user.project_access, + { + "phs000991": ["read", "read-storage"], + "phs000961": ["read", "read-storage"], + "phs000279": ["read", "read-storage"], + "phs000286": ["read", "read-storage"], + "phs000289": ["read", "read-storage"], + "phs000298": ["read", "read-storage"], + }, + ) diff --git a/tests/ras/test_ras.py b/tests/ras/test_ras.py index 122a92158..1747915b8 100644 --- a/tests/ras/test_ras.py +++ b/tests/ras/test_ras.py @@ -8,6 +8,9 @@ from fence.config import config from fence.models import User, UpstreamRefreshToken, GA4GHVisaV1 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient +from fence.config import config + +from tests.dbgap_sync.conftest import add_visa_manually from fence.job.visa_update_cronjob import Visa_Token_Update import tests.utils @@ -21,44 +24,6 @@ def add_test_user(db_session, username="admin_user", id="5678", is_admin=True): return test_user -def add_visa_manually(db_session, user, rsa_private_key, kid): - - headers = {"kid": kid} - - decoded_visa = { - "iss": "https://stsstg.nih.gov", - "sub": "abcde12345aspdij", - "iat": int(time.time()), - "exp": int(time.time()) + 1000, - "scope": "openid ga4gh_passport_v1 email profile", - "jti": "jtiajoidasndokmasdl", - "txn": "sapidjspa.asipidja", - "name": "", - "ga4gh_visa_v1": { - "type": "https://ras.nih.gov/visas/v1", - "asserted": int(time.time()), - "value": "https://nig/passport/dbgap", - "source": "https://ncbi/gap", - }, - } - - encoded_visa = jwt.encode( - decoded_visa, key=rsa_private_key, headers=headers, algorithm="RS256" - ).decode("utf-8") - - visa = GA4GHVisaV1( - user=user, - source=decoded_visa["ga4gh_visa_v1"]["source"], - type=decoded_visa["ga4gh_visa_v1"]["type"], - asserted=int(decoded_visa["ga4gh_visa_v1"]["asserted"]), - expires=int(decoded_visa["exp"]), - ga4gh_visa=encoded_visa, - ) - - db_session.add(visa) - db_session.commit() - - def add_refresh_token(db_session, user): refresh_token = "abcde1234567kposjdas" expires = int(time.time()) + 1000 @@ -383,6 +348,10 @@ def test_visa_update_cronjob( username = "no_visa_{}".format(j) test_user = add_test_user(db_session, username, j + n_users) + query_visas = db_session.query(GA4GHVisaV1).all() + + assert len(query_visas) == n_users + new_visa = { "iss": "https://stsstg.nih.gov", "sub": "abcde12345aspdij",