[HOLD] move backoff settings, send email, and create client to config

tests/ci_commands_script.sh → bin/ci_commands_script.sh

@@ -2,4 +2,4 @@
 
 mkdir -p /var/tmp/uwsgi_flask_metrics/ || true
 export PROMETHEUS_MULTIPROC_DIR="/var/tmp/uwsgi_flask_metrics/"
-poetry run pytest -vv --cov=fence --cov-report xml tests
+poetry run pytest -vv --cov=fence --cov-report xml ../tests

fence/__init__.py

@@ -18,14 +18,8 @@
 logger = get_logger(__name__, log_level="debug")
 
 # Load the configuration *before* importing modules that rely on it
-from fence.config import config
+from fence.config import config, get_SQLAlchemyDriver
 from fence.settings import CONFIG_SEARCH_FOLDERS
-
-config.load(
-    config_path=os.environ.get("FENCE_CONFIG_PATH"),
-    search_folders=CONFIG_SEARCH_FOLDERS,
-)
-
 from fence.auth import logout, build_redirect_url
 from fence.metrics import metrics
 from fence.blueprints.data.indexd import S3IndexedFileLocation
@@ -48,7 +42,6 @@
 from fence.resources.storage import StorageManager
 from fence.resources.user.user_session import UserSessionInterface
 from fence.error_handler import get_error_response
-from fence.utils import get_SQLAlchemyDriver
 import fence.blueprints.admin
 import fence.blueprints.data
 import fence.blueprints.login

fence/blueprints/data/indexd.py

@@ -6,7 +6,6 @@
 from urllib.parse import urlparse, ParseResult, urlunparse
 from datetime import datetime, timedelta
 
-from sqlalchemy.sql.functions import user
 from cached_property import cached_property
 import gen3cirrus
 from gen3cirrus import GoogleCloudManager

fence/config-default.yaml

@@ -545,6 +545,7 @@ dbGaP:
     # providing a list of additional_allowed_project_id_patterns allows usersyn to
     # read any filename that matches the patterns in the list.
     allow_non_dbGaP_whitelist: false
+    # Additional allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects'
     additional_allowed_project_id_patterns: []
     # parse out the consent from the dbgap accession number such that something
     # like "phs000123.v1.p1.c2" becomes "phs000123.c2".
@@ -602,8 +603,6 @@ dbGaP:
       # 'studyZ': ['/orgD/']
     # Allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects'
     allowed_project_id_patterns: []
-    # Additional allowed patterns for project_ids. The default value in usersync is 'phs(\d{6}) for dbgap projects'
-    additional_allowed_project_id_patterns: []
 # Regex to match an assession number that has consent information in forms like:
 #   phs00301123.c999
 #   phs000123.v3.p1.c3

fence/config.py

@@ -1,11 +1,35 @@
 import os
+from functools import wraps
+
+import bcrypt
+import flask
+import requests
+from werkzeug.datastructures import ImmutableMultiDict
 from yaml import safe_load as yaml_load
 import urllib.parse
 
 import gen3cirrus
 from gen3config import Config
 
 from cdislogging import get_logger
+from fence.settings import CONFIG_SEARCH_FOLDERS
+from fence.utils import log_backoff_retry, log_backoff_giveup, exception_do_not_retry, generate_client_credentials, \
+    logger
+from fence.models import Client, User, query_for_user
+from fence.errors import NotFound
+from userdatamodel import Base
+from sqlalchemy import (
+    Integer,
+    String,
+    Column,
+    text,
+    event,
+    ForeignKey)
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.dialects.postgresql import JSONB
+from userdatamodel.models import (
+    IdentityProvider,
+    User)
 
 logger = get_logger(__name__)
 
@@ -77,8 +101,8 @@ def post_process(self):
         # NOTE: use when fence will be deployed in such a way that fence will
         #       only receive traffic from internal clients, and can safely use HTTP
         if (
-            self._configs.get("AUTHLIB_INSECURE_TRANSPORT")
-            and "AUTHLIB_INSECURE_TRANSPORT" not in os.environ
+                self._configs.get("AUTHLIB_INSECURE_TRANSPORT")
+                and "AUTHLIB_INSECURE_TRANSPORT" not in os.environ
         ):
             os.environ["AUTHLIB_INSECURE_TRANSPORT"] = "true"
 
@@ -100,11 +124,11 @@ def post_process(self):
         #       billing rights (in other words, only use it when interacting with buckets
         #       Fence is aware of)
         if self._configs.get("BILLING_PROJECT_FOR_SA_CREDS") or self._configs.get(
-            "BILLING_PROJECT_FOR_SIGNED_URLS"
+                "BILLING_PROJECT_FOR_SIGNED_URLS"
         ):
             if (
-                "USER_ALLOWED_SCOPES" in self._configs
-                and "google_credentials" in self._configs["USER_ALLOWED_SCOPES"]
+                    "USER_ALLOWED_SCOPES" in self._configs
+                    and "google_credentials" in self._configs["USER_ALLOWED_SCOPES"]
             ):
                 logger.warning(
                     "Configuration does not restrict end-user access to billing. Correcting. "
@@ -117,8 +141,8 @@ def post_process(self):
                 self._configs["USER_ALLOWED_SCOPES"].remove("google_credentials")
 
             if (
-                "SESSION_ALLOWED_SCOPES" in self._configs
-                and "google_credentials" in self._configs["SESSION_ALLOWED_SCOPES"]
+                    "SESSION_ALLOWED_SCOPES" in self._configs
+                    and "google_credentials" in self._configs["SESSION_ALLOWED_SCOPES"]
             ):
                 logger.warning(
                     "Configuration does not restrict end-user access to billing. Correcting. "
@@ -131,8 +155,8 @@ def post_process(self):
                 self._configs["SESSION_ALLOWED_SCOPES"].remove("google_credentials")
 
         if (
-            not self._configs["ENABLE_VISA_UPDATE_CRON"]
-            and self._configs["GLOBAL_PARSE_VISAS_ON_LOGIN"] is not False
+                not self._configs["ENABLE_VISA_UPDATE_CRON"]
+                and self._configs["GLOBAL_PARSE_VISAS_ON_LOGIN"] is not False
         ):
             raise Exception(
                 "Visa parsing on login is enabled but `ENABLE_VISA_UPDATE_CRON` is disabled!"
@@ -169,3 +193,236 @@ def _validate_parent_child_studies(dbgap_configs):
 
 
 config = FenceConfig(DEFAULT_CFG_PATH)
+config.load(config_path=os.environ.get("FENCE_CONFIG_PATH"), search_folders=CONFIG_SEARCH_FOLDERS,)
+DEFAULT_BACKOFF_SETTINGS = {
+    "on_backoff": log_backoff_retry,
+    "on_giveup": log_backoff_giveup,
+    # todo: config does not have this key but it probably should. why broken?
+    "max_tries": config.get("DEFAULT_BACKOFF_SETTINGS_MAX_TRIES", 3),
+    "giveup": exception_do_not_retry,
+}
+
+
+def send_email(from_email, to_emails, subject, text, smtp_domain):
+    """
+    Send email to group of emails using mail gun api.
+
+    https://app.mailgun.com/
+
+    Args:
+        from_email(str): from email
+        to_emails(list): list of emails to receive the messages
+        text(str): the text message
+        smtp_domain(dict): smtp domain server
+
+            {
+                "smtp_hostname": "smtp.mailgun.org",
+                "default_login": "[email protected]",
+                "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net",
+                "smtp_password": "password", # pragma: allowlist secret
+                "api_key": "api key" # pragma: allowlist secret
+            }
+
+    Returns:
+        Http response
+
+    Exceptions:
+        KeyError
+
+    """
+    if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get(
+            smtp_domain
+    ).get("smtp_password"):
+        raise NotFound(
+            "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or "
+            "smtp_password was not provided. "
+            "Cannot send email.".format(smtp_domain)
+        )
+
+    api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "")
+    email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages"
+
+    return requests.post(
+        email_url,
+        auth=("api", api_key),
+        data={"from": from_email, "to": to_emails, "subject": subject, "text": text},
+    )
+
+
+def create_client(
+        DB,
+        username=None,
+        urls=[],
+        name="",
+        description="",
+        auto_approve=False,
+        is_admin=False,
+        grant_types=None,
+        confidential=True,
+        arborist=None,
+        policies=None,
+        allowed_scopes=None,
+        expires_in=None,
+):
+    client_id, client_secret, hashed_secret = generate_client_credentials(confidential)
+    if arborist is not None:
+        arborist.create_client(client_id, policies)
+    driver = get_SQLAlchemyDriver(DB)
+    auth_method = "client_secret_basic" if confidential else "none"
+
+    allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"]
+    if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])):
+        raise ValueError(
+            "Each allowed scope must be one of: {}".format(
+                config["CLIENT_ALLOWED_SCOPES"]
+            )
+        )
+
+    if "openid" not in allowed_scopes:
+        allowed_scopes.append("openid")
+        logger.warning('Adding required "openid" scope to list of allowed scopes.')
+
+    with driver.session as s:
+        user = None
+        if username:
+            user = query_for_user(session=s, username=username)
+            if not user:
+                user = User(username=username, is_admin=is_admin)
+                s.add(user)
+
+        if s.query(Client).filter(Client.name == name).first():
+            if arborist is not None:
+                arborist.delete_client(client_id)
+            raise Exception("client {} already exists".format(name))
+
+        client = Client(
+            client_id=client_id,
+            client_secret=hashed_secret,
+            user=user,
+            redirect_uris=urls,
+            allowed_scopes=" ".join(allowed_scopes),
+            description=description,
+            name=name,
+            auto_approve=auto_approve,
+            grant_types=grant_types,
+            is_confidential=confidential,
+            token_endpoint_auth_method=auth_method,
+            expires_in=expires_in,
+        )
+        s.add(client)
+        s.commit()
+
+    return client_id, client_secret
+
+
+def hash_secret(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        has_secret = "client_secret" in flask.request.form
+        has_client_id = "client_id" in flask.request.form
+        if flask.request.form and has_secret and has_client_id:
+            form = flask.request.form.to_dict()
+            with flask.current_app.db.session as session:
+                client = (
+                    session.query(Client)
+                    .filter(Client.client_id == form["client_id"])
+                    .first()
+                )
+                if client:
+                    form["client_secret"] = bcrypt.hashpw(
+                        form["client_secret"].encode("utf-8"),
+                        client.client_secret.encode("utf-8"),
+                    ).decode("utf-8")
+                flask.request.form = ImmutableMultiDict(form)
+
+        return f(*args, **kwargs)
+
+    return wrapper
+
+
+def get_SQLAlchemyDriver(db_conn_url):
+    from userdatamodel.driver import SQLAlchemyDriver
+
+    # override userdatamodel's `setup_db` function which creates tables
+    # and runs database migrations, because Alembic handles that now.
+    # TODO move userdatamodel code to Fence and remove dependencies to it
+    SQLAlchemyDriver.setup_db = lambda _: None
+    return SQLAlchemyDriver(db_conn_url)
+
+
+def get_issuer_to_idp():
+    possibly_matching_idps = [IdentityProvider.ras]
+    issuer_to_idp = {}
+
+    oidc = config.get("OPENID_CONNECT", {})
+    for idp in possibly_matching_idps:
+        discovery_url = oidc.get(idp, {}).get("discovery_url")
+        if discovery_url:
+            for allowed_issuer in config["GA4GH_VISA_ISSUER_ALLOWLIST"]:
+                if discovery_url.startswith(allowed_issuer):
+                    issuer_to_idp[allowed_issuer] = idp
+                    break
+
+    return issuer_to_idp
+
+
+class IssSubPairToUser(Base):
+    # issuer & sub pair mapping to Gen3 User sub
+
+    __tablename__ = "iss_sub_pair_to_user"
+
+    iss = Column(String(), primary_key=True)
+    sub = Column(String(), primary_key=True)
+
+    fk_to_User = Column(
+        Integer, ForeignKey(User.id, ondelete="CASCADE"), nullable=False
+    )  #  foreign key for User table
+    user = relationship(
+        "User",
+        backref=backref(
+            "iss_sub_pairs",
+            cascade="all, delete-orphan",
+            passive_deletes=True,
+        ),
+    )
+
+    # dump whatever idp provides in here
+    extra_info = Column(JSONB(), server_default=text("'{}'"))
+
+
+@event.listens_for(IssSubPairToUser.__table__, "after_create")
+def populate_iss_sub_pair_to_user_table(target, connection, **kw):
+    """
+    Populate iss_sub_pair_to_user table using User table's id_from_idp
+    column.
+    """
+    issuer_to_idp = get_issuer_to_idp()
+    for issuer, idp_name in IssSubPairToUser.ISSUER_TO_IDP.items():
+        logger.info(
+            'Attempting to populate iss_sub_pair_to_user table for users with "{}" idp and "{}" issuer'.format(
+                idp_name, issuer
+            )
+        )
+        transaction = connection.begin()
+        try:
+            connection.execute(
+                text(
+                    """
+                    WITH identity_provider_id AS (SELECT id FROM identity_provider WHERE name=:idp_name)
+                    INSERT INTO iss_sub_pair_to_user (iss, sub, "fk_to_User", extra_info)
+                    SELECT :iss, id_from_idp, id, additional_info
+                    FROM "User"
+                    WHERE idp_id IN (SELECT * FROM identity_provider_id) AND id_from_idp IS NOT NULL;
+                    """
+                ),
+                idp_name=idp_name,
+                iss=issuer,
+            )
+        except Exception as e:
+            transaction.rollback()
+            logger.warning(
+                "Could not populate iss_sub_pair_to_user table: {}".format(e)
+            )
+        else:
+            transaction.commit()
+            logger.info("Population was successful")

fence/jwt/validate.py

@@ -153,7 +153,7 @@ def validate_jwt(
                     attempt_refresh=attempt_refresh,
                     **kwargs
                 )
-            except Error as e:
+            except Exception as e:
                 raise JWTError("Invalid refresh token: {}".format(e))
         elif unverified_claims.get("pur") == "api_key" and isinstance(
             e, JWTAudienceError
@@ -170,7 +170,7 @@ def validate_jwt(
                     attempt_refresh=attempt_refresh,
                     **kwargs
                 )
-            except Error as e:
+            except Exception as e:
                 raise JWTError("Invalid API key: {}".format(e))
         else:
             ##### end refresh token, API key patch block #####