diff --git a/helm-quarry/prod-config.yaml b/helm-quarry/prod-config.yaml index fd94572..fc5f684 100644 Binary files a/helm-quarry/prod-config.yaml and b/helm-quarry/prod-config.yaml differ diff --git a/helm-quarry/values.yaml b/helm-quarry/values.yaml index 0d1205d..f164041 100644 --- a/helm-quarry/values.yaml +++ b/helm-quarry/values.yaml @@ -1,7 +1,7 @@ web: repository: 'quay.io/wikimedia-quarry/quarry' - tag: pr-65 # web tag managed by github actions + tag: pr-61 # web tag managed by github actions worker: repository: 'quay.io/wikimedia-quarry/quarry' - tag: pr-65 # worker tag managed by github actions \ No newline at end of file + tag: pr-61 # worker tag managed by github actions diff --git a/quarry/default_config.yaml b/quarry/default_config.yaml index 8762037..377b1db 100644 --- a/quarry/default_config.yaml +++ b/quarry/default_config.yaml @@ -40,6 +40,11 @@ TOOLS_DB_PORT: 3306 TOOLS_DB_USER: '' TOOLS_DB_PASSWORD: '' +QUARRY_P_HOST: 'db' +QUARRY_P_PORT: 3306 +QUARRY_P_USER: '' +QUARRY_P_PASSWORD: '' + OUTPUT_PATH_TEMPLATE: '/results/%s/%s/%s.sqlite' REDIS_HOST: 'redis' REDIS_PORT: 6379 diff --git a/quarry/web/replica.py b/quarry/web/replica.py index 1c80da0..b914c43 100644 --- a/quarry/web/replica.py +++ b/quarry/web/replica.py @@ -13,6 +13,7 @@ def __init__(self, config): def _db_name_mangler(self): self.is_tools_db = False + self.is_quarry_p = False if self.dbname == "": raise ReplicaConnectionException( "Attempting connection before a database is selected" @@ -20,6 +21,9 @@ def _db_name_mangler(self): if "__" in self.dbname and self.dbname.endswith("_p"): self.is_tools_db = True self.database_p = self.dbname + elif self.dbname == "quarry" or self.dbname == "quarry_p": + self.is_quarry_p = True + self.database_p = "quarry_p" elif self.dbname == "meta" or self.dbname == "meta_p": self.database_name = "s7" self.database_p = "meta_p" @@ -41,6 +45,8 @@ def _db_name_mangler(self): def get_host_name(self): if self.is_tools_db: return self.config["TOOLS_DB_HOST"] + if self.is_quarry_p: + return self.config["DB_HOST"] if self.config["REPLICA_DOMAIN"]: return f"{self.database_name}.{self.config['REPLICA_DOMAIN']}" return self.database_name @@ -62,7 +68,12 @@ def connection(self, db): self.dbname = db self._db_name_mangler() host = self.get_host_name() - conf_prefix = "TOOLS_DB" if self.is_tools_db else "REPLICA" + if self.is_tools_db: + conf_prefix = "TOOLS_DB" + elif self.is_quarry_p: + conf_prefix = "QUARRY_P" + else: + conf_prefix = "REPLICA" port = self.config[f"{conf_prefix}_PORT"] connect_opts = { "db": self.database_p, diff --git a/quarry/web/utils/__init__.py b/quarry/web/utils/__init__.py index 28f1859..efed242 100644 --- a/quarry/web/utils/__init__.py +++ b/quarry/web/utils/__init__.py @@ -2,7 +2,7 @@ VALID_DB_NAMES = re.compile( - r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry|s\d+__\w+_p)$" + r"^(?:(?:(?:centralauth|meta|[0-9a-z_]*wik[a-z]+)(?:_p)?)|quarry(?:_p)?|s\d+__\w+_p)$" ) diff --git a/quarry/web/worker.py b/quarry/web/worker.py index 607258d..7a6057e 100644 --- a/quarry/web/worker.py +++ b/quarry/web/worker.py @@ -27,8 +27,13 @@ def get_replag(cur): - cur.execute("SELECT lag FROM heartbeat_p.heartbeat;") - return int(cur.fetchall()[0][0]) + cur.execute("SELECT * FROM information_schema.tables WHERE table_schema='heartbeat_p' and table_name='heartbeat';") + if cur.rowcount: + cur.execute("SELECT lag FROM heartbeat_p.heartbeat;") + return int(cur.fetchall()[0][0]) + else: + # there is not a heartbeat table on this database + return 0 @worker_process_init.connect diff --git a/schema.sql b/schema.sql index 4d40e10..c388ab5 100644 --- a/schema.sql +++ b/schema.sql @@ -59,3 +59,9 @@ CREATE TABLE IF NOT EXISTS star( CREATE INDEX IF NOT EXISTS star_user_id_index ON star(user_id); CREATE INDEX IF NOT EXISTS star_query_id_index ON star(query_id); CREATE UNIQUE INDEX IF NOT EXISTS star_user_query_index ON star(user_id, query_id); + +CREATE DATABASE IF NOT EXISTS quarry_p CHARACTER SET utf8; +CREATE VIEW IF NOT EXISTS quarry_p.query AS SELECT * FROM quarry.query; +CREATE VIEW IF NOT EXISTS quarry_p.query_revision AS SELECT * FROM quarry.query_revision; +CREATE VIEW IF NOT EXISTS quarry_p.query_run AS SELECT * FROM quarry.query_run; +CREATE VIEW IF NOT EXISTS quarry_p.star AS SELECT * FROM quarry.star;