From dbf95284c7f7b5420fce01794d3d6c8e639213de Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 10 Jun 2020 18:48:37 -0300 Subject: [PATCH 1/3] Allow empty redis password --- helpers/config.py | 19 ++++++- helpers/template.py | 55 ++++++++++++++++++- .../enketo_express/config.json.tpl | 8 ++- .../envfiles/databases.txt.tpl | 4 +- .../kobo-deployments/envfiles/kobocat.txt.tpl | 2 +- .../kobo-deployments/envfiles/kpi.txt.tpl | 2 +- 6 files changed, 82 insertions(+), 8 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 1c45e7f..777d5a6 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -1332,11 +1332,28 @@ def __questions_raven(self): def __questions_redis(self): CLI.colored_print("Redis password?", CLI.COLOR_SUCCESS) self.__config["redis_password"] = CLI.get_response( - r"~^.{8,}$", + r"~^.{8,}|$", self.__config.get("redis_password"), to_lower=False, error_msg='Too short. 8 characters minimum.') + if not self.__config["redis_password"]: + CLI.colored_print("╔═════════════════════════════════════════════════╗", + CLI.COLOR_WARNING) + CLI.colored_print("║ WARNING! it's STRONGLY recommended to set a ║", + CLI.COLOR_WARNING) + CLI.colored_print("║ password for Redis as well. ║", + CLI.COLOR_WARNING) + CLI.colored_print("╚═════════════════════════════════════════════════╝", + CLI.COLOR_WARNING) + + CLI.colored_print("Do you want to continue?", CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + + if CLI.get_response([Config.TRUE, Config.FALSE], Config.FALSE) == Config.FALSE: + self.__questions_redis() + def __questions_reverse_proxy(self): if self.is_secure: diff --git a/helpers/template.py b/helpers/template.py index b523b04..f432a03 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -5,6 +5,7 @@ import fnmatch import json import os +import re import stat import sys try: @@ -284,7 +285,7 @@ def __read_unique_id(destination_directory): def __write_templates(template_variables_, root_, destination_directory_, filenames_): for filename in fnmatch.filter(filenames_, '*.tpl'): with open(os.path.join(root_, filename), "r") as template: - t = PyTemplate(template.read()) + t = ExtendedPyTemplate(template.read(), template_variables_) with open(os.path.join(destination_directory_, filename[:-4]), "w") as f: f.write(t.substitute(template_variables_)) @@ -305,3 +306,55 @@ def __write_unique_id(cls, destination_directory, unique_id): return False return True + + +class ExtendedPyTemplate(PyTemplate): + """ + Basic class to add conditional substitution to `string.Template` + + Usage example: + ``` + { + "host": "redis-cache.kobo.local", + "port": "6379"{% if REDIS_PASSWORD %},{% endif REDIS_PASSWORD %} + {% if REDIS_PASSWORD %} + "password": ${REDIS_PASSWORD} + {% endif REDIS_PASSWORD %} + } + ``` + + If `REDIS_PASSWORD` equals '123456', output would be: + ``` + { + "host": "redis-cache.kobo.local", + "port": "6379", + "password": '123456' + } + ``` + + If `REDIS_PASSWORD` equals '' (or `False` or `None`), output would be: + ``` + { + "host": "redis-cache.kobo.local", + "port": "6379" + + } + ``` + + """ + IF_PATTERN = '{{% if {} %}}' + ENDIF_PATTERN = '{{% endif {} %}}' + + def __init__(self, template, template_variables_): + for key, value in template_variables_.items(): + if self.IF_PATTERN.format(key) in template: + if value: + if_pattern = r'{}\s*'.format(self.IF_PATTERN.format(key)) + endif_pattern = r'\s*{}'.format(self.ENDIF_PATTERN.format(key)) + template = re.sub(if_pattern, '', template) + template = re.sub(endif_pattern, '', template) + else: + pattern = r'{}(.|\s)*?{}'.format(self.IF_PATTERN.format(key), + self.ENDIF_PATTERN.format(key)) + template = re.sub(pattern, '', template) + super(ExtendedPyTemplate, self).__init__(template) diff --git a/templates/kobo-deployments/enketo_express/config.json.tpl b/templates/kobo-deployments/enketo_express/config.json.tpl index 6ad3d15..99cdb60 100644 --- a/templates/kobo-deployments/enketo_express/config.json.tpl +++ b/templates/kobo-deployments/enketo_express/config.json.tpl @@ -41,13 +41,17 @@ "redis": { "cache": { "host": "redis-cache.${PRIVATE_DOMAIN_NAME}", - "port": "${REDIS_CACHE_PORT}", + "port": "${REDIS_CACHE_PORT}"{% if REDIS_PASSWORD %},{% endif REDIS_PASSWORD %} + {% if REDIS_PASSWORD %} "password": ${REDIS_PASSWORD_JS_ENCODED} + {% endif REDIS_PASSWORD %} }, "main": { "host": "redis-main.${PRIVATE_DOMAIN_NAME}", - "port": "${REDIS_MAIN_PORT}", + "port": "${REDIS_MAIN_PORT}"{% if REDIS_PASSWORD %},{% endif REDIS_PASSWORD %} + {% if REDIS_PASSWORD %} "password": ${REDIS_PASSWORD_JS_ENCODED} + {% endif REDIS_PASSWORD %} } }, "google": { diff --git a/templates/kobo-deployments/envfiles/databases.txt.tpl b/templates/kobo-deployments/envfiles/databases.txt.tpl index 3c9ef97..60ad910 100644 --- a/templates/kobo-deployments/envfiles/databases.txt.tpl +++ b/templates/kobo-deployments/envfiles/databases.txt.tpl @@ -49,6 +49,6 @@ ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE} # Default Redis backup schedule is weekly at 03:00 AM UTC on Sunday. ${USE_BACKUP}REDIS_BACKUP_SCHEDULE=${REDIS_BACKUP_SCHEDULE} -REDIS_SESSION_URL=redis://:${REDIS_PASSWORD_URL_ENCODED}@redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/2 -REDIS_LOCK_URL=redis://:${REDIS_PASSWORD_URL_ENCODED}@redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/3 +REDIS_SESSION_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/2 +REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/3 REDIS_PASSWORD=${REDIS_PASSWORD} diff --git a/templates/kobo-deployments/envfiles/kobocat.txt.tpl b/templates/kobo-deployments/envfiles/kobocat.txt.tpl index 078a246..de847e6 100644 --- a/templates/kobo-deployments/envfiles/kobocat.txt.tpl +++ b/templates/kobo-deployments/envfiles/kobocat.txt.tpl @@ -5,7 +5,7 @@ ${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True DJANGO_SETTINGS_MODULE=onadata.settings.kc_environ ENKETO_VERSION=Express -KOBOCAT_BROKER_URL=redis://:${REDIS_PASSWORD_URL_ENCODED}@redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/2 +KOBOCAT_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/2 KOBOCAT_CELERY_LOG_FILE=/srv/logs/celery.log # Default KoBoCAT media backup schedule is weekly at 12:00 AM UTC on Sunday. diff --git a/templates/kobo-deployments/envfiles/kpi.txt.tpl b/templates/kobo-deployments/envfiles/kpi.txt.tpl index 511bab2..1d01252 100644 --- a/templates/kobo-deployments/envfiles/kpi.txt.tpl +++ b/templates/kobo-deployments/envfiles/kpi.txt.tpl @@ -4,7 +4,7 @@ ${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True ENKETO_VERSION=Express KPI_PREFIX=/ -KPI_BROKER_URL=redis://:${REDIS_PASSWORD_URL_ENCODED}@redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1 +KPI_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD_URL_ENCODED}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1 KPI_MONGO_HOST=mongo.${PRIVATE_DOMAIN_NAME} KPI_MONGO_PORT=${MONGO_PORT} From 3671f00f69c29c21387bb77c4e0642d41d2f75e1 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 10 Jun 2020 19:04:48 -0300 Subject: [PATCH 2/3] Bumped version numbers --- helpers/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 777d5a6..1562647 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -10,7 +10,7 @@ import sys import time from datetime import datetime -from random import choice, randint +from random import choice from helpers.cli import CLI from helpers.network import Network @@ -28,9 +28,9 @@ class Config: DEFAULT_PROXY_PORT = "8080" DEFAULT_NGINX_PORT = "80" DEFAULT_NGINX_HTTPS_PORT = "443" - KOBO_DOCKER_BRANCH = '2.020.24' - KOBO_INSTALL_BRANCH = 'master' - KOBO_INSTALL_VERSION = '2.3.0' + KOBO_DOCKER_BRANCH = 'allow-empty-redis-password' + KOBO_INSTALL_BRANCH = 'allow-empty-redis-password' + KOBO_INSTALL_VERSION = '2.4.0' # Maybe overkill. Use this class as a singleton to get the same configuration # for each instantiation. From 8708a8f6a3c1c2b58c24ff72a717407d6042fad5 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 10 Jun 2020 20:19:01 -0300 Subject: [PATCH 3/3] Create dynamic uwsgi pass timeout based on Harakiri timeout --- helpers/config.py | 6 +++--- helpers/template.py | 1 + .../kobo-docker/docker-compose.frontend.override.yml.tpl | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 1c45e7f..4c0df48 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -28,9 +28,9 @@ class Config: DEFAULT_PROXY_PORT = "8080" DEFAULT_NGINX_PORT = "80" DEFAULT_NGINX_HTTPS_PORT = "443" - KOBO_DOCKER_BRANCH = '2.020.24' - KOBO_INSTALL_BRANCH = 'master' - KOBO_INSTALL_VERSION = '2.3.0' + KOBO_DOCKER_BRANCH = 'dynamic-uwsgi-pass-timeout' + KOBO_INSTALL_BRANCH = 'dynamic-uwsgi-pass-timeout' + KOBO_INSTALL_VERSION = '2.4.1' # Maybe overkill. Use this class as a singleton to get the same configuration # for each instantiation. diff --git a/helpers/template.py b/helpers/template.py index b523b04..b90af6d 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -199,6 +199,7 @@ def _get_value(property_, true_value="", false_value="#", "UWSGI_SOFT_LIMIT": int(config.get("uwsgi_soft_limit")) * 1024 * 1024, "UWSGI_HARAKIRI": config.get("uwsgi_harakiri"), "UWSGI_WORKER_RELOAD_MERCY": config.get("uwsgi_worker_reload_mercy"), + "UWSGI_PASS_TIMEOUT": int(config.get("uwsgi_harakiri")) + 10, "POSTGRES_REPLICATION_PASSWORD": config.get("postgres_replication_password"), "WSGI_SERVER": "runserver_plus" if config_object.dev_mode else "uWSGI", "USE_X_FORWARDED_HOST": "" if config_object.dev_mode else "#", diff --git a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl index bc0741f..0b0b9f9 100644 --- a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl @@ -62,6 +62,7 @@ services: nginx: environment: - NGINX_PUBLIC_PORT=${NGINX_PUBLIC_PORT} + - UWSGI_PASS_TIMEOUT=${UWSGI_PASS_TIMEOUT} ports: - ${NGINX_EXPOSED_PORT}:80 ${USE_EXTRA_HOSTS}extra_hosts: