diff --git a/helpers/command.py b/helpers/command.py index e147f42..40e638d 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -265,7 +265,7 @@ def start_maintenance(cls): "-f", "docker-compose.maintenance.yml", "-f", "docker-compose.maintenance.override.yml", "-p", config_object.get_prefix("maintenance"), - "up", "-d", "maintenance"] + "up", "-d"] CLI.run_command(frontend_command, config.get("kobodocker_path")) CLI.colored_print("Maintenance mode has been started", @@ -313,9 +313,7 @@ def start(cls, frontend_only=False): sys.exit(1) # Start the back-end containers - if not frontend_only: - if not config_object.multi_servers or \ - config_object.primary_backend or config_object.secondary_backend: + if not frontend_only and config_object.backend: backend_role = config.get("backend_server_role", "primary") backend_command = ["docker-compose", @@ -329,12 +327,13 @@ def start(cls, frontend_only=False): "up", "-d"] CLI.run_command(backend_command, config.get("kobodocker_path")) - # If this was previously a shared-database setup, migrate to separate - # databases for KPI and KoBoCAT - migrate_single_to_two_databases() - # Start the front-end containers - if not config_object.multi_servers or config_object.frontend: + if config_object.frontend: + + # If this was previously a shared-database setup, migrate to separate + # databases for KPI and KoBoCAT + migrate_single_to_two_databases() + frontend_command = ["docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", @@ -367,9 +366,12 @@ def start(cls, frontend_only=False): "It can take a few minutes.", CLI.COLOR_SUCCESS) cls.info() else: - CLI.colored_print(("Backend server should be up & running! " - "Please look at docker logs for further " - "information"), CLI.COLOR_WARNING) + CLI.colored_print( + ("{} backend server is starting up and should be " + "up & running soon!\nPlease look at docker logs for " + "further information: `python3 run.py -cb logs -f`".format( + config.get('backend_server_role'))), + CLI.COLOR_WARNING) @classmethod def stop(cls, output=True, frontend_only=False): @@ -405,21 +407,19 @@ def stop(cls, output=True, frontend_only=False): "down"] CLI.run_command(proxy_command, config_object.get_letsencrypt_repo_path()) - if not frontend_only: - if not config_object.multi_servers or config_object.primary_backend: - - backend_role = config.get("backend_server_role", "primary") + if not frontend_only and config_object.backend: + backend_role = config.get("backend_server_role", "primary") - backend_command = [ - "docker-compose", - "-f", - "docker-compose.backend.{}.yml".format(backend_role), - "-f", - "docker-compose.backend.{}.override.yml".format(backend_role), - "-p", config_object.get_prefix("backend"), - "down" - ] - CLI.run_command(backend_command, config.get("kobodocker_path")) + backend_command = [ + "docker-compose", + "-f", + "docker-compose.backend.{}.yml".format(backend_role), + "-f", + "docker-compose.backend.{}.override.yml".format(backend_role), + "-p", config_object.get_prefix("backend"), + "down" + ] + CLI.run_command(backend_command, config.get("kobodocker_path")) if output: CLI.colored_print("KoBoToolbox has been stopped", CLI.COLOR_SUCCESS) diff --git a/helpers/config.py b/helpers/config.py index 993f19e..a8f3313 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -15,10 +15,13 @@ from helpers.cli import CLI from helpers.network import Network -from helpers.singleton import Singleton +from helpers.singleton import Singleton, with_metaclass -class Config: +# Use this class as a singleton to get the same configuration +# for each instantiation. +class Config(with_metaclass(Singleton)): + CONFIG_FILE = ".run.conf" UNIQUE_ID_FILE = ".uniqid" UPSERT_DB_USERS_TRIGGER_FILE = ".upsert_db_users" @@ -29,12 +32,8 @@ class Config: DEFAULT_PROXY_PORT = "8080" DEFAULT_NGINX_PORT = "80" DEFAULT_NGINX_HTTPS_PORT = "443" - KOBO_DOCKER_BRANCH = '2.020.34a' - KOBO_INSTALL_VERSION = '3.1.3' - - # Maybe overkill. Use this class as a singleton to get the same configuration - # for each instantiation. - __metaclass__ = Singleton + KOBO_DOCKER_BRANCH = '2.020.37' + KOBO_INSTALL_VERSION = '3.2.1' def __init__(self): self.__config = self.read_config() @@ -49,10 +48,6 @@ def advanced_options(self): """ return self.__config.get("advanced") == Config.TRUE - @property - def block_common_http_ports(self): - return self.use_letsencrypt or self.__config.get("block_common_http_ports") == Config.TRUE - def auto_detect_network(self): """ Tries to detect new ip @@ -75,6 +70,15 @@ def aws(self): """ return self.__config.get("use_aws") == Config.TRUE + @property + def backend(self): + return not self.multi_servers or self.primary_backend or \ + self.secondary_backend + + @property + def block_common_http_ports(self): + return self.use_letsencrypt or self.__config.get("block_common_http_ports") == Config.TRUE + @property def expose_backend_ports(self): return self.__config.get("expose_backend_ports") == Config.TRUE @@ -145,26 +149,7 @@ def build(self): sys.exit(1) else: - config = self.get_config_template() - config.update(self.__config) - - # If the configuration came from a previous version that had a - # single Postgres database, we need to make sure the new - # `kc_postgres_db` is set to the name of that single database, - # *not* the default from `get_config_template()` - if ( - self.__config.get("postgres_db") - and not self.__config.get("kc_postgres_db") - ): - config["kc_postgres_db"] = self.__config["postgres_db"] - - # Force update user's config to use new terminology. - backend_role = config.get('backend_server_role') - if backend_role in ['master', 'slave']: - config['backend_server_role'] = 'primary' \ - if backend_role == 'master' else 'secondary' - - self.__config = config + self.__config = self.__get_upgraded_config() self.__welcome() self.__create_directory() @@ -177,7 +162,7 @@ def build(self): self.__questions_multi_servers() if self.multi_servers: self.__questions_roles() - if self.frontend: + if self.frontend or self.secondary_backend: self.__questions_private_routes() else: self.__reset(private_dns=True) @@ -210,9 +195,9 @@ def build(self): self.__questions_backup() - self.__config = config self.write_config() - return config + + return self.__config @property def dev_mode(self): @@ -349,6 +334,7 @@ def get_config_template(cls): "uwsgi_soft_limit": "128", "uwsgi_harakiri": "120", "uwsgi_worker_reload_mercy": "120", + "backup_from_primary": Config.TRUE, } def get_service_names(self): @@ -575,7 +561,9 @@ def __clone_repo(self, repo_path, repo_name): def __detect_network(self): self.__config["local_interface_ip"] = Network.get_primary_ip() - self.__config["primary_backend_ip"] = self.__config["local_interface_ip"] + + if self.frontend: + self.__config["primary_backend_ip"] = self.__config["local_interface_ip"] if self.advanced_options: CLI.colored_print("Please choose which network interface you want to use?", CLI.COLOR_SUCCESS) @@ -610,7 +598,38 @@ def __detect_network(self): self.__config["local_interface"] = response self.__config["local_interface_ip"] = interfaces[self.__config.get("local_interface")] - self.__config["primary_backend_ip"] = self.__config.get("local_interface_ip") + + if self.frontend: + self.__config["primary_backend_ip"] = self.__config.get("local_interface_ip") + + def __get_upgraded_config(self): + """ + Sometimes during upgrades, some keys are changed/deleted/added. + This method helps to get a compliant dict to expected config + + :return: dict + """ + + upgraded_config = self.get_config_template() + upgraded_config.update(self.__config) + + # If the configuration came from a previous version that had a + # single Postgres database, we need to make sure the new + # `kc_postgres_db` is set to the name of that single database, + # *not* the default from `get_config_template()` + if ( + self.__config.get("postgres_db") + and not self.__config.get("kc_postgres_db") + ): + upgraded_config["kc_postgres_db"] = self.__config["postgres_db"] + + # Force update user's config to use new terminology. + backend_role = upgraded_config.get('backend_server_role') + if backend_role in ['master', 'slave']: + upgraded_config['backend_server_role'] = 'primary' \ + if backend_role == 'master' else 'secondary' + + return upgraded_config def __questions_advanced_options(self): """ @@ -644,6 +663,74 @@ def __questions_aws(self): self.__config["aws_secret_key"] = "" self.__config["aws_bucket_name"] = "" + def __questions_aws_backup_settings(self): + + self.__config["aws_backup_bucket_name"] = CLI.colored_input( + "AWS Backups bucket name", CLI.COLOR_SUCCESS, + self.__config.get("aws_backup_bucket_name", "")) + + if self.__config["aws_backup_bucket_name"] != "": + + backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE + + CLI.colored_print("How many yearly backups to keep?", CLI.COLOR_SUCCESS) + self.__config["aws_backup_yearly_retention"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_backup_yearly_retention")) + + CLI.colored_print("How many monthly backups to keep?", CLI.COLOR_SUCCESS) + self.__config["aws_backup_monthly_retention"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_backup_monthly_retention")) + + CLI.colored_print("How many weekly backups to keep?", CLI.COLOR_SUCCESS) + self.__config["aws_backup_weekly_retention"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_backup_weekly_retention")) + + CLI.colored_print("How many daily backups to keep?", CLI.COLOR_SUCCESS) + self.__config["aws_backup_daily_retention"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_backup_daily_retention")) + + if (not self.multi_servers or + (self.primary_backend and backup_from_primary) or + (self.secondary_backend and not backup_from_primary)): + CLI.colored_print("PostgresSQL backup minimum size (in MB)?", + CLI.COLOR_SUCCESS) + CLI.colored_print( + "Files below this size will be ignored when rotating backups.", + CLI.COLOR_INFO) + self.__config["aws_postgres_backup_minimum_size"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_postgres_backup_minimum_size")) + + if self.primary_backend or not self.multi_servers: + CLI.colored_print("MongoDB backup minimum size (in MB)?", + CLI.COLOR_SUCCESS) + CLI.colored_print( + "Files below this size will be ignored when rotating backups.", + CLI.COLOR_INFO) + self.__config["aws_mongo_backup_minimum_size"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_mongo_backup_minimum_size")) + + CLI.colored_print("Redis backup minimum size (in MB)?", + CLI.COLOR_SUCCESS) + CLI.colored_print( + "Files below this size will be ignored when rotating backups.", + CLI.COLOR_INFO) + self.__config["aws_redis_backup_minimum_size"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_redis_backup_minimum_size")) + + CLI.colored_print("Chunk size of multipart uploads (in MB)?", + CLI.COLOR_SUCCESS) + self.__config["aws_backup_upload_chunk_size"] = CLI.get_response( + r"~^\d+$", self.__config.get("aws_backup_upload_chunk_size")) + + CLI.colored_print("Use AWS LifeCycle deletion rule?", + CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + self.__config["aws_backup_bucket_deletion_rule_enabled"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("aws_backup_bucket_deletion_rule_enabled", + Config.FALSE)) + def __questions_backup(self): """ Asks all questions about backups. @@ -688,22 +775,28 @@ def __questions_backup(self): if self.backend_questions: - CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) - self.__config["postgres_backup_schedule"] = CLI.get_response( - "~{}".format(schedule_regex_pattern), - self.__config.get( - "postgres_backup_schedule", - "0 2 * * 0")) - if self.primary_backend: - CLI.colored_print("Run backups from primary backend server?", CLI.COLOR_SUCCESS) + CLI.colored_print("Run PostgreSQL backup from primary backend server?", + CLI.COLOR_SUCCESS) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") - self.__config["backup_from_primary"] = CLI.get_response([Config.TRUE, Config.FALSE], - self.__config.get( - "backup_from_primary", - Config.TRUE)) + self.__config["backup_from_primary"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("backup_from_primary", Config.TRUE)) + + backup_from_primary = self.__config["backup_from_primary"] == Config.TRUE + if (not self.multi_servers or + (self.primary_backend and backup_from_primary) or + (self.secondary_backend and not backup_from_primary)): + CLI.colored_print("PostgreSQL backup schedule?", CLI.COLOR_SUCCESS) + self.__config["postgres_backup_schedule"] = CLI.get_response( + "~{}".format(schedule_regex_pattern), + self.__config.get( + "postgres_backup_schedule", + "0 2 * * 0")) + if self.primary_backend or not self.multi_servers: + CLI.colored_print("MongoDB backup schedule?", CLI.COLOR_SUCCESS) self.__config["mongo_backup_schedule"] = CLI.get_response( "~{}".format(schedule_regex_pattern), @@ -717,58 +810,10 @@ def __questions_backup(self): self.__config.get( "redis_backup_schedule", "0 3 * * 0")) + if self.aws: - self.__config["aws_backup_bucket_name"] = CLI.colored_input("AWS Backups bucket name", - CLI.COLOR_SUCCESS, - self.__config.get( - "aws_backup_bucket_name", - "")) - if self.__config["aws_backup_bucket_name"] != "": - CLI.colored_print("How many yearly backups to keep?", CLI.COLOR_SUCCESS) - self.__config["aws_backup_yearly_retention"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_backup_yearly_retention", "2")) - - CLI.colored_print("How many monthly backups to keep?", CLI.COLOR_SUCCESS) - self.__config["aws_backup_monthly_retention"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_backup_monthly_retention", "12")) - - CLI.colored_print("How many weekly backups to keep?", CLI.COLOR_SUCCESS) - self.__config["aws_backup_weekly_retention"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_backup_weekly_retention", "4")) - - CLI.colored_print("How many daily backups to keep?", CLI.COLOR_SUCCESS) - self.__config["aws_backup_daily_retention"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_backup_daily_retention", "30")) - - CLI.colored_print("MongoDB backup minimum size (in MB)?", CLI.COLOR_SUCCESS) - CLI.colored_print("Files below this size will be ignored when rotating backups.", - CLI.COLOR_INFO) - self.__config["aws_mongo_backup_minimum_size"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_mongo_backup_minimum_size", "50")) - - CLI.colored_print("PostgresSQL backup minimum size (in MB)?", CLI.COLOR_SUCCESS) - CLI.colored_print("Files below this size will be ignored when rotating backups.", - CLI.COLOR_INFO) - self.__config["aws_postgres_backup_minimum_size"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_postgres_backup_minimum_size", "50")) - - CLI.colored_print("Redis backup minimum size (in MB)?", CLI.COLOR_SUCCESS) - CLI.colored_print("Files below this size will be ignored when rotating backups.", - CLI.COLOR_INFO) - self.__config["aws_redis_backup_minimum_size"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_redis_backup_minimum_size", "5")) - - CLI.colored_print("Chunk size of multipart uploads (in MB)?", CLI.COLOR_SUCCESS) - self.__config["aws_backup_upload_chunk_size"] = CLI.get_response( - r"~^\d+$", self.__config.get("aws_backup_upload_chunk_size", "15")) - - CLI.colored_print("Use AWS LifeCycle deletion rule?", CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - self.__config["aws_backup_bucket_deletion_rule_enabled"] = CLI.get_response( - [Config.TRUE, Config.FALSE], - self.__config.get("aws_backup_bucket_deletion_rule_enabled", - Config.FALSE)) + self.__questions_aws_backup_settings() + else: self.__config["use_backup"] = Config.FALSE @@ -787,37 +832,45 @@ def __questions_dev_mode(self): if self.local_install: # NGinX different port CLI.colored_print("Web server port?", CLI.COLOR_SUCCESS) - self.__config["exposed_nginx_docker_port"] = CLI.get_response(r"~^\d+$", - self.__config.get( - "exposed_nginx_docker_port", - Config.DEFAULT_NGINX_PORT)) + self.__config["exposed_nginx_docker_port"] = CLI.get_response( + r"~^\d+$", self.__config.get("exposed_nginx_docker_port", + Config.DEFAULT_NGINX_PORT)) CLI.colored_print("Developer mode?", CLI.COLOR_SUCCESS) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") - self.__config["dev_mode"] = CLI.get_response([Config.TRUE, Config.FALSE], - self.__config.get("dev_mode", Config.FALSE)) + self.__config["dev_mode"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("dev_mode", Config.FALSE)) self.__config["staging_mode"] = Config.FALSE else: CLI.colored_print("Staging mode?", CLI.COLOR_SUCCESS) CLI.colored_print("\t1) Yes") CLI.colored_print("\t2) No") - self.__config["staging_mode"] = CLI.get_response([Config.TRUE, Config.FALSE], - self.__config.get("staging_mode", Config.FALSE)) + self.__config["staging_mode"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("staging_mode", Config.FALSE)) self.__config["dev_mode"] = Config.FALSE if self.dev_mode or self.staging_mode: - CLI.colored_print("╔═══════════════════════════════════════════════════════════╗", CLI.COLOR_WARNING) - CLI.colored_print("║ Where are the files located locally? It can be absolute ║", CLI.COLOR_WARNING) - CLI.colored_print("║ or relative to the directory of `kobo-docker`. ║", CLI.COLOR_WARNING) - CLI.colored_print("║ Leave empty if you don't need to overload the repository. ║", CLI.COLOR_WARNING) - CLI.colored_print("╚═══════════════════════════════════════════════════════════╝", CLI.COLOR_WARNING) - self.__config["kc_path"] = CLI.colored_input("KoBoCat files location", CLI.COLOR_SUCCESS, - self.__config.get("kc_path")) + CLI.colored_print("╔═══════════════════════════════════════════════════════════╗", + CLI.COLOR_WARNING) + CLI.colored_print("║ Where are the files located locally? It can be absolute ║", + CLI.COLOR_WARNING) + CLI.colored_print("║ or relative to the directory of `kobo-docker`. ║", + CLI.COLOR_WARNING) + CLI.colored_print("║ Leave empty if you don't need to overload the repository. ║", + CLI.COLOR_WARNING) + CLI.colored_print("╚═══════════════════════════════════════════════════════════╝", + CLI.COLOR_WARNING) + self.__config["kc_path"] = CLI.colored_input( + "KoBoCat files location", CLI.COLOR_SUCCESS, + self.__config.get("kc_path")) self.__clone_repo(self.__config["kc_path"], "kobocat") - self.__config["kpi_path"] = CLI.colored_input("KPI files location", CLI.COLOR_SUCCESS, - self.__config.get("kpi_path")) + self.__config["kpi_path"] = CLI.colored_input( + "KPI files location", CLI.COLOR_SUCCESS, + self.__config.get("kpi_path")) self.__clone_repo(self.__config["kpi_path"], "kpi") # Create an unique id to build fresh image when starting containers @@ -838,15 +891,17 @@ def __questions_dev_mode(self): CLI.colored_print("Enable DEBUG?", CLI.COLOR_SUCCESS) CLI.colored_print("\t1) True") CLI.colored_print("\t2) False") - self.__config["debug"] = CLI.get_response([Config.TRUE, Config.FALSE], - self.__config.get("debug", Config.TRUE)) + self.__config["debug"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("debug", Config.TRUE)) # Frontend development CLI.colored_print("How do you want to run `npm`?", CLI.COLOR_SUCCESS) CLI.colored_print("\t1) From within the container") CLI.colored_print("\t2) Locally") - self.__config["npm_container"] = CLI.get_response([Config.TRUE, Config.FALSE], - self.__config.get("npm_container", Config.TRUE)) + self.__config["npm_container"] = CLI.get_response( + [Config.TRUE, Config.FALSE], + self.__config.get("npm_container", Config.TRUE)) else: # Force reset paths self.__reset(dev=True, reset_nginx_port=self.staging_mode) @@ -947,90 +1002,93 @@ def _round_nearest_quarter(dt): def __questions_mongo(self): """ - Mongo credentials. + Ask for MongoDB credentials only when server is for: + - primary backend + - single server installation """ - mongo_user_username = self.__config["mongo_user_username"] - mongo_user_password = self.__config["mongo_user_password"] - mongo_root_username = self.__config["mongo_root_username"] - mongo_root_password = self.__config["mongo_root_password"] - - CLI.colored_print("MongoDB root's username?", - CLI.COLOR_SUCCESS) - self.__config["mongo_root_username"] = CLI.get_response( - r"~^\w+$", - self.__config.get("mongo_root_username"), - to_lower=False) - - CLI.colored_print("MongoDB root's password?", CLI.COLOR_SUCCESS) - self.__config["mongo_root_password"] = CLI.get_response( - r"~^.{8,}$", - self.__config.get("mongo_root_password"), - to_lower=False, - error_msg='Too short. 8 characters minimum.') + if self.primary_backend or not self.multi_servers: + mongo_user_username = self.__config["mongo_user_username"] + mongo_user_password = self.__config["mongo_user_password"] + mongo_root_username = self.__config["mongo_root_username"] + mongo_root_password = self.__config["mongo_root_password"] - CLI.colored_print("MongoDB user's username?", - CLI.COLOR_SUCCESS) - self.__config["mongo_user_username"] = CLI.get_response( - r"~^\w+$", - self.__config.get("mongo_user_username"), - to_lower=False) - - CLI.colored_print("MongoDB user's password?", CLI.COLOR_SUCCESS) - self.__config["mongo_user_password"] = CLI.get_response( - r"~^.{8,}$", - self.__config.get("mongo_user_password"), - to_lower=False, - error_msg='Too short. 8 characters minimum.') - - if (self.__config.get("mongo_secured") != Config.TRUE or - mongo_user_username != self.__config.get("mongo_user_username") or - mongo_user_password != self.__config.get("mongo_user_password") or - mongo_root_username != self.__config.get("mongo_root_username") or - mongo_root_password != self.__config.get("mongo_root_password")) and \ - not self.first_time: - - # Because chances are high we cannot communicate with DB - # (e.g ports not exposed, containers down), we delegate the task - # to MongoDB container to update (create/delete) users. - # (see. `kobo-docker/mongo/upsert_users.sh`) - # We have to transmit old users (and their respective DB) to - # MongoDB to let it know which users need to be deleted. + CLI.colored_print("MongoDB root's username?", + CLI.COLOR_SUCCESS) + self.__config["mongo_root_username"] = CLI.get_response( + r"~^\w+$", + self.__config.get("mongo_root_username"), + to_lower=False) + + CLI.colored_print("MongoDB root's password?", CLI.COLOR_SUCCESS) + self.__config["mongo_root_password"] = CLI.get_response( + r"~^.{8,}$", + self.__config.get("mongo_root_password"), + to_lower=False, + error_msg='Too short. 8 characters minimum.') - # `content` will be read by MongoDB container at next boot - # It should contains users to delete if any. - # Its format should be: `` - content = '' + CLI.colored_print("MongoDB user's username?", + CLI.COLOR_SUCCESS) + self.__config["mongo_user_username"] = CLI.get_response( + r"~^\w+$", + self.__config.get("mongo_user_username"), + to_lower=False) + + CLI.colored_print("MongoDB user's password?", CLI.COLOR_SUCCESS) + self.__config["mongo_user_password"] = CLI.get_response( + r"~^.{8,}$", + self.__config.get("mongo_user_password"), + to_lower=False, + error_msg='Too short. 8 characters minimum.') + + if (self.__config.get("mongo_secured") != Config.TRUE or + mongo_user_username != self.__config.get("mongo_user_username") or + mongo_user_password != self.__config.get("mongo_user_password") or + mongo_root_username != self.__config.get("mongo_root_username") or + mongo_root_password != self.__config.get("mongo_root_password")) and \ + not self.first_time: + + # Because chances are high we cannot communicate with DB + # (e.g ports not exposed, containers down), we delegate the task + # to MongoDB container to update (create/delete) users. + # (see. `kobo-docker/mongo/upsert_users.sh`) + # We have to transmit old users (and their respective DB) to + # MongoDB to let it know which users need to be deleted. + + # `content` will be read by MongoDB container at next boot + # It should contains users to delete if any. + # Its format should be: `` + content = '' + + if (mongo_user_username != self.__config.get("mongo_user_username") or + mongo_root_username != self.__config.get("mongo_root_username")): - if (mongo_user_username != self.__config.get("mongo_user_username") or - mongo_root_username != self.__config.get("mongo_root_username")): + CLI.colored_print("╔══════════════════════════════════════════════════════╗", + CLI.COLOR_WARNING) + CLI.colored_print("║ MongoDB root's and/or user's usernames have changed! ║", + CLI.COLOR_WARNING) + CLI.colored_print("╚══════════════════════════════════════════════════════╝", + CLI.COLOR_WARNING) + CLI.colored_print("Do you want to remove old users?", CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") + delete_users = CLI.get_response([Config.TRUE, Config.FALSE], Config.TRUE) - CLI.colored_print("╔══════════════════════════════════════════════════════╗", - CLI.COLOR_WARNING) - CLI.colored_print("║ MongoDB root's and/or user's usernames have changed! ║", - CLI.COLOR_WARNING) - CLI.colored_print("╚══════════════════════════════════════════════════════╝", - CLI.COLOR_WARNING) - CLI.colored_print("Do you want to remove old users?", CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") - delete_users = CLI.get_response([Config.TRUE, Config.FALSE], Config.TRUE) - - if delete_users == Config.TRUE: - usernames_by_db = { - mongo_user_username: 'formhub', - mongo_root_username: 'admin' - } - for username, db in usernames_by_db.items(): - if username != "": - content += "{cr}{username}\t{db}".format( - cr="\n" if content else "", - username=username, - db=db - ) + if delete_users == Config.TRUE: + usernames_by_db = { + mongo_user_username: 'formhub', + mongo_root_username: 'admin' + } + for username, db in usernames_by_db.items(): + if username != "": + content += "{cr}{username}\t{db}".format( + cr="\n" if content else "", + username=username, + db=db + ) - self.__write_upsert_db_users_trigger_file(content, 'mongo') + self.__write_upsert_db_users_trigger_file(content, 'mongo') - self.__config["mongo_secured"] = Config.TRUE + self.__config["mongo_secured"] = Config.TRUE def __questions_multi_servers(self): """ @@ -1166,6 +1224,25 @@ def __questions_postgres(self): self.__config.get("postgres_settings", Config.FALSE)) if self.__config["postgres_settings"] == Config.TRUE: + + # pgconfig.org API is often unresponsive and make kobo-install hang forever. + # A docker image is available, let's use it instead. + # (Hope docker hub is not down too). + + # Find an open port. + open_port = 9080 + while True: + if not Network.is_port_open(open_port): + break + open_port += 1 + + # Start pgconfig.org API docker image + docker_command = ['docker', 'run', '--rm', '-p', + '127.0.0.1:{}:8080'.format(open_port), + '-d', '--name', 'pgconfig_container', + 'sebastianwebber/pgconfig-api'] + CLI.run_command(docker_command) + CLI.colored_print("Total Memory in GB?", CLI.COLOR_SUCCESS) self.__config["postgres_ram"] = CLI.get_response(r"~^\d+$", self.__config.get("postgres_ram")) @@ -1202,21 +1279,32 @@ def __questions_postgres(self): else: self.__config["postgres_profile"] = "Mixed" - # use pgconfig.org API to build postgres config - endpoint = "https://api.pgconfig.org/v1/tuning/get-config?environment_name={profile}" \ - "&format=conf&include_pgbadger=false&max_connections={max_connections}&" \ - "pg_version=9.5&total_ram={ram}GB&drive_type={drive_type}".format( - profile=self.__config["postgres_profile"], - ram=self.__config["postgres_ram"], - max_connections=self.__config["postgres_max_connections"], - drive_type=self.__config["postgres_hard_drive_type"].upper() - ) - response = Network.curl(endpoint) - if response: - self.__config["postgres_settings_content"] = re.sub(r"(log|lc_).+(\n|$)", "", response) - else: - # If no response from API, keep defaults - self.__config["postgres_settings"] = Config.FALSE + endpoint = "http://127.0.0.1:{open_port}/v1/tuning/get-config?environment_name={profile}" \ + "&format=conf&include_pgbadger=false&max_connections={max_connections}&" \ + "pg_version=9.5&total_ram={ram}GB&drive_type={drive_type}".format( + open_port=open_port, + profile=self.__config["postgres_profile"], + ram=self.__config["postgres_ram"], + max_connections=self.__config["postgres_max_connections"], + drive_type=self.__config["postgres_hard_drive_type"].upper() + ) + response = Network.curl(endpoint) + if response: + self.__config["postgres_settings_content"] = re.sub( + r"(log|lc_).+(\n|$)", "", response) + else: + if self.__config["postgres_settings_content"] == '': + CLI.colored_print("Use default settings.", + CLI.COLOR_INFO) + # If no response from API, keep defaults + self.__config["postgres_settings"] = Config.FALSE + else: + CLI.colored_print("\nKeep current settings.", + CLI.COLOR_INFO) + + # Stop container + docker_command = ['docker', 'stop', '-t', '0', 'pgconfig_container'] + CLI.run_command(docker_command) def __questions_ports(self): """ @@ -1299,7 +1387,7 @@ def __questions_private_routes(self): Config.FALSE)) if self.__config["use_private_dns"] == Config.FALSE: - CLI.colored_print("IP address (IPv4) of backend server?", CLI.COLOR_SUCCESS) + CLI.colored_print("IP address (IPv4) of primary backend server?", CLI.COLOR_SUCCESS) self.__config["primary_backend_ip"] = CLI.get_response( r"~\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", self.__config.get("primary_backend_ip", self.__primary_ip)) @@ -1354,29 +1442,35 @@ def __questions_raven(self): self.__config["kpi_raven_js"] = "" def __questions_redis(self): - CLI.colored_print("Redis password?", CLI.COLOR_SUCCESS) - self.__config["redis_password"] = CLI.get_response( - r"~^.{8,}|$", - self.__config.get("redis_password"), - to_lower=False, - error_msg='Too short. 8 characters minimum.') + """ + Ask for redis password only when server is for: + - primary backend + - single server installation + """ + if self.primary_backend or not self.multi_servers: + CLI.colored_print("Redis password?", CLI.COLOR_SUCCESS) + self.__config["redis_password"] = CLI.get_response( + r"~^.{8,}|$", + self.__config.get("redis_password"), + to_lower=False, + error_msg='Too short. 8 characters minimum.') - if not self.__config["redis_password"]: - CLI.colored_print("╔═════════════════════════════════════════════════╗", - CLI.COLOR_WARNING) - CLI.colored_print("║ WARNING! it's STRONGLY recommended to set a ║", - CLI.COLOR_WARNING) - CLI.colored_print("║ password for Redis as well. ║", - CLI.COLOR_WARNING) - CLI.colored_print("╚═════════════════════════════════════════════════╝", - CLI.COLOR_WARNING) + if not self.__config["redis_password"]: + CLI.colored_print("╔═════════════════════════════════════════════════╗", + CLI.COLOR_WARNING) + CLI.colored_print("║ WARNING! it's STRONGLY recommended to set a ║", + CLI.COLOR_WARNING) + CLI.colored_print("║ password for Redis as well. ║", + CLI.COLOR_WARNING) + CLI.colored_print("╚═════════════════════════════════════════════════╝", + CLI.COLOR_WARNING) - CLI.colored_print("Do you want to continue?", CLI.COLOR_SUCCESS) - CLI.colored_print("\t1) Yes") - CLI.colored_print("\t2) No") + CLI.colored_print("Do you want to continue?", CLI.COLOR_SUCCESS) + CLI.colored_print("\t1) Yes") + CLI.colored_print("\t2) No") - if CLI.get_response([Config.TRUE, Config.FALSE], Config.FALSE) == Config.FALSE: - self.__questions_redis() + if CLI.get_response([Config.TRUE, Config.FALSE], Config.FALSE) == Config.FALSE: + self.__questions_redis() def __questions_reverse_proxy(self): diff --git a/helpers/singleton.py b/helpers/singleton.py index d67a5f6..3ac501d 100644 --- a/helpers/singleton.py +++ b/helpers/singleton.py @@ -2,6 +2,20 @@ from __future__ import print_function, unicode_literals +# Copy this method from `six` library to avoid import +# Remove it when dropping Python2 support +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + class Singleton(type): _instances = {} diff --git a/helpers/template.py b/helpers/template.py index 788e6a7..098f080 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import print_function, unicode_literals -import binascii import fnmatch import json import os diff --git a/readme.md b/readme.md index 959db29..8e6295f 100644 --- a/readme.md +++ b/readme.md @@ -174,8 +174,3 @@ or $kobo-install> sudo apt install tox $kobo-install> tox ``` - - -## To-Do - -- Handle secondary backend diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl index 8649e89..0560cde 100644 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl @@ -10,3 +10,6 @@ services: ${OVERRIDE_POSTGRES_SETTINGS} - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf ports: - ${POSTGRES_PORT}:5432 + ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: + ${ADD_BACKEND_EXTRA_HOSTS}- postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS}- primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index 3dc4fc6..2206ef5 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -37,7 +37,7 @@ KPI_DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASSWORD_URL_ENCODED}@pos # Replication KOBO_POSTGRES_REPLICATION_USER=kobo_replication KOBO_POSTGRES_REPLICATION_PASSWORD=${POSTGRES_REPLICATION_PASSWORD} -#KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME} +KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME} # Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday. ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE} diff --git a/templates/kobo-env/postgres/secondary/postgres.conf.tpl b/templates/kobo-env/postgres/secondary/postgres.conf.tpl new file mode 100644 index 0000000..aad98af --- /dev/null +++ b/templates/kobo-env/postgres/secondary/postgres.conf.tpl @@ -0,0 +1,26 @@ +##################################################################################### +# SECONDARY SPECIFIC +# If file must be appended to shared/postgres.conf +##################################################################################### +#------------------------------------------------------------------------------------ +# TUNING +#------------------------------------------------------------------------------------ +# These settings are based on server configuration +# https://www.pgconfig.org/#/tuning +# DB Version: 9.5 +# OS Type: linux +# App profile: ${POSTGRES_APP_PROFILE} +# Hard-drive: SSD +# Total Memory (RAM): ${POSTGRES_RAM}GB + +${POSTGRES_SETTINGS} + +#------------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------------ +hot_standby_feedback = on + +# https://stackoverflow.com/a/33282856 +# https://stackoverflow.com/a/34404303 +max_standby_streaming_delay = -1 +max_standby_archive_delay = -1 diff --git a/tests/test_config.py b/tests/test_config.py index f131f6f..4d3e3f2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,54 +1,30 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import json import pytest import os import shutil import tempfile import time - - try: - from unittest.mock import patch, mock_open, MagicMock - builtin_open = "builtins.open" + from unittest.mock import patch, MagicMock except ImportError: - from mock import patch, mock_open, MagicMock - builtin_open = "__builtin__.open" + from mock import patch, MagicMock from helpers.cli import CLI from helpers.config import Config +from .utils import ( + read_config, + write_trigger_upsert_db_users, +) -def reset_config(config_object): - - config_dict = dict(Config.get_config_template()) - config_dict["kobodocker_path"] = "/tmp" - config_object.__config = config_dict - - -def write_trigger_upsert_db_users(*args): - content = args[1] - with open("/tmp/upsert_db_users", "w") as f: - f.write(content) - - -def test_read_config(overrides=None): - - config_dict = dict(Config.get_config_template()) - config_dict["kobodocker_path"] = "/tmp" - if overrides is not None: - config_dict.update(overrides) - with patch(builtin_open, mock_open(read_data=json.dumps(config_dict))) as mock_file: - config_object = Config() - config_object.read_config() - assert config_object.get_config().get("kobodocker_path") == config_dict.get("kobodocker_path") - - return config_object +def test_read_config(): + config_object = read_config() def test_advanced_options(): - config_object = test_read_config() + config_object = read_config() with patch.object(CLI, "colored_input", return_value=Config.TRUE) as mock_ci: config_object._Config__questions_advanced_options() assert config_object.advanced_options @@ -59,7 +35,7 @@ def test_advanced_options(): def test_installation(): - config_object = test_read_config() + config_object = read_config() with patch.object(CLI, "colored_input", return_value=Config.FALSE) as mock_ci: config_object._Config__questions_installation_type() assert not config_object.local_install @@ -75,7 +51,7 @@ def test_installation(): @patch("helpers.config.Config._Config__clone_repo", MagicMock(return_value=True)) def test_staging_mode(): - config_object = test_read_config() + config_object = read_config() kc_repo_path = tempfile.mkdtemp() kpi_repo_path = tempfile.mkdtemp() @@ -122,7 +98,7 @@ def test_dev_mode(): def test_server_roles_questions(): - config_object = test_read_config() + config_object = read_config() assert config_object.frontend_questions assert config_object.backend_questions @@ -143,7 +119,7 @@ def test_server_roles_questions(): def test_use_https(): - config_object = test_read_config() + config_object = read_config() assert config_object.is_secure @@ -160,7 +136,7 @@ def test_use_https(): @patch("helpers.config.Config._Config__clone_repo", MagicMock(return_value=True)) def test_proxy_letsencrypt(): - config_object = test_read_config() + config_object = read_config() assert config_object.proxy assert config_object.use_letsencrypt @@ -182,7 +158,7 @@ def test_proxy_letsencrypt(): def test_proxy_no_letsencrypt_advanced(): - config_object = test_read_config() + config_object = read_config() # Force advanced options config_object._Config__config["advanced"] = Config.TRUE assert config_object.advanced_options @@ -200,7 +176,7 @@ def test_proxy_no_letsencrypt_advanced(): def test_proxy_no_letsencrypt(): - config_object = test_read_config() + config_object = read_config() assert config_object.proxy assert config_object.use_letsencrypt @@ -215,7 +191,7 @@ def test_proxy_no_letsencrypt(): def test_proxy_no_letsencrypt_retains_custom_nginx_proxy_port(): CUSTOM_PROXY_PORT = 9090 - config_object = test_read_config(overrides={ + config_object = read_config(overrides={ 'advanced': Config.TRUE, 'use_letsencrypt': Config.FALSE, 'nginx_proxy_port': str(CUSTOM_PROXY_PORT), @@ -230,7 +206,7 @@ def test_proxy_no_letsencrypt_retains_custom_nginx_proxy_port(): def test_no_proxy_no_ssl(): - config_object = test_read_config() + config_object = read_config() assert config_object.is_secure assert config_object.get_config().get("nginx_proxy_port") == Config.DEFAULT_PROXY_PORT @@ -249,7 +225,7 @@ def test_no_proxy_no_ssl(): def test_proxy_no_ssl_advanced(): - config_object = test_read_config() + config_object = read_config() # Force advanced options config_object._Config__config["advanced"] = Config.TRUE assert config_object.advanced_options @@ -281,7 +257,7 @@ def test_proxy_no_ssl_advanced(): def test_port_allowed(): - config_object = test_read_config() + config_object = read_config() # Use let's encrypt by default assert not config_object._Config__is_port_allowed(Config.DEFAULT_NGINX_PORT) assert not config_object._Config__is_port_allowed("443") @@ -295,7 +271,7 @@ def test_port_allowed(): def test_create_directory(): - config_object = test_read_config() + config_object = read_config() destination_path = tempfile.mkdtemp() with patch("helpers.cli.CLI.colored_input") as mock_colored_input: @@ -309,7 +285,7 @@ def test_create_directory(): @patch('helpers.config.Config.write_config', new=lambda *a, **k: None) def test_maintenance(): - config_object = test_read_config() + config_object = read_config() # First time with pytest.raises(SystemExit) as pytest_wrapped_e: @@ -334,7 +310,7 @@ def test_maintenance(): def test_exposed_ports(): - config_object = test_read_config() + config_object = read_config() with patch.object(CLI, "colored_input", return_value=Config.TRUE) as mock_ci: # Choose multi servers options config_object._Config__questions_multi_servers() @@ -367,7 +343,7 @@ def test_exposed_ports(): @patch('helpers.config.Config.write_config', new=lambda *a, **k: None) def test_force_secure_mongo(): - config_object = test_read_config() + config_object = read_config() config_ = config_object.get_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: @@ -400,7 +376,7 @@ def test_force_secure_mongo(): @patch('helpers.config.Config._Config__write_upsert_db_users_trigger_file', new=write_trigger_upsert_db_users) def test_secure_mongo_advanced_options(): - config_object = test_read_config() + config_object = read_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: mock_ci.side_effect = iter([ "root", @@ -415,7 +391,7 @@ def test_secure_mongo_advanced_options(): @patch('helpers.config.Config._Config__write_upsert_db_users_trigger_file', new=write_trigger_upsert_db_users) def test_update_mongo_passwords(): - config_object = test_read_config() + config_object = read_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: config_object._Config__first_time = False config_object._Config__config["mongo_root_username"] = 'root' @@ -435,7 +411,7 @@ def test_update_mongo_passwords(): @patch('helpers.config.Config._Config__write_upsert_db_users_trigger_file', new=write_trigger_upsert_db_users) def test_update_mongo_usernames(): - config_object = test_read_config() + config_object = read_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: config_object._Config__first_time = False config_object._Config__config["mongo_root_username"] = 'root' @@ -466,7 +442,7 @@ def test_update_postgres_password(): When password changes, file must contain `` Users should not be deleted if they already exist. """ - config_object = test_read_config() + config_object = read_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: config_object._Config__first_time = False config_object._Config__config["postgres_user"] = 'user' @@ -496,7 +472,7 @@ def test_update_postgres_username(): When username changes, file must contain `` """ - config_object = test_read_config() + config_object = read_config() with patch("helpers.cli.CLI.colored_input") as mock_ci: config_object._Config__first_time = False config_object._Config__config["postgres_user"] = 'user' @@ -524,12 +500,22 @@ def test_update_postgres_db_name_from_single_database(): With two databases, KoBoCat has its own database. We ensure that `kc_postgres_db` gets `postgres_db` value. """ - config_object = test_read_config() + config_object = read_config() config = config_object.get_config() old_db_name = "postgres_db_kobo" config_object._Config__config["postgres_db"] = old_db_name - assert config.get("kc_postgres_db") == config_object.get_config_template()["kc_postgres_db"] + del config_object._Config__config['kc_postgres_db'] assert "postgres_db" in config - config = config_object._Config__upgrade_kc_db(config) + assert "kc_postgres_db" not in config + config = config_object._Config__get_upgraded_config() assert config.get("kc_postgres_db") == old_db_name - assert "postgres_db" not in config + + +def test_new_terminology(): + """ + Ensure config uses `primary` instead of `master` + """ + config_object = read_config() + config_object._Config__config["backend_server_role"] = 'master' + config = config_object._Config__get_upgraded_config() + assert config.get("backend_server_role") == 'primary' diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..d9d6018 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +try: + from unittest.mock import patch, MagicMock + builtin_open = "builtins.open" +except ImportError: + from mock import patch, MagicMock + builtin_open = "__builtin__.open" + +from helpers.command import Command +from helpers.config import Config +from .utils import ( + read_config, + run_command, + MockDocker, +) + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_trivial(): + config_object = read_config() + Command.start() + mock_docker = MockDocker() + expected_containers = MockDocker.FRONTEND_CONTAINERS + \ + MockDocker.PRIMARY_BACKEND_CONTAINERS + \ + MockDocker.LETSENCRYPT + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_no_letsencrypt(): + config_object = read_config() + config_object._Config__config['use_letsencrypt'] = Config.FALSE + Command.start() + mock_docker = MockDocker() + expected_containers = MockDocker.FRONTEND_CONTAINERS + \ + MockDocker.PRIMARY_BACKEND_CONTAINERS + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_frontend(): + config_object = read_config() + Command.start(frontend_only=True) + mock_docker = MockDocker() + expected_containers = MockDocker.FRONTEND_CONTAINERS + \ + MockDocker.LETSENCRYPT + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_primary_backend(): + config_object = read_config() + config_object._Config__config['backend_server_role'] = 'primary' + config_object._Config__config['server_role'] = 'backend' + config_object._Config__config['multi'] = Config.TRUE + + Command.start() + mock_docker = MockDocker() + expected_containers = MockDocker.PRIMARY_BACKEND_CONTAINERS + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_secondary_backend(): + config_object = read_config() + config_object._Config__config['backend_server_role'] = 'secondary' + config_object._Config__config['server_role'] = 'backend' + config_object._Config__config['multi'] = Config.TRUE + + mock_docker = MockDocker() + Command.start() + expected_containers = MockDocker.SECONDARY_BACKEND_CONTAINERS + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + + +@patch('helpers.network.Network.is_port_open', + MagicMock(return_value=False)) +@patch('helpers.command.migrate_single_to_two_databases', + MagicMock(return_value=None)) +@patch('helpers.command.Command.info', + MagicMock(return_value=True)) +@patch('helpers.cli.CLI.run_command', + new=run_command) +def test_toggle_maintenance(): + config_object = read_config() + mock_docker = MockDocker() + Command.start() + expected_containers = MockDocker.FRONTEND_CONTAINERS + \ + MockDocker.PRIMARY_BACKEND_CONTAINERS + \ + MockDocker.LETSENCRYPT + assert sorted(mock_docker.ps()) == sorted(expected_containers) + + config_object._Config__config['maintenance_enabled'] = True + Command.start() + maintenance_containers = MockDocker.PRIMARY_BACKEND_CONTAINERS + \ + MockDocker.MAINTENANCE_CONTAINERS + \ + MockDocker.LETSENCRYPT + assert sorted(mock_docker.ps()) == sorted(maintenance_containers) + config_object._Config__config['maintenance_enabled'] = False + Command.start() + assert sorted(mock_docker.ps()) == sorted(expected_containers) + Command.stop() + assert len(mock_docker.ps()) == 0 + del mock_docker + diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..1998a6f --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import json +try: + from unittest.mock import patch, mock_open + builtin_open = "builtins.open" +except ImportError: + from mock import patch, mock_open + builtin_open = "__builtin__.open" + +from six import with_metaclass + +from helpers.config import Config +from helpers.singleton import Singleton + + +def read_config(overrides=None): + + config_dict = dict(Config.get_config_template()) + config_dict["kobodocker_path"] = "/tmp" + if overrides is not None: + config_dict.update(overrides) + with patch(builtin_open, mock_open(read_data=json.dumps(config_dict))) as mock_file: + config_object = Config() + config_object.read_config() + assert config_object.get_config().get("kobodocker_path") == config_dict.get("kobodocker_path") + + return config_object + + +def reset_config(config_object): + + config_dict = dict(Config.get_config_template()) + config_dict["kobodocker_path"] = "/tmp" + config_object.__config = config_dict + + +def run_command(command, cwd=None, polling=False): + if 'docker-compose' != command[0]: + raise Exception('Command: `{}` is not implemented!'.format(command[0])) + + mock_docker = MockDocker() + return mock_docker.compose(command, cwd) + + +def write_trigger_upsert_db_users_mock(*args): + content = args[1] + with open("/tmp/upsert_db_users", "w") as f: + f.write(content) + + +class MockDocker(with_metaclass(Singleton)): + + PRIMARY_BACKEND_CONTAINERS = ['primary_postgres', 'mongo', 'redis_main', 'redis_cache'] + SECONDARY_BACKEND_CONTAINERS = ['secondary_postgres'] + FRONTEND_CONTAINERS = ['nginx', 'kobocat', 'kpi', 'enketo_express'] + MAINTENANCE_CONTAINERS = ['maintenance', 'kobocat', 'kpi', 'enketo_express'] + LETSENCRYPT = ['letsencrypt_nginx', 'certbot'] + + def __init__(self): + self.__containers = [] + + def ps(self): + return self.__containers + + def compose(self, command, cwd): + config_object = Config() + letsencrypt = cwd == config_object.get_letsencrypt_repo_path() + + if command[-2] == 'config': + return "\n".join([c for c in self.FRONTEND_CONTAINERS if c != 'nginx']) + if command[-2] == 'up': + if letsencrypt: + self.__containers += self.LETSENCRYPT + elif 'primary' in command[2]: + self.__containers += self.PRIMARY_BACKEND_CONTAINERS + elif 'secondary' in command[2]: + self.__containers += self.SECONDARY_BACKEND_CONTAINERS + elif 'maintenance' in command[2]: + self.__containers += self.MAINTENANCE_CONTAINERS + elif 'frontend' in command[2]: + self.__containers += self.FRONTEND_CONTAINERS + elif command[-1] == 'down': + try: + if letsencrypt: + for container in self.LETSENCRYPT: + self.__containers.remove(container) + elif 'primary' in command[2]: + for container in self.PRIMARY_BACKEND_CONTAINERS: + self.__containers.remove(container) + elif 'secondary' in command[2]: + for container in self.SECONDARY_BACKEND_CONTAINERS: + self.__containers.remove(container) + elif 'maintenance' in command[2]: + for container in self.MAINTENANCE_CONTAINERS: + self.__containers.remove(container) + elif 'frontend' in command[2]: + for container in self.FRONTEND_CONTAINERS: + self.__containers.remove(container) + except ValueError: + # Try to take a container down but was not up before. + pass + + return True diff --git a/tox.ini b/tox.ini index dd450ba..6cc2a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ # content of: tox.ini , put in same dir as setup.py [tox] skipsdist=True -envlist = py27,py37 +envlist = py27,py37,py38 [testenv] deps = -rrequirements_py3_tests.txt