[AAP-34936] Manage Django Settings with Dynaconf

.coveragerc

@@ -19,6 +19,7 @@ exclude_also =
 branch = True
 omit =
     awx/main/migrations/*
+    awx/settings/*
 source =
     .
 source_pkgs =

awx/__init__.py

@@ -62,7 +62,8 @@
 
 def prepare_env():
     # Update the default settings environment variable based on current mode.
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings')
+    os.environ.setdefault('AWX_MODE', MODE)
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/api/generics.py

@@ -161,7 +161,7 @@ def get_view_description(view, html=False):
 
 
 def get_default_schema():
-    if settings.SETTINGS_MODULE == 'awx.settings.development':
+    if settings.DYNACONF.is_development_mode:
         from awx.api.swagger import schema_view
 
         return schema_view

awx/settings/__init__.py

@@ -1,2 +1,100 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
+import os
+from ansible_base.lib.dynamic_config import (
+    factory,
+    export,
+    load_envvars,
+    load_standard_settings_files,
+    validate,
+)
+from .functions import merge_application_name, toggle_feature_flags
+
+# Create a the standard DYNACONF instance which will come with DAB defaults
+# This loads defaults.py and environment specific file e.g: development_defaults.py
+DYNACONF = factory(
+    __name__,
+    "AWX",
+    environments=("development", "production", "quiet", "kube"),
+    settings_files=["defaults.py"],
+)
+
+# Store snapshot before loading any custom config file
+DEFAULTS_SNAPSHOT = {}
+if DYNACONF.is_development_mode:
+    DYNACONF.set(
+        "DEFAULTS_SNAPSHOT",
+        DYNACONF.as_dict(internal=False),  # should use deepcopy here?
+        loader_identifier="awx.settings:DEFAULTS_SNAPSHOT",
+    )
+##########################################################################################
+#  Any settings loaded after this point will be marked as as a read_only database setting
+##########################################################################################
+
+# Load extra config from the now deprecated /etc/tower/ directory
+# This is only for backwards compatibility and will be removed in the future
+# Load settings from any .py files in the global conf.d directory specified in
+# the environment, defaulting to /etc/tower/conf.d/.
+# Load remaining settings from the global settings file specified in the
+# environment, defaulting to /etc/tower/settings.py.
+# Attempt to load settings from /etc/tower/settings.py first, followed by
+# /etc/tower/conf.d/*.py.
+settings_dir = os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/')
+settings_files_path = os.path.join(settings_dir, '*.py')
+settings_file_path = os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py')
+DYNACONF.load_file(settings_file_path)
+DYNACONF.load_file(settings_files_path)
+
+# Load new standard settings files from
+#  /etc/ansible-automation-platform/ and /etc/ansible-automation-platform/awx/
+# this is to allow for a smoother transition from legacy (above) to new settings standard paths
+load_standard_settings_files(DYNACONF)
+
+# Load optional development only settings
+if DYNACONF.get_environ("AWX_KUBE_DEVEL"):
+    DYNACONF.load_file("kube_defaults.py")
+else:
+    DYNACONF.load_file("local_*.py")
+
+# Check at least one required setting file has been loaded
+if "production" in DYNACONF.current_env.lower():
+    required_settings_paths = [
+        os.path.dirname(settings_file_path),
+        "/etc/ansible-automation-platform/",
+    ]
+    # check if at least one file has been loaded any of the required paths
+    # use DYNACONF._loaded_files to check any filename inside the paths
+    # if not loaded then raise an ImproperlyConfigured error
+    for path in required_settings_paths:
+        if any(path in f for f in DYNACONF._loaded_files):
+            break
+    else:
+        from django.core.exceptions import ImproperlyConfigured
+
+        msg = 'No AWX configuration found at %s.' % required_settings_paths
+        msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
+        msg += 'specify an alternate path.'
+        raise ImproperlyConfigured(msg)
+
+# Load envvars at the end to allow them to override everything loaded so far
+load_envvars(DYNACONF)
+
+# This must run after all custom settings are imported
+DYNACONF.update(
+    merge_application_name(DYNACONF),
+    loader_identifier="awx.settings:merge_application_name",
+    merge=True,
+)
+
+# Toggle feature flags based on installer settings
+DYNACONF.update(
+    toggle_feature_flags(DYNACONF),
+    loader_identifier="awx.settings:toggle_feature_flags",
+    merge=True,
+)
+
+# Update django.conf.settings with DYNACONF keys.
+export(__name__, DYNACONF)
+
+# Validate the settings according to the validators registered
+validate(DYNACONF)

awx/settings/application_name.py

@@ -25,6 +25,7 @@ def get_application_name(CLUSTER_HOST_ID, function=''):
 
 
 def set_application_name(DATABASES, CLUSTER_HOST_ID, function=''):
+    """In place modification of DATABASES to set the application name for the connection."""
     # If settings files were not properly passed DATABASES could be {} at which point we don't need to set the app name.
     if not DATABASES or 'default' not in DATABASES:
         return

awx/settings/defaults.py

@@ -9,9 +9,6 @@
 import socket
 from datetime import timedelta
 
-from split_settings.tools import include
-
-
 DEBUG = True
 SQL_DEBUG = DEBUG
 
@@ -1018,10 +1015,6 @@
 ANSIBLE_BASE_RESOURCE_CONFIG_MODULE = 'awx.resource_api'
 ANSIBLE_BASE_PERMISSION_MODEL = 'main.Permission'
 
-from ansible_base.lib import dynamic_config  # noqa: E402
-
-include(os.path.join(os.path.dirname(dynamic_config.__file__), 'dynamic_settings.py'))
-
 # Add a postfix to the API URL patterns
 # example if set to '' API pattern will be /api
 # example if set to 'controller' API pattern will be /api AND /api/controller

awx/settings/development.py

@@ -1,126 +1,13 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved.
-
-# Development settings for AWX project.
-
-# Python
+# This file exists for backwards compatibility only
+# the current way of running AWX is to point settings to
+# awx/settings/__init__.py as the entry point for the settings
+# that is done by exporting: export DJANGO_SETTINGS_MODULE=awx.settings
 import os
-import socket
-import copy
-import sys
-import traceback
-
-# Centos-7 doesn't include the svg mime type
-# /usr/lib64/python/mimetypes.py
-import mimetypes
-
-# Django Split Settings
-from split_settings.tools import optional, include
-
-# Load default settings.
-from .defaults import *  # NOQA
-
-# awx-manage shell_plus --notebook
-NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
-
-# print SQL queries in shell_plus
-SHELL_PLUS_PRINT_SQL = False
-
-# show colored logs in the dev environment
-# to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
-COLOR_LOGS = True
-LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
-
-ALLOWED_HOSTS = ['*']
-
-mimetypes.add_type("image/svg+xml", ".svg", True)
-mimetypes.add_type("image/svg+xml", ".svgz", True)
-
-# Disallow sending session cookies over insecure connections
-SESSION_COOKIE_SECURE = False
-
-# Disallow sending csrf cookies over insecure connections
-CSRF_COOKIE_SECURE = False
-
-# Disable Pendo on the UI for development/test.
-# Note: This setting may be overridden by database settings.
-PENDO_TRACKING_STATE = "off"
-INSIGHTS_TRACKING_STATE = False
-
-# debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
-
-INSTALLED_APPS += ['drf_yasg', 'debug_toolbar']  # NOQA
-
-MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE  # NOQA
-
-DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
-
-# Configure a default UUID for development only.
-SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
-INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
-
-# Ansible base virtualenv paths and enablement
-# only used for deprecated fields and management commands for them
-BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
-
-CLUSTER_HOST_ID = socket.gethostname()
-
-AWX_CALLBACK_PROFILE = True
-
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-# Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
-# Allows user to trigger task managers directly for debugging and profiling purposes.
-# Only works in combination with settings.SETTINGS_MODULE == 'awx.settings.development'
-AWX_DISABLE_TASK_MANAGERS = False
-
-# Needed for launching runserver in debug mode
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-
-# Store a snapshot of default settings at this point before loading any
-# customizable config files.
-this_module = sys.modules[__name__]
-local_vars = dir(this_module)
-DEFAULTS_SNAPSHOT = {}  # define after we save local_vars so we do not snapshot the snapshot
-for setting in local_vars:
-    if setting.isupper():
-        DEFAULTS_SNAPSHOT[setting] = copy.deepcopy(getattr(this_module, setting))
-
-del local_vars  # avoid temporary variables from showing up in dir(settings)
-del this_module
-#
-###############################################################################################
-#
-#  Any settings defined after this point will be marked as as a read_only database setting
-#
-################################################################################################
-
-# If there is an `/etc/tower/settings.py`, include it.
-# If there is a `/etc/tower/conf.d/*.py`, include them.
-include(optional('/etc/tower/settings.py'), scope=locals())
-include(optional('/etc/tower/conf.d/*.py'), scope=locals())
-
-# If any local_*.py files are present in awx/settings/, use them to override
-# default settings for development.  If not present, we can still run using
-# only the defaults.
-# this needs to stay at the bottom of this file
-try:
-    if os.getenv('AWX_KUBE_DEVEL', False):
-        include(optional('development_kube.py'), scope=locals())
-    else:
-        include(optional('local_*.py'), scope=locals())
-except ImportError:
-    traceback.print_exc()
-    sys.exit(1)
-
-# The below runs AFTER all of the custom settings are imported
-# because conf.d files will define DATABASES and this should modify that
-from .application_name import set_application_name
 
-set_application_name(DATABASES, CLUSTER_HOST_ID)  # NOQA
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings")
+os.environ.setdefault("AWX_MODE", "development")
 
-del set_application_name
+from ansible_base.lib.dynamic_config import export
+from . import DYNACONF  # noqa
 
-# Set the value of any feature flags that are defined in the local settings
-for feature in list(FLAGS.keys()):  # noqa: F405
-    if feature in locals():
-        FLAGS[feature][0]['value'] = locals()[feature]  # noqa: F405
+export(__name__, DYNACONF)

awx/settings/development_defaults.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+
+# Development settings for AWX project.
+
+# Python
+import os
+import socket
+
+# Centos-7 doesn't include the svg mime type
+# /usr/lib64/python/mimetypes.py
+import mimetypes
+
+# awx-manage shell_plus --notebook
+NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
+
+# print SQL queries in shell_plus
+SHELL_PLUS_PRINT_SQL = False
+
+# show colored logs in the dev environment
+# to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
+COLOR_LOGS = True
+LOGGING__handlers__console = '@merge {"()": "awx.main.utils.handlers.ColorHandler"}'
+
+ALLOWED_HOSTS = ['*']
+
+mimetypes.add_type("image/svg+xml", ".svg", True)
+mimetypes.add_type("image/svg+xml", ".svgz", True)
+
+# Disallow sending session cookies over insecure connections
+SESSION_COOKIE_SECURE = False
+
+# Disallow sending csrf cookies over insecure connections
+CSRF_COOKIE_SECURE = False
+
+# Disable Pendo on the UI for development/test.
+# Note: This setting may be overridden by database settings.
+PENDO_TRACKING_STATE = "off"
+INSIGHTS_TRACKING_STATE = False
+
+# debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
+INSTALLED_APPS = "@merge drf_yasg,debug_toolbar"
+MIDDLEWARE = "@insert 0 debug_toolbar.middleware.DebugToolbarMiddleware"
+
+DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
+
+# Configure a default UUID for development only.
+SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
+INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
+
+# Ansible base virtualenv paths and enablement
+# only used for deprecated fields and management commands for them
+BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
+
+CLUSTER_HOST_ID = socket.gethostname()
+
+AWX_CALLBACK_PROFILE = True
+
+# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
+# Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
+# Allows user to trigger task managers directly for debugging and profiling purposes.
+# Only works in combination with settings.SETTINGS_MODULE == 'awx.settings.development'
+AWX_DISABLE_TASK_MANAGERS = False
+
+# Needed for launching runserver in debug mode
+# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================

awx/settings/development_kube.py

@@ -1,4 +1,13 @@
-BROADCAST_WEBSOCKET_SECRET = '🤖starscream🤖'
-BROADCAST_WEBSOCKET_PORT = 8052
-BROADCAST_WEBSOCKET_VERIFY_CERT = False
-BROADCAST_WEBSOCKET_PROTOCOL = 'http'
+# This file exists for backwards compatibility only
+# the current way of running AWX is to point settings to
+# awx/settings/__init__.py as the entry point for the settings
+# that is done by exporting: export DJANGO_SETTINGS_MODULE=awx.settings
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings")
+os.environ.setdefault("AWX_MODE", "development,kube")
+
+from ansible_base.lib.dynamic_config import export
+from . import DYNACONF  # noqa
+
+export(__name__, DYNACONF)