Attempt to fix CI Error

awx/settings/__init__.py

@@ -6,16 +6,19 @@
     factory,
     export,
     load_envvars,
-    load_python_file_with__injected_context,
+    load_python_file_with_injected_context,
     load_standard_settings_files,
-    validate,
+    toggle_feature_flags,
+)
+from .functions import (
+    assert_production_settings,
+    merge_application_name,
+    add_backwards_compatibility,
+    load_extra_development_files,
 )
-from .functions import merge_application_name, toggle_feature_flags, add_backwards_compatibility
-
 
 add_backwards_compatibility()
 
-
 # Create a the standard DYNACONF instance which will come with DAB defaults
 # This loads defaults.py and environment specific file e.g: development_defaults.py
 DYNACONF = factory(
@@ -26,67 +29,42 @@
 )
 
 # Store snapshot before loading any custom config file
-if DYNACONF.is_development_mode:
-    DYNACONF.set(
-        "DEFAULTS_SNAPSHOT",
-        copy.deepcopy(DYNACONF.as_dict(internal=False)),
-        loader_identifier="awx.settings:DEFAULTS_SNAPSHOT",
-    )
-##########################################################################################
-#  Any settings loaded after this point will be marked as as a read_only database setting
-##########################################################################################
+DYNACONF.set(
+    "DEFAULTS_SNAPSHOT",
+    copy.deepcopy(DYNACONF.as_dict(internal=False)),
+    loader_identifier="awx.settings:DEFAULTS_SNAPSHOT",
+)
+
+#############################################################################################
+# Settings loaded before this point will be allowed to be overridden by the database settings
+# Any settings loaded after this point will be marked as as a read_only database setting
+#############################################################################################
 
-# Load extra config from the now deprecated /etc/tower/ directory
-# This is only for backwards compatibility and will be removed in the future
-# Load settings from any .py files in the global conf.d directory specified in
-# the environment, defaulting to /etc/tower/conf.d/.
-# Load remaining settings from the global settings file specified in the
-# environment, defaulting to /etc/tower/settings.py.
-# Attempt to load settings from /etc/tower/settings.py first, followed by
-# /etc/tower/conf.d/*.py.
+# Load extra settings files from the following directories
+#  /etc/tower/conf.d/ and /etc/tower/
+# this is the legacy location, kept for backwards compatibility
 settings_dir = os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/')
 settings_files_path = os.path.join(settings_dir, '*.py')
 settings_file_path = os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py')
+load_python_file_with_injected_context(settings_files_path, settings=DYNACONF)
+load_python_file_with_injected_context(settings_file_path, settings=DYNACONF)
 
-# The following should have been loaded using `DYNACONF.load_file` but we are
-# using `load_python_file_with__injected_context` to maintain backwards compatibility
-load_python_file_with__injected_context(settings_files_path, settings=DYNACONF)
-load_python_file_with__injected_context(settings_file_path, settings=DYNACONF)
-
-
-# Load new standard settings files from
-#  /etc/ansible-automation-platform/ and /etc/ansible-automation-platform/awx/
-# this is to allow for a smoother transition from legacy (above) to new settings standard paths
+# Load extra settings files from the following directories
+# /etc/ansible-automation-platform/{settings,flags,.secrets}.yaml
+# and /etc/ansible-automation-platform/awx/{settings,flags,.secrets}.yaml
+# this is the new standard location for all services
 load_standard_settings_files(DYNACONF)
 
-# Load optional development only settings
-if DYNACONF.get_environ("AWX_KUBE_DEVEL"):
-    load_python_file_with__injected_context("kube_defaults.py", settings=DYNACONF)
-else:
-    load_python_file_with__injected_context("local_*.py", settings=DYNACONF)
+# Load optional development only settings files
+load_extra_development_files(DYNACONF)
 
 # Check at least one setting file has been loaded in production mode
-if "production" in DYNACONF.current_env.lower():  # pragma: no cover
-    required_settings_paths = [
-        os.path.dirname(settings_file_path),
-        "/etc/ansible-automation-platform/",
-        settings_dir,
-    ]
-    for path in required_settings_paths:
-        if any([path in os.path.dirname(f) for f in DYNACONF._loaded_files]):
-            break
-    else:
-        from django.core.exceptions import ImproperlyConfigured
-
-        msg = 'No AWX configuration found at %s.' % required_settings_paths
-        msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
-        msg += 'specify an alternate path.'
-        raise ImproperlyConfigured(msg)
+assert_production_settings(DYNACONF, settings_dir, settings_file_path)
 
 # Load envvars at the end to allow them to override everything loaded so far
 load_envvars(DYNACONF)
 
-# This must run after all custom settings are imported
+# This must run after all custom settings are loaded
 DYNACONF.update(
     merge_application_name(DYNACONF),
     loader_identifier="awx.settings:merge_application_name",
@@ -100,8 +78,5 @@
     merge=True,
 )
 
-# Update django.conf.settings with DYNACONF keys.
+# Update django.conf.settings with DYNACONF values
 export(__name__, DYNACONF)
-
-# Validate the settings according to the validators registered
-validate(DYNACONF)

awx/settings/defaults.py

@@ -1018,6 +1018,10 @@
 ANSIBLE_BASE_RESOURCE_CONFIG_MODULE = 'awx.resource_api'
 ANSIBLE_BASE_PERMISSION_MODEL = 'main.Permission'
 
+# Defaults to be overridden by DAB
+SPECTACULAR_SETTINGS = {}
+OAUTH2_PROVIDER = {}
+
 # Add a postfix to the API URL patterns
 # example if set to '' API pattern will be /api
 # example if set to 'controller' API pattern will be /api AND /api/controller

awx/settings/development_defaults.py

@@ -58,18 +58,6 @@
 
 AWX_CALLBACK_PROFILE = True
 
-
-# this modifies FLAGS set by defaults
-# NOTE: on dynaconf 3.3.0 this can be replaced with
-# FLAGS__FEATURE_INDIRECT_NODE_COUNTING_ENABLED__0__value = True
-# for now we use hooks
-@post_hook
-def change_flags(settings):
-    _flags = settings.FLAGS
-    _flags['FEATURE_INDIRECT_NODE_COUNTING_ENABLED'] = [{'condition': 'boolean', 'value': True}]
-    return {'FLAGS': _flags}
-
-
 # ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
 # Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
 # Allows user to trigger task managers directly for debugging and profiling purposes.
@@ -78,3 +66,11 @@ def change_flags(settings):
 
 # Needed for launching runserver in debug mode
 # ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
+
+
+# This modifies FLAGS set by defaults, must be deferred to run later
+@post_hook
+def set_dev_flags(settings):
+    defaults_flags = settings.get("FLAGS", {})
+    defaults_flags['FEATURE_INDIRECT_NODE_COUNTING_ENABLED'] = [{'condition': 'boolean', 'value': True}]
+    return {'FLAGS': defaults_flags}

awx/settings/functions.py

@@ -1,30 +1,11 @@
 import os
 from typing import Any
+from ansible_base.lib.dynamic_config import load_python_file_with_injected_context
 from dynaconf import Dynaconf
 from dynaconf.utils.functional import empty
 from .application_name import get_application_name
 
 
-def toggle_feature_flags(settings: Dynaconf) -> dict[str, Any]:
-    """Toggle FLAGS based on installer settings.
-    FLAGS is a django-flags formatted dictionary.
-        FLAGS={
-            "FEATURE_SOME_PLATFORM_FLAG_ENABLED": [
-                {"condition": "boolean", "value": False, "required": True},
-                {"condition": "before date", "value": "2022-06-01T12:00Z"},
-            ]
-        }
-    Installers will place `FEATURE_SOME_PLATFORM_FLAG_ENABLED=True/False` in the settings file.
-    This function will update the value in the index 0 in FLAGS with the installer value.
-    """
-    data = {}
-    for feature_name, feature_content in settings.get("FLAGS", {}).items():
-        if (installer_value := settings.get(feature_name, empty)) is not empty:
-            feature_content[0]["value"] = installer_value
-            data[f"FLAGS__{feature_name}"] = feature_content
-    return data
-
-
 def merge_application_name(settings):
     """Return a dynaconf merge dict to set the application name for the connection."""
     data = {}
@@ -68,3 +49,40 @@ def add_backwards_compatibility():
             if _mode_fragment not in _mode:
                 _mode.append(_mode_fragment)
             os.environ["AWX_MODE"] = ",".join(_mode)
+
+
+def load_extra_development_files(settings: Dynaconf):
+    """Load optional development only settings files."""
+    if not settings.is_development_mode:
+        return
+
+    if settings.get_environ("AWX_KUBE_DEVEL"):
+        load_python_file_with_injected_context("kube_defaults.py", settings=settings)
+    else:
+        load_python_file_with_injected_context("local_*.py", settings=settings)
+
+
+def assert_production_settings(settings: Dynaconf, settings_dir: str, settings_file_path: str):  # pragma: no cover
+    """Ensure at least one setting file has been loaded in production mode.
+    Current systems will require /etc/tower/settings.py and
+    new systems will require /etc/ansible-automation-platform/*.yaml
+    """
+    if "production" not in settings.current_env.lower():
+        return
+
+    required_settings_paths = [
+        os.path.dirname(settings_file_path),
+        "/etc/ansible-automation-platform/",
+        settings_dir,
+    ]
+
+    for path in required_settings_paths:
+        if any([path in os.path.dirname(f) for f in settings._loaded_files]):
+            break
+    else:
+        from django.core.exceptions import ImproperlyConfigured  # noqa
+
+        msg = 'No AWX configuration found at %s.' % required_settings_paths
+        msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
+        msg += 'specify an alternate path.'
+        raise ImproperlyConfigured(msg)

requirements/requirements.in

@@ -22,7 +22,6 @@ django-guid
 django-oauth-toolkit<2.0.0      # Version 2.0.0 has breaking changes that will need to be worked out before upgrading
 django-polymorphic
 django-solo
-django-split-settings
 djangorestframework>=3.15.0
 djangorestframework-yaml
 dynaconf<4

requirements/requirements.txt

@@ -158,10 +158,6 @@ django-polymorphic==3.1.0
     # via -r /awx_devel/requirements/requirements.in
 django-solo==2.4.0
     # via -r /awx_devel/requirements/requirements.in
-django-split-settings==1.3.2
-    # via
-    #   -r /awx_devel/requirements/requirements.in
-    #   django-ansible-base
 djangorestframework==3.15.2
     # via
     #   -r /awx_devel/requirements/requirements.in

requirements/requirements_git.txt

@@ -4,5 +4,4 @@ git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner
 awx-plugins-core @ git+https://github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core[credentials-github-app]
 # django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags]
 django-ansible-base @ git+https://github.com/rochacbruno/django-ansible-base@dynaconf_settings#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags]
-awx-plugins-core @ git+https://github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core
 awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git

tools/docker-compose/ansible/roles/sources/templates/local_settings.py.j2

@@ -3,14 +3,9 @@
 
 # Local Django settings for AWX project.
 
-# This file is loaded by Dynaconf from awx/settings.py 
-# dynaconf reads only the UPPER case variables from this file
-# To customize existing settings use dynaconf merging syntax
-# e.g: 
-# DATABASES__default__NAME = 'custom_db_name'
-# LOGGING__handlers__console__level = 'DEBUG'
-# 
-# For more complex conditionals use Dynaconf Post Hook.
+# All variables defined in awx/settings/development.py will already be loaded
+# into the global namespace before this file is loaded, to allow for reading
+# and updating the default settings as needed.
 
 ###############################################################################
 # MISC PROJECT SETTINGS
@@ -40,49 +35,30 @@ OPTIONAL_API_URLPATTERN_PREFIX = '{{ api_urlpattern_prefix }}'
 # WARNING also logs 4xx responses.
 
 # Enable the following lines to turn on lots of permissions-related logging.
-# LOGGING__loggers = {
-#     "dynaconf_merge": True,
-#     'awx.main.access': {'level': 'DEBUG'},
-#     'awx.main.signals': {'level': 'DEBUG'},
-#     'awx.main.permissions': {'level': 'DEBUG'},
-#     # Enable the following line to turn on database settings logging.
-#     # "awx.conf": {'level': 'DEBUG'}
-# }
+# LOGGING['loggers']['awx.main.access']['level'] = 'DEBUG'
+# LOGGING['loggers']['awx.main.signals']['level'] = 'DEBUG'
+# LOGGING['loggers']['awx.main.permissions']['level'] = 'DEBUG'
+
+# Enable the following line to turn on database settings logging.
+# LOGGING['loggers']['awx.conf']['level'] = 'DEBUG'
 
 {% if enable_otel|bool %}
-LOGGING__handlers__otel__class = 'awx.main.utils.handlers.OTLPHandler'
-LOGGING__handlers__otel__endpoint = 'http://otel:4317'
-
-from dynaconf import post_hook
-
-
-@post_hook
-def set_log_handlers(settings) -> dict:
-    """Defers the setting of log handlers until after the settings are fully loaded.
-    this function is registered as a dynaconf hook, 
-    this must return a dict that will be merged into the settings.
-    NOTE: `settings` is a read-only that contains the settings loaded so far. 
-    """
-    data = {}
-    if (logging := settings.get('LOGGING')) is not None:
-        loggers = logging.get('loggers', {})
-
-        # Add otel log handler to all log handlers where propagate is False
-        for logger in loggers.values():
-            if logger.get('propagate', True):
-                continue
-            if (handlers := logger.get('handlers')) is not None:
-                if 'otel' not in handlers:
-                    handlers.append('otel')
-
-
-        # Everything without explicit propagate=False ends up logging to 'awx' so add it
-        if (handlers := loggers.get('awx', {}).get('handlers')) is not None:
-            if 'otel' not in handlers:
-                handlers.append('otel')
-
-        data['LOGGING'] = logging
-    return data
+LOGGING['handlers']['otel'] |= {
+    'class': 'awx.main.utils.handlers.OTLPHandler',
+    'endpoint': 'http://otel:4317',
+}
+# Add otel log handler to all log handlers where propagate is False
+for name in LOGGING['loggers'].keys():
+    if not LOGGING['loggers'][name].get('propagate', True):
+        handler = LOGGING['loggers'][name].get('handlers', [])
+        if 'otel' not in handler:
+            LOGGING['loggers'][name].get('handlers', []).append('otel')
+
+# Everything without explicit propagate=False ends up logging to 'awx' so add it
+handler = LOGGING['loggers']['awx'].get('handlers', [])
+if 'otel' not in handler:
+    LOGGING['loggers']['awx'].get('handlers', []).append('otel')
+
 {% endif %}
 
 BROADCAST_WEBSOCKET_PORT = 8013