[AAP-34936] Manage Django Settings with Dynaconf

.coveragerc

@@ -19,6 +19,8 @@ exclude_also =
 branch = True
 omit =
     awx/main/migrations/*
+    awx/settings/defaults.py
+    awx/settings/*_defaults.py
 source =
     .
 source_pkgs =

awx/__init__.py

@@ -62,7 +62,8 @@ def version_file():
 
 def prepare_env():
     # Update the default settings environment variable based on current mode.
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings')
+    os.environ.setdefault('AWX_MODE', MODE)
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/api/generics.py

@@ -161,7 +161,7 @@ def get_view_description(view, html=False):
 
 
 def get_default_schema():
-    if settings.SETTINGS_MODULE == 'awx.settings.development':
+    if settings.DYNACONF.is_development_mode:
         from awx.api.swagger import schema_view
 
         return schema_view

awx/main/tests/unit/test_settings.py

@@ -1,6 +1,3 @@
-from split_settings.tools import include
-
-
 LOCAL_SETTINGS = (
     'ALLOWED_HOSTS',
     'BROADCAST_WEBSOCKET_PORT',
@@ -16,13 +13,14 @@
 
 
 def test_postprocess_auth_basic_enabled():
-    locals().update({'__file__': __file__})
+    """The final loaded settings should have basic auth enabled."""
+    from awx.settings import REST_FRAMEWORK
 
-    include('../../../settings/defaults.py', scope=locals())
-    assert 'awx.api.authentication.LoggedBasicAuthentication' in locals()['REST_FRAMEWORK']['DEFAULT_AUTHENTICATION_CLASSES']
+    assert 'awx.api.authentication.LoggedBasicAuthentication' in REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES']
 
 
 def test_default_settings():
+    """Ensure that all default settings are present in the snapshot."""
     from django.conf import settings
 
     for k in dir(settings):
@@ -31,3 +29,64 @@ def test_default_settings():
         default_val = getattr(settings.default_settings, k, None)
         snapshot_val = settings.DEFAULTS_SNAPSHOT[k]
         assert default_val == snapshot_val, f'Setting for {k} does not match shapshot:\nsnapshot: {snapshot_val}\ndefault: {default_val}'
+
+
+def test_django_conf_settings_is_awx_settings():
+    """Ensure that the settings loaded from dynaconf are the same as the settings delivered to django."""
+    from django.conf import settings
+    from awx.settings import REST_FRAMEWORK
+
+    assert settings.REST_FRAMEWORK == REST_FRAMEWORK
+
+
+def test_dynaconf_is_awx_settings():
+    """Ensure that the settings loaded from dynaconf are the same as the settings delivered to django."""
+    from django.conf import settings
+    from awx.settings import REST_FRAMEWORK
+
+    assert settings.DYNACONF.REST_FRAMEWORK == REST_FRAMEWORK
+
+
+def test_development_settings_can_be_directly_imported(monkeypatch):
+    """Ensure that the development settings can be directly imported."""
+    monkeypatch.setenv('AWX_MODE', 'development')
+    from django.conf import settings
+    from awx.settings.development import REST_FRAMEWORK
+    from awx.settings.development import DEBUG  # actually set on defaults.py and not overridden in development.py
+
+    assert settings.REST_FRAMEWORK == REST_FRAMEWORK
+    assert DEBUG is True
+
+
+def test_toggle_feature_flags():
+    """Ensure that the toggle_feature_flags function works as expected."""
+    from awx.settings.functions import toggle_feature_flags
+
+    settings = {
+        "FLAGS": {
+            "FEATURE_SOME_PLATFORM_FLAG_ENABLED": [
+                {"condition": "boolean", "value": False, "required": True},
+                {"condition": "before date", "value": "2022-06-01T12:00Z"},
+            ]
+        },
+        "FEATURE_SOME_PLATFORM_FLAG_ENABLED": True,
+    }
+    assert toggle_feature_flags(settings) == {
+        "FLAGS__FEATURE_SOME_PLATFORM_FLAG_ENABLED": [
+            {"condition": "boolean", "value": True, "required": True},
+            {"condition": "before date", "value": "2022-06-01T12:00Z"},
+        ]
+    }
+
+
+def test_merge_application_name():
+    """Ensure that the merge_application_name function works as expected."""
+    from awx.settings.functions import merge_application_name
+
+    settings = {
+        "DATABASES__default__ENGINE": "django.db.backends.postgresql",
+        "CLUSTER_HOST_ID": "test-cluster-host-id",
+    }
+    result = merge_application_name(settings)["DATABASES__default__OPTIONS__application_name"]
+    assert result.startswith("awx-")
+    assert "test-cluster" in result

awx/settings/__init__.py

@@ -1,2 +1,109 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
+import os
+from ansible_base.lib.dynamic_config import (
+    factory,
+    export,
+    load_envvars,
+    load_python_file_with__injected_context,
+    load_standard_settings_files,
+    validate,
+)
+from .functions import merge_application_name, toggle_feature_flags, add_backwards_compatibility
+
+
+add_backwards_compatibility()
+
+
+# Create a the standard DYNACONF instance which will come with DAB defaults
+# This loads defaults.py and environment specific file e.g: development_defaults.py
+DYNACONF = factory(
+    __name__,
+    "AWX",
+    environments=("development", "production", "quiet", "kube"),
+    settings_files=["defaults.py"],
+)
+
+# Store snapshot before loading any custom config file
+DEFAULTS_SNAPSHOT = {}
+if DYNACONF.is_development_mode:
+    DYNACONF.set(
+        "DEFAULTS_SNAPSHOT",
+        DYNACONF.as_dict(internal=False),  # should use deepcopy here?
+        loader_identifier="awx.settings:DEFAULTS_SNAPSHOT",
+    )
+##########################################################################################
+#  Any settings loaded after this point will be marked as as a read_only database setting
+##########################################################################################
+
+# Load extra config from the now deprecated /etc/tower/ directory
+# This is only for backwards compatibility and will be removed in the future
+# Load settings from any .py files in the global conf.d directory specified in
+# the environment, defaulting to /etc/tower/conf.d/.
+# Load remaining settings from the global settings file specified in the
+# environment, defaulting to /etc/tower/settings.py.
+# Attempt to load settings from /etc/tower/settings.py first, followed by
+# /etc/tower/conf.d/*.py.
+settings_dir = os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/')
+settings_files_path = os.path.join(settings_dir, '*.py')
+settings_file_path = os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py')
+
+# The following should have been loaded using `DYNACONF.load_file` but we are
+# using `load_python_file_with__injected_context` to maintain backwards compatibility
+load_python_file_with__injected_context(settings_files_path, settings=DYNACONF)
+load_python_file_with__injected_context(settings_file_path, settings=DYNACONF)
+
+
+# Load new standard settings files from
+#  /etc/ansible-automation-platform/ and /etc/ansible-automation-platform/awx/
+# this is to allow for a smoother transition from legacy (above) to new settings standard paths
+load_standard_settings_files(DYNACONF)
+
+# Load optional development only settings
+if DYNACONF.get_environ("AWX_KUBE_DEVEL"):
+    load_python_file_with__injected_context("kube_defaults.py", settings=DYNACONF)
+else:
+    load_python_file_with__injected_context("local_*.py", settings=DYNACONF)
+
+# Check at least one required setting file has been loaded
+if "production" in DYNACONF.current_env.lower():  # pragma: no cover
+    required_settings_paths = [
+        os.path.dirname(settings_file_path),
+        "/etc/ansible-automation-platform/",
+    ]
+    # check if at least one file has been loaded any of the required paths
+    # use DYNACONF._loaded_files to check any filename inside the paths
+    # if not loaded then raise an ImproperlyConfigured error
+    for path in required_settings_paths:
+        if any(path in f for f in DYNACONF._loaded_files):
+            break
+    else:
+        from django.core.exceptions import ImproperlyConfigured
+
+        msg = 'No AWX configuration found at %s.' % required_settings_paths
+        msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
+        msg += 'specify an alternate path.'
+        raise ImproperlyConfigured(msg)
+
+# Load envvars at the end to allow them to override everything loaded so far
+load_envvars(DYNACONF)
+
+# This must run after all custom settings are imported
+DYNACONF.update(
+    merge_application_name(DYNACONF),
+    loader_identifier="awx.settings:merge_application_name",
+    merge=True,
+)
+
+# Toggle feature flags based on installer settings
+DYNACONF.update(
+    toggle_feature_flags(DYNACONF),
+    loader_identifier="awx.settings:toggle_feature_flags",
+    merge=True,
+)
+
+# Update django.conf.settings with DYNACONF keys.
+export(__name__, DYNACONF)
+
+# Validate the settings according to the validators registered
+validate(DYNACONF)

awx/settings/application_name.py

@@ -25,6 +25,7 @@ def get_application_name(CLUSTER_HOST_ID, function=''):
 
 
 def set_application_name(DATABASES, CLUSTER_HOST_ID, function=''):
+    """In place modification of DATABASES to set the application name for the connection."""
     # If settings files were not properly passed DATABASES could be {} at which point we don't need to set the app name.
     if not DATABASES or 'default' not in DATABASES:
         return

awx/settings/defaults.py

@@ -9,9 +9,6 @@
 import socket
 from datetime import timedelta
 
-from split_settings.tools import include
-
-
 DEBUG = True
 SQL_DEBUG = DEBUG
 
@@ -1011,17 +1008,12 @@
     }
 }
 
-
 # django-ansible-base
 ANSIBLE_BASE_TEAM_MODEL = 'main.Team'
 ANSIBLE_BASE_ORGANIZATION_MODEL = 'main.Organization'
 ANSIBLE_BASE_RESOURCE_CONFIG_MODULE = 'awx.resource_api'
 ANSIBLE_BASE_PERMISSION_MODEL = 'main.Permission'
 
-from ansible_base.lib import dynamic_config  # noqa: E402
-
-include(os.path.join(os.path.dirname(dynamic_config.__file__), 'dynamic_settings.py'))
-
 # Add a postfix to the API URL patterns
 # example if set to '' API pattern will be /api
 # example if set to 'controller' API pattern will be /api AND /api/controller

awx/settings/development.py

@@ -1,126 +1,13 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved.
-
-# Development settings for AWX project.
-
-# Python
+# This file exists for backwards compatibility only
+# the current way of running AWX is to point settings to
+# awx/settings/__init__.py as the entry point for the settings
+# that is done by exporting: export DJANGO_SETTINGS_MODULE=awx.settings
 import os
-import socket
-import copy
-import sys
-import traceback
-
-# Centos-7 doesn't include the svg mime type
-# /usr/lib64/python/mimetypes.py
-import mimetypes
-
-# Django Split Settings
-from split_settings.tools import optional, include
-
-# Load default settings.
-from .defaults import *  # NOQA
-
-# awx-manage shell_plus --notebook
-NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
-
-# print SQL queries in shell_plus
-SHELL_PLUS_PRINT_SQL = False
-
-# show colored logs in the dev environment
-# to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
-COLOR_LOGS = True
-LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
-
-ALLOWED_HOSTS = ['*']
-
-mimetypes.add_type("image/svg+xml", ".svg", True)
-mimetypes.add_type("image/svg+xml", ".svgz", True)
-
-# Disallow sending session cookies over insecure connections
-SESSION_COOKIE_SECURE = False
-
-# Disallow sending csrf cookies over insecure connections
-CSRF_COOKIE_SECURE = False
-
-# Disable Pendo on the UI for development/test.
-# Note: This setting may be overridden by database settings.
-PENDO_TRACKING_STATE = "off"
-INSIGHTS_TRACKING_STATE = False
-
-# debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
-
-INSTALLED_APPS += ['drf_yasg', 'debug_toolbar']  # NOQA
-
-MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE  # NOQA
-
-DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
-
-# Configure a default UUID for development only.
-SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
-INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
-
-# Ansible base virtualenv paths and enablement
-# only used for deprecated fields and management commands for them
-BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
-
-CLUSTER_HOST_ID = socket.gethostname()
-
-AWX_CALLBACK_PROFILE = True
-
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-# Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
-# Allows user to trigger task managers directly for debugging and profiling purposes.
-# Only works in combination with settings.SETTINGS_MODULE == 'awx.settings.development'
-AWX_DISABLE_TASK_MANAGERS = False
-
-# Needed for launching runserver in debug mode
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-
-# Store a snapshot of default settings at this point before loading any
-# customizable config files.
-this_module = sys.modules[__name__]
-local_vars = dir(this_module)
-DEFAULTS_SNAPSHOT = {}  # define after we save local_vars so we do not snapshot the snapshot
-for setting in local_vars:
-    if setting.isupper():
-        DEFAULTS_SNAPSHOT[setting] = copy.deepcopy(getattr(this_module, setting))
-
-del local_vars  # avoid temporary variables from showing up in dir(settings)
-del this_module
-#
-###############################################################################################
-#
-#  Any settings defined after this point will be marked as as a read_only database setting
-#
-################################################################################################
-
-# If there is an `/etc/tower/settings.py`, include it.
-# If there is a `/etc/tower/conf.d/*.py`, include them.
-include(optional('/etc/tower/settings.py'), scope=locals())
-include(optional('/etc/tower/conf.d/*.py'), scope=locals())
-
-# If any local_*.py files are present in awx/settings/, use them to override
-# default settings for development.  If not present, we can still run using
-# only the defaults.
-# this needs to stay at the bottom of this file
-try:
-    if os.getenv('AWX_KUBE_DEVEL', False):
-        include(optional('development_kube.py'), scope=locals())
-    else:
-        include(optional('local_*.py'), scope=locals())
-except ImportError:
-    traceback.print_exc()
-    sys.exit(1)
-
-# The below runs AFTER all of the custom settings are imported
-# because conf.d files will define DATABASES and this should modify that
-from .application_name import set_application_name
 
-set_application_name(DATABASES, CLUSTER_HOST_ID)  # NOQA
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings")
+os.environ.setdefault("AWX_MODE", "development")
 
-del set_application_name
+from ansible_base.lib.dynamic_config import export
+from . import DYNACONF  # noqa
 
-# Set the value of any feature flags that are defined in the local settings
-for feature in list(FLAGS.keys()):  # noqa: F405
-    if feature in locals():
-        FLAGS[feature][0]['value'] = locals()[feature]  # noqa: F405
+export(__name__, DYNACONF)