Improve thread-safety with provider classes

This change works around an issue we've seen with Dynaconf loading fresh
settings in a threaded environment.
Additionally, bumped the concurrency of functional tests

broker/providers/__init__.py

@@ -16,6 +16,7 @@ class Provider(PickleSafe):
     # _checkout_options = [click.option("--workflow", type=str, help="Help text")]
     _checkout_options = []
     _execute_options = []
+    _fresh_settings = settings.dynaconf_clone()
 
     def __init__(self, **kwargs):
         self._construct_params = []
@@ -36,8 +37,7 @@ def _validate_settings(self, instance_name=None):
         """
         instance_name = instance_name or getattr(self, "instance", None)
         section_name = self.__class__.__name__.upper()
-        # make sure each instance isn't loading values from another
-        fresh_settings = settings.get_fresh(section_name)
+        fresh_settings = self._fresh_settings.get(section_name).copy()
         instance, default = None, False
         for candidate in fresh_settings.instances:
             logger.debug(f"Checking {instance_name} against {candidate}")

tests/functional/test_containers.py

@@ -78,7 +78,7 @@ def test_container_e2e():
 
 
 def test_container_e2e_mp():
-    with Broker(container_host="ubi8:latest", _count=2) as c_hosts:
+    with Broker(container_host="ubi8:latest", _count=7) as c_hosts:
         for c_host in c_hosts:
             assert c_host._cont_inst.top()['Processes']
             res = c_host.execute("hostname")

tests/functional/test_satlab.py

@@ -71,3 +71,12 @@ def test_tower_host():
         r_host.session.sftp_write("broker_settings.yaml", "/tmp/fake/")
         res = r_host.execute("ls /tmp/fake")
         assert "broker_settings.yaml" in res.stdout
+
+def test_tower_host_mp():
+    with Broker(workflow="deploy-base-rhel", _count=3) as r_hosts:
+        for r_host in r_hosts:
+            res = r_host.execute("hostname")
+            assert res.stdout.strip() == r_host.hostname
+            r_host.session.sftp_write("broker_settings.yaml", "/tmp/fake/")
+            res = r_host.execute("ls /tmp/fake")
+            assert "broker_settings.yaml" in res.stdout