Make the ConfigLoader work independently of the Scheduler

This is an early preparation step for removing the RPC calls between
zuul-web and the scheduler.

We want to format the status JSON and do the job freezing (job freezing
API) directly in zuul-web without utilising the scheduler via RPC. In
order to make this work, zuul-web must instantiate a ConfigLoader.
Currently this would require a scheduler instance which is not available
in zuul-web, thus we have to make this parameter optional.

Change-Id: I41214086aaa9d822ab888baf001972d2846528be

tests/unit/test_client.py

@@ -57,7 +57,7 @@ class BaseClientTestCase(BaseTestCase):
 
 
 class TestTenantValidationClient(BaseClientTestCase):
-    config_with_zk = False
+    config_with_zk = True
 
     def test_client_tenant_conf_check(self):
         self.config.set(

tests/unit/test_configloader.py

@@ -11,18 +11,52 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-from configparser import ConfigParser
-
 import fixtures
 import logging
 import textwrap
 import testtools
+from collections import defaultdict
+from configparser import ConfigParser
 
 from zuul import model
-from zuul.configloader import AuthorizationRuleParser, safe_load_yaml
+from zuul.lib.ansible import AnsibleManager
+from zuul.configloader import (
+    AuthorizationRuleParser, ConfigLoader, safe_load_yaml
+)
+from zuul.model import Abide, MergeRequest, SourceContext
+from zuul.zk.locks import tenant_read_lock
 
 from tests.base import ZuulTestCase
-from zuul.model import MergeRequest, SourceContext
+
+
+class TestConfigLoader(ZuulTestCase):
+    tenant_config_file = 'config/single-tenant/main.yaml'
+
+    def test_update_system_config(self):
+        """Test if the system config can be updated without a scheduler."""
+        sched = self.scheds.first.sched
+
+        # Get the current system config before instantiating a ConfigLoader.
+        unparsed_abide, zuul_globals = sched.system_config_cache.get()
+        ansible_manager = AnsibleManager(
+            default_version=zuul_globals.default_ansible_version)
+        loader = ConfigLoader(
+            sched.connections, self.zk_client, zuul_globals, sched.statsd,
+            keystorage=sched.keystore)
+        abide = Abide()
+        loader.loadTPCs(abide, unparsed_abide)
+        loader.loadAdminRules(abide, unparsed_abide)
+
+        for tenant_name in unparsed_abide.tenants:
+            tlock = tenant_read_lock(self.zk_client, tenant_name)
+            # Consider all caches valid (min. ltime -1)
+            min_ltimes = defaultdict(lambda: defaultdict(lambda: -1))
+            with tlock:
+                tenant = loader.loadTenant(
+                    abide, tenant_name, ansible_manager, unparsed_abide,
+                    min_ltimes=min_ltimes)
+
+                self.assertEqual(tenant.name, tenant_name)
 
 
 class TenantParserTestCase(ZuulTestCase):

zuul/cmd/client.py

@@ -31,6 +31,7 @@ import urllib.parse
 import zuul.rpcclient
 import zuul.cmd
 from zuul.lib.config import get_default
+from zuul.model import SystemAttributes
 from zuul.zk import ZooKeeperClient
 from zuul.lib.keystorage import KeyStorage
 
@@ -830,9 +831,12 @@ class Client(zuul.cmd.ZuulApp):
                 self.connections = connections
                 self.unparsed_config_cache = None
 
-        sched = SchedulerConfig(self.config, self.connections)
+        zk_client = ZooKeeperClient.fromConfig(self.config)
+        zk_client.connect()
+        zuul_globals = SystemAttributes.fromConfig(self.config)
         loader = configloader.ConfigLoader(
-            sched.connections, sched, None, None)
+            self.connections, zk_client, zuul_globals)
+        sched = SchedulerConfig(self.config, self.connections)
         tenant_config, script = sched._checkTenantSourceConf(self.config)
         unparsed_abide = loader.readConfig(tenant_config, from_script=script)
         try:

zuul/configloader.py

@@ -33,6 +33,7 @@ import zuul.manager.serial
 from zuul.lib.logutil import get_annotated_logger
 from zuul.lib.re2util import filter_allowed_disallowed
 from zuul.lib.varnames import check_varnames
+from zuul.zk.config_cache import UnparsedConfigCache
 from zuul.zk.semaphore import SemaphoreHandler
 
 
@@ -1447,13 +1448,17 @@ class ParseContext(object):
 
 
 class TenantParser(object):
-    def __init__(self, connections, scheduler, merger, keystorage):
+    def __init__(self, connections, zk_client, scheduler, merger, keystorage,
+                 zuul_globals, statsd):
         self.log = logging.getLogger("zuul.TenantParser")
         self.connections = connections
+        self.zk_client = zk_client
         self.scheduler = scheduler
         self.merger = merger
         self.keystorage = keystorage
-        self.unparsed_config_cache = self.scheduler.unparsed_config_cache
+        self.globals = zuul_globals
+        self.statsd = statsd
+        self.unparsed_config_cache = UnparsedConfigCache(self.zk_client)
 
     classes = vs.Any('pipeline', 'job', 'semaphore', 'project',
                      'project-template', 'nodeset', 'secret', 'queue')
@@ -1534,7 +1539,7 @@ class TenantParser(object):
             tenant.authorization_rules = conf['admin-rules']
         if conf.get('authentication-realm') is not None:
             tenant.default_auth_realm = conf['authentication-realm']
-        tenant.web_root = conf.get('web-root', self.scheduler.globals.web_root)
+        tenant.web_root = conf.get('web-root', self.globals.web_root)
         if tenant.web_root and not tenant.web_root.endswith('/'):
             tenant.web_root += '/'
         tenant.allowed_triggers = conf.get('allowed-triggers')
@@ -1603,8 +1608,7 @@ class TenantParser(object):
         tenant.layout = self._parseLayout(
             tenant, parsed_config, loading_errors, layout_uuid)
         tenant.semaphore_handler = SemaphoreHandler(
-            self.scheduler.zk_client, self.scheduler.statsd,
-            tenant.name, tenant.layout
+            self.zk_client, self.statsd, tenant.name, tenant.layout
         )
 
         return tenant
@@ -1791,7 +1795,10 @@ class TenantParser(object):
 
                 extra_config_files = abide.getExtraConfigFiles(project.name)
                 extra_config_dirs = abide.getExtraConfigDirs(project.name)
-                ltime = self.scheduler.zk_client.getCurrentLtime()
+                if not self.merger:
+                    raise RuntimeError(
+                        "Cannot load config files without a merger client.")
+                ltime = self.zk_client.getCurrentLtime()
                 job = self.merger.getFiles(
                     project.source.connection.connection_name,
                     project.name, branch,
@@ -2205,8 +2212,12 @@ class TenantParser(object):
 
         self._addLayoutItems(layout, tenant, data)
 
-        for pipeline in layout.pipelines.values():
-            pipeline.manager._postConfig(layout)
+        # Only call the postConfig hook if we have a scheduler as this will
+        # change data in ZooKeeper. In case we are in a zuul-web context,
+        # we don't want to do that.
+        if self.scheduler:
+            for pipeline in layout.pipelines.values():
+                pipeline.manager._postConfig(layout)
 
         return layout
 
@@ -2214,13 +2225,17 @@ class TenantParser(object):
 class ConfigLoader(object):
     log = logging.getLogger("zuul.ConfigLoader")
 
-    def __init__(self, connections, scheduler, merger, keystorage):
+    def __init__(self, connections, zk_client, zuul_globals, statsd=None,
+                 scheduler=None, merger=None, keystorage=None):
         self.connections = connections
+        self.zk_client = zk_client
+        self.globals = zuul_globals
         self.scheduler = scheduler
         self.merger = merger
         self.keystorage = keystorage
-        self.tenant_parser = TenantParser(connections, scheduler,
-                                          merger, self.keystorage)
+        self.tenant_parser = TenantParser(
+            connections, zk_client, scheduler, merger, keystorage,
+            zuul_globals, statsd)
         self.admin_rule_parser = AuthorizationRuleParser()
 
     def expandConfigPath(self, config_path):

zuul/manager/__init__.py

@@ -65,13 +65,15 @@ class PipelineManager(metaclass=ABCMeta):
         self.ref_filters = []
         # Cached dynamic layouts (layout uuid -> layout)
         self._layout_cache = {}
-        self.sql = self.sched.sql
         # A small local cache to avoid hitting the ZK-based connection
         # change cache for multiple hits in the same pipeline run.
         self._change_cache = {}
         # Current ZK context when the pipeline is locked
         self.current_context = None
 
+        if sched:
+            self.sql = sched.sql
+
     def __str__(self):
         return "<%s %s>" % (self.__class__.__name__, self.pipeline.name)
 
@@ -896,7 +898,8 @@ class PipelineManager(metaclass=ABCMeta):
         # Late import to break an import loop
         import zuul.configloader
         loader = zuul.configloader.ConfigLoader(
-            self.sched.connections, self.sched, None, None)
+            self.sched.connections, self.sched.zk_client, self.sched.globals,
+            self.sched.statsd, self.sched)
 
         log.debug("Loading dynamic layout")
 

zuul/scheduler.py

@@ -768,7 +768,8 @@ class Scheduler(threading.Thread):
             self.primeSystemConfig()
 
         loader = configloader.ConfigLoader(
-            self.connections, self, self.merger, self.keystore)
+            self.connections, self.zk_client, self.globals, self.statsd, self,
+            self.merger, self.keystore)
         new_tenants = (set(self.unparsed_abide.tenants)
                        - self.abide.tenants.keys())
 
@@ -993,7 +994,8 @@ class Scheduler(threading.Thread):
         # Consider all caches valid (min. ltime -1)
         min_ltimes = defaultdict(lambda: defaultdict(lambda: -1))
         loader = configloader.ConfigLoader(
-            self.connections, self, self.merger, self.keystore)
+            self.connections, self.zk_client, self.globals, self.statsd, self,
+            self.merger, self.keystore)
         with self.layout_lock:
             self.log.debug("Updating local layout of tenant %s ", tenant_name)
             layout_state = self.tenant_layout_state.get(tenant_name)
@@ -1044,7 +1046,8 @@ class Scheduler(threading.Thread):
             start = time.monotonic()
 
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger, self.keystore)
+                self.connections, self.zk_client, self.globals, self.statsd,
+                self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)
             unparsed_abide = loader.readConfig(tenant_config,
                                                from_script=script)
@@ -1098,7 +1101,8 @@ class Scheduler(threading.Thread):
                 default_version=self.globals.default_ansible_version)
 
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger, self.keystore)
+                self.connections, self.zk_client, self.globals, self.statsd,
+                self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)
             old_unparsed_abide = self.unparsed_abide
             self.unparsed_abide = loader.readConfig(
@@ -1188,7 +1192,8 @@ class Scheduler(threading.Thread):
                 branch_cache_min_ltimes[connection_name] = ltime
 
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger, self.keystore)
+                self.connections, self.zk_client, self.globals, self.statsd,
+                self, self.merger, self.keystore)
             old_tenant = self.abide.tenants.get(event.tenant_name)
             loader.loadTPCs(self.abide, self.unparsed_abide,
                             [event.tenant_name])
@@ -1655,7 +1660,8 @@ class Scheduler(threading.Thread):
     def primeSystemConfig(self):
         with self.layout_lock:
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger, self.keystore)
+                self.connections, self.zk_client, self.globals, self.statsd,
+                self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)
             self.unparsed_abide = loader.readConfig(
                 tenant_config, from_script=script)
@@ -1670,7 +1676,8 @@ class Scheduler(threading.Thread):
             self.ansible_manager = AnsibleManager(
                 default_version=self.globals.default_ansible_version)
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger, self.keystore)
+                self.connections, self.zk_client, self.globals, self.statsd,
+                self, self.merger, self.keystore)
             loader.loadTPCs(self.abide, self.unparsed_abide)
             loader.loadAdminRules(self.abide, self.unparsed_abide)