Cache secret/SSH keys from Zookeeper

Loading secret/SSH keys from Zookeeper can be expensive in case a tenant
contains a lot of projects. Since the keys were also loaded from
Zookeeper during reconfigurations, this can significantly increase the
reconfig time (around +30s for 500 projects).

In order to cache the key we need to instantiate a single keystore
instance in the scheduler instead of creating a new one for each config
loader.

This doesn't speed up key loading during scheduler startup but works on
cached data for reconfigurations.

Change-Id: I377777907ce325d5aaac7b18db1d3eda499d6b4f

tests/unit/test_v3.py

@@ -3564,7 +3564,7 @@ class TestProjectKeys(ZuulTestCase):
             with open(os.path.join(FIXTURE_DIR, fn)) as i:
                 test_keys.append(i.read())
 
-        keystore = self.scheds.first.sched.getKeyStorage()
+        keystore = self.scheds.first.sched.keystore
         private_secrets_key, public_secrets_key = (
             keystore.getProjectSecretsKeys("gerrit", "org/project")
         )

zuul/lib/keystorage.py

@@ -20,6 +20,7 @@ import os
 import tempfile
 import time
 
+import cachetools
 import kazoo
 import paramiko
 
@@ -268,6 +269,7 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
         self.password_bytes = password.encode("utf-8")
         self.backup = backup
 
+    @cachetools.cached(cache={})
     def getProjectSSHKeys(self, connection_name, project_name):
         key_project_name = strings.unique_project_name(project_name)
         key_path = self.SSH_PATH.format(connection_name, key_project_name)
@@ -335,6 +337,7 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
         data = json.dumps(keydata).encode("utf-8")
         self.kazoo_client.create(key_path, value=data, makepath=True)
 
+    @cachetools.cached(cache={})
     def getProjectSecretsKeys(self, connection_name, project_name):
         key_project_name = strings.unique_project_name(project_name)
         key_path = self.SECRETS_PATH.format(connection_name, key_project_name)

zuul/scheduler.py

@@ -213,6 +213,11 @@ class Scheduler(threading.Thread):
 
     def start(self):
         super(Scheduler, self).start()
+        self.keystore = ZooKeeperKeyStorage(
+            self.zk_client,
+            password=self._get_key_store_password(),
+            backup=FileKeyStorage(self._get_key_dir()))
+
         self._command_running = True
         self.log.debug("Starting command processor")
         self.command_socket.start()
@@ -665,12 +670,6 @@ class Scheduler(threading.Thread):
                             "current mode is %o" % (key_dir, mode))
         return key_dir
 
-    def getKeyStorage(self):
-        file_key_store = FileKeyStorage(self._get_key_dir())
-        return ZooKeeperKeyStorage(self.zk_client,
-                                   password=self._get_key_store_password(),
-                                   backup=file_key_store)
-
     def _checkTenantSourceConf(self, config):
         tenant_config = None
         script = False
@@ -714,8 +713,7 @@ class Scheduler(threading.Thread):
                 connection.clearCache()
 
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger,
-                self.getKeyStorage())
+                self.connections, self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)
             self.unparsed_abide = loader.readConfig(
                 tenant_config, from_script=script)
@@ -759,8 +757,7 @@ class Scheduler(threading.Thread):
                 default_version=default_ansible_version)
 
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger,
-                self.getKeyStorage())
+                self.connections, self, self.merger, self.keystore)
             tenant_config, script = self._checkTenantSourceConf(self.config)
             old_unparsed_abide = self.unparsed_abide
             self.unparsed_abide = loader.readConfig(
@@ -814,8 +811,7 @@ class Scheduler(threading.Thread):
                                                     branch_name)
             old_tenant = self.abide.tenants[event.tenant_name]
             loader = configloader.ConfigLoader(
-                self.connections, self, self.merger,
-                self.getKeyStorage())
+                self.connections, self, self.merger, self.keystore)
             abide = loader.reloadTenant(
                 self.abide, old_tenant, self.ansible_manager)
             tenant = abide.tenants[event.tenant_name]