Move key_store_password to keystore section in zuul.conf

This is likely to be needed by executors as well since passing
decrypted secrets to the executors via zookeeper has the same
encrypted-at-rest concerns as they keystore itself.  To avoid
confusion around executors needing a zuul.conf with a scheduler
section, start a new keystore section which we can later indicate
is used by schedulers and executors.  It also makes it convenient
to add new options (like those dealing with rotation, or even
using an external keystore).

Also change some log levels from debug to info where it's useful
for the operator to know that the backup keystore was used (or
a key was generated).

Change-Id: If2491bbe4eb80b76435a274cf5354a4918315e65

doc/source/discussion/components.rst

@@ -111,11 +111,13 @@ An example ``zuul.conf``:
    [zookeeper]
    hosts=zk1.example.com,zk2.example.com,zk3.example.com
 
+   [keystore]
+   password=MY_SECRET_PASSWORD
+
    [web]
    status_url=https://zuul.example.com/status
 
    [scheduler]
-   key_store_password=MY_SECRET_PASSWORD
    log_config=/etc/zuul/scheduler-logging.yaml
 
 A minimal Zuul system may consist of a :ref:`scheduler` and
@@ -303,13 +305,16 @@ The following sections of ``zuul.conf`` are used by the scheduler:
 
       .. TODO: is this effectively required?
 
-.. attr:: scheduler
+.. attr:: keystore
 
-   .. attr:: key_store_password
+   .. attr:: password
       :required:
 
       Encryption password for private data stored in Zookeeper.
 
+
+.. attr:: scheduler
+
    .. attr:: command_socket
       :default: /var/lib/zuul/scheduler.socket
 

doc/source/examples/etc_zuul/zuul.conf

@@ -10,8 +10,10 @@ tls_cert=/var/certs/certs/client.pem
 tls_key=/var/certs/keys/clientkey.pem
 tls_ca=/var/certs/certs/cacert.pem
 
+[keystore]
+password=secret
+
 [scheduler]
-key_store_password=secret
 tenant_config=/etc/zuul/main.yaml
 
 [connection "gerrit"]

doc/source/howtos/installation.rst

@@ -131,8 +131,10 @@ service in Zuul, and a connection to Gerrit.
 
 **zuul.conf**::
 
+    [keystore]
+    password=secret
+
     [scheduler]
-    key_store_password=secret
     tenant_config=/etc/zuul/main.yaml
 
     [gearman_server]

doc/source/howtos/zuul-from-scratch.rst

@@ -72,8 +72,10 @@ to appropriate values for your setup.
    [web]
    listen_address=0.0.0.0
 
+   [keystore]
+   password=secret
+
    [scheduler]
-   key_store_password=secret
    tenant_config=/etc/zuul/main.yaml
    EOF"
 

etc/zuul.conf-sample

@@ -18,8 +18,10 @@ start=true
 ;ssl_key=/path/to/server.key
 ;port=4730
 
+[keystore]
+password=secret
+
 [scheduler]
-key_store_password=secret
 tenant_config=/etc/zuul/main.yaml
 log_config=/etc/zuul/logging.conf
 pidfile=/var/run/zuul/zuul.pid

releasenotes/notes/zookeeper-key-storage-a1ad32aa8d63b05f.yaml

@@ -3,12 +3,12 @@ features:
   - |
     Project secrets keys and SSH keys are now stored in Zookeeper. All private
     data will be encrypted at rest, which requires a new mandatory setting
-    :attr:`scheduler.key_store_password` in ``zuul.conf``.
+    :attr:`keystore.password` in ``zuul.conf``.
 
     For backup purposes the secrets keys and SSH keys will still exist on the
     local filesystem of the scheduler as before.
 upgrade:
   - |
     As project secrets keys and SSH keys are stored encrypted in Zookeeper the
-    new :attr:`scheduler.key_store_password` option in ``zuul.conf`` is
+    new :attr:`keystore.password` option in ``zuul.conf`` is
     required. Please add it to your configuration.

tests/base.py

@@ -4254,8 +4254,8 @@ class ZuulTestCase(BaseTestCase):
         self.config.set(
             'scheduler', 'command_socket',
             os.path.join(self.test_root, 'scheduler.socket'))
-        if not self.config.has_option("scheduler", "key_store_password"):
+        if not self.config.has_option("keystore", "password"):
-            self.config.set("scheduler", "key_store_password",
+            self.config.set("keystore", "password",
                             uuid.uuid4().hex)
         self.config.set('merger', 'git_dir', self.merger_src_root)
         self.config.set('executor', 'git_dir', self.executor_src_root)
@@ -4417,7 +4417,8 @@ class ZuulTestCase(BaseTestCase):
         config.read(os.path.join(FIXTURE_DIR, config_file))
 
         sections = [
-            'zuul', 'scheduler', 'executor', 'merger', 'web', 'zookeeper'
+            'zuul', 'scheduler', 'executor', 'merger', 'web', 'zookeeper',
+            'keystore'
         ]
         for section in sections:
             if not config.has_section(section):

zuul/lib/keystorage.py

@@ -278,15 +278,15 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
             if self.backup and self.backup.hasProjectSSHKeys(
                 connection_name, project_name
             ):
-                self.log.debug("Using SSH key for %s/%s from backup key store",
+                self.log.info("Using SSH key for %s/%s from backup key store",
-                               connection_name, project_name)
+                              connection_name, project_name)
                 pk, _ = self.backup.getProjectSSHKeys(connection_name,
                                                       project_name)
                 with io.StringIO(pk) as o:
                     key = paramiko.RSAKey.from_private_key(o)
             else:
-                self.log.debug("Generating a new SSH key for %s/%s",
+                self.log.info("Generating a new SSH key for %s/%s",
-                               connection_name, project_name)
+                              connection_name, project_name)
                 key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
 
             try:
@@ -330,14 +330,14 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
                            connection_name, project_name)
             if self.backup and self.backup.hasProjectSecretsKeys(
                     connection_name, project_name):
-                self.log.debug(
+                self.log.info(
                     "Using secrets key for %s/%s from backup key store",
                     connection_name, project_name)
                 private_key, public_key = self.backup.getProjectSecretsKeys(
                     connection_name, project_name)
             else:
-                self.log.debug("Generating a new secrets key for %s/%s",
+                self.log.info("Generating a new secrets key for %s/%s",
-                               connection_name, project_name)
+                              connection_name, project_name)
                 private_key, public_key = encryption.generate_rsa_keypair()
 
             pem_private_key = encryption.serialize_rsa_private_key(

zuul/scheduler.py

@@ -595,7 +595,7 @@ class Scheduler(threading.Thread):
 
     def _get_key_store_password(self):
         try:
-            return self.config["scheduler"]["key_store_password"]
+            return self.config["keystore"]["password"]
         except KeyError:
             raise RuntimeError("No key store password configured!")