From 3647139920f977c4db0bd371366e41ca64b57060 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 12 Apr 2021 13:53:51 -0700 Subject: [PATCH] Move key_store_password to keystore section in zuul.conf This is likely to be needed by executors as well since passing decrypted secrets to the executors via zookeeper has the same encrypted-at-rest concerns as they keystore itself. To avoid confusion around executors needing a zuul.conf with a scheduler section, start a new keystore section which we can later indicate is used by schedulers and executors. It also makes it convenient to add new options (like those dealing with rotation, or even using an external keystore). Also change some log levels from debug to info where it's useful for the operator to know that the backup keystore was used (or a key was generated). Change-Id: If2491bbe4eb80b76435a274cf5354a4918315e65 --- doc/source/discussion/components.rst | 11 ++++++++--- doc/source/examples/etc_zuul/zuul.conf | 4 +++- doc/source/howtos/installation.rst | 4 +++- doc/source/howtos/zuul-from-scratch.rst | 4 +++- etc/zuul.conf-sample | 4 +++- .../zookeeper-key-storage-a1ad32aa8d63b05f.yaml | 4 ++-- tests/base.py | 7 ++++--- zuul/lib/keystorage.py | 14 +++++++------- zuul/scheduler.py | 2 +- 9 files changed, 34 insertions(+), 20 deletions(-) diff --git a/doc/source/discussion/components.rst b/doc/source/discussion/components.rst index f934fd86f4..d726968548 100644 --- a/doc/source/discussion/components.rst +++ b/doc/source/discussion/components.rst @@ -111,11 +111,13 @@ An example ``zuul.conf``: [zookeeper] hosts=zk1.example.com,zk2.example.com,zk3.example.com + [keystore] + password=MY_SECRET_PASSWORD + [web] status_url=https://zuul.example.com/status [scheduler] - key_store_password=MY_SECRET_PASSWORD log_config=/etc/zuul/scheduler-logging.yaml A minimal Zuul system may consist of a :ref:`scheduler` and @@ -303,13 +305,16 @@ The following sections of ``zuul.conf`` are used by the scheduler: .. TODO: is this effectively required? -.. attr:: scheduler +.. attr:: keystore - .. attr:: key_store_password + .. attr:: password :required: Encryption password for private data stored in Zookeeper. + +.. attr:: scheduler + .. attr:: command_socket :default: /var/lib/zuul/scheduler.socket diff --git a/doc/source/examples/etc_zuul/zuul.conf b/doc/source/examples/etc_zuul/zuul.conf index 043a7f6d9d..476c57280a 100644 --- a/doc/source/examples/etc_zuul/zuul.conf +++ b/doc/source/examples/etc_zuul/zuul.conf @@ -10,8 +10,10 @@ tls_cert=/var/certs/certs/client.pem tls_key=/var/certs/keys/clientkey.pem tls_ca=/var/certs/certs/cacert.pem +[keystore] +password=secret + [scheduler] -key_store_password=secret tenant_config=/etc/zuul/main.yaml [connection "gerrit"] diff --git a/doc/source/howtos/installation.rst b/doc/source/howtos/installation.rst index a6dc49b867..53e787cc6b 100644 --- a/doc/source/howtos/installation.rst +++ b/doc/source/howtos/installation.rst @@ -131,8 +131,10 @@ service in Zuul, and a connection to Gerrit. **zuul.conf**:: + [keystore] + password=secret + [scheduler] - key_store_password=secret tenant_config=/etc/zuul/main.yaml [gearman_server] diff --git a/doc/source/howtos/zuul-from-scratch.rst b/doc/source/howtos/zuul-from-scratch.rst index 5e964ae0fb..7ba3d7a2e4 100644 --- a/doc/source/howtos/zuul-from-scratch.rst +++ b/doc/source/howtos/zuul-from-scratch.rst @@ -72,8 +72,10 @@ to appropriate values for your setup. [web] listen_address=0.0.0.0 + [keystore] + password=secret + [scheduler] - key_store_password=secret tenant_config=/etc/zuul/main.yaml EOF" diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample index 3184982549..4a83e04ed8 100644 --- a/etc/zuul.conf-sample +++ b/etc/zuul.conf-sample @@ -18,8 +18,10 @@ start=true ;ssl_key=/path/to/server.key ;port=4730 +[keystore] +password=secret + [scheduler] -key_store_password=secret tenant_config=/etc/zuul/main.yaml log_config=/etc/zuul/logging.conf pidfile=/var/run/zuul/zuul.pid diff --git a/releasenotes/notes/zookeeper-key-storage-a1ad32aa8d63b05f.yaml b/releasenotes/notes/zookeeper-key-storage-a1ad32aa8d63b05f.yaml index a3c713fedd..5a475d1765 100644 --- a/releasenotes/notes/zookeeper-key-storage-a1ad32aa8d63b05f.yaml +++ b/releasenotes/notes/zookeeper-key-storage-a1ad32aa8d63b05f.yaml @@ -3,12 +3,12 @@ features: - | Project secrets keys and SSH keys are now stored in Zookeeper. All private data will be encrypted at rest, which requires a new mandatory setting - :attr:`scheduler.key_store_password` in ``zuul.conf``. + :attr:`keystore.password` in ``zuul.conf``. For backup purposes the secrets keys and SSH keys will still exist on the local filesystem of the scheduler as before. upgrade: - | As project secrets keys and SSH keys are stored encrypted in Zookeeper the - new :attr:`scheduler.key_store_password` option in ``zuul.conf`` is + new :attr:`keystore.password` option in ``zuul.conf`` is required. Please add it to your configuration. diff --git a/tests/base.py b/tests/base.py index fcca048a28..a992260b2b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -4254,8 +4254,8 @@ class ZuulTestCase(BaseTestCase): self.config.set( 'scheduler', 'command_socket', os.path.join(self.test_root, 'scheduler.socket')) - if not self.config.has_option("scheduler", "key_store_password"): - self.config.set("scheduler", "key_store_password", + if not self.config.has_option("keystore", "password"): + self.config.set("keystore", "password", uuid.uuid4().hex) self.config.set('merger', 'git_dir', self.merger_src_root) self.config.set('executor', 'git_dir', self.executor_src_root) @@ -4417,7 +4417,8 @@ class ZuulTestCase(BaseTestCase): config.read(os.path.join(FIXTURE_DIR, config_file)) sections = [ - 'zuul', 'scheduler', 'executor', 'merger', 'web', 'zookeeper' + 'zuul', 'scheduler', 'executor', 'merger', 'web', 'zookeeper', + 'keystore' ] for section in sections: if not config.has_section(section): diff --git a/zuul/lib/keystorage.py b/zuul/lib/keystorage.py index 29c5906ec7..3bb6de8016 100644 --- a/zuul/lib/keystorage.py +++ b/zuul/lib/keystorage.py @@ -278,15 +278,15 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage): if self.backup and self.backup.hasProjectSSHKeys( connection_name, project_name ): - self.log.debug("Using SSH key for %s/%s from backup key store", - connection_name, project_name) + self.log.info("Using SSH key for %s/%s from backup key store", + connection_name, project_name) pk, _ = self.backup.getProjectSSHKeys(connection_name, project_name) with io.StringIO(pk) as o: key = paramiko.RSAKey.from_private_key(o) else: - self.log.debug("Generating a new SSH key for %s/%s", - connection_name, project_name) + self.log.info("Generating a new SSH key for %s/%s", + connection_name, project_name) key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE) try: @@ -330,14 +330,14 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage): connection_name, project_name) if self.backup and self.backup.hasProjectSecretsKeys( connection_name, project_name): - self.log.debug( + self.log.info( "Using secrets key for %s/%s from backup key store", connection_name, project_name) private_key, public_key = self.backup.getProjectSecretsKeys( connection_name, project_name) else: - self.log.debug("Generating a new secrets key for %s/%s", - connection_name, project_name) + self.log.info("Generating a new secrets key for %s/%s", + connection_name, project_name) private_key, public_key = encryption.generate_rsa_keypair() pem_private_key = encryption.serialize_rsa_private_key( diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 572258b4b8..f43d15aed9 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -595,7 +595,7 @@ class Scheduler(threading.Thread): def _get_key_store_password(self): try: - return self.config["scheduler"]["key_store_password"] + return self.config["keystore"]["password"] except KeyError: raise RuntimeError("No key store password configured!")