From b62fa3313d5b3939c428040456089e8a1df8c11a Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 12 Mar 2020 09:48:01 -0700 Subject: [PATCH] Add ZooKeeper TLS support Change-Id: I009d9f90b32881aaef2d0694da6ff28074f48f8e --- doc/source/configuration.rst | 25 +++++++++++++++++++ nodepool/builder.py | 6 ++++- nodepool/cmd/config_validator.py | 7 ++++++ nodepool/cmd/nodepoolcmd.py | 6 ++++- nodepool/config.py | 12 +++++++++ nodepool/launcher.py | 5 +++- nodepool/zk.py | 15 +++++++++-- .../notes/zk-tls-885f8f0ea9f2efa2.yaml | 12 +++++++++ 8 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/zk-tls-885f8f0ea9f2efa2.yaml diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 451f4bcd8..e4e7fd898 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -122,6 +122,31 @@ Options relative to the supplied root path, is also optional and has no default. +.. attr:: zookeeper-tls + :type: dict + + To use TLS connections with Zookeeper, provide this dictionary with + the following keys: + + .. attr:: cert + :type: string + :required: + + The path to the PEM encoded certificate. + + .. attr:: key + :type: string + :required: + + The path to the PEM encoded key. + + .. attr:: ca + :type: string + :required: + + The path to the PEM encoded CA certificate. + + .. attr:: labels :type: list diff --git a/nodepool/builder.py b/nodepool/builder.py index ba5e63986..d19f5b54c 100644 --- a/nodepool/builder.py +++ b/nodepool/builder.py @@ -1372,7 +1372,11 @@ class NodePoolBuilder(object): # All worker threads share a single ZooKeeper instance/connection. self.zk = zk.ZooKeeper(enable_cache=False) - self.zk.connect(list(self._config.zookeeper_servers.values())) + self.zk.connect( + list(self._config.zookeeper_servers.values()), + tls_cert=self._config.zookeeper_tls_cert, + tls_key=self._config.zookeeper_tls_key, + tls_ca=self._config.zookeeper_tls_ca) self.log.debug('Starting listener for build jobs') diff --git a/nodepool/cmd/config_validator.py b/nodepool/cmd/config_validator.py index 4fd12228b..171a7c57b 100644 --- a/nodepool/cmd/config_validator.py +++ b/nodepool/cmd/config_validator.py @@ -59,6 +59,12 @@ class ConfigValidator: 'listen_address': str, } + zk_tls = dict( + cert=v.Required(str), + key=v.Required(str), + ca=v.Required(str), + ) + top_level = { 'webapp': webapp, 'elements-dir': str, @@ -70,6 +76,7 @@ class ConfigValidator: 'port': int, 'chroot': str, }], + 'zookeeper-tls': zk_tls, 'providers': list, 'labels': [label], 'diskimages': [diskimage], diff --git a/nodepool/cmd/nodepoolcmd.py b/nodepool/cmd/nodepoolcmd.py index 1bba30f82..82d12872b 100644 --- a/nodepool/cmd/nodepoolcmd.py +++ b/nodepool/cmd/nodepoolcmd.py @@ -370,7 +370,11 @@ class NodePoolCmd(NodepoolApp): 'list', 'delete', 'request-list', 'info', 'erase'): self.zk = zk.ZooKeeper(enable_cache=False) - self.zk.connect(list(config.zookeeper_servers.values())) + self.zk.connect( + list(config.zookeeper_servers.values()), + tls_cert=config.zookeeper_tls_cert, + tls_key=config.zookeeper_tls_key, + tls_ca=config.zookeeper_tls_ca) self.pool.setConfig(config) self.args.func() diff --git a/nodepool/config.py b/nodepool/config.py index 74fc2045f..b3a87ff7f 100644 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -36,6 +36,9 @@ class Config(ConfigValue): self.providers = {} self.provider_managers = {} self.zookeeper_servers = {} + self.zookeeper_tls_cert = None + self.zookeeper_tls_key = None + self.zookeeper_tls_ca = None self.elementsdir = None self.imagesdir = None self.build_log_dir = None @@ -83,6 +86,13 @@ class Config(ConfigValue): 'listen_address': webapp_cfg.get('listen_address', '0.0.0.0') } + def setZooKeeperTLS(self, zk_tls): + if not zk_tls: + return + self.zookeeper_tls_cert = zk_tls.get('cert') + self.zookeeper_tls_key = zk_tls.get('key') + self.zookeeper_tls_ca = zk_tls.get('ca') + def setZooKeeperServers(self, zk_cfg): if not zk_cfg: return @@ -262,6 +272,7 @@ def loadConfig(config_path): newconfig.setDiskImages(config.get('diskimages')) newconfig.setLabels(config.get('labels')) newconfig.setProviders(config.get('providers')) + newconfig.setZooKeeperTLS(config.get('zookeeper-tls')) # Ensure at least qcow2 is set to be passed to qemu-img. # Note that this needs to be after setting the providers as they @@ -286,3 +297,4 @@ def loadSecureConfig(config, secure_config_path): config.setZooKeeperServers(secure.get('zookeeper-servers')) config.setSecureDiskimageEnv( secure.get('diskimages', []), secure_config_path) + config.setZooKeeperTLS(secure.get('zookeeper-tls')) diff --git a/nodepool/launcher.py b/nodepool/launcher.py index 688e14cf4..0fe5ae054 100644 --- a/nodepool/launcher.py +++ b/nodepool/launcher.py @@ -940,7 +940,10 @@ class NodePool(threading.Thread): if not self.zk and configured: self.log.debug("Connecting to ZooKeeper servers") self.zk = zk.ZooKeeper() - self.zk.connect(configured) + self.zk.connect(configured, + tls_cert=config.zookeeper_tls_cert, + tls_key=config.zookeeper_tls_key, + tls_ca=config.zookeeper_tls_ca) else: self.log.debug("Detected ZooKeeper server changes") self.zk.resetHosts(configured) diff --git a/nodepool/zk.py b/nodepool/zk.py index 82b58bccf..2768f540a 100644 --- a/nodepool/zk.py +++ b/nodepool/zk.py @@ -945,7 +945,8 @@ class ZooKeeper(object): def resetLostFlag(self): self._became_lost = False - def connect(self, host_list, read_only=False): + def connect(self, host_list, read_only=False, tls_cert=None, + tls_key=None, tls_ca=None): ''' Establish a connection with ZooKeeper cluster. @@ -956,11 +957,21 @@ class ZooKeeper(object): :py:class:`~nodepool.zk.ZooKeeperConnectionConfig` objects (one per server) defining the ZooKeeper cluster servers. :param bool read_only: If True, establishes a read-only connection. + :param str tls_key: Path to TLS key + :param str tls_cert: Path to TLS cert + :param str tls_ca: Path to TLS CA cert ''' if self.client is None: hosts = buildZooKeeperHosts(host_list) - self.client = KazooClient(hosts=hosts, read_only=read_only) + args = dict(hosts=hosts, + read_only=read_only) + if tls_key: + args['use_ssl'] = True + args['keyfile'] = tls_key + args['certfile'] = tls_cert + args['ca'] = tls_ca + self.client = KazooClient(**args) self.client.add_listener(self._connection_listener) # Manually retry initial connection attempt while True: diff --git a/releasenotes/notes/zk-tls-885f8f0ea9f2efa2.yaml b/releasenotes/notes/zk-tls-885f8f0ea9f2efa2.yaml new file mode 100644 index 000000000..7ddfe2db1 --- /dev/null +++ b/releasenotes/notes/zk-tls-885f8f0ea9f2efa2.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Support for encrypted connections to ZooKeeper has been added. + + Before enabling, ensure that both Zuul and Nodepool software + versions support encrypted connections. See the Zuul release + notes, documentation, and associated helper scripts for more + information. + + Both Zuul and Nodepool may need to be restarted together with the + new configuration.