Add ZooKeeper TLS support

Change-Id: I009d9f90b32881aaef2d0694da6ff28074f48f8e

doc/source/configuration.rst

@@ -122,6 +122,31 @@ Options
       relative to the supplied root path, is also optional and has no
       default.
 
+.. attr:: zookeeper-tls
+   :type: dict
+
+   To use TLS connections with Zookeeper, provide this dictionary with
+   the following keys:
+
+   .. attr:: cert
+      :type: string
+      :required:
+
+      The path to the PEM encoded certificate.
+
+   .. attr:: key
+      :type: string
+      :required:
+
+      The path to the PEM encoded key.
+
+   .. attr:: ca
+      :type: string
+      :required:
+
+      The path to the PEM encoded CA certificate.
+
+
 .. attr:: labels
    :type: list
 

nodepool/builder.py

@@ -1372,7 +1372,11 @@ class NodePoolBuilder(object):
 
             # All worker threads share a single ZooKeeper instance/connection.
             self.zk = zk.ZooKeeper(enable_cache=False)
-            self.zk.connect(list(self._config.zookeeper_servers.values()))
+            self.zk.connect(
+                list(self._config.zookeeper_servers.values()),
+                tls_cert=self._config.zookeeper_tls_cert,
+                tls_key=self._config.zookeeper_tls_key,
+                tls_ca=self._config.zookeeper_tls_ca)
 
             self.log.debug('Starting listener for build jobs')
 

nodepool/cmd/config_validator.py

@@ -59,6 +59,12 @@ class ConfigValidator:
             'listen_address': str,
         }
 
+        zk_tls = dict(
+            cert=v.Required(str),
+            key=v.Required(str),
+            ca=v.Required(str),
+        )
+
         top_level = {
             'webapp': webapp,
             'elements-dir': str,
@@ -70,6 +76,7 @@ class ConfigValidator:
                 'port': int,
                 'chroot': str,
             }],
+            'zookeeper-tls': zk_tls,
             'providers': list,
             'labels': [label],
             'diskimages': [diskimage],

nodepool/cmd/nodepoolcmd.py

@@ -370,7 +370,11 @@ class NodePoolCmd(NodepoolApp):
                                  'list', 'delete',
                                  'request-list', 'info', 'erase'):
             self.zk = zk.ZooKeeper(enable_cache=False)
-            self.zk.connect(list(config.zookeeper_servers.values()))
+            self.zk.connect(
+                list(config.zookeeper_servers.values()),
+                tls_cert=config.zookeeper_tls_cert,
+                tls_key=config.zookeeper_tls_key,
+                tls_ca=config.zookeeper_tls_ca)
 
         self.pool.setConfig(config)
         self.args.func()

nodepool/config.py

@@ -36,6 +36,9 @@ class Config(ConfigValue):
         self.providers = {}
         self.provider_managers = {}
         self.zookeeper_servers = {}
+        self.zookeeper_tls_cert = None
+        self.zookeeper_tls_key = None
+        self.zookeeper_tls_ca = None
         self.elementsdir = None
         self.imagesdir = None
         self.build_log_dir = None
@@ -83,6 +86,13 @@ class Config(ConfigValue):
             'listen_address': webapp_cfg.get('listen_address', '0.0.0.0')
         }
 
+    def setZooKeeperTLS(self, zk_tls):
+        if not zk_tls:
+            return
+        self.zookeeper_tls_cert = zk_tls.get('cert')
+        self.zookeeper_tls_key = zk_tls.get('key')
+        self.zookeeper_tls_ca = zk_tls.get('ca')
+
     def setZooKeeperServers(self, zk_cfg):
         if not zk_cfg:
             return
@@ -262,6 +272,7 @@ def loadConfig(config_path):
     newconfig.setDiskImages(config.get('diskimages'))
     newconfig.setLabels(config.get('labels'))
     newconfig.setProviders(config.get('providers'))
+    newconfig.setZooKeeperTLS(config.get('zookeeper-tls'))
 
     # Ensure at least qcow2 is set to be passed to qemu-img.
     # Note that this needs to be after setting the providers as they
@@ -286,3 +297,4 @@ def loadSecureConfig(config, secure_config_path):
     config.setZooKeeperServers(secure.get('zookeeper-servers'))
     config.setSecureDiskimageEnv(
         secure.get('diskimages', []), secure_config_path)
+    config.setZooKeeperTLS(secure.get('zookeeper-tls'))

nodepool/launcher.py

@@ -940,7 +940,10 @@ class NodePool(threading.Thread):
         if not self.zk and configured:
             self.log.debug("Connecting to ZooKeeper servers")
             self.zk = zk.ZooKeeper()
-            self.zk.connect(configured)
+            self.zk.connect(configured,
+                            tls_cert=config.zookeeper_tls_cert,
+                            tls_key=config.zookeeper_tls_key,
+                            tls_ca=config.zookeeper_tls_ca)
         else:
             self.log.debug("Detected ZooKeeper server changes")
             self.zk.resetHosts(configured)

nodepool/zk.py

@@ -945,7 +945,8 @@ class ZooKeeper(object):
     def resetLostFlag(self):
         self._became_lost = False
 
-    def connect(self, host_list, read_only=False):
+    def connect(self, host_list, read_only=False, tls_cert=None,
+                tls_key=None, tls_ca=None):
         '''
         Establish a connection with ZooKeeper cluster.
 
@@ -956,11 +957,21 @@ class ZooKeeper(object):
             :py:class:`~nodepool.zk.ZooKeeperConnectionConfig` objects
             (one per server) defining the ZooKeeper cluster servers.
         :param bool read_only: If True, establishes a read-only connection.
+        :param str tls_key: Path to TLS key
+        :param str tls_cert: Path to TLS cert
+        :param str tls_ca: Path to TLS CA cert
 
         '''
         if self.client is None:
             hosts = buildZooKeeperHosts(host_list)
-            self.client = KazooClient(hosts=hosts, read_only=read_only)
+            args = dict(hosts=hosts,
+                        read_only=read_only)
+            if tls_key:
+                args['use_ssl'] = True
+                args['keyfile'] = tls_key
+                args['certfile'] = tls_cert
+                args['ca'] = tls_ca
+            self.client = KazooClient(**args)
             self.client.add_listener(self._connection_listener)
             # Manually retry initial connection attempt
             while True:

releasenotes/notes/zk-tls-885f8f0ea9f2efa2.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Support for encrypted connections to ZooKeeper has been added.
+
+    Before enabling, ensure that both Zuul and Nodepool software
+    versions support encrypted connections.  See the Zuul release
+    notes, documentation, and associated helper scripts for more
+    information.
+
+    Both Zuul and Nodepool may need to be restarted together with the
+    new configuration.