Add Zookeeper TLS support

This creates TLS certs for Zookeeper, uses them inside the ZK
quorum, and configures Nodepool and Zuul to use them as well.

A full system restart of all ZK-related components will be required
after merging this patch.

Change-Id: I0cb96a989f3d2c7e0563ce8899f2a5945ea225b3

inventory/service/group_vars/zookeeper.yaml

@@ -3,8 +3,12 @@ zookeeper_group: zookeeper
 zookeeper_uid: 10001
 zookeeper_gid: 10001
 iptables_extra_allowed_groups:
+  # Insecure (TODO: remove)
   - {'protocol': 'tcp', 'port': '2181', 'group': 'nodepool'}
   - {'protocol': 'tcp', 'port': '2181', 'group': 'zuul'}
+  # Secure
+  - {'protocol': 'tcp', 'port': '2281', 'group': 'nodepool'}
+  - {'protocol': 'tcp', 'port': '2281', 'group': 'zuul'}
   # Zookeeper election
   - {'protocol': 'tcp', 'port': '2888', 'group': 'zookeeper'}
   # Zookeeper leader

playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py

@@ -31,7 +31,7 @@ def main():
         for host in p['zk_group']:
             zk_hosts.append(dict(
                 host=p['hostvars'][host]['ansible_host'],
-                port=2181
+                port=2281
             ))
         module.exit_json(hosts=zk_hosts, changed=True)
     except Exception as e:

playbooks/roles/nodepool-base/tasks/main.yaml

@@ -26,6 +26,14 @@
     group: '{{ nodepool_group }}'
     mode: 0755
 
+- name: Generate ZooKeeper TLS cert
+  include_role:
+    name: zk-ca
+  vars:
+    zk_ca_cert_dir: /etc/nodepool
+    zk_ca_cert_dir_owner: '{{ nodepool_user }}'
+    zk_ca_cert_dir_group: '{{ nodepool_group }}'
+
 - name: Create nodepool log dir
   file:
     name: /var/log/nodepool
@@ -62,6 +70,10 @@
   vars:
     new_config:
       zookeeper-servers: '{{ zk_hosts.hosts }}'
+      zookeeper-tls:
+        cert: "/etc/nodepool/certs/cert.pem"
+        key: "/etc/nodepool/keys/key.pem"
+        ca: "/etc/nodepool/certs/cacert.pem"
   set_fact:
     nodepool_config: "{{ nodepool_config | combine(new_config) }}"
 

playbooks/roles/zk-ca/README.rst

@@ -0,0 +1,4 @@
+Generate TLS certs for ZooKeeper
+
+This will copy the certs to the remote node into the /etc/zuul
+directory by default.

playbooks/roles/zk-ca/defaults/main.yaml

@@ -0,0 +1,5 @@
+zk_ca_root: /var/zk-ca
+zk_ca_server: "{{ inventory_hostname }}"
+zk_ca_cert_dir: /etc/zuul
+zk_ca_cert_dir_owner: 10001
+zk_ca_cert_dir_group: 10001

playbooks/roles/zk-ca/tasks/main.yaml

@@ -0,0 +1,49 @@
+- name: Ensure zk-ca directory exists
+  delegate_to: localhost
+  file:
+    path: "{{ zk_ca_root }}"
+    state: directory
+
+# Run this in flock so that we can run it in plays for multiple target
+# hosts in parallel while serializing access to the CA files.
+- name: Run zk-ca.sh
+  delegate_to: localhost
+  script: "zk-ca.sh {{ zk_ca_root }} {{ zk_ca_server }}"
+  args:
+    executable: "flock {{ zk_ca_root }}/lock"
+
+- name: Ensure cert dir exists
+  file:
+    path: "{{ zk_ca_cert_dir }}/certs"
+    state: directory
+    owner: "{{ zk_ca_cert_dir_owner }}"
+    group: "{{ zk_ca_cert_dir_group }}"
+    mode: '0755'
+
+- name: Ensure keys dir exists
+  file:
+    path: "{{ zk_ca_cert_dir }}/keys"
+    state: directory
+    owner: "{{ zk_ca_cert_dir_owner }}"
+    group: "{{ zk_ca_cert_dir_group }}"
+    mode: '0700'
+
+- name: Copy TLS cacert into place
+  copy:
+    src: "/var/zk-ca/certs/cacert.pem"
+    dest: "{{ zk_ca_cert_dir }}/certs/cacert.pem"
+
+- name: Copy TLS cert into place
+  copy:
+    src: "/var/zk-ca/certs/{{ inventory_hostname }}.pem"
+    dest: "{{ zk_ca_cert_dir }}/certs/cert.pem"
+
+- name: Copy TLS key into place
+  copy:
+    src: "/var/zk-ca/keys/{{ inventory_hostname }}key.pem"
+    dest: "{{ zk_ca_cert_dir }}/keys/key.pem"
+
+- name: Copy TLS keystore into place
+  copy:
+    src: "/var/zk-ca/keystores/{{ inventory_hostname }}.pem"
+    dest: "{{ zk_ca_cert_dir }}/keys/keystore.pem"

playbooks/roles/zk-ca/zk-ca.sh

@@ -0,0 +1,104 @@
+#!/bin/sh -e
+
+# Copyright 2020 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Manage a CA for Zookeeper
+
+CAROOT=$1
+SERVER=$2
+
+SUBJECT='/C=US/ST=California/L=Oakland/O=Company Name/OU=Org'
+TOOLSDIR=$(dirname $0)
+# Unlike the zk-ca.sh in zuul, we use the system openssl.cnf
+CONFIG=""
+
+make_ca() {
+    mkdir $CAROOT/demoCA
+    mkdir $CAROOT/demoCA/reqs
+    mkdir $CAROOT/demoCA/newcerts
+    mkdir $CAROOT/demoCA/crl
+    mkdir $CAROOT/demoCA/private
+    chmod 700 $CAROOT/demoCA/private
+    touch $CAROOT/demoCA/index.txt
+    touch $CAROOT/demoCA/index.txt.attr
+    mkdir $CAROOT/certs
+    mkdir $CAROOT/keys
+    mkdir $CAROOT/keystores
+    chmod 700 $CAROOT/keys
+    chmod 700 $CAROOT/keystores
+
+    openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=caroot" \
+            -keyout $CAROOT/demoCA/private/cakey.pem \
+            -out $CAROOT/demoCA/reqs/careq.pem
+    openssl ca $CONFIG -create_serial -days 3560 -batch -selfsign -extensions v3_ca \
+            -out $CAROOT/demoCA/cacert.pem \
+            -keyfile $CAROOT/demoCA/private/cakey.pem \
+            -infiles $CAROOT/demoCA/reqs/careq.pem
+    cp $CAROOT/demoCA/cacert.pem $CAROOT/certs
+}
+
+make_client() {
+    openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=client" \
+            -keyout $CAROOT/keys/clientkey.pem \
+            -out $CAROOT/demoCA/reqs/clientreq.pem
+    openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
+            -out $CAROOT/certs/client.pem \
+            -infiles $CAROOT/demoCA/reqs/clientreq.pem
+}
+
+make_server() {
+    openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=$SERVER" \
+            -keyout $CAROOT/keys/${SERVER}key.pem \
+            -out $CAROOT/demoCA/reqs/${SERVER}req.pem
+    openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
+            -out $CAROOT/certs/$SERVER.pem \
+            -infiles $CAROOT/demoCA/reqs/${SERVER}req.pem
+    cat $CAROOT/certs/$SERVER.pem $CAROOT/keys/${SERVER}key.pem \
+        > $CAROOT/keystores/$SERVER.pem
+}
+
+help() {
+    echo "$0 CAROOT [SERVER]"
+    echo
+    echo "  CAROOT is the path to a directory in which to store the CA"
+    echo "         and certificates."
+    echo "  SERVER is the FQDN of a server for which a certificate should"
+    echo "         be generated"
+}
+
+if [ ! -d "$CAROOT" ]; then
+    echo "CAROOT must be a directory"
+    help
+    exit 1
+fi
+
+cd $CAROOT
+CAROOT=`pwd`
+
+if [ ! -d "$CAROOT/demoCA" ]; then
+    echo 'Generate CA'
+    make_ca
+    echo 'Generate client certificate'
+    make_client
+fi
+
+if [ -f "$CAROOT/certs/$SERVER.pem" ]; then
+    echo "Certificate for $SERVER already exists"
+    exit 0
+fi
+
+if [ "$SERVER" != "" ]; then
+    make_server
+fi

playbooks/roles/zookeeper/files/zookeeper-compose/docker-compose.yaml

@@ -12,3 +12,4 @@ services:
       - "/var/zookeeper/data:/data"
       - "/var/zookeeper/datalog:/datalog"
       - "/var/zookeeper/logs:/logs"
+      - "/var/zookeeper/tls:/tls"

playbooks/roles/zookeeper/tasks/main.yaml

@@ -27,6 +27,14 @@
     - data
     - datalog
     - logs
+    - tls
+- name: Generate ZooKeeper TLS cert
+  include_role:
+    name: zk-ca
+  vars:
+    zk_ca_cert_dir: /var/zookeeper/tls
+    zk_ca_cert_dir_owner: 10001
+    zk_ca_cert_dir_group: 10001
 - name: Write config
   template:
     src: zoo.cfg.j2

playbooks/roles/zookeeper/templates/zoo.cfg.j2

@@ -22,7 +22,13 @@ autopurge.purgeInterval=6
 maxClientCnxns=60
 standaloneEnabled=true
 admin.enableServer=true
-clientPort=2181
+secureClientPort=2281
+ssl.keyStore.location=/tls/keys/keystore.pem
+ssl.trustStore.location=/tls/certs/cacert.pem
 {% for host in groups['zookeeper'] %}
-server.{{ loop.index }}={{ (hostvars[host].ansible_default_ipv4.address) }}:2888:3888
+server.{{ loop.index }}={{ (hostvars[host].public_v4) }}:2888:3888
 {% endfor %}
+sslQuorum=true
+serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
+ssl.quorum.keyStore.location=/tls/keys/keystore.pem
+ssl.quorum.trustStore.location=/tls/certs/cacert.pem

playbooks/roles/zuul/tasks/main.yaml

@@ -21,6 +21,13 @@
     owner: "{{ zuul_user }}"
     group: "{{ zuul_group }}"
 
+- name: Generate ZooKeeper TLS cert
+  include_role:
+    name: zk-ca
+  vars:
+    zk_ca_cert_dir_owner: "{{ zuul_user_id }}"
+    zk_ca_cert_dir_group: "{{ zuul_group_id }}"
+
 - name: Create Zuul SSL dir
   file:
     state: directory

playbooks/roles/zuul/templates/zuul.conf.j2

@@ -28,7 +28,11 @@ relative_priority=true
 user=zuul
 
 [zookeeper]
-hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].ansible_default_ipv4.address) }}{% if not loop.last %},{% endif %}{% endfor %}
+hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].public_v4) }}:2281{% if not loop.last %},{% endif %}{% endfor %}
+
+tls_cert=/etc/zuul/certs/cert.pem
+tls_key=/etc/zuul/keys/key.pem
+tls_ca=/etc/zuul/certs/cacert.pem
 
 session_timeout=40
 

testinfra/test_zookeeper.py

@@ -22,5 +22,5 @@ def test_id_file(host):
     assert myid.content == b'1\n'
 
 def test_zk_listening(host):
-    zk = host.socket("tcp://0.0.0.0:2181")
+    zk = host.socket("tcp://0.0.0.0:2281")
     assert zk.is_listening