Add Zookeeper TLS support

This creates TLS certs for Zookeeper, uses them inside the ZK
quorum, and configures Nodepool and Zuul to use them as well.

A full system restart of all ZK-related components will be required
after merging this patch.

Change-Id: I0cb96a989f3d2c7e0563ce8899f2a5945ea225b3
This commit is contained in:
James E. Blair 2020-04-15 15:03:36 -07:00
parent 7f7c155555
commit 29825ac18b
13 changed files with 209 additions and 5 deletions

View File

@ -3,8 +3,12 @@ zookeeper_group: zookeeper
zookeeper_uid: 10001
zookeeper_gid: 10001
iptables_extra_allowed_groups:
# Insecure (TODO: remove)
- {'protocol': 'tcp', 'port': '2181', 'group': 'nodepool'}
- {'protocol': 'tcp', 'port': '2181', 'group': 'zuul'}
# Secure
- {'protocol': 'tcp', 'port': '2281', 'group': 'nodepool'}
- {'protocol': 'tcp', 'port': '2281', 'group': 'zuul'}
# Zookeeper election
- {'protocol': 'tcp', 'port': '2888', 'group': 'zookeeper'}
# Zookeeper leader

View File

@ -31,7 +31,7 @@ def main():
for host in p['zk_group']:
zk_hosts.append(dict(
host=p['hostvars'][host]['ansible_host'],
port=2181
port=2281
))
module.exit_json(hosts=zk_hosts, changed=True)
except Exception as e:

View File

@ -26,6 +26,14 @@
group: '{{ nodepool_group }}'
mode: 0755
- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir: /etc/nodepool
zk_ca_cert_dir_owner: '{{ nodepool_user }}'
zk_ca_cert_dir_group: '{{ nodepool_group }}'
- name: Create nodepool log dir
file:
name: /var/log/nodepool
@ -62,6 +70,10 @@
vars:
new_config:
zookeeper-servers: '{{ zk_hosts.hosts }}'
zookeeper-tls:
cert: "/etc/nodepool/certs/cert.pem"
key: "/etc/nodepool/keys/key.pem"
ca: "/etc/nodepool/certs/cacert.pem"
set_fact:
nodepool_config: "{{ nodepool_config | combine(new_config) }}"

View File

@ -0,0 +1,4 @@
Generate TLS certs for ZooKeeper
This will copy the certs to the remote node into the /etc/zuul
directory by default.

View File

@ -0,0 +1,5 @@
zk_ca_root: /var/zk-ca
zk_ca_server: "{{ inventory_hostname }}"
zk_ca_cert_dir: /etc/zuul
zk_ca_cert_dir_owner: 10001
zk_ca_cert_dir_group: 10001

View File

@ -0,0 +1,49 @@
- name: Ensure zk-ca directory exists
delegate_to: localhost
file:
path: "{{ zk_ca_root }}"
state: directory
# Run this in flock so that we can run it in plays for multiple target
# hosts in parallel while serializing access to the CA files.
- name: Run zk-ca.sh
delegate_to: localhost
script: "zk-ca.sh {{ zk_ca_root }} {{ zk_ca_server }}"
args:
executable: "flock {{ zk_ca_root }}/lock"
- name: Ensure cert dir exists
file:
path: "{{ zk_ca_cert_dir }}/certs"
state: directory
owner: "{{ zk_ca_cert_dir_owner }}"
group: "{{ zk_ca_cert_dir_group }}"
mode: '0755'
- name: Ensure keys dir exists
file:
path: "{{ zk_ca_cert_dir }}/keys"
state: directory
owner: "{{ zk_ca_cert_dir_owner }}"
group: "{{ zk_ca_cert_dir_group }}"
mode: '0700'
- name: Copy TLS cacert into place
copy:
src: "/var/zk-ca/certs/cacert.pem"
dest: "{{ zk_ca_cert_dir }}/certs/cacert.pem"
- name: Copy TLS cert into place
copy:
src: "/var/zk-ca/certs/{{ inventory_hostname }}.pem"
dest: "{{ zk_ca_cert_dir }}/certs/cert.pem"
- name: Copy TLS key into place
copy:
src: "/var/zk-ca/keys/{{ inventory_hostname }}key.pem"
dest: "{{ zk_ca_cert_dir }}/keys/key.pem"
- name: Copy TLS keystore into place
copy:
src: "/var/zk-ca/keystores/{{ inventory_hostname }}.pem"
dest: "{{ zk_ca_cert_dir }}/keys/keystore.pem"

104
playbooks/roles/zk-ca/zk-ca.sh Executable file
View File

@ -0,0 +1,104 @@
#!/bin/sh -e
# Copyright 2020 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage a CA for Zookeeper
CAROOT=$1
SERVER=$2
SUBJECT='/C=US/ST=California/L=Oakland/O=Company Name/OU=Org'
TOOLSDIR=$(dirname $0)
# Unlike the zk-ca.sh in zuul, we use the system openssl.cnf
CONFIG=""
make_ca() {
mkdir $CAROOT/demoCA
mkdir $CAROOT/demoCA/reqs
mkdir $CAROOT/demoCA/newcerts
mkdir $CAROOT/demoCA/crl
mkdir $CAROOT/demoCA/private
chmod 700 $CAROOT/demoCA/private
touch $CAROOT/demoCA/index.txt
touch $CAROOT/demoCA/index.txt.attr
mkdir $CAROOT/certs
mkdir $CAROOT/keys
mkdir $CAROOT/keystores
chmod 700 $CAROOT/keys
chmod 700 $CAROOT/keystores
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=caroot" \
-keyout $CAROOT/demoCA/private/cakey.pem \
-out $CAROOT/demoCA/reqs/careq.pem
openssl ca $CONFIG -create_serial -days 3560 -batch -selfsign -extensions v3_ca \
-out $CAROOT/demoCA/cacert.pem \
-keyfile $CAROOT/demoCA/private/cakey.pem \
-infiles $CAROOT/demoCA/reqs/careq.pem
cp $CAROOT/demoCA/cacert.pem $CAROOT/certs
}
make_client() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=client" \
-keyout $CAROOT/keys/clientkey.pem \
-out $CAROOT/demoCA/reqs/clientreq.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/client.pem \
-infiles $CAROOT/demoCA/reqs/clientreq.pem
}
make_server() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=$SERVER" \
-keyout $CAROOT/keys/${SERVER}key.pem \
-out $CAROOT/demoCA/reqs/${SERVER}req.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/$SERVER.pem \
-infiles $CAROOT/demoCA/reqs/${SERVER}req.pem
cat $CAROOT/certs/$SERVER.pem $CAROOT/keys/${SERVER}key.pem \
> $CAROOT/keystores/$SERVER.pem
}
help() {
echo "$0 CAROOT [SERVER]"
echo
echo " CAROOT is the path to a directory in which to store the CA"
echo " and certificates."
echo " SERVER is the FQDN of a server for which a certificate should"
echo " be generated"
}
if [ ! -d "$CAROOT" ]; then
echo "CAROOT must be a directory"
help
exit 1
fi
cd $CAROOT
CAROOT=`pwd`
if [ ! -d "$CAROOT/demoCA" ]; then
echo 'Generate CA'
make_ca
echo 'Generate client certificate'
make_client
fi
if [ -f "$CAROOT/certs/$SERVER.pem" ]; then
echo "Certificate for $SERVER already exists"
exit 0
fi
if [ "$SERVER" != "" ]; then
make_server
fi

View File

@ -12,3 +12,4 @@ services:
- "/var/zookeeper/data:/data"
- "/var/zookeeper/datalog:/datalog"
- "/var/zookeeper/logs:/logs"
- "/var/zookeeper/tls:/tls"

View File

@ -27,6 +27,14 @@
- data
- datalog
- logs
- tls
- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir: /var/zookeeper/tls
zk_ca_cert_dir_owner: 10001
zk_ca_cert_dir_group: 10001
- name: Write config
template:
src: zoo.cfg.j2

View File

@ -22,7 +22,13 @@ autopurge.purgeInterval=6
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
clientPort=2181
secureClientPort=2281
ssl.keyStore.location=/tls/keys/keystore.pem
ssl.trustStore.location=/tls/certs/cacert.pem
{% for host in groups['zookeeper'] %}
server.{{ loop.index }}={{ (hostvars[host].ansible_default_ipv4.address) }}:2888:3888
server.{{ loop.index }}={{ (hostvars[host].public_v4) }}:2888:3888
{% endfor %}
sslQuorum=true
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
ssl.quorum.keyStore.location=/tls/keys/keystore.pem
ssl.quorum.trustStore.location=/tls/certs/cacert.pem

View File

@ -21,6 +21,13 @@
owner: "{{ zuul_user }}"
group: "{{ zuul_group }}"
- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir_owner: "{{ zuul_user_id }}"
zk_ca_cert_dir_group: "{{ zuul_group_id }}"
- name: Create Zuul SSL dir
file:
state: directory

View File

@ -28,7 +28,11 @@ relative_priority=true
user=zuul
[zookeeper]
hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].ansible_default_ipv4.address) }}{% if not loop.last %},{% endif %}{% endfor %}
hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].public_v4) }}:2281{% if not loop.last %},{% endif %}{% endfor %}
tls_cert=/etc/zuul/certs/cert.pem
tls_key=/etc/zuul/keys/key.pem
tls_ca=/etc/zuul/certs/cacert.pem
session_timeout=40

View File

@ -22,5 +22,5 @@ def test_id_file(host):
assert myid.content == b'1\n'
def test_zk_listening(host):
zk = host.socket("tcp://0.0.0.0:2181")
zk = host.socket("tcp://0.0.0.0:2281")
assert zk.is_listening