Browse Source

Zookeeper: listen on plain and TLS ports

To prepare for switching to TLS, set up TLS certs for Zookeeper and
all of Nodepool and Zuul, but do not have them connect over TLS yet.
We have observed problems with Kazoo using TLS in production.  This
will let us run the ZK quorum using TLS internally, and have Zuul
and Nodepool connect over plaintext while also exposing the TLS
client port so that we can perform some more production tests.

Change-Id: If93b27f5b55be42be1cf6ee23258127fab5ce9ea
changes/09/736309/1
James E. Blair 3 weeks ago
parent
commit
a514aa0f98
12 changed files with 203 additions and 2 deletions
  1. +4
    -0
      inventory/service/group_vars/zookeeper.yaml
  2. +8
    -0
      playbooks/roles/nodepool-base/tasks/main.yaml
  3. +4
    -0
      playbooks/roles/zk-ca/README.rst
  4. +5
    -0
      playbooks/roles/zk-ca/defaults/main.yaml
  5. +49
    -0
      playbooks/roles/zk-ca/tasks/main.yaml
  6. +104
    -0
      playbooks/roles/zk-ca/zk-ca.sh
  7. +1
    -0
      playbooks/roles/zookeeper/files/zookeeper-compose/docker-compose.yaml
  8. +8
    -0
      playbooks/roles/zookeeper/tasks/main.yaml
  9. +8
    -1
      playbooks/roles/zookeeper/templates/zoo.cfg.j2
  10. +7
    -0
      playbooks/roles/zuul/tasks/main.yaml
  11. +1
    -1
      playbooks/roles/zuul/templates/zuul.conf.j2
  12. +4
    -0
      testinfra/test_zookeeper.py

+ 4
- 0
inventory/service/group_vars/zookeeper.yaml View File

@@ -3,8 +3,12 @@ zookeeper_group: zookeeper
zookeeper_uid: 10001
zookeeper_gid: 10001
iptables_extra_allowed_groups:
# Insecure
- {'protocol': 'tcp', 'port': '2181', 'group': 'nodepool'}
- {'protocol': 'tcp', 'port': '2181', 'group': 'zuul'}
# Secure
- {'protocol': 'tcp', 'port': '2281', 'group': 'nodepool'}
- {'protocol': 'tcp', 'port': '2281', 'group': 'zuul'}
# Zookeeper election
- {'protocol': 'tcp', 'port': '2888', 'group': 'zookeeper'}
# Zookeeper leader


+ 8
- 0
playbooks/roles/nodepool-base/tasks/main.yaml View File

@@ -26,6 +26,14 @@
group: '{{ nodepool_group }}'
mode: 0755

- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir: /etc/nodepool
zk_ca_cert_dir_owner: '{{ nodepool_user }}'
zk_ca_cert_dir_group: '{{ nodepool_group }}'

- name: Create nodepool log dir
file:
name: /var/log/nodepool


+ 4
- 0
playbooks/roles/zk-ca/README.rst View File

@@ -0,0 +1,4 @@
Generate TLS certs for ZooKeeper

This will copy the certs to the remote node into the /etc/zuul
directory by default.

+ 5
- 0
playbooks/roles/zk-ca/defaults/main.yaml View File

@@ -0,0 +1,5 @@
zk_ca_root: /var/zk-ca
zk_ca_server: "{{ inventory_hostname }}"
zk_ca_cert_dir: /etc/zuul
zk_ca_cert_dir_owner: 10001
zk_ca_cert_dir_group: 10001

+ 49
- 0
playbooks/roles/zk-ca/tasks/main.yaml View File

@@ -0,0 +1,49 @@
- name: Ensure zk-ca directory exists
delegate_to: localhost
file:
path: "{{ zk_ca_root }}"
state: directory

# Run this in flock so that we can run it in plays for multiple target
# hosts in parallel while serializing access to the CA files.
- name: Run zk-ca.sh
delegate_to: localhost
script: "zk-ca.sh {{ zk_ca_root }} {{ zk_ca_server }}"
args:
executable: "flock {{ zk_ca_root }}/lock"

- name: Ensure cert dir exists
file:
path: "{{ zk_ca_cert_dir }}/certs"
state: directory
owner: "{{ zk_ca_cert_dir_owner }}"
group: "{{ zk_ca_cert_dir_group }}"
mode: '0755'

- name: Ensure keys dir exists
file:
path: "{{ zk_ca_cert_dir }}/keys"
state: directory
owner: "{{ zk_ca_cert_dir_owner }}"
group: "{{ zk_ca_cert_dir_group }}"
mode: '0700'

- name: Copy TLS cacert into place
copy:
src: "/var/zk-ca/certs/cacert.pem"
dest: "{{ zk_ca_cert_dir }}/certs/cacert.pem"

- name: Copy TLS cert into place
copy:
src: "/var/zk-ca/certs/{{ inventory_hostname }}.pem"
dest: "{{ zk_ca_cert_dir }}/certs/cert.pem"

- name: Copy TLS key into place
copy:
src: "/var/zk-ca/keys/{{ inventory_hostname }}key.pem"
dest: "{{ zk_ca_cert_dir }}/keys/key.pem"

- name: Copy TLS keystore into place
copy:
src: "/var/zk-ca/keystores/{{ inventory_hostname }}.pem"
dest: "{{ zk_ca_cert_dir }}/keys/keystore.pem"

+ 104
- 0
playbooks/roles/zk-ca/zk-ca.sh View File

@@ -0,0 +1,104 @@
#!/bin/sh -e

# Copyright 2020 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# Manage a CA for Zookeeper

CAROOT=$1
SERVER=$2

SUBJECT='/C=US/ST=California/L=Oakland/O=Company Name/OU=Org'
TOOLSDIR=$(dirname $0)
# Unlike the zk-ca.sh in zuul, we use the system openssl.cnf
CONFIG=""

make_ca() {
mkdir $CAROOT/demoCA
mkdir $CAROOT/demoCA/reqs
mkdir $CAROOT/demoCA/newcerts
mkdir $CAROOT/demoCA/crl
mkdir $CAROOT/demoCA/private
chmod 700 $CAROOT/demoCA/private
touch $CAROOT/demoCA/index.txt
touch $CAROOT/demoCA/index.txt.attr
mkdir $CAROOT/certs
mkdir $CAROOT/keys
mkdir $CAROOT/keystores
chmod 700 $CAROOT/keys
chmod 700 $CAROOT/keystores

openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=caroot" \
-keyout $CAROOT/demoCA/private/cakey.pem \
-out $CAROOT/demoCA/reqs/careq.pem
openssl ca $CONFIG -create_serial -days 3560 -batch -selfsign -extensions v3_ca \
-out $CAROOT/demoCA/cacert.pem \
-keyfile $CAROOT/demoCA/private/cakey.pem \
-infiles $CAROOT/demoCA/reqs/careq.pem
cp $CAROOT/demoCA/cacert.pem $CAROOT/certs
}

make_client() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=client" \
-keyout $CAROOT/keys/clientkey.pem \
-out $CAROOT/demoCA/reqs/clientreq.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/client.pem \
-infiles $CAROOT/demoCA/reqs/clientreq.pem
}

make_server() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=$SERVER" \
-keyout $CAROOT/keys/${SERVER}key.pem \
-out $CAROOT/demoCA/reqs/${SERVER}req.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/$SERVER.pem \
-infiles $CAROOT/demoCA/reqs/${SERVER}req.pem
cat $CAROOT/certs/$SERVER.pem $CAROOT/keys/${SERVER}key.pem \
> $CAROOT/keystores/$SERVER.pem
}

help() {
echo "$0 CAROOT [SERVER]"
echo
echo " CAROOT is the path to a directory in which to store the CA"
echo " and certificates."
echo " SERVER is the FQDN of a server for which a certificate should"
echo " be generated"
}

if [ ! -d "$CAROOT" ]; then
echo "CAROOT must be a directory"
help
exit 1
fi

cd $CAROOT
CAROOT=`pwd`

if [ ! -d "$CAROOT/demoCA" ]; then
echo 'Generate CA'
make_ca
echo 'Generate client certificate'
make_client
fi

if [ -f "$CAROOT/certs/$SERVER.pem" ]; then
echo "Certificate for $SERVER already exists"
exit 0
fi

if [ "$SERVER" != "" ]; then
make_server
fi

+ 1
- 0
playbooks/roles/zookeeper/files/zookeeper-compose/docker-compose.yaml View File

@@ -12,3 +12,4 @@ services:
- "/var/zookeeper/data:/data"
- "/var/zookeeper/datalog:/datalog"
- "/var/zookeeper/logs:/logs"
- "/var/zookeeper/tls:/tls"

+ 8
- 0
playbooks/roles/zookeeper/tasks/main.yaml View File

@@ -27,6 +27,14 @@
- data
- datalog
- logs
- tls
- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir: /var/zookeeper/tls
zk_ca_cert_dir_owner: 10001
zk_ca_cert_dir_group: 10001
- name: Write config
template:
src: zoo.cfg.j2


+ 8
- 1
playbooks/roles/zookeeper/templates/zoo.cfg.j2 View File

@@ -23,6 +23,13 @@ maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
clientPort=2181
secureClientPort=2281
ssl.keyStore.location=/tls/keys/keystore.pem
ssl.trustStore.location=/tls/certs/cacert.pem
{% for host in groups['zookeeper'] %}
server.{{ loop.index }}={{ (hostvars[host].ansible_default_ipv4.address) }}:2888:3888
server.{{ loop.index }}={{ (hostvars[host].public_v4) }}:2888:3888
{% endfor %}
sslQuorum=true
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
ssl.quorum.keyStore.location=/tls/keys/keystore.pem
ssl.quorum.trustStore.location=/tls/certs/cacert.pem

+ 7
- 0
playbooks/roles/zuul/tasks/main.yaml View File

@@ -21,6 +21,13 @@
owner: "{{ zuul_user }}"
group: "{{ zuul_group }}"

- name: Generate ZooKeeper TLS cert
include_role:
name: zk-ca
vars:
zk_ca_cert_dir_owner: "{{ zuul_user_id }}"
zk_ca_cert_dir_group: "{{ zuul_group_id }}"

- name: Create Zuul SSL dir
file:
state: directory


+ 1
- 1
playbooks/roles/zuul/templates/zuul.conf.j2 View File

@@ -28,7 +28,7 @@ relative_priority=true
user=zuul

[zookeeper]
hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].ansible_default_ipv4.address) }}{% if not loop.last %},{% endif %}{% endfor %}
hosts={% for host in groups['zookeeper'] %}{{ (hostvars[host].public_v4) }}:2181{% if not loop.last %},{% endif %}{% endfor %}

session_timeout=40



+ 4
- 0
testinfra/test_zookeeper.py View File

@@ -24,3 +24,7 @@ def test_id_file(host):
def test_zk_listening(host):
zk = host.socket("tcp://0.0.0.0:2181")
assert zk.is_listening

def test_zk_listening_ssl(host):
zk = host.socket("tcp://0.0.0.0:2281")
assert zk.is_listening

Loading…
Cancel
Save