zuul/zuul-operator
Code Issues Proposed changes

Add TLS configuration to ZooKeeper service

Browse Source 
This change enables ZooKeeper TLS configuration when no
zookeeper external connection is provided.

Depends-On: https://review.opendev.org/712816
Depends-On: https://review.opendev.org/712531
Depends-On: https://review.opendev.org/712733
Change-Id: I2bef362bf7fba16fcd99229417c3e30494b2988d
This commit is contained in:
Tristan Cacqueray 2020-03-12 17:00:01 +00:00
parent c6d35be4d6
commit 5b6ee968b9
5 changed files with 622 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
conf/zuul/resources.dhall
View File

@ -12,8 +12,13 @@ The resources expect secrets to be created by the zuul ansible role:
* `client.pem`
* `client.key`
* `${name}-registry-tls` with:
* `${name}-zookeeper-tls` with:
* `ca.crt`
* `tls.crt`
* `tls.key`
* `zk.pem` the keystore
* `${name}-registry-tls` with:
* `cert.pem`
* `cert.key`
* `secret` a password
@ -277,7 +282,7 @@ let {- This method renders the zuul.conf
ssl_key=/etc/zuul-gearman/server.key
[zookeeper]
hosts=${zk-hosts}
${zk-hosts}
[merger]
git_user_email=${merger-email}
@ -311,9 +316,30 @@ let {- This method renders the zuul.conf
let mkNodepoolConf =
\(zk-host : Text)
-> ''
zookeeper-servers:
- host: ${zk-host}
port: 2181
${zk-host}
''
let {- This method renders the zoo.cfg
-} mkZookeeperConf =
\(keystore-password : Text)
-> ''
dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
server.1=0.0.0.0:2888:3888
# TLS configuration
secureClientPort=2281
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
ssl.keyStore.location=/conf/zk.pem
ssl.trustStore.location=/conf/ca.pem
''
in \(input : Input)
@ -518,14 +544,74 @@ in \(input : Input)
)
}
let zk-hosts =
let {- TODO: generate random password -} default-zk-keystorepassword =
"keystorepassword"
let zk-conf =
merge
{ None = "zk"
, Some = \(some : UserSecret) -> "%(ZUUL_ZK_HOSTS)"
{ None =
[ Volume::{
, name = "${input.name}-secret-zk"
, dir = "/conf-tls"
, files =
[ { path = "zoo.cfg"
, content = mkZookeeperConf default-zk-keystorepassword
}
]
}
]
, Some = \(some : UserSecret) -> [] : List Volume.Type
}
input.zookeeper
let zk-hosts-secret-env =
let zk-client-conf =
merge
{ None =
[ Volume::{
, name = "${input.name}-zookeeper-tls"
, dir = "/etc/zookeeper-tls"
}
]
, Some = \(some : UserSecret) -> [] : List Volume.Type
}
input.zookeeper
let zk-hosts-zuul =
merge
{ None =
''
hosts=zk:2281
tls_cert=/etc/zookeeper-tls/tls.crt
tls_key=/etc/zookeeper-tls/tls.key
tls_ca=/etc/zookeeper-tls/ca.crt
''
, Some = \(some : UserSecret) -> "hosts=%(ZUUL_ZK_HOSTS)"
}
input.zookeeper
let zk-hosts-nodepool =
merge
{ None =
''
zookeeper-servers:
- host: zk
port: 2281
zookeeper-tls:
cert: /etc/zookeeper-tls/tls.crt
key: /etc/zookeeper-tls/tls.key
ca: /etc/zookeeper-tls/ca.crt
''
, Some =
\(some : UserSecret)
-> ''
zookeeper-servers:
- hosts: %(ZUUL_ZK_HOSTS)"
''
}
input.zookeeper
let {- Add support for TLS protected external zookeeper service
-} zk-hosts-secret-env =
merge
{ None = [] : List Kubernetes.EnvVar.Type
, Some =
@ -562,9 +648,8 @@ in \(input : Input)
[ { path = "nodepool.yaml"
, content =
''
zookeeper-servers:
- host: ${zk-hosts}
port: 2181
${zk-hosts-nodepool}
webapp:
port: 5000
@ -578,7 +663,10 @@ in \(input : Input)
, name = input.name ++ "-secret-zuul"
, dir = "/etc/zuul"
, files =
[ { path = "zuul.conf", content = mkZuulConf input zk-hosts } ]
[ { path = "zuul.conf"
, content = mkZuulConf input zk-hosts-zuul
}
]
}
let etc-zuul-registry =
@ -618,7 +706,9 @@ in \(input : Input)
, name = input.name ++ "-secret-nodepool"
, dir = "/etc/nodepool"
, files =
[ { path = "nodepool.yaml", content = mkNodepoolConf zk-hosts }
[ { path = "nodepool.yaml"
, content = mkNodepoolConf zk-hosts-nodepool
}
]
}
@ -685,27 +775,42 @@ in \(input : Input)
, ZooKeeper =
merge
{ None = KubernetesComponent::{
, Service = Some (mkService "zk" "zk" 2181)
, Service = Some (mkService "zk" "zk" 2281)
, StatefulSet = Some
( mkStatefulSet
Component::{
, name = "zk"
, count = 1
, data-dir = zk-volumes
, volumes = zk-conf # zk-client-conf
, claim-size = 1
, container = Kubernetes.Container::{
, name = "zk"
, command = Some
[ "sh"
, "-c"
, "cp /conf-tls/zoo.cfg /conf/ && "
++ "cp /etc/zookeeper-tls/zk.pem /conf/zk.pem && "
++ "cp /etc/zookeeper-tls/ca.crt /conf/ca.pem && "
++ "chown zookeeper /conf/zoo.cfg /conf/zk.pem /conf/ca.pem && "
++ "exec /docker-entrypoint.sh zkServer.sh start-foreground"
]
, image = Some
"docker.io/library/zookeeper"
, imagePullPolicy = Some "IfNotPresent"
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "zk"
, containerPort = 2181
, containerPort = 2281
}
]
, volumeMounts = Some
(mkVolumeMount zk-volumes)
( mkVolumeMount
( zk-volumes
# zk-conf
# zk-client-conf
)
)
}
}
)
@ -763,7 +868,8 @@ in \(input : Input)
, dir = "/etc/zuul-executor"
}
let zuul-volumes = [ etc-zuul, gearman-config ]
let zuul-volumes =
[ etc-zuul, gearman-config ] # zk-client-conf
let web-volumes = zuul-volumes
@ -1052,6 +1158,7 @@ in \(input : Input)
[ etc-nodepool, nodepool-config ]
# openstack-config
# kubernetes-config
# zk-client-conf
let shard-config =
"cat /etc/nodepool/nodepool.yaml /etc/nodepool-config/*.yaml > /var/lib/nodepool/config.yaml; "
@ -1119,10 +1226,13 @@ in \(input : Input)
{ apiVersion = "v1"
, kind = "List"
, items =
[ mkSecret etc-zuul
, mkSecret etc-nodepool
, mkSecret etc-zuul-registry
]
Prelude.List.map
Volume.Type
Kubernetes.Resource
mkSecret
( zk-conf
# [ etc-zuul, etc-nodepool, etc-zuul-registry ]
)
# mkUnion Components.Backend.Database
# mkUnion Components.Backend.ZooKeeper
# mkUnion Components.Zuul.Scheduler

352
roles/zuul-ensure-zookeeper-tls/files/openssl.cnf Normal file
View File

@ -0,0 +1,352 @@
#
# OpenSSL example configuration file.
# This is mostly being used for generation of certificate requests.
#
# Note that you can include other files from the main configuration
# file using the .include directive.
#.include filename
# This definition stops the following lines choking if HOME isn't
# defined.
HOME = .
RANDFILE = $ENV::HOME/.rnd
# Extra OBJECT IDENTIFIER info:
#oid_file = $ENV::HOME/.oid
oid_section = new_oids
# To use this configuration file with the "-extfile" option of the
# "openssl x509" utility, name here the section containing the
# X.509v3 extensions to use:
# extensions =
# (Alternatively, use a configuration file that has only
# X.509v3 extensions in its main [= default] section.)
[ new_oids ]
# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
# Add a simple OID like this:
# testoid1=1.2.3.4
# Or use config file substitution like this:
# testoid2=${testoid1}.5.6
# Policies used by the TSA examples.
tsa_policy1 = 1.2.3.4.1
tsa_policy2 = 1.2.3.4.5.6
tsa_policy3 = 1.2.3.4.5.7
####################################################################
[ ca ]
default_ca = CA_default # The default ca section
####################################################################
[ CA_default ]
dir = ./demoCA # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
# several certs with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
certificate = $dir/cacert.pem # The CA certificate
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
# must be commented out to leave a V1 CRL
crl = $dir/crl.pem # The current CRL
private_key = $dir/private/cakey.pem# The private key
RANDFILE = $dir/private/.rand # private random number file
x509_extensions = usr_cert # The extensions to add to the cert
# Comment out the following two lines for the "traditional"
# (and highly broken) format.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
# Extension copying option: use with caution.
# copy_extensions = copy
# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
# so this is commented out by default to leave a V1 CRL.
# crlnumber must also be commented out to leave a V1 CRL.
# crl_extensions = crl_ext
default_days = 365 # how long to certify for
default_crl_days= 30 # how long before next CRL
default_md = default # use public key default MD
preserve = no # keep passed DN ordering
# A few difference way of specifying how similar the request should look
# For type CA, the listed attributes must be the same, and the optional
# and supplied fields are just that :-)
policy = policy_match
# For the CA policy
[ policy_match ]
countryName = match
stateOrProvinceName = match
organizationName = match
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
# For the 'anything' policy
# At this point in time, you must list all acceptable 'object'
# types.
[ policy_anything ]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
####################################################################
[ req ]
default_bits = 2048
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
attributes = req_attributes
x509_extensions = v3_ca # The extensions to add to the self signed cert
# Passwords for private keys if not present they will be prompted for
# input_password = secret
# output_password = secret
# This sets a mask for permitted string types. There are several options.
# default: PrintableString, T61String, BMPString.
# pkix : PrintableString, BMPString (PKIX recommendation before 2004)
# utf8only: only UTF8Strings (PKIX recommendation after 2004).
# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
# MASK:XXXX a literal mask value.
# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
string_mask = utf8only
# req_extensions = v3_req # The extensions to add to a certificate request
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_default = AU
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_default = Some-State
localityName = Locality Name (eg, city)
0.organizationName = Organization Name (eg, company)
0.organizationName_default = Internet Widgits Pty Ltd
# we can do this but it is not needed normally :-)
#1.organizationName = Second Organization Name (eg, company)
#1.organizationName_default = World Wide Web Pty Ltd
organizationalUnitName = Organizational Unit Name (eg, section)
#organizationalUnitName_default =
commonName = Common Name (e.g. server FQDN or YOUR name)
commonName_max = 64
emailAddress = Email Address
emailAddress_max = 64
# SET-ex3 = SET extension number 3
[ req_attributes ]
challengePassword = A challenge password
challengePassword_min = 4
challengePassword_max = 20
unstructuredName = An optional company name
[ usr_cert ]
# These extensions are added when 'ca' signs a request.
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This is required for TSA certificates.
# extendedKeyUsage = critical,timeStamping
[ v3_req ]
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
# Extensions for a typical CA
# PKIX recommendation.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
basicConstraints = critical,CA:true
# Key usage: this is typical for a CA certificate. However since it will
# prevent it being used as an test self-signed certificate it is best
# left out by default.
# keyUsage = cRLSign, keyCertSign
# Some might want this also
# nsCertType = sslCA, emailCA
# Include email address in subject alt name: another PKIX recommendation
# subjectAltName=email:copy
# Copy issuer details
# issuerAltName=issuer:copy
# DER hex encoding of an extension: beware experts only!
# obj=DER:02:03
# Where 'obj' is a standard or added object
# You can even override a supported extension:
# basicConstraints= critical, DER:30:03:01:01:FF
[ crl_ext ]
# CRL extensions.
# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
# issuerAltName=issuer:copy
authorityKeyIdentifier=keyid:always
[ proxy_cert_ext ]
# These extensions should be added when creating a proxy certificate
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This really needs to be in place for it to be a proxy certificate.
proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
####################################################################
[ tsa ]
default_tsa = tsa_config1 # the default TSA section
[ tsa_config1 ]
# These are used by the TSA reply generation only.
dir = ./demoCA # TSA root directory
serial = $dir/tsaserial # The current serial number (mandatory)
crypto_device = builtin # OpenSSL engine to use for signing
signer_cert = $dir/tsacert.pem # The TSA signing certificate
# (optional)
certs = $dir/cacert.pem # Certificate chain to include in reply
# (optional)
signer_key = $dir/private/tsakey.pem # The TSA private key (optional)
signer_digest = sha256 # Signing digest to use. (Optional)
default_policy = tsa_policy1 # Policy if request did not specify it
# (optional)
other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional)
digests = sha1, sha256, sha384, sha512 # Acceptable message digests (mandatory)
accuracy = secs:1, millisecs:500, microsecs:100 # (optional)
clock_precision_digits = 0 # number of digits after dot. (optional)
ordering = yes # Is ordering defined for timestamps?
# (optional, default: no)
tsa_name = yes # Must the TSA name be included in the reply?
# (optional, default: no)
ess_cert_id_chain = no # Must the ESS cert id chain be included?
# (optional, default: no)
ess_cert_id_alg = sha1 # algorithm to compute certificate
# identifier (optional, default: sha1)

103
roles/zuul-ensure-zookeeper-tls/files/zk-ca.sh Executable file
View File

@ -0,0 +1,103 @@
#!/bin/sh -e
# Copyright 2020 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage a CA for Zookeeper
CAROOT=$1
SERVER=$2
SUBJECT='/C=US/ST=California/L=Oakland/O=Company Name/OU=Org'
TOOLSDIR=$(dirname $0)
CONFIG="-config $TOOLSDIR/openssl.cnf"
make_ca() {
mkdir $CAROOT/demoCA
mkdir $CAROOT/demoCA/reqs
mkdir $CAROOT/demoCA/newcerts
mkdir $CAROOT/demoCA/crl
mkdir $CAROOT/demoCA/private
chmod 700 $CAROOT/demoCA/private
touch $CAROOT/demoCA/index.txt
touch $CAROOT/demoCA/index.txt.attr
mkdir $CAROOT/certs
mkdir $CAROOT/keys
mkdir $CAROOT/keystores
chmod 700 $CAROOT/keys
chmod 700 $CAROOT/keystores
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=caroot" \
-keyout $CAROOT/demoCA/private/cakey.pem \
-out $CAROOT/demoCA/reqs/careq.pem
openssl ca $CONFIG -create_serial -days 3560 -batch -selfsign -extensions v3_ca \
-out $CAROOT/demoCA/cacert.pem \
-keyfile $CAROOT/demoCA/private/cakey.pem \
-infiles $CAROOT/demoCA/reqs/careq.pem
cp $CAROOT/demoCA/cacert.pem $CAROOT/certs
}
make_client() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=client" \
-keyout $CAROOT/keys/clientkey.pem \
-out $CAROOT/demoCA/reqs/clientreq.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/client.pem \
-infiles $CAROOT/demoCA/reqs/clientreq.pem
}
make_server() {
openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=$SERVER" \
-keyout $CAROOT/keys/${SERVER}key.pem \
-out $CAROOT/demoCA/reqs/${SERVER}req.pem
openssl ca $CONFIG -batch -policy policy_anything -days 3560 \
-out $CAROOT/certs/$SERVER.pem \
-infiles $CAROOT/demoCA/reqs/${SERVER}req.pem
cat $CAROOT/certs/$SERVER.pem $CAROOT/keys/${SERVER}key.pem \
> $CAROOT/keystores/$SERVER.pem
}
help() {
echo "$0 CAROOT [SERVER]"
echo
echo " CAROOT is the path to a directory in which to store the CA"
echo " and certificates."
echo " SERVER is the FQDN of a server for which a certificate should"
echo " be generated"
}
if [ ! -d "$CAROOT" ]; then
echo "CAROOT must be a directory"
help
exit 1
fi
cd $CAROOT
CAROOT=`pwd`
if [ ! -d "$CAROOT/demoCA" ]; then
echo 'Generate CA'
make_ca
echo 'Generate client certificate'
make_client
fi
if [ -f "$CAROOT/certs/$SERVER.pem" ]; then
echo "Certificate for $SERVER already exists"
exit 0
fi
if [ "$SERVER" != "" ]; then
make_server
fi

30
roles/zuul-ensure-zookeeper-tls/tasks/main.yaml Normal file
View File

@ -0,0 +1,30 @@
- name: Check if zookeeper tls cert is already created
set_fact:
zookeeper_certs: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-zookeeper-tls') }}"
- name: Generate and store certs
when: zookeeper_certs.data is not defined
block:
- name: Generate certs
command: "sh -c 'mkdir -p zk-ca; {{ role_path }}/files/zk-ca.sh zk-ca/ {{ item }}'"
loop:
# TODO: support multiple zk pod
- zk
args:
creates: zk-ca/keys/clientkey.pem
- name: Create k8s secret
k8s:
state: "{{ state }}"
namespace: "{{ namespace }}"
definition:
apiVersion: v1
kind: Secret
metadata:
name: "{{ zuul_name }}-zookeeper-tls"
stringData:
ca.crt: "{{ lookup('file', 'zk-ca/demoCA/cacert.pem') }}"
tls.crt: "{{ lookup('file', 'zk-ca/certs/client.pem') }}"
tls.key: "{{ lookup('file', 'zk-ca/keys/clientkey.pem') }}"
data:
zk.pem: "{{ lookup('file', 'zk-ca/keystores/zk.pem') | b64encode }}"

5
roles/zuul/tasks/main.yaml
View File

@ -4,6 +4,11 @@
- zuul-lookup-conf
- zuul-ensure-gearman-tls
- include_role:
name: zuul-ensure-zookeeper-tls
# when the user does not provide a zookeeper
when: raw_spec['zookeeper'] is not defined
- include_role:
name: zuul-ensure-registry-tls
when: (raw_spec['registry']['count'] | default(0)) | int > 0