diff --git a/conf/zuul/resources.dhall b/conf/zuul/resources.dhall index c9f882a..81cf29a 100644 --- a/conf/zuul/resources.dhall +++ b/conf/zuul/resources.dhall @@ -12,8 +12,13 @@ The resources expect secrets to be created by the zuul ansible role: * `client.pem` * `client.key` -* `${name}-registry-tls` with: +* `${name}-zookeeper-tls` with: + * `ca.crt` + * `tls.crt` + * `tls.key` + * `zk.pem` the keystore +* `${name}-registry-tls` with: * `cert.pem` * `cert.key` * `secret` a password @@ -277,7 +282,7 @@ let {- This method renders the zuul.conf ssl_key=/etc/zuul-gearman/server.key [zookeeper] - hosts=${zk-hosts} + ${zk-hosts} [merger] git_user_email=${merger-email} @@ -311,9 +316,30 @@ let {- This method renders the zuul.conf let mkNodepoolConf = \(zk-host : Text) -> '' - zookeeper-servers: - - host: ${zk-host} - port: 2181 + ${zk-host} + '' + +let {- This method renders the zoo.cfg + -} mkZookeeperConf = + \(keystore-password : Text) + -> '' + dataDir=/data + dataLogDir=/datalog + tickTime=2000 + initLimit=5 + syncLimit=2 + autopurge.snapRetainCount=3 + autopurge.purgeInterval=0 + maxClientCnxns=60 + standaloneEnabled=true + admin.enableServer=true + server.1=0.0.0.0:2888:3888 + + # TLS configuration + secureClientPort=2281 + serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory + ssl.keyStore.location=/conf/zk.pem + ssl.trustStore.location=/conf/ca.pem '' in \(input : Input) @@ -518,14 +544,74 @@ in \(input : Input) ) } - let zk-hosts = + let {- TODO: generate random password -} default-zk-keystorepassword = + "keystorepassword" + + let zk-conf = merge - { None = "zk" - , Some = \(some : UserSecret) -> "%(ZUUL_ZK_HOSTS)" + { None = + [ Volume::{ + , name = "${input.name}-secret-zk" + , dir = "/conf-tls" + , files = + [ { path = "zoo.cfg" + , content = mkZookeeperConf default-zk-keystorepassword + } + ] + } + ] + , Some = \(some : UserSecret) -> [] : List Volume.Type } input.zookeeper - let zk-hosts-secret-env = + let zk-client-conf = + merge + { None = + [ Volume::{ + , name = "${input.name}-zookeeper-tls" + , dir = "/etc/zookeeper-tls" + } + ] + , Some = \(some : UserSecret) -> [] : List Volume.Type + } + input.zookeeper + + let zk-hosts-zuul = + merge + { None = + '' + hosts=zk:2281 + tls_cert=/etc/zookeeper-tls/tls.crt + tls_key=/etc/zookeeper-tls/tls.key + tls_ca=/etc/zookeeper-tls/ca.crt + '' + , Some = \(some : UserSecret) -> "hosts=%(ZUUL_ZK_HOSTS)" + } + input.zookeeper + + let zk-hosts-nodepool = + merge + { None = + '' + zookeeper-servers: + - host: zk + port: 2281 + zookeeper-tls: + cert: /etc/zookeeper-tls/tls.crt + key: /etc/zookeeper-tls/tls.key + ca: /etc/zookeeper-tls/ca.crt + '' + , Some = + \(some : UserSecret) + -> '' + zookeeper-servers: + - hosts: %(ZUUL_ZK_HOSTS)" + '' + } + input.zookeeper + + let {- Add support for TLS protected external zookeeper service + -} zk-hosts-secret-env = merge { None = [] : List Kubernetes.EnvVar.Type , Some = @@ -562,9 +648,8 @@ in \(input : Input) [ { path = "nodepool.yaml" , content = '' - zookeeper-servers: - - host: ${zk-hosts} - port: 2181 + ${zk-hosts-nodepool} + webapp: port: 5000 @@ -578,7 +663,10 @@ in \(input : Input) , name = input.name ++ "-secret-zuul" , dir = "/etc/zuul" , files = - [ { path = "zuul.conf", content = mkZuulConf input zk-hosts } ] + [ { path = "zuul.conf" + , content = mkZuulConf input zk-hosts-zuul + } + ] } let etc-zuul-registry = @@ -618,7 +706,9 @@ in \(input : Input) , name = input.name ++ "-secret-nodepool" , dir = "/etc/nodepool" , files = - [ { path = "nodepool.yaml", content = mkNodepoolConf zk-hosts } + [ { path = "nodepool.yaml" + , content = mkNodepoolConf zk-hosts-nodepool + } ] } @@ -685,27 +775,42 @@ in \(input : Input) , ZooKeeper = merge { None = KubernetesComponent::{ - , Service = Some (mkService "zk" "zk" 2181) + , Service = Some (mkService "zk" "zk" 2281) , StatefulSet = Some ( mkStatefulSet Component::{ , name = "zk" , count = 1 , data-dir = zk-volumes + , volumes = zk-conf # zk-client-conf , claim-size = 1 , container = Kubernetes.Container::{ , name = "zk" + , command = Some + [ "sh" + , "-c" + , "cp /conf-tls/zoo.cfg /conf/ && " + ++ "cp /etc/zookeeper-tls/zk.pem /conf/zk.pem && " + ++ "cp /etc/zookeeper-tls/ca.crt /conf/ca.pem && " + ++ "chown zookeeper /conf/zoo.cfg /conf/zk.pem /conf/ca.pem && " + ++ "exec /docker-entrypoint.sh zkServer.sh start-foreground" + ] , image = Some "docker.io/library/zookeeper" , imagePullPolicy = Some "IfNotPresent" , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "zk" - , containerPort = 2181 + , containerPort = 2281 } ] , volumeMounts = Some - (mkVolumeMount zk-volumes) + ( mkVolumeMount + ( zk-volumes + # zk-conf + # zk-client-conf + ) + ) } } ) @@ -763,7 +868,8 @@ in \(input : Input) , dir = "/etc/zuul-executor" } - let zuul-volumes = [ etc-zuul, gearman-config ] + let zuul-volumes = + [ etc-zuul, gearman-config ] # zk-client-conf let web-volumes = zuul-volumes @@ -1052,6 +1158,7 @@ in \(input : Input) [ etc-nodepool, nodepool-config ] # openstack-config # kubernetes-config + # zk-client-conf let shard-config = "cat /etc/nodepool/nodepool.yaml /etc/nodepool-config/*.yaml > /var/lib/nodepool/config.yaml; " @@ -1119,10 +1226,13 @@ in \(input : Input) { apiVersion = "v1" , kind = "List" , items = - [ mkSecret etc-zuul - , mkSecret etc-nodepool - , mkSecret etc-zuul-registry - ] + Prelude.List.map + Volume.Type + Kubernetes.Resource + mkSecret + ( zk-conf + # [ etc-zuul, etc-nodepool, etc-zuul-registry ] + ) # mkUnion Components.Backend.Database # mkUnion Components.Backend.ZooKeeper # mkUnion Components.Zuul.Scheduler diff --git a/roles/zuul-ensure-zookeeper-tls/files/openssl.cnf b/roles/zuul-ensure-zookeeper-tls/files/openssl.cnf new file mode 100644 index 0000000..7d1a8bb --- /dev/null +++ b/roles/zuul-ensure-zookeeper-tls/files/openssl.cnf @@ -0,0 +1,352 @@ +# +# OpenSSL example configuration file. +# This is mostly being used for generation of certificate requests. +# + +# Note that you can include other files from the main configuration +# file using the .include directive. +#.include filename + +# This definition stops the following lines choking if HOME isn't +# defined. +HOME = . +RANDFILE = $ENV::HOME/.rnd + +# Extra OBJECT IDENTIFIER info: +#oid_file = $ENV::HOME/.oid +oid_section = new_oids + +# To use this configuration file with the "-extfile" option of the +# "openssl x509" utility, name here the section containing the +# X.509v3 extensions to use: +# extensions = +# (Alternatively, use a configuration file that has only +# X.509v3 extensions in its main [= default] section.) + +[ new_oids ] + +# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. +# Add a simple OID like this: +# testoid1=1.2.3.4 +# Or use config file substitution like this: +# testoid2=${testoid1}.5.6 + +# Policies used by the TSA examples. +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several certs with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem# The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extensions to add to the cert + +# Comment out the following two lines for the "traditional" +# (and highly broken) format. +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +# Extension copying option: use with caution. +# copy_extensions = copy + +# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs +# so this is commented out by default to leave a V1 CRL. +# crlnumber must also be commented out to leave a V1 CRL. +# crl_extensions = crl_ext + +default_days = 365 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +# A few difference way of specifying how similar the request should look +# For type CA, the listed attributes must be the same, and the optional +# and supplied fields are just that :-) +policy = policy_match + +# For the CA policy +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +# For the 'anything' policy +# At this point in time, you must list all acceptable 'object' +# types. +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca # The extensions to add to the self signed cert + +# Passwords for private keys if not present they will be prompted for +# input_password = secret +# output_password = secret + +# This sets a mask for permitted string types. There are several options. +# default: PrintableString, T61String, BMPString. +# pkix : PrintableString, BMPString (PKIX recommendation before 2004) +# utf8only: only UTF8Strings (PKIX recommendation after 2004). +# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). +# MASK:XXXX a literal mask value. +# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = AU +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Some-State + +localityName = Locality Name (eg, city) + +0.organizationName = Organization Name (eg, company) +0.organizationName_default = Internet Widgits Pty Ltd + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = World Wide Web Pty Ltd + +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_max = 64 + +# SET-ex3 = SET extension number 3 + +[ req_attributes ] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 + +unstructuredName = An optional company name + +[ usr_cert ] + +# These extensions are added when 'ca' signs a request. + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This is required for TSA certificates. +# extendedKeyUsage = critical,timeStamping + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v3_ca ] + + +# Extensions for a typical CA + + +# PKIX recommendation. + +subjectKeyIdentifier=hash + +authorityKeyIdentifier=keyid:always,issuer + +basicConstraints = critical,CA:true + +# Key usage: this is typical for a CA certificate. However since it will +# prevent it being used as an test self-signed certificate it is best +# left out by default. +# keyUsage = cRLSign, keyCertSign + +# Some might want this also +# nsCertType = sslCA, emailCA + +# Include email address in subject alt name: another PKIX recommendation +# subjectAltName=email:copy +# Copy issuer details +# issuerAltName=issuer:copy + +# DER hex encoding of an extension: beware experts only! +# obj=DER:02:03 +# Where 'obj' is a standard or added object +# You can even override a supported extension: +# basicConstraints= critical, DER:30:03:01:01:FF + +[ crl_ext ] + +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. + +# issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always + +[ proxy_cert_ext ] +# These extensions should be added when creating a proxy certificate + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This really needs to be in place for it to be a proxy certificate. +proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo + +#################################################################### +[ tsa ] + +default_tsa = tsa_config1 # the default TSA section + +[ tsa_config1 ] + +# These are used by the TSA reply generation only. +dir = ./demoCA # TSA root directory +serial = $dir/tsaserial # The current serial number (mandatory) +crypto_device = builtin # OpenSSL engine to use for signing +signer_cert = $dir/tsacert.pem # The TSA signing certificate + # (optional) +certs = $dir/cacert.pem # Certificate chain to include in reply + # (optional) +signer_key = $dir/private/tsakey.pem # The TSA private key (optional) +signer_digest = sha256 # Signing digest to use. (Optional) +default_policy = tsa_policy1 # Policy if request did not specify it + # (optional) +other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) +digests = sha1, sha256, sha384, sha512 # Acceptable message digests (mandatory) +accuracy = secs:1, millisecs:500, microsecs:100 # (optional) +clock_precision_digits = 0 # number of digits after dot. (optional) +ordering = yes # Is ordering defined for timestamps? + # (optional, default: no) +tsa_name = yes # Must the TSA name be included in the reply? + # (optional, default: no) +ess_cert_id_chain = no # Must the ESS cert id chain be included? + # (optional, default: no) +ess_cert_id_alg = sha1 # algorithm to compute certificate + # identifier (optional, default: sha1) diff --git a/roles/zuul-ensure-zookeeper-tls/files/zk-ca.sh b/roles/zuul-ensure-zookeeper-tls/files/zk-ca.sh new file mode 100755 index 0000000..70b77dd --- /dev/null +++ b/roles/zuul-ensure-zookeeper-tls/files/zk-ca.sh @@ -0,0 +1,103 @@ +#!/bin/sh -e + +# Copyright 2020 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage a CA for Zookeeper + +CAROOT=$1 +SERVER=$2 + +SUBJECT='/C=US/ST=California/L=Oakland/O=Company Name/OU=Org' +TOOLSDIR=$(dirname $0) +CONFIG="-config $TOOLSDIR/openssl.cnf" + +make_ca() { + mkdir $CAROOT/demoCA + mkdir $CAROOT/demoCA/reqs + mkdir $CAROOT/demoCA/newcerts + mkdir $CAROOT/demoCA/crl + mkdir $CAROOT/demoCA/private + chmod 700 $CAROOT/demoCA/private + touch $CAROOT/demoCA/index.txt + touch $CAROOT/demoCA/index.txt.attr + mkdir $CAROOT/certs + mkdir $CAROOT/keys + mkdir $CAROOT/keystores + chmod 700 $CAROOT/keys + chmod 700 $CAROOT/keystores + + openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=caroot" \ + -keyout $CAROOT/demoCA/private/cakey.pem \ + -out $CAROOT/demoCA/reqs/careq.pem + openssl ca $CONFIG -create_serial -days 3560 -batch -selfsign -extensions v3_ca \ + -out $CAROOT/demoCA/cacert.pem \ + -keyfile $CAROOT/demoCA/private/cakey.pem \ + -infiles $CAROOT/demoCA/reqs/careq.pem + cp $CAROOT/demoCA/cacert.pem $CAROOT/certs +} + +make_client() { + openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=client" \ + -keyout $CAROOT/keys/clientkey.pem \ + -out $CAROOT/demoCA/reqs/clientreq.pem + openssl ca $CONFIG -batch -policy policy_anything -days 3560 \ + -out $CAROOT/certs/client.pem \ + -infiles $CAROOT/demoCA/reqs/clientreq.pem +} + +make_server() { + openssl req $CONFIG -new -nodes -subj "$SUBJECT/CN=$SERVER" \ + -keyout $CAROOT/keys/${SERVER}key.pem \ + -out $CAROOT/demoCA/reqs/${SERVER}req.pem + openssl ca $CONFIG -batch -policy policy_anything -days 3560 \ + -out $CAROOT/certs/$SERVER.pem \ + -infiles $CAROOT/demoCA/reqs/${SERVER}req.pem + cat $CAROOT/certs/$SERVER.pem $CAROOT/keys/${SERVER}key.pem \ + > $CAROOT/keystores/$SERVER.pem +} + +help() { + echo "$0 CAROOT [SERVER]" + echo + echo " CAROOT is the path to a directory in which to store the CA" + echo " and certificates." + echo " SERVER is the FQDN of a server for which a certificate should" + echo " be generated" +} + +if [ ! -d "$CAROOT" ]; then + echo "CAROOT must be a directory" + help + exit 1 +fi + +cd $CAROOT +CAROOT=`pwd` + +if [ ! -d "$CAROOT/demoCA" ]; then + echo 'Generate CA' + make_ca + echo 'Generate client certificate' + make_client +fi + +if [ -f "$CAROOT/certs/$SERVER.pem" ]; then + echo "Certificate for $SERVER already exists" + exit 0 +fi + +if [ "$SERVER" != "" ]; then + make_server +fi diff --git a/roles/zuul-ensure-zookeeper-tls/tasks/main.yaml b/roles/zuul-ensure-zookeeper-tls/tasks/main.yaml new file mode 100644 index 0000000..bed1865 --- /dev/null +++ b/roles/zuul-ensure-zookeeper-tls/tasks/main.yaml @@ -0,0 +1,30 @@ +- name: Check if zookeeper tls cert is already created + set_fact: + zookeeper_certs: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-zookeeper-tls') }}" + +- name: Generate and store certs + when: zookeeper_certs.data is not defined + block: + - name: Generate certs + command: "sh -c 'mkdir -p zk-ca; {{ role_path }}/files/zk-ca.sh zk-ca/ {{ item }}'" + loop: + # TODO: support multiple zk pod + - zk + args: + creates: zk-ca/keys/clientkey.pem + + - name: Create k8s secret + k8s: + state: "{{ state }}" + namespace: "{{ namespace }}" + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ zuul_name }}-zookeeper-tls" + stringData: + ca.crt: "{{ lookup('file', 'zk-ca/demoCA/cacert.pem') }}" + tls.crt: "{{ lookup('file', 'zk-ca/certs/client.pem') }}" + tls.key: "{{ lookup('file', 'zk-ca/keys/clientkey.pem') }}" + data: + zk.pem: "{{ lookup('file', 'zk-ca/keystores/zk.pem') | b64encode }}" diff --git a/roles/zuul/tasks/main.yaml b/roles/zuul/tasks/main.yaml index 974ec36..8605702 100644 --- a/roles/zuul/tasks/main.yaml +++ b/roles/zuul/tasks/main.yaml @@ -4,6 +4,11 @@ - zuul-lookup-conf - zuul-ensure-gearman-tls +- include_role: + name: zuul-ensure-zookeeper-tls + # when the user does not provide a zookeeper + when: raw_spec['zookeeper'] is not defined + - include_role: name: zuul-ensure-registry-tls when: (raw_spec['registry']['count'] | default(0)) | int > 0