Add CA certificates Secret and mount it

Our containerized gates started failing recently. Turns out some default
configuation was changed and `tls-proxy` service was added. This option
makes all OpenStack endpoints use HTTPS. This includes creation of a
DevStack CA certificates bundle that then will be configured to be
verified when connecting to OpenStack APIs. This works well with
non-containerized deployment as the bundle is available locally in
/opt/stack/data and our `[neutron]` section sets `cafile` option to
point there.

Things are different in containerized deployment use case as we need a
way to pass those certificates into the container. Effectively - we had
no CA certificates support for containerized deployments either in
DevStack or production.

This commit adds that support by including new Kuryr Kubernetes resource
definition - `kuryr-certificates` Secret. It is supposed to hold CA
certificate under `kuryr-ca-bundle.crt` key. kuryr-controller DaemonSet
definition was modified to mount the certificate into /etc/ssl/certs.

Changes also include implementing support for that in DevStack plugin
(placing the certificate in the secret and setting the `[neutron]cafile`
config option to point to that certificate).

Closes-Bug: 1758061
Change-Id: I7ac9d05868994cfc2a1aef4a8cd6c2148895e9c8

devstack/lib/kuryr_kubernetes

@@ -368,6 +368,31 @@ EOF
     cat $cni_conf_path | indent >> "${output_dir}/config_map.yml"
 }
 
+function generate_kuryr_certificates_secret() {
+    local output_dir
+    local certs_bundle_path
+    output_dir=$1
+    certs_bundle_path=${2:-""}
+
+    mkdir -p "$output_dir"
+    rm -f ${output_dir}/certificates_secret.yml
+
+    if [ -n $certs_bundle_path ]; then
+        CA_CERT=`cat $certs_bundle_path | base64 -w0`
+    fi
+
+    cat >> "${output_dir}/certificates_secret.yml" << EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kuryr-certificates
+  namespace: kube-system
+type: Opaque
+data:
+  kuryr-ca-bundle.crt: $CA_CERT
+EOF
+}
+
 function generate_kuryr_service_account() {
     output_dir=$1
     mkdir -p "$output_dir"
@@ -446,6 +471,9 @@ spec:
         - name: config-volume
           mountPath: "/etc/kuryr/kuryr.conf"
           subPath: kuryr.conf
+        - name: certificates-volume
+          mountPath: "/etc/ssl/certs"
+          readOnly: true
         readinessProbe:
           httpGet:
             path: /ready
@@ -464,6 +492,9 @@ EOF
       - name: config-volume
         configMap:
           name: kuryr-config
+      - name: certificates-volume
+        secret:
+          secretName: kuryr-certificates
       restartPolicy: Always
 EOF
 }

devstack/plugin.sh

@@ -135,6 +135,9 @@ function generate_containerized_kuryr_resources {
     # kuryr-controller and kuryr-cni will have tokens in different dirs.
     KURYR_CNI_CONFIG=${KURYR_CONFIG}-cni
     cp $KURYR_CONFIG $KURYR_CNI_CONFIG
+    # NOTE(dulek): In the container the CA bundle will be mounted in a standard
+    # directory, so we need to modify that.
+    iniset "$KURYR_CONFIG" neutron cafile /etc/ssl/certs/kuryr-ca-bundle.crt
     iniset "$KURYR_CONFIG" kubernetes token_file /var/run/secrets/kubernetes.io/serviceaccount/token
     iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     iniset "$KURYR_CNI_CONFIG" kubernetes token_file /etc/kuryr/token
@@ -143,6 +146,7 @@ function generate_containerized_kuryr_resources {
     # Generate kuryr resources in k8s formats.
     local output_dir="${DATA_DIR}/kuryr-kubernetes"
     generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CNI_CONFIG
+    generate_kuryr_certificates_secret $output_dir $SSL_BUNDLE_FILE
     generate_kuryr_service_account $output_dir
     generate_controller_deployment $output_dir $KURYR_HEALTH_SERVER_PORT
     generate_cni_daemon_set $output_dir $KURYR_CNI_HEALTH_SERVER_PORT $cni_daemon $CNI_BIN_DIR $CNI_CONF_DIR
@@ -153,6 +157,9 @@ function run_containerized_kuryr_resources {
     /usr/local/bin/kubectl create -f \
         "${k8s_data_dir}/config_map.yml" \
         || die $LINENO "Failed to create kuryr-kubernetes ConfigMap."
+    /usr/local/bin/kubectl create -f \
+        "${k8s_data_dir}/certificates_secret.yml" \
+        || die $LINENO "Failed to create kuryr-kubernetes certificates Secret."
     /usr/local/bin/kubectl create -f \
         "${k8s_data_dir}/service_account.yml" \
         || die $LINENO "Failed to create kuryr-kubernetes ServiceAccount."

doc/source/installation/containerized.rst

@@ -39,12 +39,15 @@ kuryr-kubernetes includes a tool that lets you generate resource definitions
 that can be used to Deploy Kuryr on Kubernetes. The script is placed in
 ``tools/generate_k8s_resource_definitions.sh`` and takes up to 3 arguments: ::
 
-    $ ./tools/generate_k8s_resource_definitions <output_dir> [<controller_conf_path>] [<cni_conf_path>]
+    $ ./tools/generate_k8s_resource_definitions <output_dir> [<controller_conf_path>] [<cni_conf_path>] [<ca_certificate_path>]
 
 * ``output_dir`` - directory where to put yaml files with definitions.
 * ``controller_conf_path`` - path to custom kuryr-controller configuration file.
 * ``cni_conf_path`` - path to custom kuryr-cni configuration file (defaults to
   ``controller_conf_path``).
+* ``ca_certificate_path`` - path to custom CA certificate for OpenStack API. It
+  will be added into Kubernetes as a ``Secret`` and mounted into
+  kuryr-controller container. Defaults to no certificate.
 
 If no path to config files is provided, script automatically generates minimal
 configuration. However some of the options should be filled by the user. You can
@@ -109,6 +112,7 @@ Deploying Kuryr resources on Kubernetes
 To deploy the files on your Kubernetes cluster run: ::
 
     $ kubectl apply -f config_map.yml -n kube-system
+    $ kubectl apply -f certificates_secret.yml -n kube-system
     $ kubectl apply -f service_account.yml -n kube-system
     $ kubectl apply -f controller_deployment.yml -n kube-system
     $ kubectl apply -f cni_ds.yml -n kube-system

tools/generate_k8s_resource_definitions.sh

@@ -20,6 +20,7 @@ source "$DIR/../devstack/lib/kuryr_kubernetes"
 OUTPUT_DIR=${1:-.}
 CONTROLLER_CONF_PATH=${2:-""}
 CNI_CONF_PATH=${3:-$CONTROLLER_CONF_PATH}
+CA_CERTIFICATE_PATH=${4:-$CA_CERTIFICATE_PATH}
 
 if [ -z $CONTROLLER_CONF_PATH ]; then
     api_root=${KURYR_K8S_API_ROOT:-https://127.0.0.1:6443}
@@ -103,6 +104,7 @@ EOF
     fi
 fi
 
+generate_kuryr_certificates_secret $OUTPUT_DIR $CA_CERTIFICATE_PATH
 generate_kuryr_configmap $OUTPUT_DIR $CONTROLLER_CONF_PATH $CNI_CONF_PATH
 generate_kuryr_service_account $OUTPUT_DIR
 health_server_port=${KURYR_HEALTH_SERVER_PORT:-8082}