From 9e7417b267fb6db2b974b07ffac3904d6e8164d6 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Robles Date: Thu, 29 Mar 2018 08:52:59 +0000 Subject: [PATCH] Mount the public TLS certificate for HAProxy on up(date|grade) on pacemaker As part of the minor update workflow and the update workflow, this changes the pacemaker haproxy bundle resource to add the needed mount for public TLS to work. This also handles the reloading of the container to fetch any new certificates and if needed, it will restart the pacemaker resource (for upgrades), since we would need pacemaker to re-create the resource. Change-Id: I850f4de17e7f7e3b46deb27119227ef76658dcb5 Closes-Bug: #1759797 (cherry picked from commit 8b85faf7e66a49977f7fd18512e1534488ca4f07) --- docker/services/pacemaker/haproxy.yaml | 140 +++++++++++++++----- puppet/extraconfig/tls/tls-cert-inject.yaml | 33 ++++- 2 files changed, 137 insertions(+), 36 deletions(-) diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml index a91c47b555..8a0dc1b193 100644 --- a/docker/services/pacemaker/haproxy.yaml +++ b/docker/services/pacemaker/haproxy.yaml @@ -28,6 +28,16 @@ parameters: description: Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. type: json + SSLCertificate: + default: '' + description: > + The content of the SSL certificate (without Key) in PEM format. + type: string + PublicSSLCertificateAutogenerated: + default: false + description: > + Whether the public SSL certificate was autogenerated or not. + type: boolean DeployedSSLCertificatePath: default: '/etc/pki/tls/private/overcloud_endpoint.pem' description: > @@ -64,6 +74,15 @@ parameters: conditions: puppet_debug_enabled: {get_param: ConfigDebug} + public_tls_enabled: + or: + - not: + equals: + - {get_param: SSLCertificate} + - "" + - equals: + - {get_param: PublicSSLCertificateAutogenerated} + - true resources: @@ -229,6 +248,58 @@ outputs: metadata_settings: get_attr: [HAProxyBase, role_data, metadata_settings] update_tasks: + - name: Set HAProxy upgrade facts + block: &haproxy_update_upgrade_facts + - name: Check for haproxy Kolla configuration + stat: + path: /var/lib/config-data/puppet-generated/haproxy + register: haproxy_kolla_config + - name: Check if haproxy is already containerized + set_fact: + haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}" + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Mount TLS cert if needed + when: + - step|int == 1 + - haproxy_containerized|bool + - is_bootstrap_node + block: + - name: Check haproxy public certificate configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']" + ignore_errors: true + register: haproxy_cert_mounted + - name: Disable the haproxy cluster resource + pacemaker_resource: + resource: haproxy-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + # rc == 6 means the configuration doesn't exist in the CIB + when: haproxy_cert_mounted.rc == 6 + - name: Set HAProxy public cert volume mount fact + set_fact: + haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath} + haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]} + - name: Add a bind mount for public certificate in the haproxy bundle + command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro + when: haproxy_cert_mounted.rc == 6 and haproxy_public_tls_enabled|bool + - name: Enable the haproxy cluster resource + pacemaker_resource: + resource: haproxy-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + when: haproxy_cert_mounted.rc == 6 - name: Haproxy fetch and retag container image for pacemaker when: step|int == 2 block: &haproxy_fetch_retag_container_tasks @@ -261,20 +332,8 @@ outputs: - name: Get docker haproxy image set_fact: docker_image_latest: *haproxy_image_pcmklatest - - name: Check for haproxy Kolla configuration - stat: - path: /var/lib/config-data/puppet-generated/haproxy - register: haproxy_kolla_config - - name: Check if haproxy is already containerized - set_fact: - haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}" - - name: get bootstrap nodeid - tags: common - command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid - register: bootstrap_node - - name: set is_bootstrap_node fact - tags: common - set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Set HAProxy upgrade facts + block: *haproxy_update_upgrade_facts - name: haproxy baremetal to container upgrade tasks when: - step|int == 1 @@ -305,7 +364,7 @@ outputs: register: output retries: 5 until: output.rc == 0 - - name: Expose HAProxy stats socket on the host + - name: Expose HAProxy stats socket on the host and mount TLS cert if needed when: - step|int == 1 - haproxy_containerized|bool @@ -315,28 +374,39 @@ outputs: command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-var-lib']" ignore_errors: true register: haproxy_stats_exposed - - name: Change haproxy stats socket configuration in pacemaker + - name: Check haproxy public certificate configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']" + ignore_errors: true + register: haproxy_cert_mounted + - name: Disable the haproxy cluster resource + pacemaker_resource: + resource: haproxy-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 # rc == 6 means the configuration doesn't exist in the CIB + when: haproxy_stats_exposed.rc == 6 or haproxy_cert_mounted.rc == 6 + - name: Add a bind mount for stats socket in the haproxy bundle + command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw when: haproxy_stats_exposed.rc == 6 - block: - - name: Disable the haproxy cluster resource - pacemaker_resource: - resource: haproxy-bundle - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Add a bind mount for stats socket in the haproxy bundle - command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw - - name: Enable the haproxy cluster resource - pacemaker_resource: - resource: haproxy-bundle - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 + - name: Set HAProxy public cert volume mount fact + set_fact: + haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath} + haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]} + - name: Add a bind mount for public certificate in the haproxy bundle + command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro + when: haproxy_cert_mounted.rc == 6 and haproxy_public_tls_enabled|bool + - name: Enable the haproxy cluster resource + pacemaker_resource: + resource: haproxy-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + when: haproxy_stats_exposed.rc == 6 or haproxy_cert_mounted.rc == 6 - name: Retag the pacemaker image if containerized when: - step|int == 3 diff --git a/puppet/extraconfig/tls/tls-cert-inject.yaml b/puppet/extraconfig/tls/tls-cert-inject.yaml index d32fc62f16..877ec71d96 100644 --- a/puppet/extraconfig/tls/tls-cert-inject.yaml +++ b/puppet/extraconfig/tls/tls-cert-inject.yaml @@ -55,7 +55,10 @@ resources: # If the HAProxy container tried to load this, it'll be a directory and # will make this fail. if [ -d ${cert_path} ]; then - rm -rf ${cert_path} + rmdir ${cert_path} + HAPROXY_TLS_UPDATE_NEEDED=1 + else + HAPROXY_TLS_UPDATE_NEEDED=0 fi cat > ${cert_path} << EOF ${cert_chain_content} @@ -75,6 +78,34 @@ resources: if [ "$haproxy_status" = "active" ]; then systemctl reload haproxy fi + pacemaker_status=$(systemctl is-active pacemaker) + # If we need an update and pacemaker is being used, we need to restart + # the pacemaker resource on the bootstrap node. We don't support the update + # in non-pacemaker cases. + if [[ $HAPROXY_TLS_UPDATE_NEEDED -eq 1 && "$pacemaker_status" == "active" ]]; then + BOOTSTRAPNODE=$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid) + MY_HOSTNAME=$(hostname) + if [[ "$BOOTSTRAPNODE" == "$MY_HOSTNAME" ]]; then + # Triggers an update + HAPROXY_RESOURCE_NAME=$(pcs status | grep container | grep haproxy | sed 's/^.*container.*: \(.*\) .*/\1/') + if [[ -n "$HAPROXY_RESOURCE_NAME" ]]; then + pcs resource restart "$HAPROXY_RESOURCE_NAME" + fi + fi + elif [[ $HAPROXY_TLS_UPDATE_NEEDED -eq 0 ]]; then + # Handles reloading HAProxy and fetching a new certificate if + # necessary + HAPROXY_CONTAINER_ID=$(docker ps | grep haproxy | awk '{print $1}') + if [[ -n "$HAPROXY_CONTAINER_ID" ]]; then + if [[ "$pacemaker_status" == "active" ]]; then + # We copy the certificate from the mount point to the desired + # path + docker exec "$HAPROXY_CONTAINER_ID" cp /var/lib/kolla/config_files/src-tls${cert_path} ${cert_path} + fi + docker kill --signal=HUP "$HAPROXY_CONTAINER_ID" + fi + fi + ControllerTLSDeployment: type: OS::Heat::SoftwareDeployment