Mount the public TLS certificate for HAProxy on up(date|grade) on pacemaker

As part of the minor update workflow and the update workflow, this changes
the pacemaker haproxy bundle resource to add the needed mount for public
TLS to work.

This also handles the reloading of the container to fetch any new certificates
and if needed, it will restart the pacemaker resource (for upgrades), since
we would need pacemaker to re-create the resource.

Change-Id: I850f4de17e7f7e3b46deb27119227ef76658dcb5
Closes-Bug: #1759797

docker/services/pacemaker/haproxy.yaml

@@ -28,6 +28,16 @@ parameters:
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
     type: json
+  SSLCertificate:
+    default: ''
+    description: >
+      The content of the SSL certificate (without Key) in PEM format.
+    type: string
+  PublicSSLCertificateAutogenerated:
+    default: false
+    description: >
+      Whether the public SSL certificate was autogenerated or not.
+    type: boolean
   DeployedSSLCertificatePath:
     default: '/etc/pki/tls/private/overcloud_endpoint.pem'
     description: >
@@ -64,6 +74,15 @@ parameters:
 
 conditions:
   puppet_debug_enabled: {get_param: ConfigDebug}
+  public_tls_enabled:
+    or:
+    - not:
+        equals:
+        - {get_param: SSLCertificate}
+        - ""
+    - equals:
+        - {get_param: PublicSSLCertificateAutogenerated}
+        - true
 
 resources:
 
@@ -229,6 +248,58 @@ outputs:
       metadata_settings:
         get_attr: [HAProxyBase, role_data, metadata_settings]
       update_tasks:
+        - name: Set HAProxy upgrade facts
+          block: &haproxy_update_upgrade_facts
+          - name: Check for haproxy Kolla configuration
+            stat:
+              path: /var/lib/config-data/puppet-generated/haproxy
+            register: haproxy_kolla_config
+          - name: Check if haproxy is already containerized
+            set_fact:
+              haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}"
+          - name: get bootstrap nodeid
+            tags: common
+            command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+            register: bootstrap_node
+          - name: set is_bootstrap_node fact
+            tags: common
+            set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Mount TLS cert if needed
+          when:
+            - step|int == 1
+            - haproxy_containerized|bool
+            - is_bootstrap_node
+          block:
+            - name: Check haproxy public certificate configuration in pacemaker
+              command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']"
+              ignore_errors: true
+              register: haproxy_cert_mounted
+            - name: Disable the haproxy cluster resource
+              pacemaker_resource:
+                resource: haproxy-bundle
+                state: disable
+                wait_for_resource: true
+              register: output
+              retries: 5
+              until: output.rc == 0
+              # rc == 6 means the configuration doesn't exist in the CIB
+              when: haproxy_cert_mounted.rc == 6
+            - name: Set HAProxy public cert volume mount fact
+              set_fact:
+                haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath}
+                haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]}
+            - name: Add a bind mount for public certificate in the haproxy bundle
+              command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro
+              when: haproxy_cert_mounted.rc == 6 and haproxy_public_tls_enabled|bool
+            - name: Enable the haproxy cluster resource
+              pacemaker_resource:
+                resource: haproxy-bundle
+                state: enable
+                wait_for_resource: true
+              register: output
+              retries: 5
+              until: output.rc == 0
+              when: haproxy_cert_mounted.rc == 6
         - name: Haproxy fetch and retag container image for pacemaker
           when: step|int == 2
           block: &haproxy_fetch_retag_container_tasks
@@ -261,20 +332,8 @@ outputs:
         - name: Get docker haproxy image
           set_fact:
             docker_image_latest: *haproxy_image_pcmklatest
-        - name: Check for haproxy Kolla configuration
-          stat:
-            path: /var/lib/config-data/puppet-generated/haproxy
-          register: haproxy_kolla_config
-        - name: Check if haproxy is already containerized
-          set_fact:
-            haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}"
-        - name: get bootstrap nodeid
-          tags: common
-          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
-          register: bootstrap_node
-        - name: set is_bootstrap_node fact
-          tags: common
-          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Set HAProxy upgrade facts
+          block: *haproxy_update_upgrade_facts
         - name: haproxy baremetal to container upgrade tasks
           when:
             - step|int == 1
@@ -305,7 +364,7 @@ outputs:
                   register: output
                   retries: 5
                   until: output.rc == 0
-        - name: Expose HAProxy stats socket on the host
+        - name: Expose HAProxy stats socket on the host and mount TLS cert if needed
           when:
             - step|int == 1
             - haproxy_containerized|bool
@@ -315,28 +374,39 @@ outputs:
               command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-var-lib']"
               ignore_errors: true
               register: haproxy_stats_exposed
-            - name: Change haproxy stats socket configuration in pacemaker
+            - name: Check haproxy public certificate configuration in pacemaker
+              command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']"
+              ignore_errors: true
+              register: haproxy_cert_mounted
+            - name: Disable the haproxy cluster resource
+              pacemaker_resource:
+                resource: haproxy-bundle
+                state: disable
+                wait_for_resource: true
+              register: output
+              retries: 5
+              until: output.rc == 0
               # rc == 6 means the configuration doesn't exist in the CIB
+              when: haproxy_stats_exposed.rc == 6 or haproxy_cert_mounted.rc == 6
+            - name: Add a bind mount for stats socket in the haproxy bundle
+              command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw
               when: haproxy_stats_exposed.rc == 6
-              block:
-                - name: Disable the haproxy cluster resource
-                  pacemaker_resource:
-                    resource: haproxy-bundle
-                    state: disable
-                    wait_for_resource: true
-                  register: output
-                  retries: 5
-                  until: output.rc == 0
-                - name: Add a bind mount for stats socket in the haproxy bundle
-                  command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw
-                - name: Enable the haproxy cluster resource
-                  pacemaker_resource:
-                    resource: haproxy-bundle
-                    state: enable
-                    wait_for_resource: true
-                  register: output
-                  retries: 5
-                  until: output.rc == 0
+            - name: Set HAProxy public cert volume mount fact
+              set_fact:
+                haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath}
+                haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]}
+            - name: Add a bind mount for public certificate in the haproxy bundle
+              command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro
+              when: haproxy_cert_mounted.rc == 6 and haproxy_public_tls_enabled|bool
+            - name: Enable the haproxy cluster resource
+              pacemaker_resource:
+                resource: haproxy-bundle
+                state: enable
+                wait_for_resource: true
+              register: output
+              retries: 5
+              until: output.rc == 0
+              when: haproxy_stats_exposed.rc == 6 or haproxy_cert_mounted.rc == 6
         - name: Retag the pacemaker image if containerized
           when:
             - step|int == 3

puppet/extraconfig/tls/tls-cert-inject.yaml

@@ -55,7 +55,10 @@ resources:
         # If the HAProxy container tried to load this, it'll be a directory and
         # will make this fail.
         if [ -d ${cert_path} ]; then
-            rm -rf ${cert_path}
+            rmdir ${cert_path}
+            HAPROXY_TLS_UPDATE_NEEDED=1
+        else
+            HAPROXY_TLS_UPDATE_NEEDED=0
         fi
         cat > ${cert_path} << EOF
         ${cert_chain_content}
@@ -75,6 +78,34 @@ resources:
         if [ "$haproxy_status" = "active" ]; then
             systemctl reload haproxy
         fi
+        pacemaker_status=$(systemctl is-active pacemaker)
+        # If we need an update and pacemaker is being used, we need to restart
+        # the pacemaker resource on the bootstrap node. We don't support the update
+        # in non-pacemaker cases.
+        if [[ $HAPROXY_TLS_UPDATE_NEEDED -eq 1 && "$pacemaker_status" == "active" ]]; then
+            BOOTSTRAPNODE=$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)
+            MY_HOSTNAME=$(hostname)
+            if [[ "$BOOTSTRAPNODE" == "$MY_HOSTNAME" ]]; then
+                # Triggers an update
+                HAPROXY_RESOURCE_NAME=$(pcs status | grep container | grep haproxy | sed 's/^.*container.*: \(.*\) .*/\1/')
+                if [[ -n "$HAPROXY_RESOURCE_NAME" ]]; then
+                    pcs resource restart "$HAPROXY_RESOURCE_NAME"
+                fi
+            fi
+        elif [[ $HAPROXY_TLS_UPDATE_NEEDED -eq 0 ]]; then
+            # Handles reloading HAProxy and fetching a new certificate if
+            # necessary
+            HAPROXY_CONTAINER_ID=$(docker ps | grep haproxy | awk '{print $1}')
+            if [[ -n "$HAPROXY_CONTAINER_ID" ]]; then
+                if [[ "$pacemaker_status" == "active" ]]; then
+                    # We copy the certificate from the mount point to the desired
+                    # path
+                    docker exec "$HAPROXY_CONTAINER_ID" cp /var/lib/kolla/config_files/src-tls${cert_path} ${cert_path}
+                fi
+                docker kill --signal=HUP "$HAPROXY_CONTAINER_ID"
+            fi
+        fi
+
 
   ControllerTLSDeployment:
     type: OS::Heat::SoftwareDeployment