Automatically backup and restore Swift rings from the undercloud

Swift rings created or updated on the overcloud nodes will now be
stored on the undercloud at the end of the deployment.  An
additional consistency check is executed before storing them,
ensuring all rings within the cluster are identical.

These rings will be retrieved (before Puppet runs) by every node
when an UPDATE is executed, and by doing this will be in a
consistent state across the cluster.

This makes it possible to add, remove or replace nodes in an
existing cluster without manual operator interaction.

Closes-Bug: 1609421
Depends-On: Ic3da38cffdd993c768bdb137c17d625dff1aa372
Change-Id: I758179182265da5160c06bb95f4c6258dc0edcd6

extraconfig/tasks/swift-ring-deploy.yaml

@@ -0,0 +1,31 @@
+heat_template_version: ocata
+
+parameters:
+  servers:
+    type: json
+  SwiftRingGetTempurl:
+    default: ''
+    description: A temporary Swift URL to download rings from.
+    type: string
+
+resources:
+  SwiftRingDeployConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: swift_ring_get_tempurl
+      config: |
+        #!/bin/sh
+        pushd /
+        curl --insecure --silent "${swift_ring_get_tempurl}" | tar xz || true
+        popd
+
+  SwiftRingDeploy:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      name: SwiftRingDeploy
+      config: {get_resource: SwiftRingDeployConfig}
+      servers:  {get_param: servers}
+      input_values:
+        swift_ring_get_tempurl: {get_param: SwiftRingGetTempurl}

extraconfig/tasks/swift-ring-update.yaml

@@ -0,0 +1,42 @@
+heat_template_version: ocata
+
+parameters:
+  servers:
+    type: json
+  SwiftRingPutTempurl:
+    default: ''
+    description: A temporary Swift URL to upload rings to.
+    type: string
+
+resources:
+  SwiftRingUpdateConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: swift_ring_put_tempurl
+      config: |
+        #!/bin/sh
+        TMP_DATA=$(mktemp -d)
+        function cleanup {
+          rm -Rf "$TMP_DATA"
+        }
+        trap cleanup EXIT
+        # sanity check in case rings are not consistent within cluster
+        swift-recon --md5 | grep -q "doesn't match" && exit 1
+        pushd ${TMP_DATA}
+        tar -cvzf swift-rings.tar.gz /etc/swift/*.builder /etc/swift/*.ring.gz /etc/swift/backups/*
+        resp=`curl --insecure --silent -X PUT "${swift_ring_put_tempurl}" --write-out "%{http_code}" --data-binary @swift-rings.tar.gz`
+        popd
+        if [ "$resp" != "201" ]; then
+            exit 1
+        fi
+
+  SwiftRingUpdate:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      name: SwiftRingUpdate
+      config: {get_resource: SwiftRingUpdateConfig}
+      servers: {get_param: servers}
+      input_values:
+        swift_ring_put_tempurl: {get_param: SwiftRingPutTempurl}

overcloud-resource-registry-puppet.j2.yaml

@@ -11,6 +11,9 @@ resource_registry:
   OS::TripleO::Tasks::UpdateWorkflow: OS::Heat::None
   OS::TripleO::Tasks::PackageUpdate: extraconfig/tasks/yum_update.yaml
 
+  OS::TripleO::Tasks::SwiftRingDeploy: extraconfig/tasks/swift-ring-deploy.yaml
+  OS::TripleO::Tasks::SwiftRingUpdate: extraconfig/tasks/swift-ring-update.yaml
+
 {% for role in roles %}
   OS::TripleO::{{role.name}}::PreNetworkConfig: OS::Heat::None
   OS::TripleO::{{role.name}}PostDeploySteps: puppet/post.yaml

puppet/puppet-steps.j2

@@ -32,6 +32,13 @@
         update_identifier: {get_param: DeployIdentifier}
   {% endif %}
 
+  {% if role.name in ['Controller', 'ObjectStorage'] %}
+  {{role.name}}SwiftRingDeploy:
+    type: OS::TripleO::Tasks::SwiftRingDeploy
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+  {% endif %}
+
   # Step through a series of configuration steps
 {% for step in range(1, 6) %}
   {{role.name}}Deployment_Step{{step}}:
@@ -85,4 +92,15 @@
       input_values:
         update_identifier: {get_param: DeployIdentifier}
   {% endif %}
+
+  {% if role.name in ['Controller', 'ObjectStorage'] %}
+  {{role.name}}SwiftRingUpdate:
+    type: OS::TripleO::Tasks::SwiftRingUpdate
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step5
+  {% endfor %}
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+  {% endif %}
 {% endfor %}