Add a Merge::Map feature.

We need to scatter gather in a few situations - determining rabbit
cluster membership, galera membership and configuring hosts for Nova
to permit live migration (which requires host->host communication).
This patch is a proof of concept for an eventual heat feature,
expressed in merge.py. The example given should work for actual use,
but I'll deliver that change separately.

Change-Id: I68e9b2471866810cc698ca3ea28ddf5bb1688d7b

examples/scale_map.yaml

@@ -0,0 +1,55 @@
+Resources:
+  ComputeUser:
+    Type: AWS::IAM::User
+    Properties:
+      Policies: [ { Ref: ComputeAccessPolicy } ]
+  GlobalAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+  NovaCompute0Key:
+    Type: FileInclude
+    Path: examples/scale_map2.yaml
+    SubKey: Resources.NovaCompute0Key
+  NovaCompute0CompletionCondition:
+    Type: FileInclude
+    Path: examples/scale_map2.yaml
+    SubKey: Resources.NovaCompute0CompletionCondition
+  NovaCompute0CompletionHandle:
+    Type: FileInclude
+    Path: examples/scale_map2.yaml
+    SubKey: Resources.NovaCompute0CompletionHandle
+  NovaCompute0Config:
+    Type: FileInclude
+    Path: examples/scale_map2.yaml
+    SubKey: Resources.NovaCompute0Config
+    Parameters:
+      AllHosts:
+        Fn::Join:
+        - "\n"
+        - Merge::Map:
+            NovaCompute0:
+              Fn::Join:
+              - ' '
+              - - Fn::Select:
+                  - 0
+                  - Fn::Select:
+                    - ctlplane
+                    - Fn::GetAtt:
+                      - NovaCompute0
+                      - networks
+                - Fn::Select:
+                  - name
+                  - Fn::GetAtt:
+                    - NovaCompute0
+                    - show
+                - Fn::Join:
+                  - '.'
+                  - - Fn::Select:
+                      - name
+                      - Fn::GetAtt:
+                        - NovaCompute0
+                        - show
+                    - 'local'
+  NovaCompute0:
+    Type: FileInclude
+    Path: examples/scale_map2.yaml
+    SubKey: Resources.NovaCompute0

examples/scale_map2.yaml

@@ -0,0 +1,54 @@
+HeatTemplateFormatVersion: '2012-12-12'
+Parameters:
+  AllHosts:
+    Type: String
+  ComputeImage:
+    Type: String
+Resources:
+  ComputeAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+    Properties:
+      AllowedResources: [ NovaCompute0 ]
+  NovaCompute0Key:
+    Type: AWS::IAM::AccessKey
+    Properties:
+      UserName:
+        Ref: ComputeUser
+  NovaCompute0CompletionCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    DependsOn: notcompute
+    Properties:
+      Handle: {Ref: NovaCompute0CompletionHandle}
+      Count: '1'
+      Timeout: '1800'
+  NovaCompute0CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute0:
+    Type: OS::Nova::Server
+    Properties:
+      image:
+        Ref: ComputeImage
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          secret_access_key:
+            Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
+          stack_name: {Ref: 'AWS::StackName'}
+          path: NovaCompute0Config.Metadata
+  NovaCompute0Config:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute0CompletionHandle
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          secret_access_key:
+            Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
+          stack_name: {Ref: 'AWS::StackName'}
+          path: NovaCompute0Config.Metadata
+      hosts:
+        Ref: AllHosts

examples/scale_map_result.yaml

@@ -0,0 +1,235 @@
+Description: examples/scale_map.yaml
+HeatTemplateFormatVersion: '2012-12-12'
+Resources:
+  ComputeUser:
+    Properties:
+      Policies:
+      - Ref: ComputeAccessPolicy
+    Type: AWS::IAM::User
+  GlobalAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+  NovaCompute0:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute0CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute0CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute0CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute0Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute0CompletionHandle
+      hosts:
+        Fn::Join:
+        - '
+
+          '
+        - - Fn::Join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - Fn::GetAtt:
+                    - NovaCompute0
+                    - networks
+              - Fn::Select:
+                - name
+                - Fn::GetAtt:
+                  - NovaCompute0
+                  - show
+              - Fn::Join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - Fn::GetAtt:
+                      - NovaCompute0
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute0Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey
+  NovaCompute1:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute1Key
+          path: NovaCompute1Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute1Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute1CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute1CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute1CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute1Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute1CompletionHandle
+      hosts:
+        Fn::Join:
+        - '
+
+          '
+        - - Fn::Join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - Fn::GetAtt:
+                    - NovaCompute1
+                    - networks
+              - Fn::Select:
+                - name
+                - Fn::GetAtt:
+                  - NovaCompute1
+                  - show
+              - Fn::Join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - Fn::GetAtt:
+                      - NovaCompute1
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute1Key
+          path: NovaCompute1Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute1Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute1Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey
+  NovaCompute2:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute2Key
+          path: NovaCompute2Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute2Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute2CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute2CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute2CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute2Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute2CompletionHandle
+      hosts:
+        Fn::Join:
+        - '
+
+          '
+        - - Fn::Join:
+            - ' '
+            - - Fn::Select:
+                - 0
+                - Fn::Select:
+                  - ctlplane
+                  - Fn::GetAtt:
+                    - NovaCompute2
+                    - networks
+              - Fn::Select:
+                - name
+                - Fn::GetAtt:
+                  - NovaCompute2
+                  - show
+              - Fn::Join:
+                - .
+                - - Fn::Select:
+                    - name
+                    - Fn::GetAtt:
+                      - NovaCompute2
+                      - show
+                  - local
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute2Key
+          path: NovaCompute2Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute2Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute2Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey

test_merge.bash

@@ -29,6 +29,7 @@ run_test "python $merge_py examples/source2.yaml" examples/source2_lib_result.ya
 run_test "python $merge_py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml
 run_test "python $merge_py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml
 run_test "python $merge_py --scale NovaCompute=3 examples/scale1.yaml" examples/scale_result.yaml
+run_test "python $merge_py --scale NovaCompute=3 examples/scale_map.yaml" examples/scale_map_result.yaml
 echo
 trap - EXIT
 exit $fail

tripleo_heat_merge/merge.py

@@ -4,6 +4,27 @@ import yaml
 import argparse
 
 
+def apply_maps(template):
+    """Apply Merge::Map within template.
+
+    Any dict {'Merge::Map': {'Foo': 'Bar', 'Baz': 'Quux'}}
+    will resolve to ['Bar', 'Quux'] - that is a dict with key
+    'Merge::Map' is replaced entirely by that dict['Merge::Map'].values().
+    """
+    if isinstance(template, dict):
+        if 'Merge::Map' in template:
+            return sorted(
+                apply_maps(value) for value in template['Merge::Map'].values()
+                )
+        else:
+            return dict((key, apply_maps(value))
+                for key, value in template.items())
+    elif isinstance(template, list):
+        return [apply_maps(item) for item in template]
+    else:
+        return template
+
+
 def apply_scaling(template, scaling, in_copies=None):
     """Apply a set of scaling operations to template.
 
@@ -301,6 +322,7 @@ def merge(templates, master_role=None, slave_roles=None,
                 end_template['Resources'][r] = rbody
 
     end_template = apply_scaling(end_template, scaling)
+    end_template = apply_maps(end_template)
 
     def fix_ref(item, old, new):
         if isinstance(item, dict):