openstack/tripleo-heat-templates
Code Issues Proposed changes

Make scaling out a feature for merge.py.

Browse Source 
This is a simple implementation designed to work with our current
non-HOT approach. We need to adjust our templates a little to support
this - though future work could make that better, it's orthogonal to
this effort.

Change-Id: I555617e5f24a5882de915f057dc02e008c81e753
This commit is contained in:
Robert Collins 2014-01-28 19:20:40 +13:00
parent 23c8aac2ba
commit c071255ad6
5 changed files with 391 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
examples/scale1.yaml Normal file
View File

@ -0,0 +1,30 @@
Resources:
ComputeUser:
Type: AWS::IAM::User
Properties:
Policies: [ { Ref: ComputeAccessPolicy } ]
GlobalAccessPolicy:
Type: OS::Heat::AccessPolicy
NovaCompute0Key:
Type: FileInclude
Path: examples/scale2.yaml
SubKey: Resources.NovaCompute0Key
NovaCompute0CompletionCondition:
Type: FileInclude
Path: examples/scale2.yaml
SubKey: Resources.NovaCompute0CompletionCondition
NovaCompute0CompletionHandle:
Type: FileInclude
Path: examples/scale2.yaml
SubKey: Resources.NovaCompute0CompletionHandle
NovaCompute0Config:
Type: FileInclude
Path: examples/scale2.yaml
SubKey: Resources.NovaCompute0Config
Parameters:
ComputeImage: "123"
RabbitPassword: "guest"
NovaCompute0:
Type: FileInclude
Path: examples/scale2.yaml
SubKey: Resources.NovaCompute0

66
examples/scale2.yaml Normal file
View File

@ -0,0 +1,66 @@
HeatTemplateFormatVersion: '2012-12-12'
Parameters:
ComputeImage:
Type: String
RabbitPassword:
Type: String
NoEcho: true
Resources:
ComputeAccessPolicy:
Type: OS::Heat::AccessPolicy
Properties:
AllowedResources: [ NovaCompute0 ]
NovaCompute0Key:
Type: AWS::IAM::AccessKey
Properties:
UserName:
Ref: ComputeUser
NovaCompute0CompletionCondition:
Type: AWS::CloudFormation::WaitCondition
DependsOn: notcompute
Properties:
Handle: {Ref: NovaCompute0CompletionHandle}
Count: '1'
Timeout: '1800'
NovaCompute0CompletionHandle:
Type: AWS::CloudFormation::WaitConditionHandle
NovaCompute0:
Type: OS::Nova::Server
Properties:
image:
Ref: ComputeImage
Metadata:
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute0Key
secret_access_key:
Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
stack_name: {Ref: 'AWS::StackName'}
path: NovaCompute0Config.Metadata
NovaCompute0Config:
Type: AWS::AutoScaling::LaunchConfiguration
Metadata:
completion-handle:
Ref: NovaCompute0CompletionHandle
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute0Key
secret_access_key:
Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
stack_name: {Ref: 'AWS::StackName'}
path: NovaCompute0Config.Metadata
neutron:
ovs:
local_ip:
Fn::Select:
- 0
- Fn::Select:
- ctlplane
- Fn::GetAtt:
- NovaCompute0
- networks
rabbit:
password: {Ref: RabbitPassword}

190
examples/scale_result.yaml Normal file
View File

@ -0,0 +1,190 @@
Description: examples/scale1.yaml
HeatTemplateFormatVersion: '2012-12-12'
Resources:
ComputeUser:
Properties:
Policies:
- Ref: ComputeAccessPolicy
Type: AWS::IAM::User
GlobalAccessPolicy:
Type: OS::Heat::AccessPolicy
NovaCompute0:
Metadata:
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute0Key
path: NovaCompute0Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute0Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
Properties:
image:
Ref: ComputeImage
Type: OS::Nova::Server
NovaCompute0CompletionCondition:
DependsOn: notcompute
Properties:
Count: '1'
Handle:
Ref: NovaCompute0CompletionHandle
Timeout: '1800'
Type: AWS::CloudFormation::WaitCondition
NovaCompute0CompletionHandle:
Type: AWS::CloudFormation::WaitConditionHandle
NovaCompute0Config:
Metadata:
completion-handle:
Ref: NovaCompute0CompletionHandle
neutron:
ovs:
local_ip:
Fn::Select:
- 0
- Fn::Select:
- ctlplane
- Fn::GetAtt:
- NovaCompute0
- networks
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute0Key
path: NovaCompute0Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute0Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
rabbit:
password: guest
Type: AWS::AutoScaling::LaunchConfiguration
NovaCompute0Key:
Properties:
UserName:
Ref: ComputeUser
Type: AWS::IAM::AccessKey
NovaCompute1:
Metadata:
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute1Key
path: NovaCompute1Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute1Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
Properties:
image:
Ref: ComputeImage
Type: OS::Nova::Server
NovaCompute1CompletionCondition:
DependsOn: notcompute
Properties:
Count: '1'
Handle:
Ref: NovaCompute1CompletionHandle
Timeout: '1800'
Type: AWS::CloudFormation::WaitCondition
NovaCompute1CompletionHandle:
Type: AWS::CloudFormation::WaitConditionHandle
NovaCompute1Config:
Metadata:
completion-handle:
Ref: NovaCompute1CompletionHandle
neutron:
ovs:
local_ip:
Fn::Select:
- 0
- Fn::Select:
- ctlplane
- Fn::GetAtt:
- NovaCompute1
- networks
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute1Key
path: NovaCompute1Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute1Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
rabbit:
password: guest
Type: AWS::AutoScaling::LaunchConfiguration
NovaCompute1Key:
Properties:
UserName:
Ref: ComputeUser
Type: AWS::IAM::AccessKey
NovaCompute2:
Metadata:
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute2Key
path: NovaCompute2Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute2Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
Properties:
image:
Ref: ComputeImage
Type: OS::Nova::Server
NovaCompute2CompletionCondition:
DependsOn: notcompute
Properties:
Count: '1'
Handle:
Ref: NovaCompute2CompletionHandle
Timeout: '1800'
Type: AWS::CloudFormation::WaitCondition
NovaCompute2CompletionHandle:
Type: AWS::CloudFormation::WaitConditionHandle
NovaCompute2Config:
Metadata:
completion-handle:
Ref: NovaCompute2CompletionHandle
neutron:
ovs:
local_ip:
Fn::Select:
- 0
- Fn::Select:
- ctlplane
- Fn::GetAtt:
- NovaCompute2
- networks
os-collect-config:
cfn:
access_key_id:
Ref: NovaCompute2Key
path: NovaCompute2Config.Metadata
secret_access_key:
Fn::GetAtt:
- NovaCompute2Key
- SecretAccessKey
stack_name:
Ref: AWS::StackName
rabbit:
password: guest
Type: AWS::AutoScaling::LaunchConfiguration
NovaCompute2Key:
Properties:
UserName:
Ref: ComputeUser
Type: AWS::IAM::AccessKey

1
test_merge.bash
View File

@ -28,6 +28,7 @@ run_test "python $merge_py examples/source.yaml" examples/source_lib_result.yaml
run_test "python $merge_py examples/source2.yaml" examples/source2_lib_result.yaml
run_test "python $merge_py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml
run_test "python $merge_py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml
run_test "python $merge_py --scale NovaCompute=3 examples/scale1.yaml" examples/scale_result.yaml
echo
trap - EXIT
exit $fail

106
tripleo_heat_merge/merge.py
View File

@ -4,6 +4,97 @@ import yaml
import argparse
def apply_scaling(template, scaling, in_copies=None):
"""Apply a set of scaling operations to template.
This is a single pass recursive function: for each call we process one
dict or list and recurse to handle children containers.
Values are handled via scale_value.
Keys in dicts are always copied per the scaling rule.
Values are either replaced or copied depending on whether the given
scaling rule is in in_copies.
"""
in_copies = dict(in_copies or {})
# Shouldn't be needed but to avoid unexpected side effects/bugs we short
# circuit no-ops.
if not scaling:
return template
if isinstance(template, dict):
new_template = {}
for key, value in template.items():
for prefix, copy_num, new_key in scale_value(
key, scaling, in_copies):
if prefix:
# e.g. Compute0, 1, Compute1Foo
in_copies[prefix] = prefix[:-1] + str(copy_num)
if isinstance(value, (dict, list)):
new_value = apply_scaling(value, scaling, in_copies)
new_template[new_key] = new_value
else:
new_values = list(scale_value(value, scaling, in_copies))
# We have nowhere to multiply a non-container value of a
# dict, so it may be copied or unchanged but not scaled.
assert len(new_values) == 1
new_template[new_key] = new_values[0][2]
if prefix:
del in_copies[prefix]
return new_template
elif isinstance(template, list):
new_template = []
for value in template:
if isinstance(value, (dict, list)):
new_template.append(apply_scaling(value, scaling, in_copies))
else:
for _, _, new_value in scale_value(value, scaling, in_copies):
new_template.append(new_value)
return new_template
else:
raise Exception("apply_scaling called with non-container %r" % template)
def scale_value(value, scaling, in_copies):
"""Scale out a value.
:param value: The value to scale (not a container).
:param scaling: The scaling map to use.
:param in_copies: What containers we're currently copying.
:return: An iterator of the new values for the value as tuples:
(prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
prefix and copy_num are only set when:
- a prefix in scaling matches value
- and that prefix is not in in_copies
"""
if isinstance(value, (str, unicode)):
for prefix, copies in scaling.items():
if not value.startswith(prefix):
continue
suffix = value[len(prefix):]
if prefix in in_copies:
# Adjust to the copy number we're on
yield None, None, in_copies[prefix] + suffix
return
else:
for n in range(copies):
yield prefix, n, prefix[:-1] + str(n) + suffix
return
yield None, None, value
else:
yield None, None, value
def parse_scaling(scaling_args):
"""Translate a list of scaling requests to a dict prefix:count."""
scaling_args = scaling_args or []
result = {}
for item in scaling_args:
key, value = item.split('=')
value = int(value)
result[key + '0'] = value
return result
def _translate_role(role, master_role, slave_roles):
if not master_role:
return role
@ -92,18 +183,27 @@ def main(argv=None):
parser.add_argument('--output',
help='File to write output to. - for stdout',
default='-')
parser.add_argument('--scale', action="append",
help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to "
"be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
"renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
"outside of any copy.")
args = parser.parse_args(argv)
templates = args.templates
scaling = parse_scaling(args.scale)
merged_template = merge(templates, args.master_role, args.slave_roles,
args.included_template_dir)
args.included_template_dir, scaling=scaling)
if args.output == '-':
out_file = sys.stdout
else:
out_file = file(args.output, 'wt')
out_file.write(merged_template)
def merge(templates, master_role=None, slave_roles=None,
included_template_dir=INCLUDED_TEMPLATE_DIR):
included_template_dir=INCLUDED_TEMPLATE_DIR,
scaling=None):
scaling = scaling or {}
errors = []
end_template={'HeatTemplateFormatVersion': '2012-12-12',
'Description': []}
@ -192,6 +292,8 @@ def merge(templates, master_role=None, slave_roles=None,
end_template['Resources'] = {}
end_template['Resources'][r] = rbody
end_template = apply_scaling(end_template, scaling)
def fix_ref(item, old, new):
if isinstance(item, dict):
copy_item = dict(item)