From 2c94431c0d3acce41ff62364afd378e57d92ec8f Mon Sep 17 00:00:00 2001 From: Hiroo Kitamura Date: Wed, 10 Mar 2021 11:04:09 +0900 Subject: [PATCH] Fix processing related to required properties In the current UT testcase and yaml, there are several cases where the required property of policy is insufficient or the definition part is incorrect. In addition, there is a bug in the code that has the translation logic under 'translator/hot/tosca/'. This patch fixes these translation logic codes, testcase and yaml. Among the fixed yaml files, the following has a line feed code of CRLF, so it has been modified to LF. * translator/tests/data/autoscaling/tosca_cluster_autoscaling.yaml * translator/tests/data/custom_types/senlin_cluster_policies.yaml * translator/tests/data/nfv/test_tosca_nfv_autoscaling_with_params.yaml In addition, NOTE description is added for 'translator/hot/tosca/tosca_policies_monitoring.py', which is one of the translation logic codes. Closes-Bug: #1918360 Change-Id: Ib9dec61007db5d354eaca86000ab3bcf6744a838 --- .../hot/tosca/tests/test_tosca_autoscaling.py | 6 +- .../tosca/tosca_cluster_policies_scaling.py | 10 +- .../hot/tosca/tosca_policies_monitoring.py | 6 + .../hot/tosca/tosca_policies_scaling.py | 32 +-- .../data/autoscaling/tosca_autoscaling.yaml | 34 ++-- .../tosca_cluster_autoscaling.yaml | 131 ++++++------ .../data/custom_types/custom_monitoring.yaml | 31 +++ .../custom_types/senlin_cluster_policies.yaml | 46 ++++- .../autoscaling/hot_autoscaling.yaml | 9 +- .../monitoring/tosca_monitoring_scaling.yaml | 12 +- .../data/nfv/test_tosca_nfv_autoscaling.yaml | 2 +- ...est_tosca_nfv_autoscaling_with_params.yaml | 188 +++++++++--------- .../tosca-vnfd-reservation-id.yaml | 2 +- 13 files changed, 294 insertions(+), 215 deletions(-) create mode 100644 translator/tests/data/custom_types/custom_monitoring.yaml diff --git a/translator/hot/tosca/tests/test_tosca_autoscaling.py b/translator/hot/tosca/tests/test_tosca_autoscaling.py index 7154ff1c..2818d7b0 100644 --- a/translator/hot/tosca/tests/test_tosca_autoscaling.py +++ b/translator/hot/tosca/tests/test_tosca_autoscaling.py @@ -37,8 +37,8 @@ class AutoscalingTest(TestCase): policy_name = list(policies[0].keys())[0] for policy in policies: tpl = policy[policy_name] - targets = tpl["targets"] properties = tpl["properties"] + targets = properties["targets"] try: nodetemplate = NodeTemplate(name, nodetemplates) toscacompute = ToscaCompute(nodetemplate) @@ -82,7 +82,6 @@ class AutoscalingTest(TestCase): - asg: type: tosca.policies.Scaling description: Simple node autoscaling - targets: [my_server_1] triggers: resize_compute: description: trigger @@ -94,6 +93,7 @@ class AutoscalingTest(TestCase): resource_type: instance comparison_operator: gt properties: + targets: [my_server_1] min_instances: 2 max_instances: 10 default_instances: 3 @@ -126,8 +126,8 @@ class AutoscalingTest(TestCase): policies: - SP1: type: tosca.policies.Scaling - targets: [VDU1] properties: + targets: [VDU1] increment: 1 cooldown: 120 min_instances: 1 diff --git a/translator/hot/tosca/tosca_cluster_policies_scaling.py b/translator/hot/tosca/tosca_cluster_policies_scaling.py index e71ba873..732a6093 100644 --- a/translator/hot/tosca/tosca_cluster_policies_scaling.py +++ b/translator/hot/tosca/tosca_cluster_policies_scaling.py @@ -66,7 +66,7 @@ class ToscaClusterAutoscaling(HotResource): hot_resources = [] hot_type = 'OS::Aodh::GnocchiAggregationByResourcesAlarm' trigger_receivers = defaultdict(list) - for node in self.policy.targets: + for node in self.policy.properties['targets']: for trigger in self.policy.entity_tpl['triggers']: for action in self.policy.\ entity_tpl['triggers'][trigger]['action']: @@ -137,7 +137,7 @@ class ToscaClusterAutoscaling(HotResource): remove_resources.append(resource) elif resource.type == 'OS::Neutron::Net': remove_resources.append(resource) - elif resource.name in self.policy.targets and \ + elif resource.name in self.policy.properties['targets'] and \ resource.type != 'OS::Senlin::Policy': props = {} del resource.properties['user_data_format'] @@ -155,11 +155,11 @@ class ToscaClusterAutoscaling(HotResource): resources.remove(remove_resource) for index, resource in enumerate(resources): - if resource.name in self.policy.targets: + if resource.name in self.policy.properties['targets']: resource.properties['properties']['networks'] = \ networks[resource.name] - for node in self.policy.targets: + for node in self.policy.properties['targets']: props = {} props["profile"] = {'get_resource': '%s' % node} temp = self.policy.entity_tpl["properties"] @@ -181,7 +181,7 @@ class ToscaClusterAutoscaling(HotResource): entity_tpl['triggers'][trigger]['action']: scale_type = self.policy.\ entity_tpl['triggers'][trigger]['action'][action]['type'] - for node in self.policy.targets: + for node in self.policy.properties['targets']: target_cluster_nodes.\ append({"get_resource": "%s_cluster" % node}) cluster_scale_type = SCALE_TYPE[scale_type] diff --git a/translator/hot/tosca/tosca_policies_monitoring.py b/translator/hot/tosca/tosca_policies_monitoring.py index f1445961..55175baa 100644 --- a/translator/hot/tosca/tosca_policies_monitoring.py +++ b/translator/hot/tosca/tosca_policies_monitoring.py @@ -28,6 +28,12 @@ ALARM_STATISTIC = {'mean': 'mean', 'median': 'median', 'summary': 'sum', class ToscaMonitoring(HotResource): '''Translate TOSCA node type tosca.policies.Monitoring''' + # NOTE: This file implements extended conversion logic other than + # the section defined at 'tosca.policies.Monitoring' in + # TOSCA_definition_1_0.yaml. Therefore, it is necessary to define + # and use toscatype that inherits this toscatype with 'derived_from'. + # This is because validation fails due to an error that an undefined + # key exists when validating using tosca-parser. toscatype = 'tosca.policies.Monitoring' def __init__(self, policy, csar_dir=None): diff --git a/translator/hot/tosca/tosca_policies_scaling.py b/translator/hot/tosca/tosca_policies_scaling.py index 6d0c6879..a78017d7 100644 --- a/translator/hot/tosca/tosca_policies_scaling.py +++ b/translator/hot/tosca/tosca_policies_scaling.py @@ -72,23 +72,23 @@ class ToscaAutoscaling(HotResource): delete_res_names = [] scale_res = [] for index, resource in enumerate(resources): - if resource.name in self.policy.targets and \ - resource.type != 'OS::Heat::AutoScalingGroup': + if resource.type != 'OS::Heat::AutoScalingGroup': temp = self.policy.entity_tpl["properties"] - props = {} - res = {} - res["min_size"] = temp["min_instances"] - res["max_size"] = temp["max_instances"] - res["desired_capacity"] = temp["default_instances"] - res["cooldown"] = temp["cooldown"] - props['type'] = resource.type - props['properties'] = resource.properties - res['resource'] = {'type': self.policy.name + '_res.yaml'} - scaling_resources = \ - HotResource(resource, - type='OS::Heat::AutoScalingGroup', - name=self.policy.name + '_group', - properties=res) + if resource.name in temp.get("targets"): + props = {} + res = {} + res["min_size"] = temp["min_instances"] + res["max_size"] = temp["max_instances"] + res["desired_capacity"] = temp["default_instances"] + res["cooldown"] = temp["cooldown"] + props['type'] = resource.type + props['properties'] = resource.properties + res['resource'] = {'type': self.policy.name + '_res.yaml'} + scaling_resources = \ + HotResource(resource, + type='OS::Heat::AutoScalingGroup', + name=self.policy.name + '_group', + properties=res) if resource.type not in SCALING_RESOURCES: delete_res_names.append(resource.name) diff --git a/translator/tests/data/autoscaling/tosca_autoscaling.yaml b/translator/tests/data/autoscaling/tosca_autoscaling.yaml index 52256ee7..c673d2c8 100644 --- a/translator/tests/data/autoscaling/tosca_autoscaling.yaml +++ b/translator/tests/data/autoscaling/tosca_autoscaling.yaml @@ -3,6 +3,11 @@ tosca_definitions_version: tosca_simple_yaml_1_0 description: > Template for deploying servers based on policies. +imports: + - ../custom_types/custom_monitoring.yaml + - ../nfv/tacker_defs.yaml + - ../nfv/tacker_nfv_defs.yaml + topology_template: node_templates: my_server_1: @@ -22,22 +27,27 @@ topology_template: version: 6.5 policies: - asg: - type: tosca.policies.Scaling + type: tosca.policies.tacker.Scaling description: Simple node autoscaling - targets: [my_server_1] - triggers: - resize_compute: - description: trigger - condition: - constraint: utilization greater_than 50% - granularity: 60 - evaluations: 1 - aggregation_method: mean - resource_type: instance - comparison_operator: gt properties: + targets: [my_server_1] min_instances: 2 max_instances: 10 default_instances: 3 increment: 1 cooldown: 60 + + - cpu_monitoring: + type: tosca.policies.Monitoring.Custom + description: Simple node monitoring + triggers: + high_cpu_usage: + description: trigger + metric: cpu_util + condition: + constraint: utilization greater_than 60% + threshold: 60 + granularity: 600 + evaluations: 1 + aggregation_method: mean + comparison_operator: gt diff --git a/translator/tests/data/autoscaling/tosca_cluster_autoscaling.yaml b/translator/tests/data/autoscaling/tosca_cluster_autoscaling.yaml index f09c7f1c..0045fa6d 100644 --- a/translator/tests/data/autoscaling/tosca_cluster_autoscaling.yaml +++ b/translator/tests/data/autoscaling/tosca_cluster_autoscaling.yaml @@ -1,64 +1,67 @@ -tosca_definitions_version: tosca_simple_yaml_1_0 - -description: > - Template for deploying servers based on policies. - -imports: - - ../custom_types/senlin_cluster_policies.yaml - -topology_template: - node_templates: - my_server_1: - type: tosca.nodes.Compute - capabilities: - host: - properties: - num_cpus: 2 - disk_size: 10 GB - mem_size: 512 MB - os: - properties: - # host Operating System image properties - architecture: x86_64 - type: Linux - distribution: RHEL - version: 6.5 - my_port_1: - type: tosca.nodes.network.Port - requirements: - - link: - node: my_network_1 - - binding: - node: my_server_1 - my_network_1: - type: tosca.nodes.network.Network - properties: - network_name: net0 - policies: - - cluster_scaling: - type: tosca.policies.Scaling.Cluster - description: Cluster node autoscaling - targets: [my_server_1] - triggers: - scale_out: - description: trigger - event_type: - type: tosca.events.resource.cpu.utilization - metric: cpu_util - implementation: Ceilometer - condition: - constraint: utilization greater_than 50% - granularity: 60 - evaluations: 1 - aggregation_method: mean - resource_type: instance - comparison_operator: gt - action: - scale_out: - type: SCALE_OUT - implementation: Senlin.webhook - properties: - min_instances: 2 - max_instances: 10 - default_instances: 3 - increment: 1 +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + Template for deploying servers based on policies. + +imports: + - ../custom_types/senlin_cluster_policies.yaml + - ../nfv/tacker_defs.yaml + - ../nfv/tacker_nfv_defs.yaml + +topology_template: + node_templates: + my_server_1: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 2 + disk_size: 10 GB + mem_size: 512 MB + os: + properties: + # host Operating System image properties + architecture: x86_64 + type: Linux + distribution: RHEL + version: 6.5 + my_port_1: + type: tosca.nodes.network.Port + requirements: + - link: + node: my_network_1 + - binding: + node: my_server_1 + my_network_1: + type: tosca.nodes.network.Network + properties: + network_name: net0 + policies: + - cluster_scaling: + type: tosca.policies.Scaling.Cluster + description: Cluster node autoscaling + properties: + targets: [my_server_1] + min_instances: 2 + max_instances: 10 + default_instances: 3 + increment: 1 + cooldown: 60 + triggers: + scale_out: + description: trigger + event_type: + type: tosca.events.resource.cpu.utilization + metric: cpu_util + implementation: Ceilometer + condition: + constraint: utilization greater_than 50% + granularity: 60 + evaluations: 1 + aggregation_method: mean + resource_type: instance + comparison_operator: gt + action: + scale_out: + type: SCALE_OUT + implementation: Senlin.webhook diff --git a/translator/tests/data/custom_types/custom_monitoring.yaml b/translator/tests/data/custom_types/custom_monitoring.yaml new file mode 100644 index 00000000..7b7ad06f --- /dev/null +++ b/translator/tests/data/custom_types/custom_monitoring.yaml @@ -0,0 +1,31 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + The TOSCA Policy Type definition that is used to expand + the monitoring policy. + +policy_types: + tosca.policies.Monitoring.Custom: + derived_from: tosca.policies.Monitoring + description: The TOSCA Policy Type definition that is used to expand + the monitoring policy. + triggers: + resize_compute: + event_type: + type: map + entry_schema: + type: string + required: true + metrics: + type: string + required: true + condition: + type: map + entry_schema: + type: string + required: false + action: + type: map + entry_schema: + type: string + required: true diff --git a/translator/tests/data/custom_types/senlin_cluster_policies.yaml b/translator/tests/data/custom_types/senlin_cluster_policies.yaml index c809ca07..b8252178 100644 --- a/translator/tests/data/custom_types/senlin_cluster_policies.yaml +++ b/translator/tests/data/custom_types/senlin_cluster_policies.yaml @@ -1,11 +1,35 @@ -tosca_definitions_version: tosca_simple_yaml_1_0 - -description: > - The TOSCA Policy Type definition that is used to govern - Senlin Policy of TOSCA nodes or groups of nodes - -policy_types: - tosca.policies.Scaling.Cluster: - derived_from: tosca.policies.Scaling - description: The TOSCA Policy Type definition that is used to govern - scaling of TOSCA nodes or groups of nodes. \ No newline at end of file +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + The TOSCA Policy Type definition that is used to govern + Senlin Policy of TOSCA nodes or groups of nodes + +imports: + - ../nfv/tacker_defs.yaml + - ../nfv/tacker_nfv_defs.yaml + +policy_types: + tosca.policies.Scaling.Cluster: + derived_from: tosca.policies.tacker.Scaling + description: The TOSCA Policy Type definition that is used to govern + scaling of TOSCA nodes or groups of nodes. + triggers: + resize_compute: + event_type: + type: map + entry_schema: + type: string + required: true + metrics: + type: string + required: true + condition: + type: map + entry_schema: + type: string + required: false + action: + type: map + entry_schema: + type: string + required: true diff --git a/translator/tests/data/hot_output/autoscaling/hot_autoscaling.yaml b/translator/tests/data/hot_output/autoscaling/hot_autoscaling.yaml index 0497eb10..b2c3c934 100644 --- a/translator/tests/data/hot_output/autoscaling/hot_autoscaling.yaml +++ b/translator/tests/data/hot_output/autoscaling/hot_autoscaling.yaml @@ -30,14 +30,15 @@ resources: adjustment_type: change_in_capacity scaling_adjustment: -1 cooldown: 60 - asg_alarm: + high_cpu_usage: type: OS::Aodh::GnocchiAggregationByResourcesAlarm properties: metric: cpu_util - description: Simple node autoscaling - granularity: 60 + description: utilization greater_than 60% + evaluation_periods: 1 + granularity: 600 aggregation_method: mean - threshold: 1 + threshold: 60 resource_type: instance comparison_operator: gt outputs: {} diff --git a/translator/tests/data/monitoring/tosca_monitoring_scaling.yaml b/translator/tests/data/monitoring/tosca_monitoring_scaling.yaml index 3460150d..fc7cfec1 100644 --- a/translator/tests/data/monitoring/tosca_monitoring_scaling.yaml +++ b/translator/tests/data/monitoring/tosca_monitoring_scaling.yaml @@ -3,6 +3,11 @@ tosca_definitions_version: tosca_simple_yaml_1_0 description: > Template for deploying servers based on policies. +imports: + - ../custom_types/custom_monitoring.yaml + - ../nfv/tacker_defs.yaml + - ../nfv/tacker_nfv_defs.yaml + topology_template: node_templates: my_server_1: @@ -22,10 +27,10 @@ topology_template: version: 6.5 policies: - asg: - type: tosca.policies.Scaling + type: tosca.policies.tacker.Scaling description: Simple node autoscaling - targets: [my_server_1] properties: + targets: [my_server_1] min_instances: 2 max_instances: 10 default_instances: 3 @@ -33,9 +38,8 @@ topology_template: cooldown: 60 - cpu_monitoring: - type: tosca.policies.Monitoring + type: tosca.policies.Monitoring.Custom description: Simple node monitoring - targets: [my_server_1] triggers: high_cpu_usage: description: trigger diff --git a/translator/tests/data/nfv/test_tosca_nfv_autoscaling.yaml b/translator/tests/data/nfv/test_tosca_nfv_autoscaling.yaml index 10c8074e..14978222 100644 --- a/translator/tests/data/nfv/test_tosca_nfv_autoscaling.yaml +++ b/translator/tests/data/nfv/test_tosca_nfv_autoscaling.yaml @@ -58,8 +58,8 @@ topology_template: policies: - SP1: type: tosca.policies.tacker.Scaling - targets: [VDU1, VDU2] properties: + targets: [VDU1, VDU2] increment: 1 cooldown: 120 min_instances: 1 diff --git a/translator/tests/data/nfv/test_tosca_nfv_autoscaling_with_params.yaml b/translator/tests/data/nfv/test_tosca_nfv_autoscaling_with_params.yaml index aee064b2..6774932a 100644 --- a/translator/tests/data/nfv/test_tosca_nfv_autoscaling_with_params.yaml +++ b/translator/tests/data/nfv/test_tosca_nfv_autoscaling_with_params.yaml @@ -1,94 +1,94 @@ -tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0 - -description: Demo example - -imports: - - tacker_defs.yaml - - tacker_nfv_defs.yaml - -metadata: - template_name: sample-tosca-vnfd - -topology_template: - inputs: - image_name: - type: string - description: Image Name - - flavor: - type: string - description: Flavor Information - - node_templates: - VDU1: - type: tosca.nodes.nfv.VDU.Tacker - properties: - image: { get_input: image_name } - flavor: { get_input: flavor } - mgmt_driver: noop - availability_zone: nova - metadata: {metering.server_group: SG1} - - CP1: - type: tosca.nodes.nfv.CP.Tacker - properties: - management: true - anti_spoofing_protection: false - requirements: - - virtualLink: - node: VL1 - - virtualBinding: - node: VDU1 - - VL1: - type: tosca.nodes.nfv.VL - properties: - network_name: net_mgmt - vendor: Tacker - - policies: - - SP: - type: tosca.policies.tacker.Scaling - targets: [VDU1] - properties: - increment: 1 - cooldown: 120 - min_instances: 1 - max_instances: 3 - default_instances: 1 - - - vdu_cpu_usage_monitoring_policy: - type: tosca.policies.tacker.Alarming - triggers: - vdu_hcpu_usage_scaling_out: - event_type: - type: tosca.events.resource.utilization - implementation: ceilometer - metric: cpu_util - condition: - threshold: 80 - constraint: utilization greater_than 80% - granularity: 60 - evaluations: 1 - aggregation_method: mean - resource_type: instance - comparison_operator: gt - metadata: SG1 - action: [SP] - - vdu_lcpu_usage_scaling_in: - event_type: - type: tosca.events.resource.utilization - implementation: ceilometer - metric: cpu_util - condition: - threshold: 10 - constraint: utilization less_than 10% - granularity: 60 - evaluations: 1 - aggregation_method: mean - resource_type: instance - comparison_operator: lt - metadata: SG1 - action: [SP] - +tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0 + +description: Demo example + +imports: + - tacker_defs.yaml + - tacker_nfv_defs.yaml + +metadata: + template_name: sample-tosca-vnfd + +topology_template: + inputs: + image_name: + type: string + description: Image Name + + flavor: + type: string + description: Flavor Information + + node_templates: + VDU1: + type: tosca.nodes.nfv.VDU.Tacker + properties: + image: { get_input: image_name } + flavor: { get_input: flavor } + mgmt_driver: noop + availability_zone: nova + metadata: {metering.server_group: SG1} + + CP1: + type: tosca.nodes.nfv.CP.Tacker + properties: + management: true + anti_spoofing_protection: false + requirements: + - virtualLink: + node: VL1 + - virtualBinding: + node: VDU1 + + VL1: + type: tosca.nodes.nfv.VL + properties: + network_name: net_mgmt + vendor: Tacker + + policies: + - SP: + type: tosca.policies.tacker.Scaling + properties: + targets: [VDU1] + increment: 1 + cooldown: 120 + min_instances: 1 + max_instances: 3 + default_instances: 1 + + - vdu_cpu_usage_monitoring_policy: + type: tosca.policies.tacker.Alarming + triggers: + vdu_hcpu_usage_scaling_out: + event_type: + type: tosca.events.resource.utilization + implementation: ceilometer + metric: cpu_util + condition: + threshold: 80 + constraint: utilization greater_than 80% + granularity: 60 + evaluations: 1 + aggregation_method: mean + resource_type: instance + comparison_operator: gt + metadata: SG1 + action: [SP] + + vdu_lcpu_usage_scaling_in: + event_type: + type: tosca.events.resource.utilization + implementation: ceilometer + metric: cpu_util + condition: + threshold: 10 + constraint: utilization less_than 10% + granularity: 60 + evaluations: 1 + aggregation_method: mean + resource_type: instance + comparison_operator: lt + metadata: SG1 + action: [SP] + diff --git a/translator/tests/data/reservation/tosca-vnfd-reservation-id.yaml b/translator/tests/data/reservation/tosca-vnfd-reservation-id.yaml index 97d2ddf4..6545147b 100644 --- a/translator/tests/data/reservation/tosca-vnfd-reservation-id.yaml +++ b/translator/tests/data/reservation/tosca-vnfd-reservation-id.yaml @@ -57,4 +57,4 @@ topology_template: min_instances: 0 max_instances: 3 default_instances: 0 - targets: [VDU1] + targets: [VDU1]