From e39f712e5184d2fccb809ba4737a27aa2618ee09 Mon Sep 17 00:00:00 2001 From: Haiyang DING Date: Tue, 27 Jan 2015 01:36:16 +0000 Subject: [PATCH] Add scalar unit type Implement scalar-unit.size type: 1. any number of spaces (including zero or none) is allowed between the scalar value and the unit value. 2. support constraints like greater_or_equal and in_range. (eg: storage_size: in_range [4 GB, 20 GB ] ) 3. default unit is 'B'(byte) according to the TOSCA profile while allowing user to set the unit of the output number as long as the input unit is a valid one 4. supported scalar-unit.size (case-insensitive): B, KB, KiB, MB, MiB, GB, GiB, TB, TiB implement BP: add-scalar-unit-type Change-Id: Idf3705312afd33d7aeadc3f8856cf728c41ec190 --- translator/toscalib/dataentity.py | 2 + .../toscalib/elements/TOSCA_definition.yaml | 16 +- translator/toscalib/elements/constraints.py | 90 ++++++-- translator/toscalib/properties.py | 22 +- translator/toscalib/tests/test_memunit.py | 167 --------------- translator/toscalib/tests/test_scalar_unit.py | 194 ++++++++++++++++++ translator/toscalib/tests/test_toscadef.py | 7 - 7 files changed, 274 insertions(+), 224 deletions(-) delete mode 100644 translator/toscalib/tests/test_memunit.py create mode 100644 translator/toscalib/tests/test_scalar_unit.py diff --git a/translator/toscalib/dataentity.py b/translator/toscalib/dataentity.py index 03f05fc..06151b4 100644 --- a/translator/toscalib/dataentity.py +++ b/translator/toscalib/dataentity.py @@ -113,6 +113,8 @@ class DataEntity(object): if entry_schema: DataEntity.validate_entry(value, entry_schema, custom_def) return value + elif type == Schema.SCALAR_UNIT_SIZE: + return Constraint.validate_scalar_unit_size(value) elif type == Schema.MAP: Constraint.validate_map(value) if entry_schema: diff --git a/translator/toscalib/elements/TOSCA_definition.yaml b/translator/toscalib/elements/TOSCA_definition.yaml index 078dba2..e2650f9 100644 --- a/translator/toscalib/elements/TOSCA_definition.yaml +++ b/translator/toscalib/elements/TOSCA_definition.yaml @@ -38,21 +38,19 @@ tosca.nodes.Compute: Number of (actual or virtual) CPUs associated with the Compute node. disk_size: required: no - type: integer + type: scalar-unit.size constraints: - - greater_or_equal: 0 + - greater_or_equal: 0 MB description: > - Size of the local disk, in Gigabytes (GB), available to applications - running on the Compute node. + Size of the local disk, available to applications running on the + Compute node. mem_size: required: no - type: integer + type: scalar-unit.size constraints: - - greater_or_equal: 0 + - greater_or_equal: 0 MB description: > - Size of memory, in Megabytes (MB), available to applications running - on the Compute node. - default: 1024 + Size of memory, available to applications running on the Compute node. os_arch: required: no default: x86_64 diff --git a/translator/toscalib/elements/constraints.py b/translator/toscalib/elements/constraints.py index 61a2f2c..4e9d4bd 100644 --- a/translator/toscalib/elements/constraints.py +++ b/translator/toscalib/elements/constraints.py @@ -12,6 +12,7 @@ import collections import datetime +import math import numbers import re import six @@ -34,12 +35,18 @@ class Schema(collections.Mapping): PROPERTY_TYPES = ( INTEGER, STRING, BOOLEAN, FLOAT, - NUMBER, TIMESTAMP, LIST, MAP + NUMBER, TIMESTAMP, LIST, MAP, SCALAR_UNIT_SIZE ) = ( 'integer', 'string', 'boolean', 'float', - 'number', 'timestamp', 'list', 'map' + 'number', 'timestamp', 'list', 'map', 'scalar-unit.size' ) + SCALAR_UNIT_SIZE_DEFAULT = 'B' + SCALAR_UNIT_SIZE_DICT = {'B': 1, 'KB': 1000, 'KIB': 1024, 'MB': 1000000, + 'MIB': 1048576, 'GB': 1000000000, + 'GIB': 1073741824, 'TB': 1000000000000, + 'TIB': 1099511627776} + def __init__(self, name, schema_dict): self.name = name if not isinstance(schema_dict, collections.Mapping): @@ -135,6 +142,16 @@ class Constraint(object): self.property_name = property_name self.property_type = property_type self.constraint_value = constraint[self.constraint_key] + self.constraint_value_msg = self.constraint_value + if self.property_type == Schema.SCALAR_UNIT_SIZE: + if isinstance(self.constraint_value, list): + self.constraint_value = [Constraint. + get_num_from_scalar_unit_size(v) + for v in self.constraint_value] + else: + self.constraint_value = (Constraint. + get_num_from_scalar_unit_size + (self.constraint_value)) # check if constraint is valid for property type if property_type not in self.valid_prop_types: msg = _('Constraint type "%(ctype)s" is not valid ' @@ -147,6 +164,9 @@ class Constraint(object): return _('Property %s could not be validated.') % self.property_name def validate(self, value): + self.value_msg = value + if self.property_type == Schema.SCALAR_UNIT_SIZE: + value = self.get_num_from_scalar_unit_size(value) if not self._is_valid(value): err_msg = self._err_msg(value) raise ValidationError(message=err_msg) @@ -190,6 +210,36 @@ class Constraint(object): return normalised == 'true' raise ValueError(_('"%s" is not a boolean') % value) + @staticmethod + def validate_scalar_unit_size(value): + regex = re.compile('(\d*)\s*(\w*)') + result = regex.match(str(value)).groups() + if result[0] and ((not result[1]) or (result[1].upper() in + Schema.SCALAR_UNIT_SIZE_DICT. + keys())): + return value + raise ValueError(_('"%s" is not a valid scalar-unit') % value) + + @staticmethod + def get_num_from_scalar_unit_size(value, unit=None): + if unit: + if unit.upper() not in Schema.SCALAR_UNIT_SIZE_DICT.keys(): + raise ValueError(_('input unit "%s" is not a valid unit') + % unit) + else: + unit = Schema.SCALAR_UNIT_SIZE_DEFAULT + Constraint.validate_scalar_unit_size(value) + regex = re.compile('(\d*)\s*(\w*)') + result = regex.match(str(value)).groups() + if not result[1]: + converted = (Constraint.str_to_num(result[0])) + if result[1].upper() in Schema.SCALAR_UNIT_SIZE_DICT.keys(): + converted = int(Constraint.str_to_num(result[0]) + * Schema.SCALAR_UNIT_SIZE_DICT[result[1].upper()] + * math.pow(Schema.SCALAR_UNIT_SIZE_DICT + [unit.upper()], -1)) + return converted + @staticmethod def str_to_num(value): '''Convert a string representation of a number into a numeric type.''' @@ -221,8 +271,8 @@ class Equal(Constraint): def _err_msg(self, value): return (_('%(pname)s: %(pvalue)s is not equal to "%(cvalue)s".') % dict(pname=self.property_name, - pvalue=value, - cvalue=self.constraint_value)) + pvalue=self.value_msg, + cvalue=self.constraint_value_msg)) class GreaterThan(Constraint): @@ -238,7 +288,7 @@ class GreaterThan(Constraint): datetime.time, datetime.datetime) valid_prop_types = (Schema.INTEGER, Schema.FLOAT, - Schema.TIMESTAMP) + Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE) def __init__(self, property_name, property_type, constraint): super(GreaterThan, self).__init__(property_name, property_type, @@ -256,8 +306,8 @@ class GreaterThan(Constraint): def _err_msg(self, value): return (_('%(pname)s: %(pvalue)s must be greater than "%(cvalue)s".') % dict(pname=self.property_name, - pvalue=value, - cvalue=self.constraint_value)) + pvalue=self.value_msg, + cvalue=self.constraint_value_msg)) class GreaterOrEqual(Constraint): @@ -273,7 +323,7 @@ class GreaterOrEqual(Constraint): datetime.time, datetime.datetime) valid_prop_types = (Schema.INTEGER, Schema.FLOAT, - Schema.TIMESTAMP) + Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE) def __init__(self, property_name, property_type, constraint): super(GreaterOrEqual, self).__init__(property_name, property_type, @@ -291,8 +341,8 @@ class GreaterOrEqual(Constraint): return (_('%(pname)s: %(pvalue)s must be greater or equal ' 'to "%(cvalue)s".') % dict(pname=self.property_name, - pvalue=value, - cvalue=self.constraint_value)) + pvalue=self.value_msg, + cvalue=self.constraint_value_msg)) class LessThan(Constraint): @@ -308,7 +358,7 @@ class LessThan(Constraint): datetime.time, datetime.datetime) valid_prop_types = (Schema.INTEGER, Schema.FLOAT, - Schema.TIMESTAMP) + Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE) def __init__(self, property_name, property_type, constraint): super(LessThan, self).__init__(property_name, property_type, @@ -326,8 +376,8 @@ class LessThan(Constraint): def _err_msg(self, value): return (_('%(pname)s: %(pvalue)s must be less than "%(cvalue)s".') % dict(pname=self.property_name, - pvalue=value, - cvalue=self.constraint_value)) + pvalue=self.value_msg, + cvalue=self.constraint_value_msg)) class LessOrEqual(Constraint): @@ -343,7 +393,7 @@ class LessOrEqual(Constraint): datetime.time, datetime.datetime) valid_prop_types = (Schema.INTEGER, Schema.FLOAT, - Schema.TIMESTAMP) + Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE) def __init__(self, property_name, property_type, constraint): super(LessOrEqual, self).__init__(property_name, property_type, @@ -362,8 +412,8 @@ class LessOrEqual(Constraint): return (_('%(pname)s: %(pvalue)s must be less or ' 'equal to "%(cvalue)s".') % dict(pname=self.property_name, - pvalue=value, - cvalue=self.constraint_value)) + pvalue=self.value_msg, + cvalue=self.constraint_value_msg)) class InRange(Constraint): @@ -379,7 +429,7 @@ class InRange(Constraint): datetime.time, datetime.datetime) valid_prop_types = (Schema.INTEGER, Schema.FLOAT, - Schema.TIMESTAMP) + Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE) def __init__(self, property_name, property_type, constraint): super(InRange, self).__init__(property_name, property_type, constraint) @@ -407,9 +457,9 @@ class InRange(Constraint): return (_('%(pname)s: %(pvalue)s is out of range ' '(min:%(vmin)s, max:%(vmax)s).') % dict(pname=self.property_name, - pvalue=value, - vmin=self.min, - vmax=self.max)) + pvalue=self.value_msg, + vmin=self.constraint_value_msg[0], + vmax=self.constraint_value_msg[1])) class ValidValues(Constraint): diff --git a/translator/toscalib/properties.py b/translator/toscalib/properties.py index 6c86365..3f4cb18 100644 --- a/translator/toscalib/properties.py +++ b/translator/toscalib/properties.py @@ -10,9 +10,6 @@ # License for the specific language governing permissions and limitations # under the License. -import re - -from translator.toscalib.common.exception import InvalidPropertyValueError from translator.toscalib.dataentity import DataEntity from translator.toscalib.elements.constraints import Schema from translator.toscalib.functions import is_function @@ -35,7 +32,7 @@ class Property(object): def __init__(self, property_name, value, schema_dict, custom_def=None): self.name = property_name - self.value = self._convert_value(value) + self.value = value self.custom_def = custom_def self.schema = Schema(property_name, schema_dict) @@ -76,20 +73,3 @@ class Property(object): if self.constraints: for constraint in self.constraints: constraint.validate(self.value) - - def _convert_value(self, value): - if self.name == 'mem_size': - mem_reader = re.compile('(\d*)\s*(\w*)') - matcher = str(value) - result = mem_reader.match(matcher).groups() - r = [] - if (result[0] != '') and (result[1] == ''): - r = int(result[0]) - elif (result[0] != '') and (result[1] == 'MB'): - r = int(result[0]) - elif (result[0] != '') and (result[1] == 'GB'): - r = int(result[0]) * 1024 - else: - raise InvalidPropertyValueError(what=self.name) - return r - return value diff --git a/translator/toscalib/tests/test_memunit.py b/translator/toscalib/tests/test_memunit.py deleted file mode 100644 index 58b4d31..0000000 --- a/translator/toscalib/tests/test_memunit.py +++ /dev/null @@ -1,167 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from translator.toscalib.common.exception import InvalidPropertyValueError -from translator.toscalib.nodetemplate import NodeTemplate -from translator.toscalib.tests.base import TestCase -import translator.toscalib.utils.yamlparser - - -class ToscaTemplateMemorySizeOutputTest(TestCase): - - scenarios = [ - ( - # tpl_snippet with mem_size given as number - 'mem_size_is_number', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1024 - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ( - # tpl_snippet with mem_size given as number+space+MB - 'mem_size_is_number_Space_MB', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1024 MB - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ( - # tpl_snippet with mem_size given as number+space+GB - 'mem_size_is_number_Space_GB', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1 GB - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ( - # tpl_snippet with mem_size given as number+GB - 'mem_size_is_number_NoSpace_GB', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1GB - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ( - # tpl_snippet with mem_size given as number+Spaces+GB - 'mem_size_is_number_Spaces_GB', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1 GB - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ( - # tpl_snippet with no mem_size given - 'mem_size_is_absent', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - # host image properties - os_type: Linux - ''', - expected=1024) - ), - ] - - def test_scenario_mem_size(self): - tpl = self.tpl_snippet - nodetemplates = (translator.toscalib.utils.yamlparser. - simple_parse(tpl))['node_templates'] - name = list(nodetemplates.keys())[0] - nodetemplate = NodeTemplate(name, nodetemplates) - for p in nodetemplate.properties: - if p.name == 'mem_size': - resolved = p.value - self.assertEqual(resolved, self.expected) - - -class ToscaTemplateMemorySizeErrorTest(TestCase): - - scenarios = [ - ( - # tpl_snippet with mem_size given as empty (error) - 'mem_size_is_empty', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: - # host image properties - os_type: Linux - ''', - err=InvalidPropertyValueError) - ), - ( - # tpl_snippet with mem_size given as number+InvalidUnit (error) - 'mem_size_unit_is_invalid', - dict(tpl_snippet=''' - node_templates: - server: - type: tosca.nodes.Compute - properties: - # compute properties (flavor) - mem_size: 1 QB - # host image properties - os_type: Linux - ''', - err=InvalidPropertyValueError) - ), - ] - - def test_scenario_mem_size_error(self): - tpl = self.tpl_snippet - nodetemplates = (translator.toscalib.utils.yamlparser. - simple_parse(tpl))['node_templates'] - name = list(nodetemplates.keys())[0] - nodetemplate = NodeTemplate(name, nodetemplates) - self.assertRaises(self.err, - nodetemplate._create_properties) diff --git a/translator/toscalib/tests/test_scalar_unit.py b/translator/toscalib/tests/test_scalar_unit.py new file mode 100644 index 0000000..06d389e --- /dev/null +++ b/translator/toscalib/tests/test_scalar_unit.py @@ -0,0 +1,194 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from translator.toscalib.common.exception import ValidationError +from translator.toscalib.elements.constraints import Constraint +from translator.toscalib.nodetemplate import NodeTemplate +from translator.toscalib.tests.base import TestCase +from translator.toscalib.utils import yamlparser + + +class ScalarUnitPositiveTest(TestCase): + + scenarios = [ + ( + # tpl_snippet with mem_size given as number + 'mem_size_is_number', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1024 + ''', + expected=1024) + ), + ( + # tpl_snippet with mem_size given as number+space+MB + 'mem_size_is_number_Space_MB', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1024 MB + ''', + expected='1024 MB') + ), + ( + # tpl_snippet with mem_size given as number+spaces+GB + 'mem_size_is_number_Space_GB', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1 GB + ''', + expected='1 GB') + ), + ( + # tpl_snippet with mem_size given as number+tiB + 'mem_size_is_number_NoSpace_GB', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1tiB + ''', + expected='1tiB') + ), + ( + # tpl_snippet with mem_size given as number+Spaces+GIB + 'mem_size_is_number_Spaces_GB', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1 GIB + ''', + expected='1 GIB') + ), + ( + # tpl_snippet with mem_size given as number+Space+tib + 'mem_size_is_number_Spaces_GB', + dict(tpl_snippet=''' + server: + type: tosca.nodes.Compute + properties: + mem_size: 1 tib + ''', + expected='1 tib') + ), + ] + + def test_scenario_scalar_unit_positive(self): + tpl = self.tpl_snippet + nodetemplates = yamlparser.simple_parse(tpl) + nodetemplate = NodeTemplate('server', nodetemplates) + for p in nodetemplate.properties: + if p.name == 'mem_size': + self.assertIsNone(p.validate()) + resolved = p.value + self.assertEqual(resolved, self.expected) + + +class GetNumFromScalarUnitSizePositive(TestCase): + + scenarios = [ + ( # Note that (1 TB) / (1 GB) = 1000 + 'Input is TB, user input is GB', + dict(InputMemSize='1 TB', + UserInputUnit='gB', + expected=1000) + ), + ( # Note that (1 Tib)/ (1 GB) = 1099 + 'Input is TiB, user input is GB', + dict(InputMemSize='1 TiB', + UserInputUnit='gB', + expected=1099) + ), + ] + + def test_scenario_get_num_from_scalar_unit_size(self): + resolved = Constraint.get_num_from_scalar_unit_size(self.InputMemSize, + self.UserInputUnit) + self.assertEqual(resolved, self.expected) + + +class GetNumFromScalarUnitSizeNegative(TestCase): + + InputMemSize = '1 GB' + UserInputUnit = 'qB' + + def test_get_num_from_scalar_unit_size_negative(self): + try: + Constraint.get_num_from_scalar_unit_size(self.InputMemSize, + self.UserInputUnit) + except Exception as error: + self.assertTrue(isinstance(error, ValueError)) + self.assertEqual('input unit "qB" is not a valid unit', + error.__str__()) + + +class ScalarUnitNegativeTest(TestCase): + + custom_def_snippet = ''' + tosca.my.nodes.Compute: + derived_from: tosca.nodes.Root + properties: + disk_size: + required: no + type: scalar-unit.size + constraints: + - greater_or_equal: 1 GB + mem_size: + required: no + type: scalar-unit.size + constraints: + - in_range: [1 MiB, 1 GiB] + ''' + custom_def = yamlparser.simple_parse(custom_def_snippet) + + # disk_size doesn't provide a value, mem_size uses an invalid unit. + def test_invalid_scalar_unit(self): + tpl_snippet = ''' + server: + type: tosca.my.nodes.Compute + properties: + disk_size: MB + mem_size: 1 QB + ''' + nodetemplates = yamlparser.simple_parse(tpl_snippet) + nodetemplate = NodeTemplate('server', nodetemplates, self.custom_def) + for p in nodetemplate.properties: + self.assertRaises(ValueError, p.validate) + + # disk_size is less than 1 GB, mem_size is not in the required range. + # Note: in the spec, the minimum value of mem_size is 1 MiB (> 1 MB) + def test_constraint_for_scalar_unit(self): + tpl_snippet = ''' + server: + type: tosca.my.nodes.Compute + properties: + disk_size: 500 MB + mem_size: 1 MB + ''' + nodetemplates = yamlparser.simple_parse(tpl_snippet) + nodetemplate = NodeTemplate('server', nodetemplates, self.custom_def) + for p in nodetemplate.properties: + if p.name == 'disk_size': + error = self.assertRaises(ValidationError, p.validate) + self.assertEqual('disk_size: 500 MB must be greater or ' + 'equal to "1 GB".', error.__str__()) + if p.name == 'mem_size': + error = self.assertRaises(ValidationError, p.validate) + self.assertEqual('mem_size: 1 MB is out of range ' + '(min:1 MiB, ' + 'max:1 GiB).', error.__str__()) diff --git a/translator/toscalib/tests/test_toscadef.py b/translator/toscalib/tests/test_toscadef.py index bb39cb3..c41e84c 100644 --- a/translator/toscalib/tests/test_toscadef.py +++ b/translator/toscalib/tests/test_toscadef.py @@ -64,10 +64,3 @@ class ToscaDefTest(TestCase): self.assertEqual(compute_type.interfaces, None) root_node = NodeType('tosca.nodes.Root') self.assertIn('tosca.interfaces.node.Lifecycle', root_node.interfaces) - - def test_default_mem_size(self): - test_value = 0 - for p_def in compute_type.properties_def: - if p_def.name == 'mem_size': - test_value = p_def.default - self.assertEqual(test_value, 1024)