Add scalar unit type

Implement scalar-unit.size type:

1. any number of spaces (including zero or none) is allowed between the
scalar value and the unit value.
2. support constraints like greater_or_equal and in_range. (eg:
storage_size: in_range [4 GB, 20 GB ] )
3. default unit is 'B'(byte) according to the TOSCA profile while
allowing user to set the unit of the output number as long as the input
unit is a valid one
4. supported scalar-unit.size (case-insensitive): B, KB, KiB, MB, MiB,
GB, GiB, TB, TiB

implement BP: add-scalar-unit-type

Change-Id: Idf3705312afd33d7aeadc3f8856cf728c41ec190

translator/toscalib/dataentity.py

@@ -113,6 +113,8 @@ class DataEntity(object):
             if entry_schema:
                 DataEntity.validate_entry(value, entry_schema, custom_def)
             return value
+        elif type == Schema.SCALAR_UNIT_SIZE:
+            return Constraint.validate_scalar_unit_size(value)
         elif type == Schema.MAP:
             Constraint.validate_map(value)
             if entry_schema:

translator/toscalib/elements/TOSCA_definition.yaml

@@ -38,21 +38,19 @@ tosca.nodes.Compute:
         Number of (actual or virtual) CPUs associated with the Compute node.
     disk_size:
       required: no
-      type: integer
+      type: scalar-unit.size
       constraints:
-        - greater_or_equal: 0
+        - greater_or_equal: 0 MB
       description: >
-        Size of the local disk, in Gigabytes (GB), available to applications
+        Size of the local disk, available to applications running on the
-        running on the Compute node.
+        Compute node.
     mem_size:
       required: no
-      type: integer
+      type: scalar-unit.size
       constraints:
-        - greater_or_equal: 0
+        - greater_or_equal: 0 MB
       description: >
-        Size of memory, in Megabytes (MB), available to applications running
+        Size of memory, available to applications running on the Compute node.
-        on the Compute node.
-      default: 1024
     os_arch:
       required: no
       default: x86_64

translator/toscalib/elements/constraints.py

@@ -12,6 +12,7 @@
 
 import collections
 import datetime
+import math
 import numbers
 import re
 import six
@@ -34,12 +35,18 @@ class Schema(collections.Mapping):
 
     PROPERTY_TYPES = (
         INTEGER, STRING, BOOLEAN, FLOAT,
-        NUMBER, TIMESTAMP, LIST, MAP
+        NUMBER, TIMESTAMP, LIST, MAP, SCALAR_UNIT_SIZE
     ) = (
         'integer', 'string', 'boolean', 'float',
-        'number', 'timestamp', 'list', 'map'
+        'number', 'timestamp', 'list', 'map', 'scalar-unit.size'
     )
 
+    SCALAR_UNIT_SIZE_DEFAULT = 'B'
+    SCALAR_UNIT_SIZE_DICT = {'B': 1, 'KB': 1000, 'KIB': 1024, 'MB': 1000000,
+                             'MIB': 1048576, 'GB': 1000000000,
+                             'GIB': 1073741824, 'TB': 1000000000000,
+                             'TIB': 1099511627776}
+
     def __init__(self, name, schema_dict):
         self.name = name
         if not isinstance(schema_dict, collections.Mapping):
@@ -135,6 +142,16 @@ class Constraint(object):
         self.property_name = property_name
         self.property_type = property_type
         self.constraint_value = constraint[self.constraint_key]
+        self.constraint_value_msg = self.constraint_value
+        if self.property_type == Schema.SCALAR_UNIT_SIZE:
+            if isinstance(self.constraint_value, list):
+                self.constraint_value = [Constraint.
+                                         get_num_from_scalar_unit_size(v)
+                                         for v in self.constraint_value]
+            else:
+                self.constraint_value = (Constraint.
+                                         get_num_from_scalar_unit_size
+                                         (self.constraint_value))
         # check if constraint is valid for property type
         if property_type not in self.valid_prop_types:
             msg = _('Constraint type "%(ctype)s" is not valid '
@@ -147,6 +164,9 @@ class Constraint(object):
         return _('Property %s could not be validated.') % self.property_name
 
     def validate(self, value):
+        self.value_msg = value
+        if self.property_type == Schema.SCALAR_UNIT_SIZE:
+            value = self.get_num_from_scalar_unit_size(value)
         if not self._is_valid(value):
             err_msg = self._err_msg(value)
             raise ValidationError(message=err_msg)
@@ -190,6 +210,36 @@ class Constraint(object):
                 return normalised == 'true'
         raise ValueError(_('"%s" is not a boolean') % value)
 
+    @staticmethod
+    def validate_scalar_unit_size(value):
+        regex = re.compile('(\d*)\s*(\w*)')
+        result = regex.match(str(value)).groups()
+        if result[0] and ((not result[1]) or (result[1].upper() in
+                                              Schema.SCALAR_UNIT_SIZE_DICT.
+                                              keys())):
+            return value
+        raise ValueError(_('"%s" is not a valid scalar-unit') % value)
+
+    @staticmethod
+    def get_num_from_scalar_unit_size(value, unit=None):
+        if unit:
+            if unit.upper() not in Schema.SCALAR_UNIT_SIZE_DICT.keys():
+                raise ValueError(_('input unit "%s" is not a valid unit')
+                                 % unit)
+        else:
+            unit = Schema.SCALAR_UNIT_SIZE_DEFAULT
+        Constraint.validate_scalar_unit_size(value)
+        regex = re.compile('(\d*)\s*(\w*)')
+        result = regex.match(str(value)).groups()
+        if not result[1]:
+            converted = (Constraint.str_to_num(result[0]))
+        if result[1].upper() in Schema.SCALAR_UNIT_SIZE_DICT.keys():
+            converted = int(Constraint.str_to_num(result[0])
+                            * Schema.SCALAR_UNIT_SIZE_DICT[result[1].upper()]
+                            * math.pow(Schema.SCALAR_UNIT_SIZE_DICT
+                                       [unit.upper()], -1))
+        return converted
+
     @staticmethod
     def str_to_num(value):
         '''Convert a string representation of a number into a numeric type.'''
@@ -221,8 +271,8 @@ class Equal(Constraint):
     def _err_msg(self, value):
         return (_('%(pname)s: %(pvalue)s is not equal to "%(cvalue)s".') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     cvalue=self.constraint_value))
+                     cvalue=self.constraint_value_msg))
 
 
 class GreaterThan(Constraint):
@@ -238,7 +288,7 @@ class GreaterThan(Constraint):
                    datetime.time, datetime.datetime)
 
     valid_prop_types = (Schema.INTEGER, Schema.FLOAT,
-                        Schema.TIMESTAMP)
+                        Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE)
 
     def __init__(self, property_name, property_type, constraint):
         super(GreaterThan, self).__init__(property_name, property_type,
@@ -256,8 +306,8 @@ class GreaterThan(Constraint):
     def _err_msg(self, value):
         return (_('%(pname)s: %(pvalue)s must be greater than "%(cvalue)s".') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     cvalue=self.constraint_value))
+                     cvalue=self.constraint_value_msg))
 
 
 class GreaterOrEqual(Constraint):
@@ -273,7 +323,7 @@ class GreaterOrEqual(Constraint):
                    datetime.time, datetime.datetime)
 
     valid_prop_types = (Schema.INTEGER, Schema.FLOAT,
-                        Schema.TIMESTAMP)
+                        Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE)
 
     def __init__(self, property_name, property_type, constraint):
         super(GreaterOrEqual, self).__init__(property_name, property_type,
@@ -291,8 +341,8 @@ class GreaterOrEqual(Constraint):
         return (_('%(pname)s: %(pvalue)s must be greater or equal '
                   'to "%(cvalue)s".') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     cvalue=self.constraint_value))
+                     cvalue=self.constraint_value_msg))
 
 
 class LessThan(Constraint):
@@ -308,7 +358,7 @@ class LessThan(Constraint):
                    datetime.time, datetime.datetime)
 
     valid_prop_types = (Schema.INTEGER, Schema.FLOAT,
-                        Schema.TIMESTAMP)
+                        Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE)
 
     def __init__(self, property_name, property_type, constraint):
         super(LessThan, self).__init__(property_name, property_type,
@@ -326,8 +376,8 @@ class LessThan(Constraint):
     def _err_msg(self, value):
         return (_('%(pname)s: %(pvalue)s must be less than "%(cvalue)s".') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     cvalue=self.constraint_value))
+                     cvalue=self.constraint_value_msg))
 
 
 class LessOrEqual(Constraint):
@@ -343,7 +393,7 @@ class LessOrEqual(Constraint):
                    datetime.time, datetime.datetime)
 
     valid_prop_types = (Schema.INTEGER, Schema.FLOAT,
-                        Schema.TIMESTAMP)
+                        Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE)
 
     def __init__(self, property_name, property_type, constraint):
         super(LessOrEqual, self).__init__(property_name, property_type,
@@ -362,8 +412,8 @@ class LessOrEqual(Constraint):
         return (_('%(pname)s: %(pvalue)s must be less or '
                   'equal to "%(cvalue)s".') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     cvalue=self.constraint_value))
+                     cvalue=self.constraint_value_msg))
 
 
 class InRange(Constraint):
@@ -379,7 +429,7 @@ class InRange(Constraint):
                    datetime.time, datetime.datetime)
 
     valid_prop_types = (Schema.INTEGER, Schema.FLOAT,
-                        Schema.TIMESTAMP)
+                        Schema.TIMESTAMP, Schema.SCALAR_UNIT_SIZE)
 
     def __init__(self, property_name, property_type, constraint):
         super(InRange, self).__init__(property_name, property_type, constraint)
@@ -407,9 +457,9 @@ class InRange(Constraint):
         return (_('%(pname)s: %(pvalue)s is out of range '
                   '(min:%(vmin)s, max:%(vmax)s).') %
                 dict(pname=self.property_name,
-                     pvalue=value,
+                     pvalue=self.value_msg,
-                     vmin=self.min,
+                     vmin=self.constraint_value_msg[0],
-                     vmax=self.max))
+                     vmax=self.constraint_value_msg[1]))
 
 
 class ValidValues(Constraint):

translator/toscalib/properties.py

@@ -10,9 +10,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import re
-
-from translator.toscalib.common.exception import InvalidPropertyValueError
 from translator.toscalib.dataentity import DataEntity
 from translator.toscalib.elements.constraints import Schema
 from translator.toscalib.functions import is_function
@@ -35,7 +32,7 @@ class Property(object):
 
     def __init__(self, property_name, value, schema_dict, custom_def=None):
         self.name = property_name
-        self.value = self._convert_value(value)
+        self.value = value
         self.custom_def = custom_def
         self.schema = Schema(property_name, schema_dict)
 
@@ -76,20 +73,3 @@ class Property(object):
         if self.constraints:
             for constraint in self.constraints:
                 constraint.validate(self.value)
-
-    def _convert_value(self, value):
-        if self.name == 'mem_size':
-            mem_reader = re.compile('(\d*)\s*(\w*)')
-            matcher = str(value)
-            result = mem_reader.match(matcher).groups()
-            r = []
-            if (result[0] != '') and (result[1] == ''):
-                r = int(result[0])
-            elif (result[0] != '') and (result[1] == 'MB'):
-                r = int(result[0])
-            elif (result[0] != '') and (result[1] == 'GB'):
-                r = int(result[0]) * 1024
-            else:
-                raise InvalidPropertyValueError(what=self.name)
-            return r
-        return value

translator/toscalib/tests/test_memunit.py

@@ -1,167 +0,0 @@
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from translator.toscalib.common.exception import InvalidPropertyValueError
-from translator.toscalib.nodetemplate import NodeTemplate
-from translator.toscalib.tests.base import TestCase
-import translator.toscalib.utils.yamlparser
-
-
-class ToscaTemplateMemorySizeOutputTest(TestCase):
-
-    scenarios = [
-        (
-            # tpl_snippet with mem_size given as number
-            'mem_size_is_number',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1024
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-        (
-            # tpl_snippet with mem_size given as number+space+MB
-            'mem_size_is_number_Space_MB',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1024 MB
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-        (
-            # tpl_snippet with mem_size given as number+space+GB
-            'mem_size_is_number_Space_GB',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1 GB
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-        (
-            # tpl_snippet with mem_size given as number+GB
-            'mem_size_is_number_NoSpace_GB',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1GB
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-        (
-            # tpl_snippet with mem_size given as number+Spaces+GB
-            'mem_size_is_number_Spaces_GB',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1     GB
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-        (
-            # tpl_snippet with no mem_size given
-            'mem_size_is_absent',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 expected=1024)
-        ),
-    ]
-
-    def test_scenario_mem_size(self):
-        tpl = self.tpl_snippet
-        nodetemplates = (translator.toscalib.utils.yamlparser.
-                         simple_parse(tpl))['node_templates']
-        name = list(nodetemplates.keys())[0]
-        nodetemplate = NodeTemplate(name, nodetemplates)
-        for p in nodetemplate.properties:
-            if p.name == 'mem_size':
-                resolved = p.value
-        self.assertEqual(resolved, self.expected)
-
-
-class ToscaTemplateMemorySizeErrorTest(TestCase):
-
-    scenarios = [
-        (
-            # tpl_snippet with mem_size given as empty (error)
-            'mem_size_is_empty',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size:
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 err=InvalidPropertyValueError)
-        ),
-        (
-            # tpl_snippet with mem_size given as number+InvalidUnit (error)
-            'mem_size_unit_is_invalid',
-            dict(tpl_snippet='''
-                 node_templates:
-                   server:
-                     type: tosca.nodes.Compute
-                     properties:
-                       # compute properties (flavor)
-                       mem_size: 1    QB
-                       # host image properties
-                       os_type: Linux
-                 ''',
-                 err=InvalidPropertyValueError)
-        ),
-    ]
-
-    def test_scenario_mem_size_error(self):
-        tpl = self.tpl_snippet
-        nodetemplates = (translator.toscalib.utils.yamlparser.
-                         simple_parse(tpl))['node_templates']
-        name = list(nodetemplates.keys())[0]
-        nodetemplate = NodeTemplate(name, nodetemplates)
-        self.assertRaises(self.err,
-                          nodetemplate._create_properties)

translator/toscalib/tests/test_scalar_unit.py

@@ -0,0 +1,194 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from translator.toscalib.common.exception import ValidationError
+from translator.toscalib.elements.constraints import Constraint
+from translator.toscalib.nodetemplate import NodeTemplate
+from translator.toscalib.tests.base import TestCase
+from translator.toscalib.utils import yamlparser
+
+
+class ScalarUnitPositiveTest(TestCase):
+
+    scenarios = [
+        (
+            # tpl_snippet with mem_size given as number
+            'mem_size_is_number',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1024
+                 ''',
+                 expected=1024)
+        ),
+        (
+            # tpl_snippet with mem_size given as number+space+MB
+            'mem_size_is_number_Space_MB',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1024 MB
+                 ''',
+                 expected='1024 MB')
+        ),
+        (
+            # tpl_snippet with mem_size given as number+spaces+GB
+            'mem_size_is_number_Space_GB',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1     GB
+                 ''',
+                 expected='1     GB')
+        ),
+        (
+            # tpl_snippet with mem_size given as number+tiB
+            'mem_size_is_number_NoSpace_GB',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1tiB
+                 ''',
+                 expected='1tiB')
+        ),
+        (
+            # tpl_snippet with mem_size given as number+Spaces+GIB
+            'mem_size_is_number_Spaces_GB',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1     GIB
+                 ''',
+                 expected='1     GIB')
+        ),
+        (
+            # tpl_snippet with mem_size given as number+Space+tib
+            'mem_size_is_number_Spaces_GB',
+            dict(tpl_snippet='''
+                 server:
+                   type: tosca.nodes.Compute
+                   properties:
+                     mem_size: 1 tib
+                 ''',
+                 expected='1 tib')
+        ),
+    ]
+
+    def test_scenario_scalar_unit_positive(self):
+        tpl = self.tpl_snippet
+        nodetemplates = yamlparser.simple_parse(tpl)
+        nodetemplate = NodeTemplate('server', nodetemplates)
+        for p in nodetemplate.properties:
+            if p.name == 'mem_size':
+                self.assertIsNone(p.validate())
+                resolved = p.value
+        self.assertEqual(resolved, self.expected)
+
+
+class GetNumFromScalarUnitSizePositive(TestCase):
+
+    scenarios = [
+        (   # Note that (1 TB) / (1 GB) = 1000
+            'Input is TB, user input is GB',
+            dict(InputMemSize='1   TB',
+                 UserInputUnit='gB',
+                 expected=1000)
+        ),
+        (   # Note that (1 Tib)/ (1 GB) = 1099
+            'Input is TiB, user input is GB',
+            dict(InputMemSize='1   TiB',
+                 UserInputUnit='gB',
+                 expected=1099)
+        ),
+    ]
+
+    def test_scenario_get_num_from_scalar_unit_size(self):
+        resolved = Constraint.get_num_from_scalar_unit_size(self.InputMemSize,
+                                                            self.UserInputUnit)
+        self.assertEqual(resolved, self.expected)
+
+
+class GetNumFromScalarUnitSizeNegative(TestCase):
+
+    InputMemSize = '1 GB'
+    UserInputUnit = 'qB'
+
+    def test_get_num_from_scalar_unit_size_negative(self):
+        try:
+            Constraint.get_num_from_scalar_unit_size(self.InputMemSize,
+                                                     self.UserInputUnit)
+        except Exception as error:
+            self.assertTrue(isinstance(error, ValueError))
+            self.assertEqual('input unit "qB" is not a valid unit',
+                             error.__str__())
+
+
+class ScalarUnitNegativeTest(TestCase):
+
+    custom_def_snippet = '''
+    tosca.my.nodes.Compute:
+      derived_from: tosca.nodes.Root
+      properties:
+        disk_size:
+          required: no
+          type: scalar-unit.size
+          constraints:
+            - greater_or_equal: 1 GB
+        mem_size:
+          required: no
+          type: scalar-unit.size
+          constraints:
+            - in_range: [1 MiB, 1 GiB]
+    '''
+    custom_def = yamlparser.simple_parse(custom_def_snippet)
+
+    # disk_size doesn't provide a value, mem_size uses an invalid unit.
+    def test_invalid_scalar_unit(self):
+        tpl_snippet = '''
+        server:
+          type: tosca.my.nodes.Compute
+          properties:
+            disk_size: MB
+            mem_size: 1 QB
+        '''
+        nodetemplates = yamlparser.simple_parse(tpl_snippet)
+        nodetemplate = NodeTemplate('server', nodetemplates, self.custom_def)
+        for p in nodetemplate.properties:
+            self.assertRaises(ValueError, p.validate)
+
+    # disk_size is less than 1 GB, mem_size is not in the required range.
+    # Note: in the spec, the minimum value of mem_size is 1 MiB (> 1 MB)
+    def test_constraint_for_scalar_unit(self):
+        tpl_snippet = '''
+        server:
+          type: tosca.my.nodes.Compute
+          properties:
+            disk_size: 500 MB
+            mem_size: 1 MB
+        '''
+        nodetemplates = yamlparser.simple_parse(tpl_snippet)
+        nodetemplate = NodeTemplate('server', nodetemplates, self.custom_def)
+        for p in nodetemplate.properties:
+            if p.name == 'disk_size':
+                error = self.assertRaises(ValidationError, p.validate)
+                self.assertEqual('disk_size: 500 MB must be greater or '
+                                 'equal to "1 GB".', error.__str__())
+            if p.name == 'mem_size':
+                error = self.assertRaises(ValidationError, p.validate)
+                self.assertEqual('mem_size: 1 MB is out of range '
+                                 '(min:1 MiB, '
+                                 'max:1 GiB).', error.__str__())

translator/toscalib/tests/test_toscadef.py

@@ -64,10 +64,3 @@ class ToscaDefTest(TestCase):
         self.assertEqual(compute_type.interfaces, None)
         root_node = NodeType('tosca.nodes.Root')
         self.assertIn('tosca.interfaces.node.Lifecycle', root_node.interfaces)
-
-    def test_default_mem_size(self):
-        test_value = 0
-        for p_def in compute_type.properties_def:
-            if p_def.name == 'mem_size':
-                test_value = p_def.default
-        self.assertEqual(test_value, 1024)