First code drop on data modeling.

This is the first code drop for the project. Covers basis project structure
and reading tosca profile into a memory model.

heat-translator/exit.py

@@ -0,0 +1,2 @@
+def write_output(output):
+    pass

heat-translator/main.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+import exit
+import os
+import sys
+from source import Source
+from tosca.translate import TOSCATranslator
+from tosca.tosca_profile import Tosca
+from tosca.validate import ToscaValidator
+
+
+def main():
+    sourcetype = sys.argv[1]
+    path = sys.argv[2]
+    #sourcetype = "tosca"
+    #path = "/heat-translator/tosca/tests/tosca.yaml"
+    if not sourcetype:
+        print("Translation type is needed. For example, 'tosca'")
+    if not path.endswith(".yaml"):
+        print "Only YAML file is supported at this time."
+        
+    if os.path.isdir(path):
+        print('Translation of directory is not supported at this time : %s' % path)
+    elif os.path.isfile(path):
+        heat_tpl = translate(sourcetype, path)
+        exit.write_output(heat_tpl)
+    else:
+        print('%s is not a valid file.' % path)
+
+def translate(sourcetype, path):
+    tpl = Source(path)
+    if sourcetype == "tosca":
+        tosca = Tosca(tpl)
+        ToscaValidator(tosca).validate()
+        return TOSCATranslator(tosca).translate()
+
+if __name__ == '__main__':
+    main()

heat-translator/source.py

@@ -0,0 +1,18 @@
+from yaml_loader import Loader
+
+class Source(object):
+    def __init__(self, path):
+        self.profile = Loader(path).load()
+            
+    def __contains__(self, key):
+        return key in self.profile
+
+    def __iter__(self):
+        return iter(self.profile)
+
+    def __len__(self):
+        return len(self.profile)
+
+    def __getitem__(self, key):
+        '''Get a section.'''
+        return self.profile[key]

heat-translator/tosca/inputs.py

@@ -0,0 +1,66 @@
+from tosca.nodetemplates.schema import Schema
+from tosca.nodetemplates.constraints import Constraint
+
+class InputParameters(object):
+    def __init__(self, inputs):
+        self.inputs = inputs
+
+    def __contains__(self, key):
+        return key in self.inputs
+
+    def __iter__(self):
+        return iter(self.inputs)
+
+    def __len__(self):
+        return len(self.inputs)
+
+    def __getitem__(self, key):
+        '''Get a input value.'''
+        return self.inputs[key]
+
+class Input(object):
+                    
+    def __init__(self, name, schema):
+        self.name = name
+        self.schema = schema
+    
+    def get_type(self):
+        return self.schema['type']
+    
+    def get_description(self):
+        if self.has_default():
+            return self.schema['description']
+        return ''
+    
+    def get_default(self):
+        if self.has_default():
+            return self.schema['default']
+        return ''
+    
+    def get_constraints(self):
+        return self.schema['constraints']
+    
+    def has_default(self):
+        '''Return whether the parameter has a default value.'''
+        return Schema.DEFAULT in self.schema
+
+    def has_description(self):
+        '''Return whether the parameter has a default value.'''
+        return Schema.DESCRIPTION in self.schema
+        
+    def validate(self):
+        self.validate_type(self.get_type())
+        self.validate_constraints(self.get_constraints())
+        
+    def validate_type(self, input_type):
+        if input_type not in Schema.TYPES:
+            raise ValueError('Invalid type %s' % type)
+    
+    def validate_constraints(self, constraints):
+        for constraint in constraints:
+            for key in constraint.keys():
+                if key not in Constraint.CONSTRAINTS:
+                    raise ValueError('Invalid constraint %s' % constraint)
+                if isinstance(key, dict): #and is_required to have a min or max or in certain range or equal to something etc.
+                    pass
+    

heat-translator/tosca/nodetemplates/constraints.py

@@ -0,0 +1,87 @@
+import numbers    
+
+class Constraint(object): 
+    
+    CONSTRAINTS = (EQUAL, GREATER_THAN,
+                   GREATER_OR_EQUAL, LESS_THAN, LESS_OR_EQUAL, IN_RANGE, VALID_VALUES,
+                   LENGTH, MIN_LENGHT, MAX_LENGTH, PATTERN) = \
+                   ('equal', 'greater_than', 'greater_or_equal',
+                     'less_than', 'less_or_equal', 'in_range', 'valid_values', 'length', 'min_length',
+                     'max_length', 'pattern')
+                    
+    def __init__(self, propertyname, value, constraint): 
+        self.propertyname = propertyname
+        self.value = value
+        self.constraint = constraint #dictionary e.g. valid_values: [ 1, 2, 4, 8 ] or greater_or_equal: 1
+    
+    def validate(self):
+        # for key in self.constraint.iterkeys():
+        for key, value in self.constraint.iteritems():
+            if key == "greater_or_equal":
+                self.validate_greater_than(value)
+        
+    def validate_equal(self):
+        pass
+
+    def validate_greater_than(self, value):
+        if self.value < value:
+            print("%s value requires to be greater than %s" % (self.propertyname, value))
+    
+    def validate_greater_or_equal(self):
+        pass
+    
+    def validate_less_than(self):
+        pass
+    
+    def validate_less_or_equal(self):
+        pass
+    
+    def validate_in_range(self):
+        pass
+    
+    def validate_valid_values(self):
+        pass
+    
+    def validate_length(self):
+        pass
+    
+    def validate_min_length(self):
+        pass
+    
+    def validate_max_length(self):
+        pass
+    
+    def validate_pattern(self):
+        pass
+
+    @staticmethod
+    def validate_integer(value):
+        if not isinstance(value, (int, long)):
+            import pdb 
+            pdb.set_trace()
+            raise TypeError('value is not an integer for %s' %value)
+        return Constraint.validate_number(value)
+    
+    @staticmethod
+    def validate_number(value):
+        return Constraint.str_to_num(value)
+
+    @staticmethod
+    def validate_string(value):
+        if not isinstance(value, basestring):
+            raise ValueError('Value must be a string %s' %value)
+        return value
+    
+    @staticmethod
+    def validate_list(self, value):
+        pass
+    
+    @staticmethod
+    def str_to_num(value):
+        """Convert a string representation of a number into a numeric type."""
+        if isinstance(value, numbers.Number):
+            return value
+        try:
+            return int(value)
+        except ValueError:
+            return float(value)

heat-translator/tosca/nodetemplates/node_template.py

@@ -0,0 +1,117 @@
+from properties import Property
+from tosca.nodetemplates.schema import Schema
+from tosca.inputs import InputParameters
+from tosca.inputs import Input
+from tosca.nodetemplates.node_templates import NodeTemplates
+
+class NodeTemplate(object):
+    def __init__(self, name, nodetemplate, tosca):
+        self.name = name
+        self.nodetemplate = nodetemplate
+        self.tosca = tosca
+    
+    def get_node(self):
+        return self.nodetemplate
+    
+    def get_name(self):
+        return self.name
+    
+    def get_type(self):
+        return self.nodetemplate['type']
+    
+    def get_properties(self):
+        return self.nodetemplate['properties']
+    
+    def validate(self):
+        self.validate_prpoerties()
+        self.validate_type()
+        
+    #TODO
+    def validate_type(self):
+        pass
+    
+    #TODO
+    def validate_relationship(self):
+        pass
+    
+    def validate_prpoerties(self):
+        '''validate that required properties for a particular nodetype is provided and matches constraints'''
+        nodetype = self.get_type()
+        required_props = Schema(nodetype).required()
+        for req in required_props:
+            if req not in self.get_properties():
+                raise ValueError('Missing required property %s' %req)
+        for prop, val in self.get_properties().iteritems():
+            if isinstance(val, dict):
+                for key, value in val.iteritems():           
+                    if key == "get_input":
+                        val = self.get_input_ref(value)
+                        break
+                    if key == "get_property":
+                        val = self.get_property_ref(value, self.tosca)
+            if val is not None:
+                Property(prop, self.get_type(), val).validate()
+            
+    def get_input_ref(self, ref):
+        ''' get input reference, for example, get_input: db_user '''
+        input_val = self.tosca.get_inputs()[ref]
+        if Input(ref, input_val).has_default():
+            return Input(ref, input_val).get_default()
+    
+    @classmethod
+    def get_property_ref(cls, ref, t):
+        ''' get property reference, for example, get_property: [ mysql, db_user ] '''
+        item = ref[0]
+        n = t.get_nodetemplates()[item]
+        c = cls(item, n, t)
+        for val in c.get_properties().itervalues():
+            if isinstance(val, dict):
+                for key, value in val.iteritems():           
+                    if key == "get_input":
+                        return c.get_input_ref(value)
+        
+    def get_relationship(self):
+        return self.nodetemplate['requirements']
+    
+    def get_hostedOn_relationship(self):
+        hosted = []
+        r = self.get_relationship()
+        for key, value in r.iteritems():
+            if key == "host":
+                hosted.append(value)
+        return hosted
+    
+    def get_dependsOn_relationship(self):
+        depends = []
+        r = self.get_relationship()
+        for key, value in r.iteritems():
+            if key == "dependency":
+                depends.append(value)
+        return depends
+    
+    def get_connectedTo_relationship(self):
+        connected = []
+        r = self.get_relationship()
+        for key, value in r.iteritems():
+            if key == "database":
+                connected.append(value)
+        return connected
+    
+    def is_hostedOn(self):
+        pass
+    
+    def is_dependsOn(self):
+        pass
+    
+    def is_connectedTo(self):
+        pass
+    
+    def get_interfaces(self):
+        return self.nodetemplate['interfaces']
+
+    def get_interface_inputs(self, interface):
+        i = self.get_interfaces()
+        for key, value in i.iteritems():
+            if key == interface:
+                return value
+            

heat-translator/tosca/nodetemplates/node_templates.py

@@ -0,0 +1,19 @@
+from tosca.tosca_profile import Tosca
+
+class NodeTemplates(object):
+    
+    def __init__(self, nodetemplates):
+        self.nodetemplates = nodetemplates
+    
+    def __contains__(self, key):
+        return key in self.nodetemplates
+
+    def __iter__(self):
+        return iter(self.nodetemplates)
+
+    def __len__(self):
+        return len(self.nodetemplates)
+
+    def __getitem__(self, key):
+        '''Get a node template value.'''
+        return self.nodetemplates[key]

heat-translator/tosca/nodetemplates/nodetype.py

@@ -0,0 +1,29 @@
+from rootnodetype import RootNodeType
+
+class NodeType(RootNodeType):
+    def __init__(self, nodetype): 
+        super(NodeType, self).__init__()
+        self.nodetype = nodetype
+    
+    ''' get properties for a given node type'''
+    def get_properties(self):
+        properties = []
+        nodetype = self.get_nodetype()
+        for prop_key, prop_vale in nodetype.iteritems():
+            if prop_key == 'properties':
+                properties = prop_vale
+        return properties
+    
+    ''' get capabilities for a given node type'''
+    def get_capabilities(self):
+        pass
+    
+    def derived_from(self):
+        pass
+    
+    def set_requirements(self):
+        pass
+    
+    def get_requirements(self):
+        return self.requirements()
+    

heat-translator/tosca/nodetemplates/nodetypeschema.yaml

@@ -0,0 +1,49 @@
+tosca_nodes_Compute:
+  num_cpus:
+    required: no
+    type: integer
+    constraints:
+       - greater_or_equal: 1
+    description: >
+       Number of (actual or virtual) CPUs associated with the Compute node.
+  disk_size: 
+    required: no
+    type: integer
+    constraints:
+       - greater_or_equal: 0
+    description: >
+      Size of the local disk, in Gigabytes (GB), available to applications running on the Compute node.
+  mem_size: 
+    required: no
+    type: integer
+    constraints:
+       - greater_or_equal: 0
+    description: >
+      Size of memory, in Megabytes (MB), available to applications running on the Compute node.
+  os_arch: 
+    required: no
+    default: x86_64 
+    type: string
+    description: >
+      The host Operating System (OS) architecture.
+  os_type: 
+    required: yes
+    type: string
+    description: >
+      The host Operating System (OS) type.
+  os_distribution: 
+    required: no
+    type: string
+    description: >
+      The host Operating System (OS) distribution. Examples of valid values for an “os_type” of “Linux” would include:  
+      debian, fedora, rhel and ubuntu.
+  os_version: 
+    required: no
+    type: integer
+    description: >
+      The host Operating System version.
+  ip_address:
+    required: no
+    type: string
+    description: >
+       The primary IP address assigned by the cloud provider that applications may use to access the Compute node.

heat-translator/tosca/nodetemplates/nodetypesdef.yaml

@@ -0,0 +1,46 @@
+tosca_nodes_Root:
+   properties:
+      - property
+   capabilities:
+      - capability1: somecapbility
+   interfaces:
+      - create
+      - start
+      - stop
+      - install
+      - delete
+         
+tosca_nodes_Compute:
+   derived_from: tosca_nodes_Root
+   properties:
+      - num_cpus
+      - disk_size
+      - mem_size
+      - os_arch
+      - os_type
+      - os_version
+      - ip_address
+   requirements:
+      - host: none
+      - database: none
+   capabilities:
+      container:
+         type: Container
+         containee_types: [tosca_nodes_SoftwareComponent]
+      
+tosca_nodes_SoftwareComponent:
+   derived_from: tosca_nodes_Root
+   requirements:
+      - host: tosca.nodes.Compute
+   capabilities:
+      containee:
+         type: Containee
+         container_types: [tosca.nodes.Compute]
+
+tosca.nodes.MySQL:
+   derived_from: tosca_nodes_SoftwareComponent
+   properties:
+      - db_user
+      - db_pwd
+      - db_root_pwd
+   

heat-translator/tosca/nodetemplates/properties.py

@@ -0,0 +1,35 @@
+from tosca.nodetemplates.schema import Schema
+from tosca.nodetemplates.constraints import Constraint
+
+class Property(object):
+    def __init__(self, name, nodetype, value):
+        self.name = name
+        self.nodetype = nodetype
+        self.value = value
+    
+    def is_required(self):
+        return Schema(self.nodetype).is_required(self.name)
+    
+    def get_name(self):
+        return self.name
+    
+    def validate(self):
+        self.validate_data_type()
+        self.validate_constraints()
+    
+    def validate_data_type(self):
+        data_type = Schema(self.nodetype).get_type(self.name)
+        if data_type == Schema.STRING:
+            return Constraint.validate_string(self.value)
+        elif data_type == Schema.INTEGER:
+            return Constraint.validate_integer(self.value)
+        elif data_type == Schema.NUMBER:
+            return Constraint.validate_number(self.value)
+    
+    def validate_constraints(self):
+        constraints = Schema(self.nodetype).get_constraints(self.name)
+        if constraints:
+            for constraint in constraints:
+                Constraint(self.name, self.value, constraint).validate()
+        pass
+    

heat-translator/tosca/nodetemplates/relationshiptype.py

@@ -0,0 +1,6 @@
+from rootnodetype import RootRelationshipType
+
+class RelatonshipType(RootRelationshipType):
+    def __init__(self, relatointype): 
+        super(RelatonshipType, self).__init__()
+        self.nodetype = relatointype

heat-translator/tosca/nodetemplates/rootnodetype.py

@@ -0,0 +1,31 @@
+from yaml_loader import Loader
+import os
+
+nodetype_def_file = os.path.dirname(os.path.abspath(__file__)) + os.sep + "nodetypesdef.yaml"
+nodetype_def = Loader(nodetype_def_file).load()
+
+class RootNodeType(object):
+    def __init__(self):
+        self.nodetypes = self._set_nodetypes()
+
+        '''set a list of node names from the nodetype definition'''
+    def _set_nodetypes(self):
+        nodetypes = []
+        if isinstance(nodetype_def, dict):
+            for key in nodetype_def.iterkeys():
+                nodetypes.append(key)
+        return nodetypes
+    
+    def get_nodetype(self, nodetype=None):
+        rtype = {}
+        if nodetype == None:
+            nodetype = self.nodetype
+        ntype = nodetype
+        ntype = ntype.replace(".", "_")
+        if ntype in self.nodetypes:
+            rtype = nodetype_def[ntype]
+        return rtype
+
+class RootRelationshipType(object):
+    def __init__(self):
+        pass  

heat-translator/tosca/nodetemplates/schema.py

@@ -0,0 +1,88 @@
+from yaml_loader import Loader
+import os
+
+schema_file = os.path.dirname(os.path.abspath(__file__)) + os.sep + "nodetypeschema.yaml"
+schema = Loader(schema_file).load()
+
+class Schema(object):
+    
+    TYPES = (
+        INTEGER,
+        STRING, NUMBER, BOOLEAN,
+        LIST
+    ) = (
+        'integer',
+        'string', 'number', 'boolean',
+        'list'
+    )
+    
+    KEYS = (
+        TYPE, DESCRIPTION, DEFAULT, CONSTRAINTS,
+    ) = (
+        'type', 'description', 'default', 'constraints'
+    )
+                    
+    def __init__(self, nodetype): 
+        self.nodetype = nodetype
+        self.nodes = self._set_nodes()
+    
+        '''set a list of node names from the schema'''
+    def _set_nodes(self):
+        sections = []
+        if isinstance(schema, dict):
+            for key in schema.iterkeys():
+                sections.append(key)
+        return sections
+    
+    def _get_section(self, section_name):
+        section = {}
+        if section_name in self.nodes:
+            return schema[section_name]
+        return section
+    
+    ''' return true if property is a required for a given node '''
+    def is_required(self, property_name):
+        return property_name in self.required()
+    
+    ''' get schemata for a given node type'''
+    def get_schemata(self):
+        ntype = self.nodetype
+        ntype = ntype.replace(".", "_")
+        return self._get_section(ntype)
+    
+    ''' get schema for a given property'''
+    def get_schema(self, property_name):
+        schema = {}
+        schemata = self.get_schemata()
+        for prop_key, prop_vale in schemata.iteritems():
+            if prop_key == property_name:
+                for attr, value in prop_vale.iteritems():
+                    schema[attr] = value
+        return schema
+    
+    def get_type(self, property_name):
+        return self.get_schema(property_name)['type']
+    
+    def get_constraints(self, property_name):
+        s = self.get_schema(property_name)
+        if 'constraints' in s:
+            s['constraints']
+    
+    def get_description(self, property_name):
+        return self.get_schema(property_name)['description']   
+    
+    def get_greater_or_equal(self, property_name):
+        pass
+    
+    def get_equal(self, property_name):
+        pass
+                
+    ''' list all the required properties for a given nodetype '''
+    def required(self):
+        required = []
+        schemata = self.get_schemata()
+        for prop_key, prop_vale in schemata.iteritems():
+            for attr, value in prop_vale.iteritems():
+                if attr == 'required' and value:
+                    required.append(prop_key)
+        return required

heat-translator/tosca/outputs.py

@@ -0,0 +1,4 @@
+
+class Outputs(object):
+    def __init__(self):
+        pass

heat-translator/tosca/tests/tosca.yaml

@@ -0,0 +1,30 @@
+tosca_definitions_version: tosca_simple_1_0
+
+description: >
+  TOSCA simple profile that just defines a single compute instance. Note, this example does not include inputs values.
+
+inputs:
+  cpus:
+    type: integer
+    description: cpu
+    constraints:
+      - valid_values: [ 1, 2, 4, 8 ]
+
+node_templates:
+  my_server:
+    type: tosca.nodes.Compute
+    properties:
+      # compute properties (flavor)
+      disk_size: 10 
+      num_cpus: { get_input: cpus }
+      mem_size: 4096   
+      # host image properties
+      os_arch: x86_64 
+      os_type: Linux  
+      os_distribution: Ubuntu  
+      os_version: 12
+
+outputs:
+  instance_ip:
+    description: The IP address of the deployed instance.
+    value: { get_property: [my_server, ip_address ] }

heat-translator/tosca/tosca_profile.py

@@ -0,0 +1,24 @@
+SECTIONS = (VERSION, DESCRIPTION, INPUTS,
+            NODE_TEMPLATES, OUTPUTS) = \
+           ('tosca_definitions_version', 'description', 'inputs',
+            'node_templates', 'outputs')
+
+class Tosca(object):
+    
+    def __init__(self, sourcedata):
+        self.sourcedata = sourcedata
+
+    def get_version(self):
+        return self.sourcedata[VERSION]
+
+    def get_description(self):
+        return self.sourcedata[DESCRIPTION]
+    
+    def get_inputs(self):
+        return self.sourcedata[INPUTS]
+
+    def get_nodetemplates(self):
+        return self.sourcedata[NODE_TEMPLATES]
+
+    def get_outputs(self):
+        return self.sourcedata[OUTPUTS]

heat-translator/tosca/translate.py

@@ -0,0 +1,34 @@
+import yaml
+
+HEAT_VERSIONS = '2013-05-23'
+
+if hasattr(yaml, 'CSafeDumper'):
+    yaml_dumper = yaml.CSafeDumper
+else:
+    yaml_dumper = yaml.SafeDumper
+
+yaml_tpl = {}
+
+class TOSCATranslator(object):
+    def __init__(self, tosca):
+        super(TOSCATranslator, self).__init__()
+        self.tosca = tosca
+ 
+    def translate(self):
+        self._translate_version()
+        self._translate_inputs()
+        self._translate_node_templates()
+        self._translate_outputs()
+
+    def _translate_version(self):
+        pass
+    
+    def _translate_inputs(self):
+        pass
+    
+    def _translate_node_templates(self):
+        pass
+            
+    def _translate_outputs(self):
+        pass
+    

heat-translator/tosca/validate.py

@@ -0,0 +1,20 @@
+from tosca.inputs import Input
+from tosca.nodetemplates.node_template import NodeTemplate
+
+class ToscaValidator():
+    def __init__(self, Tosca):
+        self.inputs = Tosca.get_inputs()
+        self.nodetemplates = Tosca.get_nodetemplates()
+        self.tosca = Tosca
+        
+    def validate(self):
+        #validate inputs
+        for name, attrs in self.inputs.iteritems():
+            if not isinstance(attrs, dict):
+                print ("The input %s has no attributes", name)
+            Input(name, attrs).validate() 
+        
+        #validate node templates
+        for nodetemplate, value in self.nodetemplates.iteritems():
+            NodeTemplate(nodetemplate, value, self.tosca).validate()
+    

heat-translator/utils.py

@@ -0,0 +1,26 @@
+import numbers
+
+def validate_integer(value):
+    if not isinstance(value, (int, long)):
+        raise TypeError('value is not an integer for %s' %value)
+    return validate_number(value)
+    
+def validate_number(value):
+    return str_to_num(value)
+
+def validate_string(value):
+    if not isinstance(value, basestring):
+        raise ValueError(_('Value must be a string'))
+    return value
+
+def validate_list(self, value):
+    pass
+    
+def str_to_num(value):
+    '''Convert a string representation of a number into a numeric type.'''
+    if isinstance(value, numbers.Number):
+        return value
+    try:
+        return int(value)
+    except ValueError:
+        return float(value)

heat-translator/yaml_loader.py

@@ -0,0 +1,20 @@
+import yaml
+
+if hasattr(yaml, 'CSafeLoader'):
+    yaml_loader = yaml.CSafeLoader
+else:
+    yaml_loader = yaml.SafeLoader
+    
+
+class Loader(object):
+    def __init__(self, file_name):
+        self.file_name = file_name
+
+    def load(self):
+        f = open(self.file_name, 'r')
+        profile = f.read() # string
+        try:
+            doc = yaml.load(profile, Loader=yaml_loader)
+        except yaml.YAMLError as error:
+            raise ValueError(error)
+        return doc