Models refactoring

- Add unit tests for models
- Avoid default method arguments with mutable values
- Simplify object serialization/unserialization
- Model objects are self-contained and do not use global functions
- Do not hardcode specific image metadata in the code
- Rename "os" key to the standard name "image_meta"
- Both keys "os" and "image_meta" are stored in the db for backward compatibility
- List of image metadata is configurable in config file

Change-Id: I2826713e438de63a49aae71cf7100288bde6bee1

almanach/api/v1/routes.py

@@ -49,7 +49,7 @@ def to_json(api_call):
             LOG.warning(e.message)
             return send_response({"error": e.message}, 400)
         except KeyError as e:
-            message = "The {param} param is mandatory for the request you have made.".format(param=e)
+            message = "The {} param is mandatory for the request you have made.".format(e)
             LOG.warning(message)
             return send_response({"error": message}, 400)
         except (TypeError, ValueError):
@@ -124,18 +124,18 @@ def create_instance(project_id):
     .. literalinclude:: ../api_examples/input/create_instance-body.json
         :language: json
     """
-    instance = jsonutils.loads(flask.request.data)
+    body = jsonutils.loads(flask.request.data)
-    LOG.info("Creating instance for tenant %s with data %s", project_id, instance)
+    LOG.info("Creating instance for tenant %s with data %s", project_id, body)
+
     instance_ctl.create_instance(
         tenant_id=project_id,
-        instance_id=instance['id'],
+        instance_id=body['id'],
-        create_date=instance['created_at'],
+        create_date=body['created_at'],
-        flavor=instance['flavor'],
+        name=body['name'],
-        os_type=instance['os_type'],
+        flavor=body['flavor'],
-        distro=instance['os_distro'],
+        image_meta=dict(distro=body['os_distro'],
-        version=instance['os_version'],
+                        version=body['os_version'],
-        name=instance['name'],
+                        os_type=body['os_type'])
-        metadata={}
     )
 
     return flask.Response(status=201)
@@ -220,14 +220,14 @@ def rebuild_instance(instance_id):
     .. literalinclude:: ../api_examples/input/rebuild_instance-body.json
         :language: json
     """
-    instance = jsonutils.loads(flask.request.data)
+    body = jsonutils.loads(flask.request.data)
-    LOG.info("Rebuilding instance with id %s with data %s", instance_id, instance)
+    LOG.info("Rebuilding instance with id %s with data %s", instance_id, body)
     instance_ctl.rebuild_instance(
         instance_id=instance_id,
-        distro=instance['distro'],
+        rebuild_date=body['rebuild_date'],
-        version=instance['version'],
+        image_meta=dict(distro=body['distro'],
-        os_type=instance['os_type'],
+                        version=body['version'],
-        rebuild_date=instance['rebuild_date'],
+                        os_type=body['os_type'])
     )
 
     return flask.Response(status=200)

almanach/collector/handlers/instance_handler.py

@@ -33,15 +33,13 @@ class InstanceHandler(base_handler.BaseHandler):
 
     def _on_instance_created(self, notification):
         self.controller.create_instance(
-            notification.payload.get("instance_id"),
+            instance_id=notification.payload.get("instance_id"),
-            notification.payload.get("tenant_id"),
+            tenant_id=notification.payload.get("tenant_id"),
-            notification.payload.get("created_at"),
+            create_date=notification.payload.get("created_at"),
-            notification.payload.get("instance_type"),
+            name=notification.payload.get("hostname"),
-            notification.payload.get("image_meta").get("os_type"),
+            flavor=notification.payload.get("instance_type"),
-            notification.payload.get("image_meta").get("distro"),
+            image_meta=notification.payload.get("image_meta"),
-            notification.payload.get("image_meta").get("version"),
+            metadata=notification.payload.get("metadata"),
-            notification.payload.get("hostname"),
-            notification.payload.get("metadata", {})
         )
 
     def _on_instance_deleted(self, notification):
@@ -58,7 +56,8 @@ class InstanceHandler(base_handler.BaseHandler):
     def _on_instance_rebuilt(self, notification):
         date = notification.context.get("timestamp")
         instance_id = notification.payload.get("instance_id")
-        distro = notification.payload.get("image_meta").get("distro")
+        self.controller.rebuild_instance(
-        version = notification.payload.get("image_meta").get("version")
+            instance_id=instance_id,
-        os_type = notification.payload.get("image_meta").get("os_type")
+            rebuild_date=date,
-        self.controller.rebuild_instance(instance_id, distro, version, os_type, date)
+            image_meta=notification.payload.get("image_meta")
+        )

almanach/core/controllers/instance_controller.py

@@ -24,24 +24,30 @@ LOG = log.getLogger(__name__)
 class InstanceController(base_controller.BaseController):
 
     def __init__(self, config, database_adapter):
+        self.config = config
         self.database_adapter = database_adapter
-        self.metadata_whitelist = config.resources.device_metadata_whitelist
 
-    def create_instance(self, instance_id, tenant_id, create_date, flavor, os_type, distro, version, name, metadata):
+    def create_instance(self, instance_id, tenant_id, create_date, name, flavor, image_meta=None, metadata=None):
         create_date = self._validate_and_parse_date(create_date)
-        LOG.info("instance %s created in project %s (flavor %s; distro %s %s %s) on %s",
+        image_meta = self._filter_image_meta(image_meta)
-                 instance_id, tenant_id, flavor, os_type, distro, version, create_date)
+        LOG.info("Instance %s created (tenant %s; flavor %s; image_meta %s) on %s",
+                 instance_id, tenant_id, flavor, image_meta, create_date)
 
         if self._fresher_entity_exists(instance_id, create_date):
             LOG.warning("instance %s already exists with a more recent entry", instance_id)
             return
 
-        filtered_metadata = self._filter_metadata_with_whitelist(metadata)
+        entity = model.Instance(
+            entity_id=instance_id,
+            project_id=tenant_id,
+            last_event=create_date,
+            start=create_date,
+            end=None,
+            name=name,
+            flavor=flavor,
+            image_meta=image_meta,
+            metadata=self._filter_metadata(metadata))
 
-        entity = model.Instance(instance_id, tenant_id, create_date, None, flavor,
-                                {"os_type": os_type, "distro": distro,
-                                 "version": version},
-                                create_date, name, filtered_metadata)
         self.database_adapter.insert_entity(entity)
 
     def delete_instance(self, instance_id, delete_date):
@@ -50,12 +56,12 @@ class InstanceController(base_controller.BaseController):
                     "InstanceId: {0} Not Found".format(instance_id))
 
         delete_date = self._validate_and_parse_date(delete_date)
-        LOG.info("instance %s deleted on %s", instance_id, delete_date)
+        LOG.info("Instance %s deleted on %s", instance_id, delete_date)
         self.database_adapter.close_active_entity(instance_id, delete_date)
 
     def resize_instance(self, instance_id, flavor, resize_date):
         resize_date = self._validate_and_parse_date(resize_date)
-        LOG.info("instance %s resized to flavor %s on %s", instance_id, flavor, resize_date)
+        LOG.info("Instance %s resized to flavor %s on %s", instance_id, flavor, resize_date)
         try:
             instance = self.database_adapter.get_active_entity(instance_id)
             if flavor != instance.flavor:
@@ -69,18 +75,16 @@ class InstanceController(base_controller.BaseController):
             LOG.error("Trying to resize an instance with id '%s' not in the database yet.", instance_id)
             raise e
 
-    def rebuild_instance(self, instance_id, distro, version, os_type, rebuild_date):
+    def rebuild_instance(self, instance_id, rebuild_date, image_meta):
         rebuild_date = self._validate_and_parse_date(rebuild_date)
         instance = self.database_adapter.get_active_entity(instance_id)
-        LOG.info("instance %s rebuilded in project %s to os %s %s %s on %s",
+        image_meta = self._filter_image_meta(image_meta)
-                 instance_id, instance.project_id, os_type, distro, version, rebuild_date)
+        LOG.info("Instance %s rebuilt for tenant %s with %s on %s",
+                 instance_id, instance.project_id, image_meta, rebuild_date)
 
-        if instance.os.distro != distro or instance.os.version != version:
+        if instance.image_meta != image_meta:
             self.database_adapter.close_active_entity(instance_id, rebuild_date)
-
+            instance.image_meta = image_meta
-            instance.os.distro = distro
-            instance.os.version = version
-            instance.os.os_type = os_type
             instance.start = rebuild_date
             instance.end = None
             instance.last_event = rebuild_date
@@ -89,5 +93,14 @@ class InstanceController(base_controller.BaseController):
     def list_instances(self, project_id, start, end):
         return self.database_adapter.get_all_entities_by_project(project_id, start, end, model.Instance.TYPE)
 
-    def _filter_metadata_with_whitelist(self, metadata):
+    def _filter_metadata(self, metadata):
-        return {key: value for key, value in metadata.items() if key in self.metadata_whitelist}
+        return self._filter(metadata, self.config.entities.instance_metadata)
+
+    def _filter_image_meta(self, image_meta):
+        return self._filter(image_meta, self.config.entities.instance_image_meta)
+
+    @staticmethod
+    def _filter(d, whitelist):
+        if d:
+            return {key: value for key, value in d.items() if key in whitelist}
+        return {}

almanach/core/controllers/volume_controller.py

@@ -26,7 +26,7 @@ class VolumeController(base_controller.BaseController):
 
     def __init__(self, config, database_adapter):
         self.database_adapter = database_adapter
-        self.volume_existence_threshold = timedelta(0, config.resources.volume_existence_threshold)
+        self.volume_existence_threshold = timedelta(0, config.entities.volume_existence_threshold)
 
     def list_volumes(self, project_id, start, end):
         return self.database_adapter.get_all_entities_by_project(project_id, start, end, model.Volume.TYPE)

almanach/core/model.py

@@ -12,12 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import abc
 import six
 
 from almanach.core import exception
 
 
+@six.add_metaclass(abc.ABCMeta)
 class Entity(object):
+
     def __init__(self, entity_id, project_id, start, end, last_event, name, entity_type):
         self.entity_id = entity_id
         self.project_id = project_id
@@ -28,7 +31,19 @@ class Entity(object):
         self.entity_type = entity_type
 
     def as_dict(self):
-        return todict(self)
+        return dict(
+            entity_id=self.entity_id,
+            project_id=self.project_id,
+            start=self.start,
+            end=self.end,
+            last_event=self.last_event,
+            name=self.name,
+            entity_type=self.entity_type,
+        )
+
+    @staticmethod
+    def from_dict(d):
+        raise NotImplementedError
 
     def __eq__(self, other):
         return (other.entity_id == self.entity_id and
@@ -44,49 +59,74 @@ class Entity(object):
 
 
 class Instance(Entity):
-    TYPE = "instance"
+    TYPE = 'instance'
 
-    def __init__(self, entity_id, project_id, start, end, flavor, os, last_event, name, metadata={}, entity_type=TYPE):
+    def __init__(self, entity_id, project_id, start, end, flavor, last_event, name, image_meta=None, metadata=None):
-        super(Instance, self).__init__(entity_id, project_id, start, end, last_event, name, entity_type)
+        super(Instance, self).__init__(entity_id, project_id, start, end, last_event, name, self.TYPE)
         self.flavor = flavor
-        self.metadata = metadata
+        self.metadata = metadata or dict()
-        self.os = OS(**os)
+        self.image_meta = image_meta or dict()
 
-    def as_dict(self):
+        # TODO(fguillot): This attribute still used by the legacy API,
-        _replace_metadata_name_with_dot_instead_of_circumflex(self)
+        # that should be removed when the new API v2 will be implemented
-        return todict(self)
+        self.os = self.image_meta
 
     def __eq__(self, other):
         return (super(Instance, self).__eq__(other) and
                 other.flavor == self.flavor and
-                other.os == self.os and
+                other.image_meta == self.image_meta and
                 other.metadata == self.metadata)
 
-    def __ne__(self, other):
+    def as_dict(self):
-        return not self.__eq__(other)
+        d = super(Instance, self).as_dict()
+        d['flavor'] = self.flavor
+        d['metadata'] = self.metadata
+        d['image_meta'] = self.image_meta
 
+        # NOTE(fguillot): we keep this key for backward compatibility
+        d['os'] = self.image_meta
+        return d
 
-class OS(object):
+    @staticmethod
-    def __init__(self, os_type, distro, version):
+    def from_dict(d):
-        self.os_type = os_type
+        return Instance(
-        self.distro = distro
+            entity_id=d.get('entity_id'),
-        self.version = version
+            project_id=d.get('project_id'),
+            start=d.get('start'),
+            end=d.get('end'),
+            last_event=d.get('last_event'),
+            name=d.get('name'),
+            flavor=d.get('flavor'),
+            image_meta=d.get('os') or d.get('image_meta'),
+            metadata=Instance._unserialize_metadata(d),
+        )
 
-    def __eq__(self, other):
+    @staticmethod
-        return (other.os_type == self.os_type and
+    def _unserialize_metadata(d):
-                other.distro == self.distro and
+        metadata = d.get('metadata')
-                other.version == self.version)
+        if metadata:
+            tmp = dict()
+            for key, value in metadata.items():
+                if '^' in key:
+                    key = key.replace('^', '.')
+                tmp[key] = value
+            metadata = tmp
+        return metadata
 
-    def __ne__(self, other):
+    def _serialize_metadata(self):
-        return not self.__eq__(other)
+        tmp = dict()
+        for key, value in self.metadata.items():
+            if '.' in key:
+                key = key.replace('.', '^')
+            tmp[key] = value
+        return tmp
 
 
 class Volume(Entity):
-    TYPE = "volume"
+    TYPE = 'volume'
 
-    def __init__(self, entity_id, project_id, start, end, volume_type, size, last_event, name, attached_to=None,
+    def __init__(self, entity_id, project_id, start, end, volume_type, size, last_event, name, attached_to=None):
-                 entity_type=TYPE):
+        super(Volume, self).__init__(entity_id, project_id, start, end, last_event, name, self.TYPE)
-        super(Volume, self).__init__(entity_id, project_id, start, end, last_event, name, entity_type)
         self.volume_type = volume_type
         self.size = size
         self.attached_to = attached_to or []
@@ -97,11 +137,30 @@ class Volume(Entity):
                 other.size == self.size and
                 other.attached_to == self.attached_to)
 
-    def __ne__(self, other):
+    def as_dict(self):
-        return not self.__eq__(other)
+        d = super(Volume, self).as_dict()
+        d['volume_type'] = self.volume_type
+        d['size'] = self.size
+        d['attached_to'] = self.attached_to
+        return d
+
+    @staticmethod
+    def from_dict(d):
+        return Volume(
+            entity_id=d.get('entity_id'),
+            project_id=d.get('project_id'),
+            start=d.get('start'),
+            end=d.get('end'),
+            last_event=d.get('last_event'),
+            name=d.get('name'),
+            volume_type=d.get('volume_type'),
+            size=d.get('size'),
+            attached_to=d.get('attached_to'),
+        )
 
 
 class VolumeType(object):
+
     def __init__(self, volume_type_id, volume_type_name):
         self.volume_type_id = volume_type_id
         self.volume_type_name = volume_type_name
@@ -109,52 +168,23 @@ class VolumeType(object):
     def __eq__(self, other):
         return other.__dict__ == self.__dict__
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
     def as_dict(self):
-        return todict(self)
+        return dict(
+            volume_type_id=self.volume_type_id,
+            volume_type_name=self.volume_type_name,
+        )
+
+    @staticmethod
+    def from_dict(d):
+        return VolumeType(volume_type_id=d['volume_type_id'],
+                          volume_type_name=d['volume_type_name'])
 
 
-def build_entity_from_dict(entity_dict):
+def get_entity_from_dict(d):
-    if entity_dict.get("entity_type") == Instance.TYPE:
+    entity_type = d.get('entity_type')
-        _replace_metadata_name_with_circumflex_instead_of_dot(entity_dict)
+    if entity_type == Instance.TYPE:
-        return Instance(**entity_dict)
+        return Instance.from_dict(d)
-    elif entity_dict.get("entity_type") == Volume.TYPE:
+    elif entity_type == Volume.TYPE:
-        return Volume(**entity_dict)
+        return Volume.from_dict(d)
     raise exception.EntityTypeNotSupportedException(
-            'Unsupported entity type: "{}"'.format(entity_dict.get("entity_type")))
+            'Unsupported entity type: "{}"'.format(entity_type))
-
-
-def todict(obj):
-    if isinstance(obj, dict) or isinstance(obj, six.text_type):
-        return obj
-    elif hasattr(obj, "__iter__"):
-        return [todict(v) for v in obj]
-    elif hasattr(obj, "__dict__"):
-        return dict([(key, todict(value))
-                     for key, value in obj.__dict__.items()
-                     if not callable(value) and not key.startswith('_')])
-    else:
-        return obj
-
-
-def _replace_metadata_name_with_dot_instead_of_circumflex(instance):
-    if instance.metadata:
-        cleaned_metadata = dict()
-        for key, value in instance.metadata.items():
-            if '.' in key:
-                key = key.replace(".", "^")
-            cleaned_metadata[key] = value
-        instance.metadata = cleaned_metadata
-
-
-def _replace_metadata_name_with_circumflex_instead_of_dot(entity_dict):
-    metadata = entity_dict.get("metadata")
-    if metadata:
-        dirty_metadata = dict()
-        for key, value in metadata.items():
-            if '^' in key:
-                key = key.replace("^", ".")
-            dirty_metadata[key] = value
-        entity_dict["metadata"] = dirty_metadata

almanach/core/opts.py

@@ -87,14 +87,16 @@ auth_opts = [
                help='Private key for private key authentication'),
 ]
 
-resource_opts = [
+entity_opts = [
     cfg.IntOpt('volume_existence_threshold',
                default=60,
                help='Volume existence threshold'),
-    cfg.ListOpt('device_metadata_whitelist',
+    cfg.ListOpt('instance_metadata',
                 default=[],
-                deprecated_for_removal=True,
+                help='List of instance metadata to include from notifications'),
-                help='Metadata to include in entity'),
+    cfg.ListOpt('instance_image_meta',
+                default=[],
+                help='List of instance image metadata to include from notifications'),
 ]
 
 CONF.register_opts(database_opts, group='database')
@@ -102,7 +104,7 @@ CONF.register_opts(api_opts, group='api')
 CONF.register_opts(collector_opts, group='collector')
 CONF.register_opts(auth_opts, group='auth')
 CONF.register_opts(keystone_opts, group='keystone_authtoken')
-CONF.register_opts(resource_opts, group='resources')
+CONF.register_opts(entity_opts, group='entities')
 
 logging.register_options(CONF)
 logging.setup(CONF, DOMAIN)
@@ -115,5 +117,5 @@ def list_opts():
         ('collector', collector_opts),
         ('auth', auth_opts),
         ('keystone_authtoken', keystone_opts),
-        ('resources', resource_opts),
+        ('entities', entity_opts),
     ]

almanach/storage/drivers/mongodb_driver.py

@@ -17,7 +17,7 @@ import pymongo
 
 from almanach.core import exception
 from almanach.core import model
-from almanach.core.model import build_entity_from_dict
+from almanach.core.model import get_entity_from_dict
 from almanach.storage.drivers import base_driver
 
 LOG = log.getLogger(__name__)
@@ -50,7 +50,7 @@ class MongoDbDriver(base_driver.BaseDriver):
         entity = self.db.entity.find_one({"entity_id": entity_id, "end": None}, {"_id": 0})
         if not entity:
             raise exception.EntityNotFoundException("Entity {} not found".format(entity_id))
-        return build_entity_from_dict(entity)
+        return get_entity_from_dict(entity)
 
     def get_all_entities_by_project(self, project_id, start, end, entity_type=None):
         args = {
@@ -65,11 +65,11 @@ class MongoDbDriver(base_driver.BaseDriver):
             args["entity_type"] = entity_type
 
         entities = list(self.db.entity.find(args, {"_id": 0}))
-        return [build_entity_from_dict(entity) for entity in entities]
+        return [get_entity_from_dict(entity) for entity in entities]
 
     def get_all_entities_by_id(self, entity_id):
         entities = self.db.entity.find({"entity_id": entity_id}, {"_id": 0})
-        return [build_entity_from_dict(entity) for entity in entities]
+        return [get_entity_from_dict(entity) for entity in entities]
 
     def get_all_entities_by_id_and_date(self, entity_id, start, end):
         entities = self.db.entity.find({
@@ -80,7 +80,7 @@ class MongoDbDriver(base_driver.BaseDriver):
                 {"end": {"$lte": end}}
             ]
         }, {"_id": 0})
-        return [build_entity_from_dict(entity) for entity in entities]
+        return [get_entity_from_dict(entity) for entity in entities]
 
     def close_active_entity(self, entity_id, end):
         self.db.entity.update({"entity_id": entity_id, "end": None}, {"$set": {"end": end, "last_event": end}})
@@ -110,9 +110,7 @@ class MongoDbDriver(base_driver.BaseDriver):
         volume_type = self.db.volume_type.find_one({"volume_type_id": volume_type_id})
         if not volume_type:
             raise exception.VolumeTypeNotFoundException(volume_type_id=volume_type_id)
-
+        return model.VolumeType.from_dict(volume_type)
-        return model.VolumeType(volume_type_id=volume_type["volume_type_id"],
-                                volume_type_name=volume_type["volume_type_name"])
 
     def delete_volume_type(self, volume_type_id):
         if volume_type_id is None:

almanach/tests/tempest/tests/api/base.py

@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from uuid import uuid4
 
 from oslo_serialization import jsonutils as json
 from tempest.common.utils import data_utils
 from tempest import config
 import tempest.test
+from uuid import uuid4
 
 from almanach.tests.tempest import clients
 
@@ -24,6 +24,7 @@ CONF = config.CONF
 
 
 class BaseAlmanachTest(tempest.test.BaseTestCase):
+
     @classmethod
     def skip_checks(cls):
         super(BaseAlmanachTest, cls).skip_checks()

almanach/tests/tempest/tests/api/test_server_creation.py

@@ -11,11 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from oslo_serialization import jsonutils as json
+from tempest.lib import exceptions
 from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
-from oslo_serialization import jsonutils as json
-from tempest.lib import exceptions
 
 
 class TestServerCreation(base.BaseAlmanachTest):

almanach/tests/tempest/tests/api/test_server_deletion.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from oslo_serialization import jsonutils as json
 from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
-from oslo_serialization import jsonutils as json
 
 
 class TestServerDeletion(base.BaseAlmanachTest):
+
     def setUp(self):
         super(base.BaseAlmanachTest, self).setUp()
 

almanach/tests/tempest/tests/api/test_server_rebuild.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from oslo_serialization import jsonutils as json
 from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
-from oslo_serialization import jsonutils as json
 
 
 class TestServerRebuild(base.BaseAlmanachTest):
+
     def setUp(self):
         super(base.BaseAlmanachTest, self).setUp()
 
@@ -30,15 +32,15 @@ class TestServerRebuild(base.BaseAlmanachTest):
         server = self.get_server_creation_payload()
         self.create_server_through_api(tenant_id, server)
 
-        rebuild_data = {
+        data = {
-            'distro': 'Ubuntu',
+            'distro': 'debian',
-            'version': '14.04',
+            'version': '8.0',
-            'os_type': 'Linux',
+            'os_type': 'linux',
             'rebuild_date': '2016-01-01T18:50:00Z'
         }
-        data = json.dumps(rebuild_data)
 
-        self.almanach_client.rebuild(server['id'], data)
+        self.almanach_client.rebuild(server['id'],
+                                     json.dumps(data))
 
         resp, response_body = self.almanach_client.get_tenant_entities(tenant_id)
 
@@ -46,11 +48,26 @@ class TestServerRebuild(base.BaseAlmanachTest):
         self.assertIsInstance(entities, list)
         self.assertEqual(2, len(entities))
 
-        rebuilded_server, initial_server = sorted(entities, key=lambda k: k['end'] if k['end'] is not None else '')
+        rebuilt_server, initial_server = sorted(entities, key=lambda k: k['end'] if k['end'] is not None else '')
 
         self.assertEqual(server['id'], initial_server['entity_id'])
-        self.assertEqual(server['os_version'], initial_server['os']['version'])
         self.assertIsNotNone(initial_server['end'])
-        self.assertEqual(server['id'], rebuilded_server['entity_id'])
+
-        self.assertEqual(rebuild_data['version'], rebuilded_server['os']['version'])
+        self.assertEqual(server['os_distro'], initial_server['os']['distro'])
-        self.assertIsNone(rebuilded_server['end'])
+        self.assertEqual(server['os_version'], initial_server['os']['version'])
+        self.assertEqual(server['os_type'], initial_server['os']['os_type'])
+
+        self.assertEqual(server['os_distro'], initial_server['image_meta']['distro'])
+        self.assertEqual(server['os_version'], initial_server['image_meta']['version'])
+        self.assertEqual(server['os_type'], initial_server['image_meta']['os_type'])
+
+        self.assertEqual(server['id'], rebuilt_server['entity_id'])
+        self.assertIsNone(rebuilt_server['end'])
+
+        self.assertEqual(data['distro'], rebuilt_server['os']['distro'])
+        self.assertEqual(data['version'], rebuilt_server['os']['version'])
+        self.assertEqual(data['os_type'], rebuilt_server['os']['os_type'])
+
+        self.assertEqual(data['distro'], rebuilt_server['image_meta']['distro'])
+        self.assertEqual(data['version'], rebuilt_server['image_meta']['version'])
+        self.assertEqual(data['os_type'], rebuilt_server['image_meta']['os_type'])

almanach/tests/tempest/tests/api/test_server_resize.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from oslo_serialization import jsonutils as json
 from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
-from oslo_serialization import jsonutils as json
 
 
 class TestServerResize(base.BaseAlmanachTest):
+
     def setUp(self):
         super(base.BaseAlmanachTest, self).setUp()
 

almanach/tests/tempest/tests/api/test_server_update.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from oslo_serialization import jsonutils as json
 from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
-from oslo_serialization import jsonutils as json
 
 
 class TestServerUpdate(base.BaseAlmanachTest):
+
     def setUp(self):
         super(base.BaseAlmanachTest, self).setUp()
 

almanach/tests/tempest/tests/api/test_volume_attach.py

@@ -11,14 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from uuid import uuid4
 
 from oslo_serialization import jsonutils as json
+from uuid import uuid4
 
 from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeAttach(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeAttach, cls).resource_setup()
@@ -38,8 +39,10 @@ class TestVolumeAttach(base.BaseAlmanachTest):
         self.assertEqual(resp.status, 200)
         self.assertIsInstance(response_body, list)
         self.assertEqual(2, len(response_body))
+
         un_attached_volume = [v for v in response_body if v['attached_to'] == []][0]
         attached_volume = [v for v in response_body if v['attached_to'] != []][0]
+
         self.assertEqual(volume['volume_id'], attached_volume['entity_id'])
         self.assertEqual(volume['volume_id'], un_attached_volume['entity_id'])
         self.assertEqual('volume', attached_volume['entity_type'])

almanach/tests/tempest/tests/api/test_volume_creation.py

@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from oslo_serialization import jsonutils as json
 
 from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeCreation(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeCreation, cls).resource_setup()

almanach/tests/tempest/tests/api/test_volume_deletion.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from oslo_serialization import jsonutils as json
 from oslo_utils import timeutils
 
@@ -18,6 +19,7 @@ from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeDeletion(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeDeletion, cls).resource_setup()

almanach/tests/tempest/tests/api/test_volume_detach.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from oslo_serialization import jsonutils as json
 from uuid import uuid4
 
@@ -18,6 +19,7 @@ from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeDetach(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeDetach, cls).resource_setup()
@@ -43,8 +45,10 @@ class TestVolumeDetach(base.BaseAlmanachTest):
         self.assertEqual(resp.status, 200)
         self.assertIsInstance(response_body, list)
         self.assertEqual(3, len(response_body))
+
         un_attached_volumes = [v for v in response_body if v['attached_to'] == []]
         attached_volume = [v for v in response_body if v['attached_to'] != []][0]
+
         self.assertEqual(volume['volume_id'], attached_volume['entity_id'])
         self.assertEqual(volume['volume_id'], un_attached_volumes[0]['entity_id'])
         self.assertEqual(volume['volume_id'], un_attached_volumes[1]['entity_id'])

almanach/tests/tempest/tests/api/test_volume_resize.py

@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from oslo_serialization import jsonutils as json
 
 from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeResize(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeResize, cls).resource_setup()

almanach/tests/tempest/tests/api/test_volume_type_creation.py

@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from oslo_serialization import jsonutils as json
 
 from almanach.tests.tempest.tests.api import base
 
 
 class TestVolumeCreation(base.BaseAlmanachTest):
+
     @classmethod
     def resource_setup(cls):
         super(TestVolumeCreation, cls).resource_setup()

almanach/tests/tempest/tests/scenario/test_server_rebuild.py

@@ -32,7 +32,8 @@ class TestServerRebuildScenario(base.BaseAlmanachScenarioTest):
         self.assertEqual(flavor['name'], entities[0]['flavor'])
         self.assertIsNotNone(entities[0]['start'])
         self.assertIsNotNone(entities[0]['end'])
-        self.assertIsNone(entities[0]['os']['distro'])
+        self.assertEqual(dict(), entities[0]['os'])
+        self.assertEqual(dict(), entities[0]['image_meta'])
 
         self.assertEqual(server['id'], entities[1]['entity_id'])
         self.assertEqual('instance', entities[1]['entity_type'])
@@ -40,6 +41,7 @@ class TestServerRebuildScenario(base.BaseAlmanachScenarioTest):
         self.assertEqual(flavor['name'], entities[1]['flavor'])
         self.assertIsNotNone(entities[1]['start'])
         self.assertIsNone(entities[1]['end'])
+        self.assertEqual('linux', entities[1]['image_meta']['distro'])
         self.assertEqual('linux', entities[1]['os']['distro'])
 
     def _rebuild_server(self):

almanach/tests/tempest/tests/scenario/test_server_resize.py

@@ -32,6 +32,8 @@ class TestServerResizeScenario(base.BaseAlmanachScenarioTest):
         self.assertEqual(initial_flavor['name'], entities[0]['flavor'])
         self.assertIsNotNone(entities[0]['start'])
         self.assertIsNotNone(entities[0]['end'])
+        self.assertEqual(dict(), entities[0]['os'])
+        self.assertEqual(dict(), entities[0]['image_meta'])
 
         self.assertEqual(server['id'], entities[1]['entity_id'])
         self.assertEqual('instance', entities[1]['entity_type'])
@@ -39,6 +41,8 @@ class TestServerResizeScenario(base.BaseAlmanachScenarioTest):
         self.assertEqual(resized_flavor['name'], entities[1]['flavor'])
         self.assertIsNotNone(entities[1]['start'])
         self.assertIsNone(entities[1]['end'])
+        self.assertEqual(dict(), entities[0]['os'])
+        self.assertEqual(dict(), entities[0]['image_meta'])
 
     def _resize_server(self):
         flavors = self.flavors_client.list_flavors()['flavors']

almanach/tests/unit/api/test_api_instance.py

@@ -57,13 +57,12 @@ class ApiInstanceTest(base_api.BaseApi):
             .with_args(tenant_id="PROJECT_ID",
                        instance_id=data["id"],
                        create_date=data["created_at"],
-                       flavor=data['flavor'],
-                       os_type=data['os_type'],
-                       distro=data['os_distro'],
-                       version=data['os_version'],
                        name=data['name'],
-                       metadata={}) \
+                       flavor=data['flavor'],
-            .once()
+                       image_meta=dict(os_type=data['os_type'],
+                                       distro=data['os_distro'],
+                                       version=data['os_version'])
+                       ).once()
 
         code, result = self.api_post(
             '/project/PROJECT_ID/instance',
@@ -105,11 +104,10 @@ class ApiInstanceTest(base_api.BaseApi):
                        instance_id=data["id"],
                        create_date=data["created_at"],
                        flavor=data['flavor'],
-                       os_type=data['os_type'],
+                       image_meta=dict(os_type=data['os_type'],
                                        distro=data['os_distro'],
-                       version=data['os_version'],
+                                       version=data['os_version']),
-                       name=data['name'],
+                       name=data['name']) \
-                       metadata={}) \
             .once() \
             .and_raise(exception.DateFormatException)
 
@@ -241,11 +239,11 @@ class ApiInstanceTest(base_api.BaseApi):
         self.instance_ctl.should_receive('rebuild_instance') \
             .with_args(
                 instance_id=instance_id,
-            distro=data.get('distro'),
+                rebuild_date=data.get('rebuild_date'),
+                image_meta=dict(distro=data.get('distro'),
                                 version=data.get('version'),
-            os_type=data.get('os_type'),
+                                os_type=data.get('os_type'))
-            rebuild_date=data.get('rebuild_date')) \
+            ).once()
-            .once()
 
         code, result = self.api_put(
             '/instance/INSTANCE_ID/rebuild',
@@ -282,7 +280,11 @@ class ApiInstanceTest(base_api.BaseApi):
         }
 
         self.instance_ctl.should_receive('rebuild_instance') \
-            .with_args(instance_id=instance_id, **data) \
+            .with_args(instance_id=instance_id,
+                       rebuild_date=data.get('rebuild_date'),
+                       image_meta=dict(distro=data.get('distro'),
+                                       version=data.get('version'),
+                                       os_type=data.get('os_type'))) \
             .once() \
             .and_raise(exception.DateFormatException)
 

almanach/tests/unit/builders/entity.py

@@ -29,7 +29,7 @@ class Builder(object):
 class EntityBuilder(Builder):
 
     def build(self):
-        return model.build_entity_from_dict(self.dict_object)
+        return model.get_entity_from_dict(self.dict_object)
 
     def with_id(self, entity_id):
         self.dict_object["entity_id"] = entity_id

almanach/tests/unit/collector/handlers/test_instance_handler.py

@@ -37,15 +37,13 @@ class InstanceHandlerTest(base.BaseTestCase):
         self.instance_handler.handle_events(notification)
 
         self.controller.create_instance.assert_called_once_with(
-            notification.payload['instance_id'],
+            instance_id=notification.payload['instance_id'],
-            notification.payload['tenant_id'],
+            tenant_id=notification.payload['tenant_id'],
-            notification.payload['created_at'],
+            create_date=notification.payload['created_at'],
-            notification.payload['instance_type'],
+            name=notification.payload['hostname'],
-            notification.payload['image_meta']['os_type'],
+            flavor=notification.payload['instance_type'],
-            notification.payload['image_meta']['distro'],
+            image_meta=notification.payload['image_meta'],
-            notification.payload['image_meta']['version'],
+            metadata=notification.payload['metadata'],
-            notification.payload['hostname'],
-            notification.payload['metadata'],
         )
 
     def test_instance_deleted(self):
@@ -87,9 +85,7 @@ class InstanceHandlerTest(base.BaseTestCase):
         self.instance_handler.handle_events(notification)
 
         self.controller.rebuild_instance.assert_called_once_with(
-            notification.payload['instance_id'],
+            instance_id=notification.payload['instance_id'],
-            notification.payload['image_meta']['distro'],
+            rebuild_date=notification.context.get("timestamp"),
-            notification.payload['image_meta']['version'],
+            image_meta=notification.payload['image_meta'],
-            notification.payload['image_meta']['os_type'],
-            notification.context.get("timestamp")
         )

almanach/tests/unit/core/controllers/test_instance_controller.py

@@ -31,6 +31,7 @@ class InstanceControllerTest(base.BaseTestCase):
 
     def setUp(self):
         super(InstanceControllerTest, self).setUp()
+        self.config.entities.instance_image_meta = ['distro', 'version', 'os_type']
         self.database_adapter = flexmock(base_driver.BaseDriver)
         self.controller = instance_controller.InstanceController(self.config, self.database_adapter)
 
@@ -47,9 +48,13 @@ class InstanceControllerTest(base.BaseTestCase):
          .should_receive("insert_entity")
          .once())
 
-        self.controller.create_instance(fake_instance.entity_id, fake_instance.project_id, fake_instance.start,
+        self.controller.create_instance(fake_instance.entity_id,
-                                        fake_instance.flavor, fake_instance.os.os_type, fake_instance.os.distro,
+                                        fake_instance.project_id,
-                                        fake_instance.os.version, fake_instance.name, fake_instance.metadata)
+                                        fake_instance.start,
+                                        fake_instance.flavor,
+                                        fake_instance.name,
+                                        fake_instance.image_meta,
+                                        fake_instance.metadata)
 
     def test_resize_instance(self):
         fake_instance = a(instance())
@@ -88,8 +93,7 @@ class InstanceControllerTest(base.BaseTestCase):
 
         self.controller.create_instance(fake_instance.entity_id, fake_instance.project_id,
                                         '2015-10-21T16:25:00.000000Z',
-                                        fake_instance.flavor, fake_instance.os.os_type, fake_instance.os.distro,
+                                        fake_instance.flavor, fake_instance.image_meta, fake_instance.metadata)
-                                        fake_instance.os.version, fake_instance.name, fake_instance.metadata)
 
     def test_instance_created_but_find_garbage(self):
         fake_instance = a(instance().with_all_dates_in_string())
@@ -105,8 +109,7 @@ class InstanceControllerTest(base.BaseTestCase):
          .once())
 
         self.controller.create_instance(fake_instance.entity_id, fake_instance.project_id, fake_instance.start,
-                                        fake_instance.flavor, fake_instance.os.os_type, fake_instance.os.distro,
+                                        fake_instance.flavor, fake_instance.image_meta, fake_instance.metadata)
-                                        fake_instance.os.version, fake_instance.name, fake_instance.metadata)
 
     def test_instance_deleted(self):
         (flexmock(self.database_adapter)
@@ -158,15 +161,11 @@ class InstanceControllerTest(base.BaseTestCase):
 
         self.controller.rebuild_instance(
             "an_instance_id",
-            "some_distro",
+            "2015-10-21T16:25:00.000000Z",
-            "some_version",
+            dict(distro="some_distro", version="some_version", os_type="some_type")
-            "some_type",
-            "2015-10-21T16:25:00.000000Z"
         )
         self.controller.rebuild_instance(
             "an_instance_id",
-            i.os.distro,
+            "2015-10-21T16:25:00.000000Z",
-            i.os.version,
+            dict(distro=i.image_meta['distro'], version=i.image_meta['version'], os_type=i.image_meta['os_type'])
-            i.os.os_type,
-            "2015-10-21T16:25:00.000000Z"
         )

almanach/tests/unit/core/test_model.py

@@ -0,0 +1,334 @@
+# Copyright 2016 Internap.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from almanach.core import exception
+from almanach.core import model
+from datetime import datetime
+import pytz
+
+from almanach.tests.unit import base
+
+
+class TestModel(base.BaseTestCase):
+
+    def test_instance_serialize(self):
+        instance = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='flavor_id',
+            image_meta=dict(os_type='linux', distro='Ubuntu', version='16.04'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata={'some_key': 'some.value', 'another^key': 'another.value'},
+        )
+
+        entry = instance.as_dict()
+        self.assertEqual('instance_id', entry['entity_id'])
+        self.assertEqual('project_id', entry['project_id'])
+        self.assertEqual('instance', entry['entity_type'])
+        self.assertEqual('hostname', entry['name'])
+        self.assertEqual('flavor_id', entry['flavor'])
+        self.assertEqual('linux', entry['os']['os_type'])
+        self.assertEqual('Ubuntu', entry['os']['distro'])
+        self.assertEqual('16.04', entry['os']['version'])
+        self.assertEqual('linux', entry['image_meta']['os_type'])
+        self.assertEqual('Ubuntu', entry['image_meta']['distro'])
+        self.assertEqual('16.04', entry['image_meta']['version'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), entry['last_event'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), entry['start'])
+        self.assertIsNone(entry['end'])
+
+    def test_instance_unserialize(self):
+        entry = {
+            'entity_id': 'instance_id',
+            'entity_type': 'instance',
+            'project_id': 'project_id',
+            'start': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'end': None,
+            'last_event': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'flavor': 'flavor_id',
+            'image_meta': {
+                'os_type': 'linux',
+                'distro': 'Ubuntu',
+                'version': '16.04',
+            },
+            'name': 'hostname'
+        }
+
+        instance = model.get_entity_from_dict(entry)
+        self.assertEqual('instance_id', instance.entity_id)
+        self.assertEqual('project_id', instance.project_id)
+        self.assertEqual('instance', instance.entity_type)
+        self.assertEqual('hostname', instance.name)
+        self.assertEqual('flavor_id', instance.flavor)
+        self.assertEqual('linux', instance.image_meta['os_type'])
+        self.assertEqual('Ubuntu', instance.image_meta['distro'])
+        self.assertEqual('16.04', instance.image_meta['version'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.last_event)
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.start)
+        self.assertIsNone(instance.end)
+
+    def test_instance_unserialize_with_legacy_os(self):
+        entry = {
+            'entity_id': 'instance_id',
+            'entity_type': 'instance',
+            'project_id': 'project_id',
+            'start': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'end': None,
+            'last_event': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'flavor': 'flavor_id',
+            'os': {
+                'os_type': 'linux',
+                'distro': 'Ubuntu',
+                'version': '16.04',
+            },
+            'name': 'hostname'
+        }
+
+        instance = model.get_entity_from_dict(entry)
+        self.assertEqual('instance_id', instance.entity_id)
+        self.assertEqual('project_id', instance.project_id)
+        self.assertEqual('instance', instance.entity_type)
+        self.assertEqual('hostname', instance.name)
+        self.assertEqual('flavor_id', instance.flavor)
+        self.assertEqual('linux', instance.image_meta['os_type'])
+        self.assertEqual('Ubuntu', instance.image_meta['distro'])
+        self.assertEqual('16.04', instance.image_meta['version'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.last_event)
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.start)
+        self.assertIsNone(instance.end)
+
+    def test_instance_unserialize_with_both_keys(self):
+        entry = {
+            'entity_id': 'instance_id',
+            'entity_type': 'instance',
+            'project_id': 'project_id',
+            'start': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'end': None,
+            'last_event': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'flavor': 'flavor_id',
+            'os': {
+                'os_type': 'linux',
+                'distro': 'Ubuntu',
+                'version': '16.04',
+            },
+            'image_meta': {
+                'os_type': 'linux',
+                'distro': 'Ubuntu',
+                'version': '16.04',
+            },
+            'name': 'hostname'
+        }
+
+        instance = model.get_entity_from_dict(entry)
+        self.assertEqual('instance_id', instance.entity_id)
+        self.assertEqual('project_id', instance.project_id)
+        self.assertEqual('instance', instance.entity_type)
+        self.assertEqual('hostname', instance.name)
+        self.assertEqual('flavor_id', instance.flavor)
+        self.assertEqual('linux', instance.image_meta['os_type'])
+        self.assertEqual('Ubuntu', instance.image_meta['distro'])
+        self.assertEqual('16.04', instance.image_meta['version'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.last_event)
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), instance.start)
+        self.assertIsNone(instance.end)
+
+    def test_instance_comparison(self):
+        instance1 = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='flavor_id',
+            image_meta=dict(os_type='linux', distro='Ubuntu', version='16.04'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata={'some_key': 'some.value', 'another^key': 'another.value'},
+        )
+
+        instance2 = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='flavor_id',
+            image_meta=dict(os_type='linux', distro='Ubuntu', version='16.04'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata={'some_key': 'some.value', 'another^key': 'another.value'},
+        )
+
+        # different image properties
+        instance3 = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='flavor_id',
+            image_meta=dict(os_type='linux', distro='Centos', version='7'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata={'some_key': 'some.value', 'another^key': 'another.value'},
+        )
+
+        # different flavor
+        instance4 = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='another_flavor',
+            image_meta=dict(os_type='linux', distro='Ubuntu', version='16.04'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata={'some_key': 'some.value', 'another^key': 'another.value'},
+        )
+
+        # different metadata
+        instance5 = model.Instance(
+            entity_id='instance_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            flavor='flavor_id',
+            image_meta=dict(os_type='linux', distro='Ubuntu', version='16.04'),
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='hostname',
+            metadata=dict(),
+        )
+
+        self.assertTrue(instance1 == instance2)
+        self.assertTrue(instance1 != instance3)
+        self.assertTrue(instance1 != instance4)
+        self.assertTrue(instance1 != instance5)
+
+    def test_volume_serialize(self):
+        volume = model.Volume(
+            entity_id='volume_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='volume_name',
+            volume_type='volume_type_id',
+            size=2,
+            attached_to=['instance_id1', 'instance_id2'],
+        )
+
+        entry = volume.as_dict()
+        self.assertEqual('volume_id', entry['entity_id'])
+        self.assertEqual('project_id', entry['project_id'])
+        self.assertEqual('volume', entry['entity_type'])
+        self.assertEqual('volume_name', entry['name'])
+        self.assertEqual('volume_type_id', entry['volume_type'])
+        self.assertEqual(2, entry['size'])
+        self.assertEqual(['instance_id1', 'instance_id2'], entry['attached_to'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), entry['last_event'])
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), entry['start'])
+        self.assertIsNone(entry['end'])
+
+    def test_volume_unserialize(self):
+        entry = {
+            'entity_id': 'volume_id',
+            'entity_type': 'volume',
+            'project_id': 'project_id',
+            'start': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'end': None,
+            'last_event': datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            'volume_type': 'volume_type_id',
+            'name': 'volume_name',
+            'size': 2,
+        }
+
+        volume = model.get_entity_from_dict(entry)
+        self.assertEqual('volume_id', volume.entity_id)
+        self.assertEqual('project_id', volume.project_id)
+        self.assertEqual('volume', volume.entity_type)
+        self.assertEqual('volume_name', volume.name)
+        self.assertEqual(2, volume.size)
+        self.assertEqual([], volume.attached_to)
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), volume.last_event)
+        self.assertEqual(datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc), volume.start)
+        self.assertIsNone(volume.end)
+
+    def test_volume_comparison(self):
+        volume1 = model.Volume(
+            entity_id='volume_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='volume_name',
+            volume_type='volume_type_id',
+            size=2,
+            attached_to=['instance_id1', 'instance_id2'],
+        )
+
+        volume2 = model.Volume(
+            entity_id='volume_id',
+            project_id='project_id',
+            start=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            end=None,
+            last_event=datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc),
+            name='volume_name',
+            volume_type='volume_type_id',
+            size=2,
+            attached_to=['instance_id1', 'instance_id2'],
+        )
+
+        self.assertTrue(volume1 == volume2)
+
+        volume2.volume_type = 'another_volume_type'
+        self.assertFalse(volume1 == volume2)
+
+        volume2.volume_type = 'volume_type_id'
+        volume2.size = 3
+        self.assertFalse(volume1 == volume2)
+
+        volume2.volume_type = 'volume_type_id'
+        volume2.size = 2
+        volume2.attached_to = []
+        self.assertFalse(volume1 == volume2)
+
+    def test_volume_type_serialize(self):
+        volume_type = model.VolumeType(
+            volume_type_id='id',
+            volume_type_name='name',
+        )
+
+        entry = volume_type.as_dict()
+        self.assertEqual('id', entry['volume_type_id'])
+        self.assertEqual('name', entry['volume_type_name'])
+
+    def test_volume_type_unserialize(self):
+        entry = dict(volume_type_id='id', volume_type_name='name')
+
+        volume_type = model.VolumeType.from_dict(entry)
+        self.assertEqual('id', volume_type.volume_type_id)
+        self.assertEqual('name', volume_type.volume_type_name)
+
+    def test_volume_type_comparison(self):
+        volume_type1 = model.VolumeType(volume_type_id='id', volume_type_name='name')
+        volume_type2 = model.VolumeType(volume_type_id='id2', volume_type_name='name2')
+        volume_type3 = model.VolumeType(volume_type_id='id', volume_type_name='name')
+
+        self.assertTrue(volume_type1 != volume_type2)
+        self.assertTrue(volume_type1 == volume_type3)
+
+    def test_unserialize_unknown_entity(self):
+        self.assertRaises(exception.EntityTypeNotSupportedException,
+                          model.get_entity_from_dict,
+                          dict(entity_type='unknown'))

almanach/tests/unit/storage/drivers/test_mongodb_driver.py

@@ -59,14 +59,14 @@ class MongoDbDriverTest(base.BaseTestCase):
 
     def test_get_active_entity(self):
         fake_entity = a(instance().with_metadata({}))
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         self.assertEqual(fake_entity, self.adapter.get_active_entity(fake_entity.entity_id))
 
     def test_get_active_entity_with_special_metadata_characters(self):
         fake_entity = a(instance().with_metadata({"a_metadata_not_sanitize": "not.sanitize",
                                                   "a_metadata^to_sanitize": "this.sanitize"}))
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         entity = self.adapter.get_active_entity(fake_entity.entity_id)
 
         expected_entity = a(instance()
@@ -87,7 +87,7 @@ class MongoDbDriverTest(base.BaseTestCase):
         fake_entity = a(instance())
         fake_entity.entity_type = "will_raise_exception"
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         self.assertRaises(exception.EntityTypeNotSupportedException,
                           self.adapter.get_active_entity,
                           fake_entity.entity_id)
@@ -115,7 +115,7 @@ class MongoDbDriverTest(base.BaseTestCase):
         ]
 
         all_entities = fake_active_entities + fake_inactive_entities
-        [self.db.entity.insert(model.todict(fake_entity)) for fake_entity in all_entities]
+        [self.db.entity.insert(fake_entity.as_dict()) for fake_entity in all_entities]
 
         self.assertEqual(4, self.adapter.count_entities())
         self.assertEqual(2, self.adapter.count_active_entities())
@@ -124,7 +124,7 @@ class MongoDbDriverTest(base.BaseTestCase):
 
     def test_get_all_entities_by_id(self):
         fake_entity = a(instance().with_id("id1").with_start(2014, 1, 1, 8, 0, 0).with_no_end())
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
 
         entries = self.adapter.get_all_entities_by_id(entity_id="id1")
         self.assertEqual(1, len(entries))
@@ -173,7 +173,7 @@ class MongoDbDriverTest(base.BaseTestCase):
               .with_project_id("project_id")),
         ]
 
-        [self.db.entity.insert(model.todict(fake_entity)) for fake_entity in fake_instances + fake_volumes]
+        [self.db.entity.insert(fake_entity.as_dict()) for fake_entity in fake_instances + fake_volumes]
 
         entities = self.adapter.get_all_entities_by_project("project_id",
                                                             datetime(2014, 1, 1, 0, 0, 0, tzinfo=pytz.utc),
@@ -217,7 +217,7 @@ class MongoDbDriverTest(base.BaseTestCase):
               .with_metadata({"a_metadata.to_sanitize": "this.sanitize"})),
         ]
 
-        [self.db.entity.insert(model.todict(fake_entity)) for fake_entity in fake_instances]
+        [self.db.entity.insert(fake_entity.as_dict()) for fake_entity in fake_instances]
 
         entities = self.adapter.get_all_entities_by_project("project_id",
                                                             datetime(2014, 1, 1, 0, 0, 0, tzinfo=pytz.utc),
@@ -265,16 +265,17 @@ class MongoDbDriverTest(base.BaseTestCase):
               .with_project_id("project_id")),
         ]
 
-        [self.db.entity.insert(model.todict(fake_entity))
+        [self.db.entity.insert(fake_entity.as_dict())
          for fake_entity in fake_entities_in_period + fake_entities_out_period]
 
         entities = self.adapter.get_all_entities_by_project("project_id",
                                                             datetime(2014, 1, 1, 6, 0, 0, tzinfo=pytz.utc),
                                                             datetime(2014, 1, 1, 9, 0, 0, tzinfo=pytz.utc))
         self.assertEqual(3, len(entities))
-        self.assertIn(fake_entities_in_period[0], entities)
+        entity_ids = [entity.entity_id for entity in entities]
-        self.assertIn(fake_entities_in_period[1], entities)
+        self.assertIn(fake_entities_in_period[0].entity_id, entity_ids)
-        self.assertIn(fake_entities_in_period[2], entities)
+        self.assertIn(fake_entities_in_period[1].entity_id, entity_ids)
+        self.assertIn(fake_entities_in_period[2].entity_id, entity_ids)
 
     def test_get_all_entities_by_id_and_date(self):
         start = datetime(2016, 3, 1, 0, 0, 0, 0, pytz.utc)
@@ -292,7 +293,7 @@ class MongoDbDriverTest(base.BaseTestCase):
               .with_no_end()),
         ]
 
-        [self.db.entity.insert(model.todict(fake_instance)) for fake_instance in instances]
+        [self.db.entity.insert(fake_instance.as_dict()) for fake_instance in instances]
         entities = self.adapter.get_all_entities_by_id_and_date("id1", start, end)
 
         self.assertEqual(1, len(entities))
@@ -302,7 +303,7 @@ class MongoDbDriverTest(base.BaseTestCase):
         fake_entity = a(instance())
         end_date = datetime(2015, 10, 21, 16, 29, 0)
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         self.adapter.close_active_entity(fake_entity.entity_id, end_date)
 
         self.assertEqual(self.db.entity.find_one({"entity_id": fake_entity.entity_id})["end"], end_date)
@@ -310,7 +311,7 @@ class MongoDbDriverTest(base.BaseTestCase):
     def test_update_closed_entity(self):
         fake_entity = a(instance().with_end(2016, 3, 2, 0, 0, 0))
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         fake_entity.flavor = "my_new_flavor"
         self.adapter.update_closed_entity(fake_entity, data={"flavor": fake_entity.flavor})
 
@@ -320,20 +321,22 @@ class MongoDbDriverTest(base.BaseTestCase):
 
     def test_update_active_entity(self):
         fake_entity = a(instance())
-        fake_entity.os.distro = "Centos"
+        fake_entity.image_meta['distro'] = "Centos"
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
-        fake_entity.os.distro = "Windows"
+        fake_entity.image_meta['distro'] = "Windows"
 
         self.adapter.update_active_entity(fake_entity)
 
         self.assertEqual(self.db.entity.find_one({"entity_id": fake_entity.entity_id})["os"]["distro"],
-                         fake_entity.os.distro)
+                         fake_entity.image_meta['distro'])
+        self.assertEqual(self.db.entity.find_one({"entity_id": fake_entity.entity_id})["image_meta"]["distro"],
+                         fake_entity.image_meta['distro'])
 
     def test_delete_active_entity(self):
         fake_entity = a(volume())
 
-        self.db.entity.insert(model.todict(fake_entity))
+        self.db.entity.insert(fake_entity.as_dict())
         self.assertEqual(1, self.db.entity.count())
 
         self.adapter.delete_active_entity(fake_entity.entity_id)
@@ -348,7 +351,7 @@ class MongoDbDriverTest(base.BaseTestCase):
 
     def test_get_volume_type(self):
         fake_volume_type = a(volume_type())
-        self.db.volume_type.insert(model.todict(fake_volume_type))
+        self.db.volume_type.insert(fake_volume_type.as_dict())
         self.assertEqual(self.adapter.get_volume_type(fake_volume_type.volume_type_id), fake_volume_type)
 
     def test_get_volume_type_that_does_not_exists(self):
@@ -360,7 +363,7 @@ class MongoDbDriverTest(base.BaseTestCase):
 
     def test_delete_volume_type(self):
         fake_volume_type = a(volume_type())
-        self.db.volume_type.insert(model.todict(fake_volume_type))
+        self.db.volume_type.insert(fake_volume_type.as_dict())
         self.assertEqual(1, self.db.volume_type.count())
         self.adapter.delete_volume_type(fake_volume_type.volume_type_id)
         self.assertEqual(0, self.db.volume_type.count())
@@ -379,7 +382,7 @@ class MongoDbDriverTest(base.BaseTestCase):
         fake_volume_types = [a(volume_type()), a(volume_type())]
 
         for fake_volume_type in fake_volume_types:
-            self.db.volume_type.insert(model.todict(fake_volume_type))
+            self.db.volume_type.insert(fake_volume_type.as_dict())
 
         self.assertEqual(len(self.adapter.list_volume_types()), 2)
 

devstack/plugin.sh

@@ -46,6 +46,8 @@ function almanach_configure {
     iniset $ALMANACH_CONF collector transport_url rabbit://$RABBIT_USERID:$RABBIT_PASSWORD@$RABBIT_HOST:5672
 
     iniset $ALMANACH_CONF database connection_url mongodb://localhost/almanach
+
+    iniset $ALMANACH_CONF entities instance_image_meta distro,version,os_type
 }
 
 function almanach_configure_external_services {