Collection of MuranoPL instance statistics for billing purposes

Implements blueprint app-catalog-billing

Change-Id: Ib3c4b479d82d476629126cb67fe94314fdfecd4f

meta/io.murano.Application/manifest.yaml

@@ -1,3 +1,14 @@
 Namespaces:
     =: io.murano
+
 Name: Application
+
+Workflow:
+  reportDeployed:
+      Body:
+        - $this.find(Environment).instanceNotifier.trackApplication($this)
+
+  reportDestroyed:
+      Body:
+        - $this.find(Environment).instanceNotifier.untrackApplication($this)
+

meta/io.murano.Environment/manifest.yaml

@@ -7,20 +7,28 @@ Name: Environment
 Properties:
   name:
     Contract: $.string().notNull()
+
   applications:
     Contract: [$.class(Application).owned().notNull()]
+
   agentListener:
     Contract: $.class(sys:AgentListener)
     Type: Runtime
+
   stack:
     Contract: $.class(sys:HeatStack)
     Type: Runtime
 
+  instanceNotifier:
+    Contract: $.class(sys:InstanceNotifier)
+    Type: Runtime
+
 Workflow:
   initialize:
     Body:
       - $this.agentListener: new(sys:AgentListener, name => $.name)
       - $this.stack: new(sys:HeatStack, name => $.name)
+      - $this.instanceNotifier: new(sys:InstanceNotifier, environment => $this)
 
   deploy:
     Body:

muranoapi/api/v1/environment_statistics.py

@@ -0,0 +1,50 @@
+#    Copyright (c) 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from muranoapi.api.v1 import request_statistics
+from muranoapi.db.services import instances
+
+from muranoapi.openstack.common.gettextutils import _  # noqa
+from muranoapi.openstack.common import log as logging
+from muranoapi.openstack.common import wsgi
+
+
+LOG = logging.getLogger(__name__)
+API_NAME = 'EnvironmentStatistics'
+
+
+class Controller(object):
+    @request_statistics.stats_count(API_NAME, 'GetForEnvironment')
+    def get_for_environment(self, request, environment_id):
+        LOG.debug(_('EnvironmentStatistics:GetForEnvironment'))
+
+        # TODO (stanlagun): Check that caller is authorized to access
+        #  tenant's statistics
+
+        return instances.InstanceStatsServices.get_environment_stats(
+            environment_id)
+
+    @request_statistics.stats_count(API_NAME, 'GetForInstance')
+    def get_for_instance(self, request, environment_id, instance_id):
+        LOG.debug(_('EnvironmentStatistics:GetForInstance'))
+
+        # TODO (stanlagun): Check that caller is authorized to access
+        #  tenant's statistics
+
+        return instances.InstanceStatsServices.get_environment_stats(
+            environment_id, instance_id)
+
+
+def create_resource():
+    return wsgi.Resource(Controller())

muranoapi/api/v1/router.py

@@ -15,6 +15,7 @@ import routes
 
 from muranoapi.api.v1 import catalog
 from muranoapi.api.v1 import deployments
+from muranoapi.api.v1 import environment_statistics
 from muranoapi.api.v1 import environments
 from muranoapi.api.v1 import services
 from muranoapi.api.v1 import sessions
@@ -120,6 +121,18 @@ class API(wsgi.Router):
                        action='deploy',
                        conditions={'method': ['POST']})
 
+        statistics_resource = environment_statistics.create_resource()
+        mapper.connect(
+            '/environments/{environment_id}/statistics/{instance_id}',
+            controller=statistics_resource,
+            action='get_for_instance',
+            conditions={'method': ['GET']})
+        mapper.connect(
+            '/environments/{environment_id}/statistics',
+            controller=statistics_resource,
+            action='get_for_environment',
+            conditions={'method': ['GET']})
+
         catalog_resource = catalog.create_resource()
         mapper.connect('/catalog/packages/{package_id}',
                        controller=catalog_resource,

muranoapi/common/server.py

@@ -23,6 +23,7 @@ from sqlalchemy import desc
 from muranoapi.common import config
 from muranoapi.common.helpers import token_sanitizer
 from muranoapi.db import models
+from muranoapi.db.services import instances
 from muranoapi.db import session
 from muranoapi.openstack.common.gettextutils import _  # noqa
 from muranoapi.openstack.common import log as logging
@@ -109,6 +110,27 @@ def notification_endpoint_wrapper(priority='info'):
     return wrapper
 
 
+@notification_endpoint_wrapper()
+def track_instance(payload):
+    LOG.debug(_('Got track instance request from orchestration '
+                'engine:\n{0}'.format(payload)))
+    instance_id = payload['instance']
+    instance_type = payload.get('instance_type', 0)
+    environment_id = payload['environment']
+    instances.InstanceStatsServices.track_instance(
+        instance_id, environment_id, instance_type)
+
+
+@notification_endpoint_wrapper()
+def untrack_instance(payload):
+    LOG.debug(_('Got untrack instance request from orchestration '
+                'engine:\n{0}'.format(payload)))
+    instance_id = payload['instance']
+    environment_id = payload['environment']
+    instances.InstanceStatsServices.destroy_instance(
+        instance_id, environment_id)
+
+
 @notification_endpoint_wrapper()
 def report_notification(report):
     LOG.debug(_('Got report from orchestration '
@@ -145,7 +167,7 @@ def _prepare_rpc_service(server_id):
 
 
 def _prepare_notification_service(server_id):
-    endpoints = [report_notification]
+    endpoints = [report_notification, track_instance, untrack_instance]
 
     transport = messaging.get_transport(config.CONF)
     s_target = target.Target(topic='murano', server=server_id)

muranoapi/db/migrate_repo/versions/005_add_instance_table.py

@@ -0,0 +1,37 @@
+#    Copyright (c) 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import schema
+from sqlalchemy import types
+
+meta = schema.MetaData()
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    table = schema.Table(
+        'instance',
+        meta,
+        schema.Column('environment_id', types.String(100), primary_key=True),
+        schema.Column('instance_id', types.String(100), primary_key=True),
+        schema.Column('instance_type', types.Integer, nullable=False),
+        schema.Column('created', types.Integer, nullable=False),
+        schema.Column('destroyed', types.Integer, nullable=True))
+    table.create()
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    table = schema.Table('instance', meta, autoload=True)
+    table.drop()

muranoapi/db/models.py

@@ -36,13 +36,6 @@ def compile_big_int_sqlite(type_, compiler, **kw):
 
 
 class ModelBase(object):
-    __protected_attributes__ = set(["created", "updated"])
-
-    created = sa.Column(sa.DateTime, default=timeutils.utcnow,
-                        nullable=False)
-    updated = sa.Column(sa.DateTime, default=timeutils.utcnow,
-                        nullable=False, onupdate=timeutils.utcnow)
-
     def save(self, session=None):
         """Save this object"""
         session = session or db_session.get_session()
@@ -51,12 +44,10 @@ class ModelBase(object):
 
     def update(self, values):
         """dict.update() behaviour."""
-        self.updated = timeutils.utcnow()
         for k, v in values.iteritems():
             self[k] = v
 
     def __setitem__(self, key, value):
-        self.updated = timeutils.utcnow()
         setattr(self, key, value)
 
     def __getitem__(self, key):
@@ -85,6 +76,24 @@ class ModelBase(object):
                     if k != '_sa_instance_state')
 
 
+class ModificationsTrackedObject(ModelBase):
+    __protected_attributes__ = set(["created", "updated"])
+
+    created = sa.Column(sa.DateTime, default=timeutils.utcnow,
+                        nullable=False)
+    updated = sa.Column(sa.DateTime, default=timeutils.utcnow,
+                        nullable=False, onupdate=timeutils.utcnow)
+
+    def update(self, values):
+        """dict.update() behaviour."""
+        self.updated = timeutils.utcnow()
+        super(ModificationsTrackedObject, self).update(values)
+
+    def __setitem__(self, key, value):
+        self.updated = timeutils.utcnow()
+        super(ModificationsTrackedObject, self).__setitem__(key, value)
+
+
 class JsonBlob(sa.TypeDecorator):
     impl = sa.Text
 
@@ -95,7 +104,7 @@ class JsonBlob(sa.TypeDecorator):
         return anyjson.deserialize(value)
 
 
-class Environment(BASE, ModelBase):
+class Environment(BASE, ModificationsTrackedObject):
     """Represents a Environment in the metadata-store"""
     __tablename__ = 'environment'
 
@@ -119,7 +128,7 @@ class Environment(BASE, ModelBase):
         return dictionary
 
 
-class Session(BASE, ModelBase):
+class Session(BASE, ModificationsTrackedObject):
     __tablename__ = 'session'
 
     id = sa.Column(sa.String(32),
@@ -141,7 +150,7 @@ class Session(BASE, ModelBase):
         return dictionary
 
 
-class Deployment(BASE, ModelBase):
+class Deployment(BASE, ModificationsTrackedObject):
     __tablename__ = 'deployment'
 
     id = sa.Column(sa.String(32),
@@ -165,7 +174,7 @@ class Deployment(BASE, ModelBase):
         return dictionary
 
 
-class Status(BASE, ModelBase):
+class Status(BASE, ModificationsTrackedObject):
     __tablename__ = 'status'
 
     id = sa.Column(sa.String(32),
@@ -186,7 +195,7 @@ class Status(BASE, ModelBase):
         return dictionary
 
 
-class ApiStats(BASE, ModelBase):
+class ApiStats(BASE, ModificationsTrackedObject):
     __tablename__ = 'apistats'
 
     id = sa.Column(sa.Integer(), primary_key=True)
@@ -223,7 +232,23 @@ package_to_tag = sa.Table('package_to_tag',
                           )
 
 
-class Package(BASE, ModelBase):
+class Instance(BASE, ModelBase):
+    __tablename__ = 'instance'
+
+    environment_id = sa.Column(
+        sa.String(100), primary_key=True, nullable=False)
+    instance_id = sa.Column(
+        sa.String(100), primary_key=True, nullable=False)
+    instance_type = sa.Column(sa.Integer, default=0, nullable=False)
+    created = sa.Column(sa.Integer, nullable=False)
+    destroyed = sa.Column(sa.Integer, nullable=True)
+
+    def to_dict(self):
+        dictionary = super(Instance, self).to_dict()
+        return dictionary
+
+
+class Package(BASE, ModificationsTrackedObject):
     """
     Represents a meta information about application package.
     """
@@ -272,7 +297,7 @@ class Package(BASE, ModelBase):
         return d
 
 
-class Category(BASE, ModelBase):
+class Category(BASE, ModificationsTrackedObject):
     """
     Represents an application categories in the datastore.
     """
@@ -284,7 +309,7 @@ class Category(BASE, ModelBase):
     name = sa.Column(sa.String(80), nullable=False, index=True, unique=True)
 
 
-class Tag(BASE, ModelBase):
+class Tag(BASE, ModificationsTrackedObject):
     """
     Represents tags in the datastore.
     """
@@ -296,7 +321,7 @@ class Tag(BASE, ModelBase):
     name = sa.Column(sa.String(80), nullable=False, unique=True)
 
 
-class Class(BASE, ModelBase):
+class Class(BASE, ModificationsTrackedObject):
     """
     Represents a class definition in the datastore.
     """
@@ -314,7 +339,7 @@ def register_models(engine):
     Creates database tables for all models with the given engine
     """
     models = (Environment, Status, Session, Deployment,
-              ApiStats, Package, Category, Class)
+              ApiStats, Package, Category, Class, Instance)
     for model in models:
         model.metadata.create_all(engine)
 

muranoapi/db/services/instances.py

@@ -0,0 +1,72 @@
+#    Copyright (c) 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy.sql import func
+
+from muranoapi.db import models
+from muranoapi.db import session as db_session
+from muranoapi.openstack.common.db import exception
+from muranoapi.openstack.common import timeutils
+
+
+UNCLASSIFIED = 0
+APPLICATION = 100
+OS_INSTANCE = 200
+
+
+class InstanceStatsServices(object):
+    @staticmethod
+    def track_instance(instance_id, environment_id, instance_type):
+        instance = models.Instance()
+        instance.instance_id = instance_id
+        instance.environment_id = environment_id
+        instance.instance_type = instance_type
+        instance.created = timeutils.utcnow_ts()
+        instance.destroyed = None
+
+        unit = db_session.get_session()
+        try:
+            with unit.begin():
+                unit.add(instance)
+        except exception.DBDuplicateEntry:
+            pass  # expected behaviour when record already exists
+
+    @staticmethod
+    def destroy_instance(instance_id, environment_id):
+        unit = db_session.get_session()
+        instance = unit.query(models.Instance).get(
+            (environment_id, instance_id))
+        if instance and not instance.destroyed:
+            instance.destroyed = timeutils.utcnow_ts()
+            instance.save(unit)
+
+    @staticmethod
+    def get_environment_stats(environment_id, instance_id=None):
+        unit = db_session.get_session()
+        now = timeutils.utcnow_ts()
+        query = unit.query(models.Instance.instance_type, func.sum(
+            func.coalesce(models.Instance.destroyed, now) -
+            models.Instance.created), func.count()).filter(
+                models.Instance.environment_id == environment_id)
+        if instance_id is not None:
+            query = query.filter(
+                models.Instance.instance_id == instance_id)
+
+        res = query.group_by(models.Instance.instance_type).all()
+
+        return [{
+                'type': int(record[0]),
+                'duration': int(record[1]),
+                'count': int(record[2])
+                } for record in res]

muranoapi/engine/system/instance_reporter.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo import messaging
+
+from muranoapi.common import config
+from muranoapi.common import uuidutils
+from muranoapi.dsl import murano_class
+from muranoapi.openstack.common import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+UNCLASSIFIED = 0
+APPLICATION = 100
+OS_INSTANCE = 200
+
+
+@murano_class.classname('io.murano.system.InstanceNotifier')
+class InstanceReportNotifier(object):
+    transport = None
+
+    def __init__(self):
+        if InstanceReportNotifier.transport is None:
+            InstanceReportNotifier.transport = \
+                messaging.get_transport(config.CONF)
+        self._notifier = messaging.Notifier(
+            InstanceReportNotifier.transport,
+            publisher_id=uuidutils.generate_uuid(),
+            topic='murano')
+
+    def initialize(self, environment):
+        self._environment_id = environment.object_id
+
+    def _track_instance(self, instance, instance_type, untrack=False):
+        payload = {
+            'instance': instance.object_id,
+            'environment': self._environment_id,
+            'instance_type': instance_type
+        }
+
+        event_type = 'murano.untrack_instance' \
+            if untrack else 'murano.track_instance'
+
+        self._notifier.info({}, event_type, payload)
+
+    def trackApplication(self, instance):
+        self._track_instance(instance, APPLICATION, False)
+
+    def untrackApplication(self, instance):
+        self._track_instance(instance, APPLICATION, True)
+
+    def trackCloudInstance(self, instance):
+        self._track_instance(instance, OS_INSTANCE, False)
+
+    def untrackCloudInstance(self, instance):
+        self._track_instance(instance, OS_INSTANCE, True)

muranoapi/engine/system/system_objects.py

@@ -15,11 +15,12 @@
 
 import inspect
 
-import muranoapi.dsl.murano_class as murano_class
-import muranoapi.engine.system.agent as agent
-import muranoapi.engine.system.agent_listener as agent_listener
-import muranoapi.engine.system.heat_stack as heat_stack
-import muranoapi.engine.system.resource_manager as resource_manager
+from muranoapi.dsl import murano_class
+from muranoapi.engine.system import agent
+from muranoapi.engine.system import agent_listener
+from muranoapi.engine.system import heat_stack
+from muranoapi.engine.system import instance_reporter
+from muranoapi.engine.system import resource_manager
 
 
 def _auto_register(class_loader):
@@ -46,3 +47,4 @@ def register(class_loader, path):
     class_loader.import_class(agent_listener.AgentListener)
     class_loader.import_class(heat_stack.HeatStack)
     class_loader.import_class(ResourceManagerWrapper)
+    class_loader.import_class(instance_reporter.InstanceReportNotifier)