rewrite interface

Change-Id: Ic02a180d62c733c4c53185a838dcb97a96e40fcc

artifice/api/web.py

@@ -1,6 +1,6 @@
 import flask
 from flask import Flask, Blueprint
-from artifice import interface, database, config
+from artifice import interface, database, config, transformers
 from artifice.sales_order import RatesFile
 from artifice.models import UsageEntry, SalesOrder, Tenant, billing
 import sqlalchemy
@@ -66,6 +66,15 @@ def generate_windows(start, end):
         start = new_start
 
 
+meter_mapping = {
+    'state':                {'type': 'virtual_machine', 'transformer': transformers.Uptime()},
+    'ip.floating':          {'type': 'floating_ip',     'transformer': transformers.GaugeMax()},
+    'volume.size':          {'type': 'volume',          'transformer': transformers.GaugeMax()},
+    'storage.objects.size': {'type': 'object_storage',  'transformer': transformers.GaugeMax()},
+    # TODO: add network usage, when we get the neutron bits for that figured out.
+}
+
+
 def collect_usage(tenant, db, session, resp, end):
     timestamp = datetime.now()
     session.begin(subtransactions=True)
@@ -84,10 +93,25 @@ def collect_usage(tenant, db, session, resp, end):
 
         try:
             print "%s %s slice %s %s" % (tenant.id, tenant.name, window_start, window_end)
-            usage = tenant.usage(window_start, window_end)
 
-            # enter all resources into the db
-            db.enter(usage.values(), window_start, window_end, timestamp)
+            for meter_name, meter_info in meter_mapping.items():
+                usage = tenant.usage(meter_name, window_start, window_end)
+                usage_by_resource = {}
+
+                for u in usage:
+                    resource_id = u['resource_id']
+                    entries = usage_by_resource.setdefault(resource_id, [])
+                    entries.append(u)
+
+                for res, entries in usage_by_resource.items():
+                    # apply the transformer.
+                    transformed = meter_info['transformer'].transform_usage(
+                        meter_name, entries, window_start, window_end)
+
+                    db.insert_resource(tenant.id, res, meter_info['type'], timestamp)
+                    db.insert_usage(tenant.id, res, transformed,
+                        window_start, window_end, timestamp)
+
             session.commit()
             resp["tenants"].append(
                 {"id": tenant.id,

artifice/database.py

@@ -22,44 +22,37 @@ class Database(object):
                                     name=tenant_name,
                                     created=timestamp
                                     ))
-            self.session.flush()
+            self.session.flush()            # can't assume deferred constraints.
 
-    def enter(self, usage, start, end, timestamp):
+    def insert_resource(self, tenant_id, resource_id, resource_type, timestamp):
+        query = self.session.query(Resource).\
+            filter(Resource.id == resource_id,
+                   Resource.tenant_id == tenant_id)
+        if query.count() == 0:
+            self.session.add(Resource(
+                id=resource_id,
+                info=resource_type,
+                tenant_id=tenant_id,
+                created=timestamp))
+            self.session.flush()            # can't assume deferred constraints.
+
+    def insert_usage(self, tenant_id, resource_id, entries, start, end, timestamp):
+        for service, volume in entries.items():
+            entry = UsageEntry(
+                service=service,
+                volume=volume,
+                resource_id=resource_id,
+                tenant_id=tenant_id,
+                start=start,
+                end=end,
+                created=timestamp)
+            self.session.add(entry)
+            print entry
+
+    def enter(self, tenant, resource, entries, timestamp):
         """Creates a new database entry for every usage strategy
            in a resource, for all the resources given"""
-
-        for resource in usage:
-            resource_id = resource.resource_id
-            tenant_id = resource.tenant_id
-            try:
-                for service, volume in resource.usage().iteritems():
-                    #  Have we seen this resource before?
-                    query = self.session.query(Resource).\
-                        filter(Resource.id == resource_id,
-                               Resource.tenant_id == tenant_id)
-                    if query.count() == 0:
-                        info = json.dumps(resource.info)
-                        self.session.add(Resource(id=resource_id,
-                                                  info=str(info),
-                                                  tenant_id=tenant_id,
-                                                  created=timestamp
-                                                  ))
-
-                    entry = UsageEntry(service=service,
-                                       volume=volume,
-                                       resource_id=resource_id,
-                                       tenant_id=tenant_id,
-                                       start=start,
-                                       end=end,
-                                       created=timestamp
-                                       )
-                    print entry
-                    self.session.add(entry)
-                    self.session.flush()
-            except TransformerValidationError:
-                # log something related to the resource usage failing
-                # transform.
-                pass
+        raise Exception("Dead!")
 
     def usage(self, start, end, tenant_id):
         """Returns a query of usage entries for a given tenant,

artifice/interface.py

@@ -5,24 +5,16 @@ from ceilometerclient.v2.client import Client as ceilometer
 from artifice.models import resources
 from constants import date_format
 import config
-from datetime import timedelta
+from datetime import timedelta, datetime
+from contextlib import contextmanager
 
-window_leadin = timedelta(minutes=10)
-
-def add_dates(start, end):
-    return [
-        {
-            "field": "timestamp",
-            "op": "ge",
-            "value": start.strftime(date_format)
-        },
-        {
-            "field": "timestamp",
-            "op": "lt",
-            "value": end.strftime(date_format)
-        }
-    ]
 
+@contextmanager
+def timed(desc):
+    start = datetime.now()
+    yield
+    end = datetime.now()
+    print "%s: %s" % (desc, end - start)
 
 class Artifice(object):
     """Produces billable artifacts"""
@@ -63,20 +55,37 @@ class Artifice(object):
         """All the tenants in our system"""
         if not self._tenancy:
             self._tenancy = []
-            for tenant in self.auth.tenants.list():
+            with timed("fetch tenant list from keystone"):
+                _tenants = self.auth.tenants.list()
+            for tenant in _tenants:
                 t = Tenant(tenant, self)
                 self._tenancy.append(t)
         return self._tenancy
 
 
-class Tenant(object):
+class InterfaceException(Exception):
+    pass
 
+window_leadin = timedelta(minutes=10)
+
+def add_dates(start, end):
+    return [
+        {
+            "field": "timestamp",
+            "op": "ge",
+            "value": start.strftime(date_format)
+        },
+        {
+            "field": "timestamp",
+            "op": "lt",
+            "value": end.strftime(date_format)
+        }
+    ]
+
+class Tenant(object):
     def __init__(self, tenant, conn):
         self.tenant = tenant
-        # Conn is the niceometer object we were instanced from
-        self.conn = conn
-        self._meters = set()
-        self._resources = None
+        self.conn = conn            # the Artifice object that produced us.
 
     @property
     def id(self):
@@ -88,182 +97,19 @@ class Tenant(object):
     def description(self):
         return self.tenant.description
 
-    def resources(self, start, end):
-        date_fields = [
-            {"field": "project_id",
-             "op": "eq",
-             "value": self.tenant.id
-             },
-        ]
-        date_fields.extend(add_dates(start, end))
-        resources = self.conn.ceilometer.resources.list(date_fields)
+    def usage(self, meter_name, start, end):
+        fields = [{'field': 'project_id', 'op': 'eq', 'value': self.tenant.id}]
+        fields.extend(add_dates(start - window_leadin, end))
 
-    def usage(self, start, end):
-        """
-        Usage is the meat of Artifice, returning a dict of location to
-        sub-information
-        """
-        vms = []
-        networks = []
-        ips = []
-        storage = []
-        volumes = []
+        with timed('fetch global usage for meter %s' % meter_name):
+            r = requests.get('%s/v2/meters/%s' % (config.ceilometer['host'], meter_name),
+                    headers={
+                        "X-Auth-Token": self.conn.auth.auth_token,
+                        "Content-Type": "application/json"
+                    },
+                    data=json.dumps({'q': fields}))
 
-        for resource in self.resources(start, end):
-            rels = [link["rel"] for link in resource.links if link["rel"] != 'self']
-            if "storage.objects" in rels:
-                storage.append(Resource(resource, self.conn))
-                pass
-            elif "network.incoming.bytes" in rels:
-                networks.append(Resource(resource, self.conn))
-            elif "volume" in rels:
-                volumes.append(Resource(resource, self.conn))
-            elif 'instance' in rels:
-                vms.append(Resource(resource, self.conn))
-            elif 'ip.floating' in rels:
-                ips.append(Resource(resource, self.conn))
-
-        region_tmpl = {
-            "vms": vms,
-            "networks": networks,
-            "objects": storage,
-            "volumes": volumes,
-            "ips": ips
-        }
-
-        return Usage(region_tmpl, start, end, self.conn)
-
-
-class Usage(object):
-    """
-    This is a dict-like object containing all the datacenters and
-    meters available in those datacenters.
-    """
-
-    def __init__(self, contents, start, end, conn):
-        self.contents = contents
-        self.start = start
-        self.end = end
-        self.conn = conn
-
-        self._vms = []
-        self._objects = []
-        self._volumes = []
-        self._networks = []
-        self._ips = []
-
-    def values(self):
-        return (self.vms + self.objects + self.volumes +
-                self.networks + self.ips)
-
-    @property
-    def vms(self):
-        if not self._vms:
-            vms = []
-            for vm in self.contents["vms"]:
-                VM = resources.VM(vm, self.start, self.end)
-                vms.append(VM)
-            self._vms = vms
-        return self._vms
-
-    @property
-    def objects(self):
-        if not self._objects:
-            objs = []
-            for object_ in self.contents["objects"]:
-                obj = resources.Object(object_, self.start, self.end)
-                objs.append(obj)
-            self._objs = objs
-        return self._objs
-
-    @property
-    def networks(self):
-        if not self._networks:
-            networks = []
-            for obj in self.contents["networks"]:
-                obj = resources.Network(obj, self.start, self.end)
-                networks.append(obj)
-            self._networks = networks
-        return self._networks
-
-    @property
-    def ips(self):
-        if not self._ips:
-            ips = []
-            for obj in self.contents["ips"]:
-                obj = resources.FloatingIP(obj, self.start, self.end)
-                ips.append(obj)
-            self._ips = ips
-        return self._ips
-
-    @property
-    def volumes(self):
-        if not self._volumes:
-            objs = []
-            for obj in self.contents["volumes"]:
-                obj = resources.Volume(obj, self.start, self.end)
-                objs.append(obj)
-            self._volumes = objs
-        return self._volumes
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        keys = self.contents.keys()
-        for key in keys:
-            yield key
-        raise StopIteration()
-
-
-class Resource(object):
-
-    def __init__(self, resource, conn):
-        self.resource = resource
-        self.conn = conn
-        self._meters = {}
-
-    def meter(self, name, start, end):
-        try:
-            for meter in self.resource.links:
-                if meter["rel"] == name:
-                    m = Meter(self, meter["href"], self.conn, start, end, name)
-                    self._meters[name] = m
-                    return m
-        except Exception as e:
-            print "If you drop exceptions on the floor i will cut you."
-            print e
-        raise AttributeError("no such meter %s" % name)
-
-
-class Meter(object):
-
-    def __init__(self, resource, link, conn, start, end, name):
-        self.resource = resource
-        self.link = link.split('?')[0]  # strip off the resource_id crap.
-        self.conn = conn
-        self.start = start
-        self.end = end
-        self.name = name
-        self.measurements = self.get_meter(start, end,
-                                           self.conn.auth.auth_token)
-
-    def get_meter(self, start, end, auth):
-        # Meter is a href; in this case, it has a set of fields with it already.
-        date_fields = add_dates(start - window_leadin, end)
-        date_fields.append({'field': 'resource_id',
-            'value': self.resource.resource.resource_id})
-
-        r = requests.get(
-            self.link,
-            headers={
-                "X-Auth-Token": auth,
-                "Content-Type": "application/json"
-            },
-            data=json.dumps({'q': date_fields})
-        )
-        result = json.loads(r.text)
-        return result
-
-    def usage(self):
-        return self.measurements
+            if r.status_code == 200:
+                return json.loads(r.text)
+            else:
+                raise InterfaceException('%d %s' % (r.status_code, r.text))

artifice/models/resources.py

@@ -1,120 +1,2 @@
 from artifice import transformers
 
-
-class BaseModelConstruct(object):
-
-    def __init__(self, raw, start, end):
-        # raw is the raw data for this
-        self._raw = raw
-        self._location = None
-        self.start = start
-        self.end = end
-
-    @property
-    def resource_id(self):
-        return self._raw.resource.resource_id
-
-    @property
-    def tenant_id(self):
-        return self._raw.resource.project_id
-
-    @property
-    def info(self):
-        return {"type": self.type}
-
-    def __getitem__(self, item):
-        return self._raw[item]
-
-    def meters(self):
-        dct = {}
-        for meter in self.relevant_meters:
-            try:
-                mtr = self._raw.meter(meter, self.start, self.end)
-                dct[meter] = mtr
-            except AttributeError:
-                # This is OK. We're not worried about non-existent meters,
-                # I think. For now, anyway.
-                pass
-        return dct
-
-    def usage(self):
-        meters = self.meters()
-        usage = self.transformer.transform_usage(meters, self.start, self.end)
-        return usage
-
-
-class VM(BaseModelConstruct):
-    relevant_meters = ['state']
-
-    transformer = transformers.Uptime()
-
-    type = "virtual_machine"
-
-    @property
-    def info(self):
-        return {"name": self.name,
-                "type": self.type}
-
-    @property
-    def memory(self):
-        return self._raw.resource.metadata["memory"]
-
-    @property
-    def cpus(self):
-        return self._raw.resource.metadata["vcpus"]
-
-    @property
-    def state(self):
-        return self._raw.resource.metadata["state"]
-
-    @property
-    def name(self):
-        return self._raw.resource.metadata["display_name"]
-
-    @property
-    def region(self):
-        return self._raw.resource.metadata["OS-EXT-AZ:availability_zone"]
-
-
-class FloatingIP(BaseModelConstruct):
-
-    relevant_meters = ["ip.floating"]
-
-    transformer = transformers.GaugeMax()
-
-    type = "floating_ip"
-
-
-class Object(BaseModelConstruct):
-
-    relevant_meters = ["storage.objects.size"]
-
-    transformer = transformers.GaugeMax()
-
-    type = "object_storage"
-
-
-class Volume(BaseModelConstruct):
-
-    relevant_meters = ["volume.size"]
-
-    transformer = transformers.GaugeMax()
-
-    type = "volume"
-
-    @property
-    def info(self):
-        return {"name": self.name,
-                "type": self.type}
-
-    @property
-    def name(self):
-        return self._raw.resource.metadata["display_name"]
-
-
-class Network(BaseModelConstruct):
-    relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"]
-
-    transformer = transformers.CumulativeRange()
-
-    type = "network_interface"

artifice/transformers.py

@@ -9,30 +9,10 @@ class TransformerValidationError(Exception):
 
 
 class Transformer(object):
+    def transform_usage(self, name, data, start, end):
+        return self._transform_usage(name, data, start, end)
 
-    meter_type = None
-    required_meters = []
-
-    def transform_usage(self, meters, start, end):
-        self.validate_meters(meters)
-        return self._transform_usage(meters, start, end)
-
-    def validate_meters(self, meters):
-        return
-        if self.meter_type is None:
-            for meter in self.required_meters:
-                if meter not in meters:
-                    raise TransformerValidationError(
-                        "Required meters: " +
-                        str(self.required_meters))
-        else:
-            for meter in meters.values():
-                if meter.type != self.meter_type:
-                    raise TransformerValidationError(
-                        "Meters must all be of type: " +
-                        self.meter_type)
-
-    def _transform_usage(self, meters, start, end):
+    def _transform_usage(self, name, data, start, end):
         raise NotImplementedError
 
 
@@ -41,9 +21,8 @@ class Uptime(Transformer):
     Transformer to calculate uptime based on states,
     which is broken apart into flavor at point in time.
     """
-    required_meters = ['state']
 
-    def _transform_usage(self, meters, start, end):
+    def _transform_usage(self, name, data, start, end):
         # get tracked states from config
         tracked = config.transformers['uptime']['tracked_states']
 
@@ -51,14 +30,12 @@ class Uptime(Transformer):
 
         usage_dict = {}
 
-        state = meters['state']
-
         def sort_and_clip_end(usage):
-            parsed = (self._parse_timestamp(s) for s in usage)
-            clipped = (s for s in parsed if s['timestamp'] < end)
+            cleaned = (self._clean_entry(s) for s in usage)
+            clipped = (s for s in cleaned if s['timestamp'] < end)
             return sorted(clipped, key=lambda x: x['timestamp'])
 
-        state = sort_and_clip_end(state.usage())
+        state = sort_and_clip_end(data)
 
         if not len(state):
             # there was no data for this period.
@@ -93,7 +70,7 @@ class Uptime(Transformer):
         # map the flavors to names on the way out
         return { helpers.flavor_name(f): v for f, v in usage_dict.items() }
 
-    def _parse_timestamp(self, entry):
+    def _clean_entry(self, entry):
         result = {
             'counter_volume': entry['counter_volume'],
             'flavor': entry['resource_metadata'].get('flavor.id',
@@ -115,41 +92,6 @@ class GaugeMax(Transformer):
     """
     meter_type = 'gauge'
 
-    def _transform_usage(self, meters, start, end):
-        usage_dict = {}
-        for name, meter in meters.iteritems():
-            usage = meter.usage()
-            max_vol = max([v["counter_volume"] for v in usage])
-            usage_dict[name] = max_vol
-        return usage_dict
-
-
-class CumulativeRange(Transformer):
-    """
-    Transformer to get the usage over a given range in a cumulative
-    metric, while taking into account that the metric can reset.
-    """
-    meter_type = 'cumulative'
-
-    def _transform_usage(self, meters, start, end):
-        usage_dict = {}
-        for name, meter in meters.iteritems():
-            measurements = meter.usage()
-            measurements = sorted(measurements, key=lambda x: x["timestamp"])
-            count = 0
-            usage = 0
-            last_measure = None
-            for measure in measurements:
-                if (last_measure is not None and
-                        (measure["counter_volume"] <
-                            last_measure["counter_volume"])):
-                    usage = usage + last_measure["counter_volume"]
-                count = count + 1
-                last_measure = measure
-
-            usage = usage + measurements[-1]["counter_volume"]
-
-            if count > 1:
-                total_usage = usage - measurements[0]["counter_volume"]
-                usage_dict[name] = total_usage
-        return usage_dict
+    def _transform_usage(self, name, data, start, end):
+        max_vol = max([v["counter_volume"] for v in data]) if len(data) else 0
+        return {name: max_vol}