openstack/cloudkitty
Code Issues Proposed changes

PEP8 and hacking compliance

Browse Source 
Change-Id: I8854027c486fdf8ee3b2ca1812f5231a50b1014e
This commit is contained in:
Stéphane Albert 2014-06-20 19:03:26 +02:00
parent 734890ea52
commit d6f0a32b94
16 changed files with 98 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cloudkitty/backend/base.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

1
cloudkitty/backend/file.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

1
cloudkitty/billing/base.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

5
cloudkitty/billing/hash.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -16,10 +17,10 @@
#
import json
from cloudkitty.billing.base import BaseBillingProcessor
from cloudkitty.billing import base
class BasicHashMap(BaseBillingProcessor):
class BasicHashMap(base.BaseBillingProcessor):
def __init__(self):
self._billing_info = {}
self._load_billing_rates()

5
cloudkitty/billing/noop.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -14,10 +15,10 @@
#
# @author: Stéphane Albert
#
from cloudkitty.billing.base import BaseBillingProcessor
from cloudkitty.billing import base
class Noop(BaseBillingProcessor):
class Noop(base.BaseBillingProcessor):
def __init__(self):
pass

8
cloudkitty/collector/base.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -14,7 +15,7 @@
#
# @author: Stéphane Albert
#
from datetime import datetime, timedelta
import datetime
import cloudkitty.utils as utils
@ -39,8 +40,9 @@ class BaseCollector(object):
@staticmethod
def last_month():
now = datetime.now()
month_end = datetime(now.year, now.month, 1) - timedelta(days=1)
now = datetime.datetime.now()
month_end = (datetime.datetime(now.year, now.month, 1) -
datetime.timedelta(days=1))
month_start = month_end.replace(day=1)
start_ts = utils.dt2ts(month_start)
end_ts = utils.dt2ts(month_end)

35
cloudkitty/collector/ceilometer.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -14,23 +15,21 @@
#
# @author: Stéphane Albert
#
from datetime import datetime
import datetime
from ceilometerclient import client as cclient
from cloudkitty.collector.base import BaseCollector
from cloudkitty.collector import base
class CeilometerCollector(BaseCollector):
class CeilometerCollector(base.BaseCollector):
def __init__(self, **kwargs):
super(CeilometerCollector, self).__init__(**kwargs)
self._resource_cache = {}
def _connect(self):
"""
Initialize connection to the Ceilometer endpoint.
"""
"""Initialize connection to the Ceilometer endpoint."""
self._conn = cclient.get_client('2', os_username=self.user,
os_password=self.password,
os_auth_url=self.keystone_url,
@ -38,27 +37,21 @@ class CeilometerCollector(BaseCollector):
os_region_name=self.region)
def gen_filter(self, op='eq', **kwargs):
"""
Generate ceilometer filter from kwargs.
"""
"""Generate ceilometer filter from kwargs."""
q_filter = []
for kwarg in kwargs:
q_filter.append({'field': kwarg, 'op': op, 'value': kwargs[kwarg]})
return q_filter
def prepend_filter(self, prepend, **kwargs):
"""
Prepend the dict key with the prepend value, useful to compose filters.
"""
"""Filter composer."""
q_filter = {}
for kwarg in kwargs:
q_filter[prepend + kwarg] = kwargs[kwarg]
return q_filter
def user_metadata_filter(self, op='eq', **kwargs):
"""
Create user_metadata filter from kwargs.
"""
"""Create user_metadata filter from kwargs."""
user_filter = {}
for kwarg in kwargs:
field = kwarg
@ -70,18 +63,13 @@ class CeilometerCollector(BaseCollector):
return self.metadata_filter(op, **user_filter)
def metadata_filter(self, op='eq', **kwargs):
"""
Create metadata filter from kwargs.
"""
"""Create metadata filter from kwargs."""
meta_filter = self.prepend_filter('metadata.', **kwargs)
return self.gen_filter(op, **meta_filter)
def get_active_instances(self, start, end=None, project_id=None,
q_filter=None):
"""
Return the number of instance that were active during the
timespan.
"""
"""Instance that were active during the timespan."""
start_iso = datetime.fromtimestamp(start).isoformat()
req_filter = self.gen_filter(op='ge', timestamp=start_iso)
if project_id:
@ -121,8 +109,9 @@ class CeilometerCollector(BaseCollector):
res_data['vcpus'] = data.metadata.get('vcpus')
res_data['memory'] = data.metadata.get('memory_mb')
res_data['image_id'] = data.metadata.get('image.id')
res_data['availability_zone'] = \
res_data['availability_zone'] = (
data.metadata.get('OS-EXT-AZ.availability_zone')
)
res_data['project_id'] = data.project_id
res_data['user_id'] = data.user_id

1
cloudkitty/config.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

36
cloudkitty/orchestrator.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# !/usr/bin/env python
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,17 +16,18 @@
#
# @author: Stéphane Albert
#
from datetime import datetime
from __future__ import print_function
import datetime
import sys
import time
from keystoneclient.v2_0 import client as kclient
from oslo.config import cfg
import cloudkitty.utils as utils
import cloudkitty.config # NOQA
from cloudkitty.state import StateManager
from cloudkitty.write_orchestrator import WriteOrchestrator
from cloudkitty import state
import cloudkitty.utils as utils
from cloudkitty import write_orchestrator as w_orch
CONF = cfg.CONF
@ -48,10 +50,10 @@ class Orchestrator(object):
region_name=CONF.auth.region,
auth_url=CONF.auth.url)
self.sm = StateManager(utils.import_class(CONF.state.backend),
CONF.state.basepath,
self.keystone.user_id,
'osrtf')
self.sm = state.StateManager(utils.import_class(CONF.state.backend),
CONF.state.basepath,
self.keystone.user_id,
'osrtf')
collector = utils.import_class(CONF.collect.collector)
self.collector = collector(user=CONF.auth.username,
@ -61,18 +63,20 @@ class Orchestrator(object):
keystone_url=CONF.auth.url,
period=CONF.collect.period)
self.wo = WriteOrchestrator(utils.import_class(CONF.output.backend),
utils.import_class(CONF.state.backend),
self.keystone.user_id,
self.sm)
w_backend = utils.import_class(CONF.output.backend)
s_backend = utils.import_class(CONF.state.backend)
self.wo = w_orch.WriteOrchestrator(w_backend,
s_backend,
self.keystone.user_id,
self.sm)
for writer in self.output_pipeline:
self.wo.add_writer(writer)
def _check_state(self):
def _get_this_month_timestamp():
now = datetime.now()
month_start = datetime(now.year, now.month, 1)
now = datetime.datetime.now()
month_start = datetime.datetime(now.year, now.month, 1)
timestamp = int(time.mktime(month_start.timetuple()))
return timestamp
@ -100,7 +104,7 @@ class Orchestrator(object):
while True:
timestamp = self._check_state()
if not timestamp:
print "Nothing left to do."
print("Nothing left to do.")
break
for service in CONF.collect.services:

17
cloudkitty/state.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -57,35 +58,27 @@ class StateManager(object):
state_file.close()
def set_state(self, timestamp):
"""
Set the current state's timestamp
"""
"""Set the current state's timestamp."""
if self._distributed:
self._load()
self._ts = timestamp
self._update()
def get_state(self):
"""
Get the state timestamp
"""
"""Get the state timestamp."""
if self._distributed:
self._load()
return self._ts
def set_metadata(self, metadata):
"""
Set metadata attached to the state
"""
"""Set metadata attached to the state."""
if self._distributed:
self._load()
self._metadata = metadata
self._update()
def get_metadata(self):
"""
Get metadata attached to the state
"""
"""Get metadata attached to the state."""
if self._distributed:
self._load()
return self._metadata

1
cloudkitty/utils.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

43
cloudkitty/write_orchestrator.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -14,24 +15,25 @@
#
# @author: Stéphane Albert
#
import datetime
import json
from datetime import datetime
from zipfile import ZipFile
import zipfile
import cloudkitty.utils as utils
class OSRTFBackend(object):
"""
Native backend for transient report storage.
"""Native backend for transient report storage.
Used to store data from the output of the billing pipeline.
"""
def __init__(self, backend):
self._backend = backend
self._osrtf = None
def open(self, filename):
self._osrtf = ZipFile(self._backend(filename, 'ab+'), 'a')
self._osrtf = zipfile.ZipFile(self._backend(filename, 'ab+'), 'a')
def _gen_filename(self, timeframe):
filename = '{}-{:02d}-{:02d}-{}-{}.json'.format(timeframe.year,
@ -48,9 +50,7 @@ class OSRTFBackend(object):
return False
def add(self, timeframe, data):
"""
Add the data to the OpenStack Report Transient Format.
"""
"""Add the data to the OpenStack Report Transient Format."""
filename = self._gen_filename(timeframe)
# We can only check for the existence of a file not rewrite or delete
# it
@ -62,13 +62,13 @@ class OSRTFBackend(object):
filename = self._gen_filename(timeframe)
data = json.loads(self._osrtf.read(filename))
return data
except:
except Exception:
pass
class WriteOrchestrator(object):
"""
Write Orchestrator:
"""Write Orchestrator:
Handle incoming data from the global orchestrator, and store them in an
intermediary data format before final transformation.
"""
@ -102,7 +102,7 @@ class WriteOrchestrator(object):
self._write_pipeline.append(writer)
def _gen_osrtf_filename(self, timeframe):
if not isinstance(timeframe, datetime):
if not isinstance(timeframe, datetime.datetime):
raise TypeError('timeframe should be of type datetime.')
date = '{}-{:02d}'.format(timeframe.year, timeframe.month)
filename = '{}-osrtf-{}.zip'.format(self._uid, date)
@ -115,15 +115,16 @@ class WriteOrchestrator(object):
def _get_state_manager_timeframe(self):
timeframe = self._sm.get_state()
self.usage_start = datetime.fromtimestamp(timeframe)
self.usage_end = datetime.fromtimestamp(timeframe + self._period)
self.usage_start = datetime.datetime.fromtimestamp(timeframe)
end_frame = timeframe + self._period
self.usage_end = datetime.datetime.fromtimestamp(end_frame)
metadata = self._sm.get_metadata()
self.total = metadata.get('total', 0)
def _filter_period(self, json_data):
"""
Detect the best usage period to extract. Removes the usage from the
json data and returns it.
"""Detect the best usage period to extract.
Removes the usage from the json data and returns it.
"""
candidate_ts = None
candidate_idx = 0
@ -190,8 +191,12 @@ class WriteOrchestrator(object):
if self.usage_start is None:
self.usage_start = usage_start
self.usage_end = usage_start + self._period
self.usage_start_dt = datetime.fromtimestamp(self.usage_start)
self.usage_end_dt = datetime.fromtimestamp(self.usage_end)
self.usage_start_dt = (
datetime.datetime.fromtimestamp(self.usage_start)
)
self.usage_end_dt = (
datetime.datetime.fromtimestamp(self.usage_end)
)
self._dispatch(data)

21
cloudkitty/writer/base.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -14,22 +15,23 @@
#
# @author: Stéphane Albert
#
from datetime import datetime
from cloudkitty.state import StateManager
import datetime
from cloudkitty import state
class BaseReportWriter(object):
"""
Base report writer.
"""
"""Base report writer."""
report_type = None
def __init__(self, write_orchestrator, user_id, backend, state_backend):
self._write_orchestrator = write_orchestrator
self._write_backend = backend
self._uid = user_id
self._sm = StateManager(state_backend, None, self._uid,
self.report_type)
self._sm = state.StateManager(state_backend,
None,
self._uid,
self.report_type)
self._report = None
self.period = 3600
@ -68,8 +70,9 @@ class BaseReportWriter(object):
def _get_state_manager_timeframe(self):
timeframe = self._sm.get_state()
self.usage_start = timeframe
self.usage_start_dt = datetime.fromtimestamp(timeframe)
self.usage_end = datetime.fromtimestamp(timeframe + self.period)
self.usage_start_dt = datetime.datetime.fromtimestamp(timeframe)
end_frame = timeframe + self._period
self.usage_end = datetime.datetime.fromtimestamp(end_frame)
metadata = self._sm.get_metadata()
self.total = metadata.get('total', 0)

10
cloudkitty/writer/osrf.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,12 +16,13 @@
# @author: Stéphane Albert
#
import json
from cloudkitty.writer.base import BaseReportWriter
from cloudkitty.writer import base
class OSRFBackend(BaseReportWriter):
"""
OpenStack Report Format Writer:
class OSRFBackend(base.BaseReportWriter):
"""OpenStack Report Format Writer:
Generates report in native format (json)
"""
report_type = 'osrf'

1
requirements.txt
View File

@ -3,3 +3,4 @@ python-keystoneclient
iso8601
oslo.config>=1.2.0
pbr>=0.6,<1.0
six>=1.7.0

3
test-requirements.txt
View File

@ -1,7 +1,8 @@
hacking>=0.9.1,<0.10
discover
testtools
testrepository
flake8
mock>=1.0
sphinx>=1.1.2,<1.2
oslosphinx
oslotest