PEP8 and hacking compliance
Change-Id: I8854027c486fdf8ee3b2ca1812f5231a50b1014e
This commit is contained in:
parent
734890ea52
commit
d6f0a32b94
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -16,10 +17,10 @@
|
||||
#
|
||||
import json
|
||||
|
||||
from cloudkitty.billing.base import BaseBillingProcessor
|
||||
from cloudkitty.billing import base
|
||||
|
||||
|
||||
class BasicHashMap(BaseBillingProcessor):
|
||||
class BasicHashMap(base.BaseBillingProcessor):
|
||||
def __init__(self):
|
||||
self._billing_info = {}
|
||||
self._load_billing_rates()
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -14,10 +15,10 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
from cloudkitty.billing.base import BaseBillingProcessor
|
||||
from cloudkitty.billing import base
|
||||
|
||||
|
||||
class Noop(BaseBillingProcessor):
|
||||
class Noop(base.BaseBillingProcessor):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -14,7 +15,7 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
from datetime import datetime, timedelta
|
||||
import datetime
|
||||
|
||||
import cloudkitty.utils as utils
|
||||
|
||||
@ -39,8 +40,9 @@ class BaseCollector(object):
|
||||
|
||||
@staticmethod
|
||||
def last_month():
|
||||
now = datetime.now()
|
||||
month_end = datetime(now.year, now.month, 1) - timedelta(days=1)
|
||||
now = datetime.datetime.now()
|
||||
month_end = (datetime.datetime(now.year, now.month, 1) -
|
||||
datetime.timedelta(days=1))
|
||||
month_start = month_end.replace(day=1)
|
||||
start_ts = utils.dt2ts(month_start)
|
||||
end_ts = utils.dt2ts(month_end)
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -14,23 +15,21 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
from datetime import datetime
|
||||
import datetime
|
||||
|
||||
from ceilometerclient import client as cclient
|
||||
|
||||
from cloudkitty.collector.base import BaseCollector
|
||||
from cloudkitty.collector import base
|
||||
|
||||
|
||||
class CeilometerCollector(BaseCollector):
|
||||
class CeilometerCollector(base.BaseCollector):
|
||||
def __init__(self, **kwargs):
|
||||
super(CeilometerCollector, self).__init__(**kwargs)
|
||||
|
||||
self._resource_cache = {}
|
||||
|
||||
def _connect(self):
|
||||
"""
|
||||
Initialize connection to the Ceilometer endpoint.
|
||||
"""
|
||||
"""Initialize connection to the Ceilometer endpoint."""
|
||||
self._conn = cclient.get_client('2', os_username=self.user,
|
||||
os_password=self.password,
|
||||
os_auth_url=self.keystone_url,
|
||||
@ -38,27 +37,21 @@ class CeilometerCollector(BaseCollector):
|
||||
os_region_name=self.region)
|
||||
|
||||
def gen_filter(self, op='eq', **kwargs):
|
||||
"""
|
||||
Generate ceilometer filter from kwargs.
|
||||
"""
|
||||
"""Generate ceilometer filter from kwargs."""
|
||||
q_filter = []
|
||||
for kwarg in kwargs:
|
||||
q_filter.append({'field': kwarg, 'op': op, 'value': kwargs[kwarg]})
|
||||
return q_filter
|
||||
|
||||
def prepend_filter(self, prepend, **kwargs):
|
||||
"""
|
||||
Prepend the dict key with the prepend value, useful to compose filters.
|
||||
"""
|
||||
"""Filter composer."""
|
||||
q_filter = {}
|
||||
for kwarg in kwargs:
|
||||
q_filter[prepend + kwarg] = kwargs[kwarg]
|
||||
return q_filter
|
||||
|
||||
def user_metadata_filter(self, op='eq', **kwargs):
|
||||
"""
|
||||
Create user_metadata filter from kwargs.
|
||||
"""
|
||||
"""Create user_metadata filter from kwargs."""
|
||||
user_filter = {}
|
||||
for kwarg in kwargs:
|
||||
field = kwarg
|
||||
@ -70,18 +63,13 @@ class CeilometerCollector(BaseCollector):
|
||||
return self.metadata_filter(op, **user_filter)
|
||||
|
||||
def metadata_filter(self, op='eq', **kwargs):
|
||||
"""
|
||||
Create metadata filter from kwargs.
|
||||
"""
|
||||
"""Create metadata filter from kwargs."""
|
||||
meta_filter = self.prepend_filter('metadata.', **kwargs)
|
||||
return self.gen_filter(op, **meta_filter)
|
||||
|
||||
def get_active_instances(self, start, end=None, project_id=None,
|
||||
q_filter=None):
|
||||
"""
|
||||
Return the number of instance that were active during the
|
||||
timespan.
|
||||
"""
|
||||
"""Instance that were active during the timespan."""
|
||||
start_iso = datetime.fromtimestamp(start).isoformat()
|
||||
req_filter = self.gen_filter(op='ge', timestamp=start_iso)
|
||||
if project_id:
|
||||
@ -121,8 +109,9 @@ class CeilometerCollector(BaseCollector):
|
||||
res_data['vcpus'] = data.metadata.get('vcpus')
|
||||
res_data['memory'] = data.metadata.get('memory_mb')
|
||||
res_data['image_id'] = data.metadata.get('image.id')
|
||||
res_data['availability_zone'] = \
|
||||
res_data['availability_zone'] = (
|
||||
data.metadata.get('OS-EXT-AZ.availability_zone')
|
||||
)
|
||||
|
||||
res_data['project_id'] = data.project_id
|
||||
res_data['user_id'] = data.user_id
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# !/usr/bin/env python
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -15,17 +16,18 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
from datetime import datetime
|
||||
from __future__ import print_function
|
||||
import datetime
|
||||
import sys
|
||||
import time
|
||||
|
||||
from keystoneclient.v2_0 import client as kclient
|
||||
from oslo.config import cfg
|
||||
|
||||
import cloudkitty.utils as utils
|
||||
import cloudkitty.config # NOQA
|
||||
from cloudkitty.state import StateManager
|
||||
from cloudkitty.write_orchestrator import WriteOrchestrator
|
||||
from cloudkitty import state
|
||||
import cloudkitty.utils as utils
|
||||
from cloudkitty import write_orchestrator as w_orch
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
@ -48,7 +50,7 @@ class Orchestrator(object):
|
||||
region_name=CONF.auth.region,
|
||||
auth_url=CONF.auth.url)
|
||||
|
||||
self.sm = StateManager(utils.import_class(CONF.state.backend),
|
||||
self.sm = state.StateManager(utils.import_class(CONF.state.backend),
|
||||
CONF.state.basepath,
|
||||
self.keystone.user_id,
|
||||
'osrtf')
|
||||
@ -61,8 +63,10 @@ class Orchestrator(object):
|
||||
keystone_url=CONF.auth.url,
|
||||
period=CONF.collect.period)
|
||||
|
||||
self.wo = WriteOrchestrator(utils.import_class(CONF.output.backend),
|
||||
utils.import_class(CONF.state.backend),
|
||||
w_backend = utils.import_class(CONF.output.backend)
|
||||
s_backend = utils.import_class(CONF.state.backend)
|
||||
self.wo = w_orch.WriteOrchestrator(w_backend,
|
||||
s_backend,
|
||||
self.keystone.user_id,
|
||||
self.sm)
|
||||
|
||||
@ -71,8 +75,8 @@ class Orchestrator(object):
|
||||
|
||||
def _check_state(self):
|
||||
def _get_this_month_timestamp():
|
||||
now = datetime.now()
|
||||
month_start = datetime(now.year, now.month, 1)
|
||||
now = datetime.datetime.now()
|
||||
month_start = datetime.datetime(now.year, now.month, 1)
|
||||
timestamp = int(time.mktime(month_start.timetuple()))
|
||||
return timestamp
|
||||
|
||||
@ -100,7 +104,7 @@ class Orchestrator(object):
|
||||
while True:
|
||||
timestamp = self._check_state()
|
||||
if not timestamp:
|
||||
print "Nothing left to do."
|
||||
print("Nothing left to do.")
|
||||
break
|
||||
|
||||
for service in CONF.collect.services:
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -57,35 +58,27 @@ class StateManager(object):
|
||||
state_file.close()
|
||||
|
||||
def set_state(self, timestamp):
|
||||
"""
|
||||
Set the current state's timestamp
|
||||
"""
|
||||
"""Set the current state's timestamp."""
|
||||
if self._distributed:
|
||||
self._load()
|
||||
self._ts = timestamp
|
||||
self._update()
|
||||
|
||||
def get_state(self):
|
||||
"""
|
||||
Get the state timestamp
|
||||
"""
|
||||
"""Get the state timestamp."""
|
||||
if self._distributed:
|
||||
self._load()
|
||||
return self._ts
|
||||
|
||||
def set_metadata(self, metadata):
|
||||
"""
|
||||
Set metadata attached to the state
|
||||
"""
|
||||
"""Set metadata attached to the state."""
|
||||
if self._distributed:
|
||||
self._load()
|
||||
self._metadata = metadata
|
||||
self._update()
|
||||
|
||||
def get_metadata(self):
|
||||
"""
|
||||
Get metadata attached to the state
|
||||
"""
|
||||
"""Get metadata attached to the state."""
|
||||
if self._distributed:
|
||||
self._load()
|
||||
return self._metadata
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -14,24 +15,25 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
import datetime
|
||||
import json
|
||||
from datetime import datetime
|
||||
from zipfile import ZipFile
|
||||
import zipfile
|
||||
|
||||
import cloudkitty.utils as utils
|
||||
|
||||
|
||||
class OSRTFBackend(object):
|
||||
"""
|
||||
Native backend for transient report storage.
|
||||
"""Native backend for transient report storage.
|
||||
|
||||
Used to store data from the output of the billing pipeline.
|
||||
"""
|
||||
|
||||
def __init__(self, backend):
|
||||
self._backend = backend
|
||||
self._osrtf = None
|
||||
|
||||
def open(self, filename):
|
||||
self._osrtf = ZipFile(self._backend(filename, 'ab+'), 'a')
|
||||
self._osrtf = zipfile.ZipFile(self._backend(filename, 'ab+'), 'a')
|
||||
|
||||
def _gen_filename(self, timeframe):
|
||||
filename = '{}-{:02d}-{:02d}-{}-{}.json'.format(timeframe.year,
|
||||
@ -48,9 +50,7 @@ class OSRTFBackend(object):
|
||||
return False
|
||||
|
||||
def add(self, timeframe, data):
|
||||
"""
|
||||
Add the data to the OpenStack Report Transient Format.
|
||||
"""
|
||||
"""Add the data to the OpenStack Report Transient Format."""
|
||||
filename = self._gen_filename(timeframe)
|
||||
# We can only check for the existence of a file not rewrite or delete
|
||||
# it
|
||||
@ -62,13 +62,13 @@ class OSRTFBackend(object):
|
||||
filename = self._gen_filename(timeframe)
|
||||
data = json.loads(self._osrtf.read(filename))
|
||||
return data
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class WriteOrchestrator(object):
|
||||
"""
|
||||
Write Orchestrator:
|
||||
"""Write Orchestrator:
|
||||
|
||||
Handle incoming data from the global orchestrator, and store them in an
|
||||
intermediary data format before final transformation.
|
||||
"""
|
||||
@ -102,7 +102,7 @@ class WriteOrchestrator(object):
|
||||
self._write_pipeline.append(writer)
|
||||
|
||||
def _gen_osrtf_filename(self, timeframe):
|
||||
if not isinstance(timeframe, datetime):
|
||||
if not isinstance(timeframe, datetime.datetime):
|
||||
raise TypeError('timeframe should be of type datetime.')
|
||||
date = '{}-{:02d}'.format(timeframe.year, timeframe.month)
|
||||
filename = '{}-osrtf-{}.zip'.format(self._uid, date)
|
||||
@ -115,15 +115,16 @@ class WriteOrchestrator(object):
|
||||
|
||||
def _get_state_manager_timeframe(self):
|
||||
timeframe = self._sm.get_state()
|
||||
self.usage_start = datetime.fromtimestamp(timeframe)
|
||||
self.usage_end = datetime.fromtimestamp(timeframe + self._period)
|
||||
self.usage_start = datetime.datetime.fromtimestamp(timeframe)
|
||||
end_frame = timeframe + self._period
|
||||
self.usage_end = datetime.datetime.fromtimestamp(end_frame)
|
||||
metadata = self._sm.get_metadata()
|
||||
self.total = metadata.get('total', 0)
|
||||
|
||||
def _filter_period(self, json_data):
|
||||
"""
|
||||
Detect the best usage period to extract. Removes the usage from the
|
||||
json data and returns it.
|
||||
"""Detect the best usage period to extract.
|
||||
|
||||
Removes the usage from the json data and returns it.
|
||||
"""
|
||||
candidate_ts = None
|
||||
candidate_idx = 0
|
||||
@ -190,8 +191,12 @@ class WriteOrchestrator(object):
|
||||
if self.usage_start is None:
|
||||
self.usage_start = usage_start
|
||||
self.usage_end = usage_start + self._period
|
||||
self.usage_start_dt = datetime.fromtimestamp(self.usage_start)
|
||||
self.usage_end_dt = datetime.fromtimestamp(self.usage_end)
|
||||
self.usage_start_dt = (
|
||||
datetime.datetime.fromtimestamp(self.usage_start)
|
||||
)
|
||||
self.usage_end_dt = (
|
||||
datetime.datetime.fromtimestamp(self.usage_end)
|
||||
)
|
||||
|
||||
self._dispatch(data)
|
||||
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -14,21 +15,22 @@
|
||||
#
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
from datetime import datetime
|
||||
from cloudkitty.state import StateManager
|
||||
import datetime
|
||||
|
||||
from cloudkitty import state
|
||||
|
||||
|
||||
class BaseReportWriter(object):
|
||||
"""
|
||||
Base report writer.
|
||||
"""
|
||||
"""Base report writer."""
|
||||
report_type = None
|
||||
|
||||
def __init__(self, write_orchestrator, user_id, backend, state_backend):
|
||||
self._write_orchestrator = write_orchestrator
|
||||
self._write_backend = backend
|
||||
self._uid = user_id
|
||||
self._sm = StateManager(state_backend, None, self._uid,
|
||||
self._sm = state.StateManager(state_backend,
|
||||
None,
|
||||
self._uid,
|
||||
self.report_type)
|
||||
self._report = None
|
||||
self.period = 3600
|
||||
@ -68,8 +70,9 @@ class BaseReportWriter(object):
|
||||
def _get_state_manager_timeframe(self):
|
||||
timeframe = self._sm.get_state()
|
||||
self.usage_start = timeframe
|
||||
self.usage_start_dt = datetime.fromtimestamp(timeframe)
|
||||
self.usage_end = datetime.fromtimestamp(timeframe + self.period)
|
||||
self.usage_start_dt = datetime.datetime.fromtimestamp(timeframe)
|
||||
end_frame = timeframe + self._period
|
||||
self.usage_end = datetime.datetime.fromtimestamp(end_frame)
|
||||
metadata = self._sm.get_metadata()
|
||||
self.total = metadata.get('total', 0)
|
||||
|
||||
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
@ -15,12 +16,13 @@
|
||||
# @author: Stéphane Albert
|
||||
#
|
||||
import json
|
||||
from cloudkitty.writer.base import BaseReportWriter
|
||||
|
||||
from cloudkitty.writer import base
|
||||
|
||||
|
||||
class OSRFBackend(BaseReportWriter):
|
||||
"""
|
||||
OpenStack Report Format Writer:
|
||||
class OSRFBackend(base.BaseReportWriter):
|
||||
"""OpenStack Report Format Writer:
|
||||
|
||||
Generates report in native format (json)
|
||||
"""
|
||||
report_type = 'osrf'
|
||||
|
@ -3,3 +3,4 @@ python-keystoneclient
|
||||
iso8601
|
||||
oslo.config>=1.2.0
|
||||
pbr>=0.6,<1.0
|
||||
six>=1.7.0
|
||||
|
@ -1,7 +1,8 @@
|
||||
hacking>=0.9.1,<0.10
|
||||
discover
|
||||
testtools
|
||||
testrepository
|
||||
flake8
|
||||
mock>=1.0
|
||||
sphinx>=1.1.2,<1.2
|
||||
oslosphinx
|
||||
oslotest
|
||||
|
Loading…
Reference in New Issue
Block a user