diff --git a/cloudkitty/backend/__init__.py b/cloudkitty/backend/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cloudkitty/backend/file.py b/cloudkitty/backend/file.py deleted file mode 100644 index 7cd6fde8..00000000 --- a/cloudkitty/backend/file.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import io - - -class FileBackend(io.FileIO): - def __init__(self, path, mode='a+'): - super(FileBackend, self).__init__(path, mode) diff --git a/cloudkitty/cli/writer.py b/cloudkitty/cli/writer.py deleted file mode 100644 index a090d98e..00000000 --- a/cloudkitty/cli/writer.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -from oslo_config import cfg -from oslo_utils import importutils as i_utils - -from cloudkitty import config # noqa -from cloudkitty import service -from cloudkitty import storage -from cloudkitty import utils as ck_utils -from cloudkitty import write_orchestrator - -CONF = cfg.CONF -CONF.import_opt('period', 'cloudkitty.collector', 'collect') -CONF.import_opt('backend', 'cloudkitty.config', 'output') -CONF.import_opt('basepath', 'cloudkitty.config', 'output') -STORAGES_NAMESPACE = 'cloudkitty.storage.backends' - - -class DBCommand(object): - - def __init__(self): - self._storage = None - self._output = None - self._load_storage_backend() - self._load_output_backend() - - def _load_storage_backend(self): - self._storage = storage.get_storage() - - def _load_output_backend(self): - backend = i_utils.import_class(CONF.output.backend) - self._output = backend - - def generate(self): - if not CONF.command.tenant: - if not CONF.command.begin: - CONF.command.begin = ck_utils.get_month_start() - if not CONF.command.end: - CONF.command.end = ck_utils.get_next_month() - tenants = self._storage.get_tenants(CONF.command.begin, - CONF.command.end) - else: - tenants = [CONF.command.tenant] - for tenant in tenants: - wo = write_orchestrator.WriteOrchestrator(self._output, - tenant, - self._storage, - CONF.output.basepath) - wo.init_writing_pipeline() - if not CONF.command.begin: - wo.restart_month() - wo.process() - - def tenants_list(self): - if not CONF.command.begin: - CONF.command.begin = ck_utils.get_month_start() - if not CONF.command.end: - CONF.command.end = ck_utils.get_next_month() - tenants = self._storage.get_tenants(CONF.command.begin, - CONF.command.end) - print('Tenant list:') - for tenant in tenants: - print(tenant) - - -def call_generate(command_object): - command_object.generate() - - -def call_tenants_list(command_object): - command_object.tenants_list() - - -def add_command_parsers(subparsers): - parser = subparsers.add_parser('generate') - parser.set_defaults(func=call_generate) - parser.add_argument('--tenant', nargs='?') - parser.add_argument('--begin', nargs='?') - parser.add_argument('--end', nargs='?') - - parser = subparsers.add_parser('tenants_list') - parser.set_defaults(func=call_tenants_list) - parser.add_argument('--begin', nargs='?') - parser.add_argument('--end', nargs='?') - - -command_opt = cfg.SubCommandOpt('command', - title='Command', - help='Available commands', - handler=add_command_parsers) - -CONF.register_cli_opt(command_opt) - - -def main(): - service.prepare_service() - command_object = DBCommand() - CONF.command.func(command_object) diff --git a/cloudkitty/common/config.py b/cloudkitty/common/config.py index a7d34c82..54ddffc1 100644 --- a/cloudkitty/common/config.py +++ b/cloudkitty/common/config.py @@ -59,8 +59,6 @@ _opts = [ cloudkitty.fetcher.source.fetcher_source_opts))), ('orchestrator', list(itertools.chain( cloudkitty.orchestrator.orchestrator_opts))), - ('output', list(itertools.chain( - cloudkitty.config.output_opts))), ('storage', list(itertools.chain( cloudkitty.storage.storage_opts))), ('storage_influxdb', list(itertools.chain( diff --git a/cloudkitty/config.py b/cloudkitty/config.py index 87d2c224..745958ec 100644 --- a/cloudkitty/config.py +++ b/cloudkitty/config.py @@ -14,24 +14,9 @@ # under the License. # from oslo_config import cfg -from oslo_db import options as db_options # noqa -from oslo_messaging import opts # noqa +from oslo_db import options as db_options -output_opts = [ - cfg.StrOpt('backend', - default='cloudkitty.backend.file.FileBackend', - help='Backend for the output manager.'), - cfg.StrOpt('basepath', - default='/var/lib/cloudkitty/states/', - help='Storage directory for the file output backend.'), - cfg.ListOpt('pipeline', - default=['osrf'], - help='Output pipeline'), ] - - -cfg.CONF.register_opts(output_opts, 'output') - # oslo.db defaults db_options.set_defaults( cfg.CONF, diff --git a/cloudkitty/write_orchestrator.py b/cloudkitty/write_orchestrator.py deleted file mode 100644 index b41d5a31..00000000 --- a/cloudkitty/write_orchestrator.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import copy - -from oslo_config import cfg -from oslo_utils import fileutils -from stevedore import named - -from cloudkitty import state -from cloudkitty import storage -from cloudkitty import storage_state -from cloudkitty import utils as ck_utils - -CONF = cfg.CONF -WRITERS_NAMESPACE = 'cloudkitty.output.writers' - - -class WriteOrchestrator(object): - """Write Orchestrator: - - Handle incoming data from the global orchestrator, and store them in an - intermediary data format before final transformation. - """ - def __init__(self, - backend, - tenant_id, - storage, - basepath=None, - period=3600): - self._backend = backend - self._tenant_id = tenant_id - self._storage = storage - self._storage_state = storage_state.StateManager() - self._basepath = basepath - if self._basepath: - fileutils.ensure_tree(self._basepath) - self._period = period - self._sm = state.DBStateManager(self._tenant_id, - 'writer_status') - self._write_pipeline = [] - - # State vars - self.usage_start = None - self.usage_end = None - - # Current total - self.total = 0 - - def init_writing_pipeline(self): - CONF.import_opt('pipeline', 'cloudkitty.config', 'output') - output_pipeline = named.NamedExtensionManager( - WRITERS_NAMESPACE, - CONF.output.pipeline) - for writer in output_pipeline: - self.add_writer(writer.plugin) - - def add_writer(self, writer_class): - writer = writer_class(self, - self._tenant_id, - self._backend, - self._basepath) - self._write_pipeline.append(writer) - - def _update_state_manager_data(self): - self._sm.set_state(self.usage_end) - metadata = {'total': self.total} - self._sm.set_metadata(metadata) - - def _load_state_manager_data(self): - timeframe = self._sm.get_state() - if timeframe: - self.usage_start = timeframe - self.usage_end = self.usage_start + self._period - metadata = self._sm.get_metadata() - if metadata: - self.total = metadata.get('total', 0) - - def _dispatch(self, data): - for service in data: - # Update totals - for entry in data[service]: - self.total += entry['rating']['price'] - # Dispatch data to writing pipeline - for backend in self._write_pipeline: - backend.append(data, self.usage_start, self.usage_end) - - def get_timeframe(self, timeframe, timeframe_end=None): - if not timeframe_end: - timeframe_end = timeframe + self._period - try: - filters = {'project_id': self._tenant_id} - data = self._storage.retrieve(begin=timeframe, - end=timeframe_end, - filters=filters, - paginate=False) - for df in data['dataframes']: - for service, resources in df['usage'].items(): - for resource in resources: - resource['desc'] = copy.deepcopy(resource['metadata']) - resource['desc'].update(resource['groupby']) - except storage.NoTimeFrame: - return None - return data - - def close(self): - for writer in self._write_pipeline: - writer.close() - - def _push_data(self): - data = self.get_timeframe(self.usage_start, self.usage_end) - if data and data['total'] > 0: - for timeframe in data['dataframes']: - self._dispatch(timeframe['usage']) - return True - else: - return False - - def _commit_data(self): - for backend in self._write_pipeline: - backend.commit() - - def reset_state(self): - self._load_state_manager_data() - self.usage_end = self._storage_state.get_last_processed_timestamp() - self._update_state_manager_data() - - def restart_month(self): - self._load_state_manager_data() - month_start = ck_utils.get_month_start() - self.usage_end = ck_utils.dt2ts(month_start) - self._update_state_manager_data() - - def process(self): - self._load_state_manager_data() - storage_state = self._storage_state.get_last_processed_timestamp( - self._tenant_id) - if not self.usage_start: - self.usage_start = storage_state - self.usage_end = self.usage_start + self._period - while storage_state > self.usage_start: - if self._push_data(): - self._commit_data() - self._update_state_manager_data() - self._load_state_manager_data() - storage_state = self._storage_state.get_last_processed_timestamp( - self._tenant_id) - self.close() diff --git a/cloudkitty/writer/__init__.py b/cloudkitty/writer/__init__.py deleted file mode 100644 index f5c1c3f6..00000000 --- a/cloudkitty/writer/__init__.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import abc - -from cloudkitty import state -from cloudkitty import utils as ck_utils - - -class BaseReportWriter(object, metaclass=abc.ABCMeta): - """Base report writer.""" - report_type = None - - def __init__(self, write_orchestrator, tenant_id, backend, basepath=None): - self._write_orchestrator = write_orchestrator - self._backend = backend - self._tenant_id = tenant_id - self._sm = state.DBStateManager(self._tenant_id, - self.report_type) - self._report = None - self._period = 3600 - - self._basepath = basepath - - # State vars - self.checked_first_line = False - self.usage_start = None - self.usage_start_dt = None - self.usage_end = None - self.usage_end_dt = None - - # Current total - self.total = 0 - - # Current usage period lines - self._usage_data = {} - - @abc.abstractmethod - def _gen_filename(self): - """Filename generation - - """ - - def _open(self): - filename = self._gen_filename() - self._report = self._backend(filename, 'wb+') - self._report.seek(0, 2) - - def _get_report_size(self): - return self._report.tell() - - @abc.abstractmethod - def _recover_state(self): - """Recover state from a last run. - - """ - - def _update_state_manager(self): - self._sm.set_state(self.usage_end) - metadata = {'total': self.total} - self._sm.set_metadata(metadata) - - def _get_state_manager_timeframe(self): - timeframe = self._sm.get_state() - self.usage_start = timeframe - self.usage_start_dt = ck_utils.ts2dt(timeframe) - self.usage_end = timeframe + self._period - self.usage_end_dt = ck_utils.ts2dt(self.usage_end) - metadata = self._sm.get_metadata() - self.total = metadata.get('total', 0) - - def get_timeframe(self, timeframe): - return self._write_orchestrator.get_timeframe(timeframe) - - @abc.abstractmethod - def _write_header(self): - """Write report headers - - """ - - @abc.abstractmethod - def _write_total(self): - """Write current total - - """ - - @abc.abstractmethod - def _write(self): - """Write report content - - """ - - def _pre_commit(self): - if self._report is None: - self._open() - if not self.checked_first_line: - if self._get_report_size() == 0: - self._write_header() - else: - self._recover_state() - self.checked_first_line = True - else: - self._recover_state() - - def _commit(self): - self._pre_commit() - - self._write() - self._update_state_manager() - - self._post_commit() - - def _post_commit(self): - self._usage_data = {} - self._write_total() - - def _update(self, data): - for service in data: - if service in self._usage_data: - self._usage_data[service].extend(data[service]) - else: - self._usage_data[service] = data[service] - # Update totals - for entry in data[service]: - self.total += entry['rating']['price'] - - def append(self, data, start, end): - # FIXME we should use the real time values - if self.usage_end is not None and start >= self.usage_end: - self.usage_start = None - - if self.usage_start is None: - self.usage_start = start - self.usage_end = start + self._period - self.usage_start_dt = ck_utils.ts2dt(self.usage_start) - self.usage_end_dt = ck_utils.ts2dt(self.usage_end) - - self._update(data) - - def commit(self): - self._commit() - - @abc.abstractmethod - def _close_file(self): - """Close report file - - """ - - def close(self): - self._close_file() diff --git a/cloudkitty/writer/csv_base.py b/cloudkitty/writer/csv_base.py deleted file mode 100644 index a9b72e78..00000000 --- a/cloudkitty/writer/csv_base.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import collections -import csv -import datetime -import os - -from cloudkitty import utils as ck_utils -from cloudkitty import writer - - -class InconsistentHeaders(Exception): - pass - - -class BaseCSVBackend(writer.BaseReportWriter): - """Report format writer: - - Generates report in csv format - """ - report_type = 'csv' - - def __init__(self, write_orchestrator, user_id, backend, basepath): - super(BaseCSVBackend, self).__init__(write_orchestrator, - user_id, - backend, - basepath) - - # Detailed transform OrderedDict - self._field_map = collections.OrderedDict() - - self._headers = [] - self._headers_len = 0 - self._extra_headers = [] - self._extra_headers_len = 0 - - # File vars - self._csv_report = None - - # State variables - self.cached_start = None - self.cached_start_str = '' - self.cached_end = None - self.cached_end_str = '' - self._crumpled = False - - # Current usage period lines - self._usage_data = [] - - def _gen_filename(self, timeframe): - filename = ('{}-{}-{:02d}.csv').format(self._tenant_id, - timeframe.year, - timeframe.month) - if self._basepath: - filename = os.path.join(self._basepath, filename) - return filename - - def _open(self): - filename = self._gen_filename(self.usage_start_dt) - self._report = self._backend(filename, 'rb+') - self._csv_report = csv.writer(self._report) - self._report.seek(0, 2) - - def _close_file(self): - if self._report is not None: - self._report.close() - - def _get_state_manager_timeframe(self): - if self.report_type is None: - raise NotImplementedError() - - def _update_state_manager(self): - if self.report_type is None: - raise NotImplementedError() - - super(BaseCSVBackend, self)._update_state_manager() - - metadata = {'total': self.total} - metadata['headers'] = self._extra_headers - self._sm.set_metadata(metadata) - - def _init_headers(self): - headers = self._field_map.keys() - for header in headers: - if ':*' in header: - continue - self._headers.append(header) - self._headers_len = len(self._headers) - - def _write_header(self): - self._csv_report.writerow(self._headers + self._extra_headers) - - def _write(self): - self._csv_report.writerows(self._usage_data) - - def _post_commit(self): - self._crumpled = False - self._usage_data = [] - self._write_total() - - def _update(self, data): - """Dispatch report data with context awareness. - - """ - if self._crumpled: - return - try: - for service in data: - for report_data in data[service]: - self._process_data(service, report_data) - self.total += report_data['rating']['price'] - except InconsistentHeaders: - self._crumple() - self._crumpled = True - - def _recover_state(self): - # Rewind 3 lines - self._report.seek(0, 2) - buf_size = self._report.tell() - if buf_size > 2000: - buf_size = 2000 - elif buf_size == 0: - return - self._report.seek(-buf_size, 2) - end_buf = self._report.read() - last_line = buf_size - for dummy in range(4): - last_line = end_buf.rfind('\n', 0, last_line) - if last_line > 0: - last_line -= len(end_buf) - 1 - else: - raise RuntimeError('Unable to recover file state.') - self._report.seek(last_line, 2) - self._report.truncate() - - def _crumple(self): - # Reset states - self._usage_data = [] - self.total = 0 - - # Recover state from file - if self._report is not None: - self._report.seek(0) - reader = csv.reader(self._report) - # Skip header - for dummy in range(2): - line = reader.next() - self.usage_start_dt = datetime.datetime.strptime( - line[0], - '%Y/%m/%d %H:%M:%S') - self.usage_start = ck_utils.dt2ts(self.usage_start_dt) - self.usage_end_dt = datetime.datetime.strptime( - line[1], - '%Y/%m/%d %H:%M:%S') - self.usage_end = ck_utils.dt2ts(self.usage_end_dt) - - # Reset file - self._report.seek(0) - self._report.truncate() - self._write_header() - - timeframe = self._write_orchestrator.get_timeframe( - self.usage_start) - start = self.usage_start - self.usage_start = None - for data in timeframe: - self.append(data['usage'], - start, - None) - self.usage_start = self.usage_end - - def _update_extra_headers(self, new_head): - self._extra_headers.append(new_head) - self._extra_headers.sort() - self._extra_headers_len += 1 - - def _allocate_extra(self, line): - for dummy in range(self._extra_headers_len): - line.append('') - - def _map_wildcard(self, base, report_data): - wildcard_line = [] - headers_changed = False - self._allocate_extra(wildcard_line) - base_section, dummy = base.split(':') - if not report_data: - return [] - for field in report_data: - col_name = base_section + ':' + field - if col_name not in self._extra_headers: - self._update_extra_headers(col_name) - headers_changed = True - else: - idx = self._extra_headers.index(col_name) - wildcard_line[idx] = report_data[field] - if headers_changed: - raise InconsistentHeaders('Headers value changed' - ', need to rebuild.') - return wildcard_line - - def _recurse_sections(self, sections, data): - if not sections.count(':'): - return data.get(sections, '') - fields = sections.split(':') - cur_data = data - for field in fields: - if field in cur_data: - cur_data = cur_data[field] - else: - return None - return cur_data - - def _process_data(self, context, report_data): - """Transform the raw json data to the final CSV values. - - """ - if not self._headers_len: - self._init_headers() - - formated_data = [] - for base, mapped in self._field_map.items(): - final_data = '' - if isinstance(mapped, str): - mapped_section, mapped_field = mapped.rsplit(':', 1) - data = self._recurse_sections(mapped_section, report_data) - if mapped_field == '*': - extra_fields = self._map_wildcard(base, data) - formated_data.extend(extra_fields) - continue - elif mapped_section in report_data: - data = report_data[mapped_section] - if mapped_field in data: - final_data = data[mapped_field] - elif mapped is not None: - final_data = mapped(context, report_data) - formated_data.append(final_data) - - self._usage_data.append(formated_data) diff --git a/cloudkitty/writer/csv_map.py b/cloudkitty/writer/csv_map.py deleted file mode 100644 index 7e33b0b3..00000000 --- a/cloudkitty/writer/csv_map.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import collections -import datetime - -from cloudkitty.writer import csv_base - - -class CSVMapped(csv_base.BaseCSVBackend): - report_type = 'csv' - - def __init__(self, write_orchestrator, user_id, backend, state_backend): - super(CSVMapped, self).__init__(write_orchestrator, - user_id, - backend, - state_backend) - - # Detailed transform dict - self._field_map = collections.OrderedDict( - [('UsageStart', self._trans_get_usage_start), - ('UsageEnd', self._trans_get_usage_end), - ('ResourceId', self._trans_res_id), - ('Operation', self._trans_operation), - ('UserId', 'desc:user_id'), - ('ProjectId', 'desc:project_id'), - ('ItemName', 'desc:name'), - ('ItemFlavor', 'desc:flavor_name'), - ('ItemFlavorId', 'desc:flavor_id'), - ('AvailabilityZone', 'desc:availability_zone'), - ('Service', self._trans_service), - ('UsageQuantity', 'vol:qty'), - ('RateValue', 'rating:price'), - ('Cost', self._trans_calc_cost), - ('user:*', 'desc:metadata:*')]) - - def _write_total(self): - lines = [[''] * self._headers_len for i in range(3)] - for i in range(len(lines)): - lines[i][1] = self._tenant_id - - lines[1][2] = self._tenant_id - - lines[0][3] = 'InvoiceTotal' - lines[1][3] = 'AccountTotal' - lines[2][3] = 'StatementTotal' - - lines[0][5] = 'Total amount for invoice' - lines[1][5] = 'Total for linked account# {}'.format(self._tenant_id) - start_month = datetime.datetime( - self.usage_start_dt.year, - self.usage_start_dt.month, - 1) - lines[2][5] = ('Total statement amount for period ' - '{} - {}').format(self._format_date(start_month), - self._get_usage_end()) - - lines[0][8] = self.total - lines[1][8] = self.total - lines[2][8] = self.total - - self._csv_report.writerows(lines) - - @staticmethod - def _format_date(raw_dt): - return raw_dt.strftime('%Y/%m/%d %H:%M:%S') - - def _get_usage_start(self): - """Get the start usage of this period. - - """ - if self.cached_start == self.usage_start: - return self.cached_start_str - else: - self.cached_start = self.usage_start - self.cached_start_str = self._format_date(self.usage_start_dt) - return self.cached_start_str - - def _get_usage_end(self): - """Get the end usage of this period. - - """ - if self.cached_start == self.usage_start and self.cached_end_str \ - and self.cached_end > self.cached_start: - return self.cached_end_str - else: - usage_end = self.usage_start_dt + datetime.timedelta( - seconds=self._period) - self.cached_end_str = self._format_date(usage_end) - return self.cached_end_str - - def _trans_get_usage_start(self, _context, _report_data): - """Dummy transformation function to comply with the standard. - - """ - return self._get_usage_start() - - def _trans_get_usage_end(self, _context, _report_data): - """Dummy transformation function to comply with the standard. - - """ - return self._get_usage_end() - - def _trans_product_name(self, context, _report_data): - """Context dependent product name translation. - - """ - if context == 'compute' or context == 'instance': - return 'Nova Computing' - else: - return context - - def _trans_operation(self, context, _report_data): - """Context dependent operation translation. - - """ - if context == 'compute' or context == 'instance': - return 'RunInstances' - - def _trans_res_id(self, context, report_data): - """Context dependent resource id transformation function. - - """ - return report_data['desc'].get('resource_id') - - def _trans_calc_cost(self, context, report_data): - """Cost calculation function. - - """ - try: - quantity = report_data['vol'].get('qty') - rate = report_data['rating'].get('price') - return str(float(quantity) * rate) - except TypeError: - pass - - def _trans_service(self, context, report_data): - return context diff --git a/cloudkitty/writer/osrf.py b/cloudkitty/writer/osrf.py deleted file mode 100644 index 97453e45..00000000 --- a/cloudkitty/writer/osrf.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import os - -from cloudkitty.utils import json -from cloudkitty import writer - - -class OSRFBackend(writer.BaseReportWriter): - """OpenStack Report Format Writer: - - Generates report in native format (json) - """ - report_type = 'osrf' - - def _gen_filename(self, timeframe): - filename = '{}-osrf-{}-{:02d}.json'.format(self._tenant_id, - timeframe.year, - timeframe.month) - if self._basepath: - filename = os.path.join(self._basepath, filename) - return filename - - def _open(self): - filename = self._gen_filename(self.usage_start_dt) - self._report = self._backend(filename, 'rb+') - self._report.seek(0, 2) - if self._report.tell(): - self._recover_state() - else: - self._report.seek(0) - - def _write_header(self): - self._report.write('[') - self._report.flush() - - def _write_total(self): - total = {'total': self.total} - self._report.write(json.dumps(total)) - self._report.write(']') - self._report.flush() - - def _recover_state(self): - # Search for last comma - self._report.seek(0, 2) - max_idx = self._report.tell() - if max_idx > 2000: - max_idx = 2000 - hay = '' - for idx in range(10, max_idx, 10): - self._report.seek(-idx, 2) - hay = self._report.read() - if hay.count(','): - break - last_comma = hay.rfind(',') - if last_comma > -1: - last_comma -= len(hay) - else: - raise RuntimeError('Unable to recover file state.') - self._report.seek(last_comma, 2) - self._report.write(', ') - self._report.truncate() - - def _close_file(self): - if self._report is not None: - self._recover_state() - self._write_total() - self._report.close() - - def _write(self): - data = {} - data['period'] = {'begin': self.usage_start_dt.isoformat(), - 'end': self.usage_end_dt.isoformat()} - data['usage'] = self._usage_data - - self._report.write(json.dumps(data)) - self._report.write(', ') - self._report.flush() diff --git a/devstack/plugin.sh b/devstack/plugin.sh index f46456d4..b33c793a 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -82,7 +82,6 @@ function is_cloudkitty_enabled { function cleanup_cloudkitty { # Clean up dirs rm -rf $CLOUDKITTY_CONF_DIR/* - rm -rf $CLOUDKITTY_OUTPUT_BASEPATH/* for i in $(find $CLOUDKITTY_ENABLED_DIR -iname '_[0-9]*.py' -printf '%f\n'); do rm -f "${CLOUDKITTY_HORIZON_ENABLED_DIR}/$i" done @@ -184,11 +183,6 @@ function configure_cloudkitty { # when starting a devstack installation, but is NOT a recommended setting iniset $CLOUDKITTY_CONF collect wait_periods 0 - # output - iniset $CLOUDKITTY_CONF output backend $CLOUDKITTY_OUTPUT_BACKEND - iniset $CLOUDKITTY_CONF output basepath $CLOUDKITTY_OUTPUT_BASEPATH - iniset $CLOUDKITTY_CONF output pipeline $CLOUDKITTY_OUTPUT_PIPELINE - # storage iniset $CLOUDKITTY_CONF storage backend $CLOUDKITTY_STORAGE_BACKEND iniset $CLOUDKITTY_CONF storage version $CLOUDKITTY_STORAGE_VERSION @@ -247,11 +241,6 @@ function create_opensearch_index { # init_cloudkitty() - Initialize CloudKitty database function init_cloudkitty { - # Delete existing cache - sudo rm -rf $CLOUDKITTY_OUTPUT_BASEPATH - sudo mkdir -p $CLOUDKITTY_OUTPUT_BASEPATH - sudo chown $STACK_USER $CLOUDKITTY_OUTPUT_BASEPATH - # (Re)create cloudkitty database recreate_database cloudkitty utf8 diff --git a/devstack/settings b/devstack/settings index d8381bbb..9acab5ba 100644 --- a/devstack/settings +++ b/devstack/settings @@ -55,11 +55,6 @@ CLOUDKITTY_STORAGE_BACKEND=${CLOUDKITTY_STORAGE_BACKEND:-"influxdb"} CLOUDKITTY_STORAGE_VERSION=${CLOUDKITTY_STORAGE_VERSION:-"2"} CLOUDKITTY_INFLUX_VERSION=${CLOUDKITTY_INFLUX_VERSION:-1} -# Set CloudKitty output info -CLOUDKITTY_OUTPUT_BACKEND=${CLOUDKITTY_OUTPUT_BACKEND:-"cloudkitty.backend.file.FileBackend"} -CLOUDKITTY_OUTPUT_BASEPATH=${CLOUDKITTY_OUTPUT_BASEPATH:-$CLOUDKITTY_REPORTS_DIR} -CLOUDKITTY_OUTPUT_PIPELINE=${CLOUDKITTY_OUTPUT_PIPELINE:-"osrf"} - # Set Cloudkitty client info GITREPO["python-cloudkittyclient"]=${CLOUDKITTYCLIENT_REPO:-${GIT_BASE}/openstack/python-cloudkittyclient.git} GITDIR["python-cloudkittyclient"]=$DEST/python-cloudkittyclient diff --git a/doc/source/admin/install/install-source.rst b/doc/source/admin/install/install-source.rst index 826806cf..699f2e36 100644 --- a/doc/source/admin/install/install-source.rst +++ b/doc/source/admin/install/install-source.rst @@ -17,7 +17,6 @@ following executables: * ``cloudkitty-processor``: Processing service (collecting and rating) * ``cloudkitty-dbsync``: Tool to create and upgrade the database schema * ``cloudkitty-storage-init``: Tool to initiate the storage backend -* ``cloudkitty-writer``: Reporting tool Install sample configuration files:: diff --git a/releasenotes/notes/remove-cloudkitty-writer-bb6e093316a00535.yaml b/releasenotes/notes/remove-cloudkitty-writer-bb6e093316a00535.yaml new file mode 100644 index 00000000..fd0e9992 --- /dev/null +++ b/releasenotes/notes/remove-cloudkitty-writer-bb6e093316a00535.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + The ``cloudkitty-writer`` CLI has been removed. The CLI has been + unmaintained for a long time and has not been functional for multiple + releases. Use the report API instead. Also, the ``[output]`` configuration + section has been removed. diff --git a/setup.cfg b/setup.cfg index 5017ccd7..5aeb5079 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,6 @@ console_scripts = cloudkitty-dbsync = cloudkitty.cli.dbsync:main cloudkitty-processor = cloudkitty.cli.processor:main cloudkitty-storage-init = cloudkitty.cli.storage:main - cloudkitty-writer = cloudkitty.cli.writer:main cloudkitty-status = cloudkitty.cli.status:main wsgi_scripts = @@ -73,7 +72,3 @@ cloudkitty.storage.v2.backends = cloudkitty.storage.hybrid.backends = gnocchi = cloudkitty.storage.v1.hybrid.backends.gnocchi:GnocchiStorage - -cloudkitty.output.writers = - osrf = cloudkitty.writer.osrf:OSRFBackend - csv = cloudkitty.writer.csv_map:CSVMapped