diff --git a/bin/nova-instance-usage-audit b/bin/nova-instance-usage-audit deleted file mode 100755 index 1a7f34d6b9a7..000000000000 --- a/bin/nova-instance-usage-audit +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 Openstack, LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Cron script to generate usage notifications for instances existing - during the audit period. - - Together with the notifications generated by compute on instance - create/delete/resize, over that time period, this allows an external - system consuming usage notification feeds to calculate instance usage - for each tenant. - - Time periods are specified as 'hour', 'month', 'day' or 'year' - - hour = previous hour. If run at 9:07am, will generate usage for 8-9am. - month = previous month. If the script is run April 1, it will generate - usages for March 1 through March 31. - day = previous day. if run on July 4th, it generates usages for July 3rd. - year = previous year. If run on Jan 1, it generates usages for - Jan 1 through Dec 31 of the previous year. -""" - -import datetime -import gettext -import os -import sys -import time -import traceback - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')): - sys.path.insert(0, POSSIBLE_TOPDIR) - -gettext.install('nova', unicode=1) -import nova.compute.utils -from nova import context -from nova import db -from nova import exception -from nova import flags -from nova.openstack.common import log as logging -from nova.openstack.common import rpc -from nova import utils - - -FLAGS = flags.FLAGS - -if __name__ == '__main__': - admin_context = context.get_admin_context() - flags.parse_args(sys.argv) - logging.setup("nova") - begin, end = utils.last_completed_audit_period() - print "Starting instance usage audit" - print "Creating usages for %s until %s" % (str(begin), str(end)) - instances = db.instance_get_active_by_window_joined(admin_context, - begin, - end) - print "Found %d instances" % len(instances) - for instance_ref in instances: - try: - nova.compute.utils.notify_usage_exists( - admin_context, instance_ref, - ignore_missing_network_data=False) - except Exception, e: - print traceback.format_exc(e) - print "Instance usage audit completed" diff --git a/etc/nova/policy.json b/etc/nova/policy.json index 7d86a9d3e188..8bc8ae4cc238 100644 --- a/etc/nova/policy.json +++ b/etc/nova/policy.json @@ -42,6 +42,7 @@ "compute_extension:floating_ips": [], "compute_extension:hosts": [["rule:admin_api"]], "compute_extension:hypervisors": [["rule:admin_api"]], + "compute_extension:instance_usage_audit_log": [["rule:admin_api"]], "compute_extension:keypairs": [], "compute_extension:multinic": [], "compute_extension:networks": [["rule:admin_api"]], diff --git a/nova/api/openstack/compute/contrib/instance_usage_audit_log.py b/nova/api/openstack/compute/contrib/instance_usage_audit_log.py new file mode 100644 index 000000000000..4a427d2a7a83 --- /dev/null +++ b/nova/api/openstack/compute/contrib/instance_usage_audit_log.py @@ -0,0 +1,71 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import datetime +from webob import exc + +from nova.api.openstack import extensions +from nova.compute import utils as compute_utils +from nova import context as nova_context +from nova import exception +from nova import flags + +FLAGS = flags.FLAGS + + +authorize = extensions.extension_authorizer('compute', + 'instance_usage_audit_log') + + +class InstanceUsageAuditLogController(object): + + def index(self, req): + context = req.environ['nova.context'] + authorize(context) + task_log = compute_utils.get_audit_task_logs(context) + return {'instance_usage_audit_logs': task_log} + + def show(self, req, id): + context = req.environ['nova.context'] + authorize(context) + try: + if '.' in id: + before_date = datetime.datetime.strptime(str(id), + "%Y-%m-%d %H:%M:%S.%f") + else: + before_date = datetime.datetime.strptime(str(id), + "%Y-%m-%d %H:%M:%S") + except ValueError: + msg = _("Invalid timestamp for date %s") % id + raise webob.exc.HTTPBadRequest(explanation=msg) + task_log = compute_utils.get_audit_task_logs(context, + before=before_date) + return {'instance_usage_audit_log': task_log} + + +class Instance_usage_audit_log(extensions.ExtensionDescriptor): + """Admin-only Task Log Monitoring""" + name = "OSInstanceUsageAuditLog" + alias = "os-instance_usage_audit_log" + namespace = "http://docs.openstack.org/ext/services/api/v1.1" + updated = "2012-07-06T01:00:00+00:00" + + def get_resources(self): + ext = extensions.ResourceExtension('os-instance_usage_audit_log', + InstanceUsageAuditLogController()) + return [ext] diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b922758f0ce1..19bd7e9ba11f 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -146,6 +146,9 @@ compute_opts = [ 'this functionality will be replaced when HostAggregates ' 'become more funtional for general grouping in Folsom. (see: ' 'http://etherpad.openstack.org/FolsomNovaHostAggregates-v2)'), + cfg.BoolOpt('instance_usage_audit', + default=False, + help="Generate periodic compute.instance.exists notifications"), ] @@ -2368,6 +2371,52 @@ class ComputeManager(manager.SchedulerDependentManager): "Will retry later.") LOG.error(msg % locals(), instance=instance) + @manager.periodic_task + def _instance_usage_audit(self, context): + if FLAGS.instance_usage_audit: + if not compute_utils.has_audit_been_run(context, self.host): + begin, end = utils.last_completed_audit_period() + instances = self.db.instance_get_active_by_window_joined( + context, + begin, + end, + host=self.host) + num_instances = len(instances) + errors = 0 + successes = 0 + LOG.info(_("Running instance usage audit for" + " host %(host)s from %(begin_time)s to " + "%(end_time)s. %(number_instances)s" + " instances.") % dict(host=self.host, + begin_time=begin, + end_time=end, + number_instances=num_instances)) + start_time = time.time() + compute_utils.start_instance_usage_audit(context, + begin, end, + self.host, num_instances) + for instance_ref in instances: + try: + compute_utils.notify_usage_exists( + context, instance_ref, + ignore_missing_network_data=False) + successes += 1 + except Exception: + LOG.exception(_('Failed to generate usage ' + 'audit for instance ' + 'on host %s') % self.host, + instance=instance) + errors += 1 + compute_utils.finish_instance_usage_audit(context, + begin, end, + self.host, errors, + "Instance usage audit ran " + "for host %s, %s instances " + "in %s seconds." % ( + self.host, + num_instances, + time.time() - start_time)) + @manager.periodic_task def _poll_bandwidth_usage(self, context, start_time=None, stop_time=None): if not start_time: diff --git a/nova/compute/utils.py b/nova/compute/utils.py index 04d8a842c18d..65a3b2d90dcb 100644 --- a/nova/compute/utils.py +++ b/nova/compute/utils.py @@ -23,7 +23,7 @@ from nova.network import model as network_model from nova import notifications from nova.openstack.common import log from nova.openstack.common.notifier import api as notifier_api - +from nova import utils FLAGS = flags.FLAGS LOG = log.getLogger(__name__) @@ -108,3 +108,81 @@ def get_nw_info_for_instance(instance): info_cache = instance['info_cache'] or {} cached_nwinfo = info_cache.get('network_info') or [] return network_model.NetworkInfo.hydrate(cached_nwinfo) + + +def has_audit_been_run(context, host, timestamp=None): + begin, end = utils.last_completed_audit_period(before=timestamp) + task_log = db.task_log_get(context, "instance_usage_audit", + begin, end, host) + if task_log: + return True + else: + return False + + +def start_instance_usage_audit(context, begin, end, host, num_instances): + db.task_log_begin_task(context, "instance_usage_audit", begin, end, host, + num_instances, "Instance usage audit started...") + + +def finish_instance_usage_audit(context, begin, end, host, errors, message): + db.task_log_end_task(context, "instance_usage_audit", begin, end, host, + errors, message) + + +def get_audit_task_logs(context, begin=None, end=None, before=None): + """Returns a full log for all instance usage audit tasks on all computes. + + :param begin: datetime beginning of audit period to get logs for, + Defaults to the beginning of the most recently completed + audit period prior to the 'before' date. + :param end: datetime ending of audit period to get logs for, + Defaults to the ending of the most recently completed + audit period prior to the 'before' date. + :param before: By default we look for the audit period most recently + completed before this datetime. Has no effect if both begin and end + are specified. + """ + defbegin, defend = utils.last_completed_audit_period(before=before) + if begin is None: + begin = defbegin + if end is None: + end = defend + task_logs = db.task_log_get_all(context, "instance_usage_audit", + begin, end) + services = db.service_get_all_by_topic(context, "compute") + hosts = set(serv['host'] for serv in services) + seen_hosts = set() + done_hosts = set() + running_hosts = set() + total_errors = 0 + total_items = 0 + for tlog in task_logs: + seen_hosts.add(tlog['host']) + if tlog['state'] == "DONE": + done_hosts.add(tlog['host']) + if tlog['state'] == "RUNNING": + running_hosts.add(tlog['host']) + total_errors += tlog['errors'] + total_items += tlog['task_items'] + log = dict((tl['host'], dict(state=tl['state'], + instances=tl['task_items'], + errors=tl['errors'], + message=tl['message'])) + for tl in task_logs) + missing_hosts = hosts - seen_hosts + overall_status = "%s hosts done. %s errors." % ( + 'ALL' if len(done_hosts) == len(hosts) + else "%s of %s" % (len(done_hosts), len(hosts)), + total_errors) + return dict(period_beginning=str(begin), + period_ending=str(end), + num_hosts=len(hosts), + num_hosts_done=len(done_hosts), + num_hosts_running=len(running_hosts), + num_hosts_not_run=len(missing_hosts), + hosts_not_run=list(missing_hosts), + total_instances=total_items, + total_errors=total_errors, + overall_status=overall_status, + log=log) diff --git a/nova/db/api.py b/nova/db/api.py index 695c083c9b08..fd4babb55fea 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -585,20 +585,26 @@ def instance_get_all_by_filters(context, filters, sort_key='created_at', sort_dir) -def instance_get_active_by_window(context, begin, end=None, project_id=None): +def instance_get_active_by_window(context, begin, end=None, project_id=None, + host=None): """Get instances active during a certain time window. - Specifying a project_id will filter for a certain project.""" - return IMPL.instance_get_active_by_window(context, begin, end, project_id) + Specifying a project_id will filter for a certain project. + Specifying a host will filter for instances on a given compute host. + """ + return IMPL.instance_get_active_by_window(context, begin, end, + project_id, host) def instance_get_active_by_window_joined(context, begin, end=None, - project_id=None): + project_id=None, host=None): """Get instances and joins active during a certain time window. - Specifying a project_id will filter for a certain project.""" + Specifying a project_id will filter for a certain project. + Specifying a host will filter for instances on a given compute host. + """ return IMPL.instance_get_active_by_window_joined(context, begin, end, - project_id) + project_id, host) def instance_get_all_by_project(context, project_id): @@ -1948,3 +1954,52 @@ def get_instance_uuid_by_ec2_id(context, instance_id): def ec2_instance_create(context, instance_ec2_id): """Create the ec2 id to instance uuid mapping on demand""" return IMPL.ec2_instance_create(context, instance_ec2_id) + + +#################### + + +def task_log_end_task(context, task_name, + period_beginning, + period_ending, + host, + errors, + message=None, + session=None): + """Mark a task as complete for a given host/time period""" + return IMPL.task_log_end_task(context, task_name, + period_beginning, + period_ending, + host, + errors, + message, + session) + + +def task_log_begin_task(context, task_name, + period_beginning, + period_ending, + host, + task_items=None, + message=None, + session=None): + """Mark a task as started for a given host/time period""" + return IMPL.task_log_begin_task(context, task_name, + period_beginning, + period_ending, + host, + task_items, + message, + session) + + +def task_log_get_all(context, task_name, period_beginning, + period_ending, host=None, state=None, session=None): + return IMPL.task_log_get_all(context, task_name, period_beginning, + period_ending, host, state, session) + + +def task_log_get(context, task_name, period_beginning, + period_ending, host, state=None, session=None): + return IMPL.task_log_get(context, task_name, period_beginning, + period_ending, host, state, session) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 93bce01c59a4..54d70b14fb7a 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1564,7 +1564,8 @@ def instance_get_all_by_filters(context, filters, sort_key, sort_dir): @require_context -def instance_get_active_by_window(context, begin, end=None, project_id=None): +def instance_get_active_by_window(context, begin, end=None, + project_id=None, host=None): """Return instances that were active during window.""" session = get_session() query = session.query(models.Instance) @@ -1575,13 +1576,15 @@ def instance_get_active_by_window(context, begin, end=None, project_id=None): query = query.filter(models.Instance.launched_at < end) if project_id: query = query.filter_by(project_id=project_id) + if host: + query = query.filter_by(host=host) return query.all() @require_admin_context def instance_get_active_by_window_joined(context, begin, end=None, - project_id=None): + project_id=None, host=None): """Return instances and joins that were active during window.""" session = get_session() query = session.query(models.Instance) @@ -1596,6 +1599,8 @@ def instance_get_active_by_window_joined(context, begin, end=None, query = query.filter(models.Instance.launched_at < end) if project_id: query = query.filter_by(project_id=project_id) + if host: + query = query.filter_by(host=host) return query.all() @@ -5189,3 +5194,89 @@ def get_instance_uuid_by_ec2_id(context, instance_id, session=None): @require_context def _ec2_instance_get_query(context, session=None): return model_query(context, models.InstanceIdMapping, session=session) + + +@require_admin_context +def task_log_get(context, task_name, period_beginning, + period_ending, host, state=None, session=None): + query = model_query(context, models.TaskLog, session=session).\ + filter_by(task_name=task_name).\ + filter_by(period_beginning=period_beginning).\ + filter_by(period_ending=period_ending).\ + filter_by(host=host) + if state is not None: + query = query.filter_by(state=state) + + return query.first() + + +@require_admin_context +def task_log_get_all(context, task_name, period_beginning, + period_ending, host=None, state=None, session=None): + query = model_query(context, models.TaskLog, session=session).\ + filter_by(task_name=task_name).\ + filter_by(period_beginning=period_beginning).\ + filter_by(period_ending=period_ending) + if host is not None: + query = query.filter_by(host=host) + if state is not None: + query = query.filter_by(state=state) + return query.all() + + +@require_admin_context +def task_log_begin_task(context, task_name, + period_beginning, + period_ending, + host, + task_items=None, + message=None, + session=None): + session = session or get_session() + with session.begin(): + task = task_log_get(context, task_name, + period_beginning, + period_ending, + host, + session=session) + if task: + #It's already run(ning)! + raise exception.TaskAlreadyRunning(task_name=task_name, host=host) + task = models.TaskLog() + task.task_name = task_name + task.period_beginning = period_beginning + task.period_ending = period_ending + task.host = host + task.state = "RUNNING" + if message: + task.message = message + if task_items: + task.task_items = task_items + task.save(session=session) + return task + + +@require_admin_context +def task_log_end_task(context, task_name, + period_beginning, + period_ending, + host, + errors, + message=None, + session=None): + session = session or get_session() + with session.begin(): + task = task_log_get(context, task_name, + period_beginning, + period_ending, + host, + session=session) + if not task: + #It's not running! + raise exception.TaskNotRunning(task_name=task_name, host=host) + task.state = "DONE" + if message: + task.message = message + task.errors = errors + task.save(session=session) + return task diff --git a/nova/db/sqlalchemy/migrate_repo/versions/108_task_log.py b/nova/db/sqlalchemy/migrate_repo/versions/108_task_log.py new file mode 100644 index 000000000000..e6aedc1a6a76 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/108_task_log.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2012 SINA Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime, Integer +from sqlalchemy import Index, MetaData, String, Table + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # create new table + task_log = Table('task_log', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', + Boolean(create_constraint=True, name=None)), + Column('id', Integer(), + primary_key=True, + nullable=False, + autoincrement=True), + Column('task_name', String(255), nullable=False), + Column('state', String(255), nullable=False), + Column('host', String(255), index=True, nullable=False), + Column('period_beginning', String(255), + index=True, nullable=False), + Column('period_ending', String(255), index=True, nullable=False), + Column('message', String(255), nullable=False), + Column('task_items', Integer()), + Column('errors', Integer()), + ) + try: + task_log.create() + except Exception: + meta.drop_all(tables=[task_log]) + raise + + if migrate_engine.name == "mysql": + migrate_engine.execute("ALTER TABLE task_log " + "Engine=InnoDB") + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + task_log = Table('task_log', meta, autoload=True) + task_log.drop() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 3359891358a5..d117d93617b9 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -1043,3 +1043,17 @@ class InstanceIdMapping(BASE, NovaBase): __tablename__ = 'instance_id_mappings' id = Column(Integer, primary_key=True, nullable=False, autoincrement=True) uuid = Column(String(36), nullable=False) + + +class TaskLog(BASE, NovaBase): + """Audit log for background periodic tasks""" + __tablename__ = 'task_log' + id = Column(Integer, primary_key=True, nullable=False, autoincrement=True) + task_name = Column(String(255), nullable=False) + state = Column(String(255), nullable=False) + host = Column(String(255)) + period_beginning = Column(String(255), default=timeutils.utcnow) + period_ending = Column(String(255), default=timeutils.utcnow) + message = Column(String(255), nullable=False) + task_items = Column(Integer(), default=0) + errors = Column(Integer(), default=0) diff --git a/nova/exception.py b/nova/exception.py index c1f417afe292..0efe8e41bd3b 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1100,6 +1100,14 @@ class CouldNotFetchImage(NovaException): message = _("Could not fetch image %(image_id)s") +class TaskAlreadyRunning(NovaException): + message = _("Task %(task_name) is already running on host %(host)") + + +class TaskNotRunning(NovaException): + message = _("Task %(task_name) is not running on host %(host)") + + def get_context_from_function_and_args(function, args, kwargs): """Find an arg of type RequestContext and return it. diff --git a/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py b/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py new file mode 100644 index 000000000000..b81052ddc35f --- /dev/null +++ b/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py @@ -0,0 +1,188 @@ +# Copyright (c) 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +from webob import exc + +from nova.api.openstack.compute.contrib import instance_usage_audit_log as ial +from nova.compute import utils as compute_utils +from nova import context +from nova import db +from nova import exception +from nova.openstack.common import timeutils +from nova import test +from nova.tests.api.openstack import fakes +from nova import utils + + +TEST_COMPUTE_SERVICES = [dict(host=name) for name in + "foo bar baz plonk".split()] + + +begin1 = datetime.datetime(2012, 7, 4, 6, 0, 0) +begin2 = end1 = datetime.datetime(2012, 7, 5, 6, 0, 0) +begin3 = end2 = datetime.datetime(2012, 7, 6, 6, 0, 0) +end3 = datetime.datetime(2012, 7, 7, 6, 0, 0) + + +#test data + + +TEST_LOGS1 = [ + #all services done, no errors. + dict(host="plonk", period_beginning=begin1, period_ending=end1, + state="DONE", errors=0, task_items=23, message="test1"), + dict(host="baz", period_beginning=begin1, period_ending=end1, + state="DONE", errors=0, task_items=17, message="test2"), + dict(host="bar", period_beginning=begin1, period_ending=end1, + state="DONE", errors=0, task_items=10, message="test3"), + dict(host="foo", period_beginning=begin1, period_ending=end1, + state="DONE", errors=0, task_items=7, message="test4"), + ] + + +TEST_LOGS2 = [ + #some still running... + dict(host="plonk", period_beginning=begin2, period_ending=end2, + state="DONE", errors=0, task_items=23, message="test5"), + dict(host="baz", period_beginning=begin2, period_ending=end2, + state="DONE", errors=0, task_items=17, message="test6"), + dict(host="bar", period_beginning=begin2, period_ending=end2, + state="RUNNING", errors=0, task_items=10, message="test7"), + dict(host="foo", period_beginning=begin2, period_ending=end2, + state="DONE", errors=0, task_items=7, message="test8"), + ] + + +TEST_LOGS3 = [ + #some errors.. + dict(host="plonk", period_beginning=begin3, period_ending=end3, + state="DONE", errors=0, task_items=23, message="test9"), + dict(host="baz", period_beginning=begin3, period_ending=end3, + state="DONE", errors=2, task_items=17, message="test10"), + dict(host="bar", period_beginning=begin3, period_ending=end3, + state="DONE", errors=0, task_items=10, message="test11"), + dict(host="foo", period_beginning=begin3, period_ending=end3, + state="DONE", errors=1, task_items=7, message="test12"), + ] + + +def fake_service_get_all_by_topic(context, topic): + assert topic == "compute" + return TEST_COMPUTE_SERVICES + + +def fake_task_log_get_all(context, task_name, begin, end): + assert task_name == "instance_usage_audit" + + if begin == begin1 and end == end1: + return TEST_LOGS1 + if begin == begin2 and end == end2: + return TEST_LOGS2 + if begin == begin3 and end == end3: + return TEST_LOGS3 + raise AssertionError("Invalid date %s to %s" % (begin, end)) + + +def fake_last_completed_audit_period(unit=None, before=None): + audit_periods = [(begin3, end3), + (begin2, end2), + (begin1, end1)] + if before is not None: + for begin, end in audit_periods: + if before > end: + return begin, end + raise AssertionError("Invalid before date %s" % (before)) + return begin1, end1 + + +class InstanceUsageAuditLogTest(test.TestCase): + def setUp(self): + super(InstanceUsageAuditLogTest, self).setUp() + self.context = context.get_admin_context() + timeutils.set_time_override(datetime.datetime(2012, 7, 5, 10, 0, 0)) + self.controller = ial.InstanceUsageAuditLogController() + + self.stubs.Set(utils, 'last_completed_audit_period', + fake_last_completed_audit_period) + self.stubs.Set(db, 'service_get_all_by_topic', + fake_service_get_all_by_topic) + self.stubs.Set(db, 'task_log_get_all', + fake_task_log_get_all) + + def tearDown(self): + super(InstanceUsageAuditLogTest, self).tearDown() + timeutils.clear_time_override() + + def test_index(self): + req = fakes.HTTPRequest.blank('/v2/fake/os-instance_usage_audit_log') + result = self.controller.index(req) + self.assertIn('instance_usage_audit_logs', result) + logs = result['instance_usage_audit_logs'] + self.assertEquals(57, logs['total_instances']) + self.assertEquals(0, logs['total_errors']) + self.assertEquals(4, len(logs['log'])) + self.assertEquals(4, logs['num_hosts']) + self.assertEquals(4, logs['num_hosts_done']) + self.assertEquals(0, logs['num_hosts_running']) + self.assertEquals(0, logs['num_hosts_not_run']) + self.assertEquals("ALL hosts done. 0 errors.", logs['overall_status']) + + def test_show(self): + req = fakes.HTTPRequest.blank( + '/v2/fake/os-instance_usage_audit_log/show') + result = self.controller.show(req, '2012-07-05 10:00:00') + self.assertIn('instance_usage_audit_log', result) + logs = result['instance_usage_audit_log'] + self.assertEquals(57, logs['total_instances']) + self.assertEquals(0, logs['total_errors']) + self.assertEquals(4, len(logs['log'])) + self.assertEquals(4, logs['num_hosts']) + self.assertEquals(4, logs['num_hosts_done']) + self.assertEquals(0, logs['num_hosts_running']) + self.assertEquals(0, logs['num_hosts_not_run']) + self.assertEquals("ALL hosts done. 0 errors.", logs['overall_status']) + + def test_show_with_running(self): + req = fakes.HTTPRequest.blank( + '/v2/fake/os-instance_usage_audit_log/show') + result = self.controller.show(req, '2012-07-06 10:00:00') + self.assertIn('instance_usage_audit_log', result) + logs = result['instance_usage_audit_log'] + self.assertEquals(57, logs['total_instances']) + self.assertEquals(0, logs['total_errors']) + self.assertEquals(4, len(logs['log'])) + self.assertEquals(4, logs['num_hosts']) + self.assertEquals(3, logs['num_hosts_done']) + self.assertEquals(1, logs['num_hosts_running']) + self.assertEquals(0, logs['num_hosts_not_run']) + self.assertEquals("3 of 4 hosts done. 0 errors.", + logs['overall_status']) + + def test_show_with_errors(self): + req = fakes.HTTPRequest.blank( + '/v2/fake/os-instance_usage_audit_log/show') + result = self.controller.show(req, '2012-07-07 10:00:00') + self.assertIn('instance_usage_audit_log', result) + logs = result['instance_usage_audit_log'] + self.assertEquals(57, logs['total_instances']) + self.assertEquals(3, logs['total_errors']) + self.assertEquals(4, len(logs['log'])) + self.assertEquals(4, logs['num_hosts']) + self.assertEquals(4, logs['num_hosts_done']) + self.assertEquals(0, logs['num_hosts_running']) + self.assertEquals(0, logs['num_hosts_not_run']) + self.assertEquals("ALL hosts done. 3 errors.", + logs['overall_status']) diff --git a/nova/tests/policy.json b/nova/tests/policy.json index 206cb574a55a..b0b3114c4984 100644 --- a/nova/tests/policy.json +++ b/nova/tests/policy.json @@ -98,6 +98,7 @@ "compute_extension:floating_ips": [], "compute_extension:hosts": [], "compute_extension:hypervisors": [], + "compute_extension:instance_usage_audit_log": [], "compute_extension:keypairs": [], "compute_extension:multinic": [], "compute_extension:networks": [], diff --git a/nova/utils.py b/nova/utils.py index b9af41fcaee0..86cbdce1d1c8 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -299,7 +299,7 @@ EASIER_PASSWORD_SYMBOLS = ('23456789', # Removed: 0, 1 'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O -def last_completed_audit_period(unit=None): +def last_completed_audit_period(unit=None, before=None): """This method gives you the most recently *completed* audit period. arguments: @@ -311,6 +311,8 @@ def last_completed_audit_period(unit=None): like so: 'day@18' This will begin the period at 18:00 UTC. 'month@15' starts a monthly period on the 15th, and year@3 begins a yearly one on March 1st. + before: Give the audit period most recently completed before + . Defaults to now. returns: 2 tuple of datetimes (begin, end) @@ -324,7 +326,10 @@ def last_completed_audit_period(unit=None): unit, offset = unit.split("@", 1) offset = int(offset) - rightnow = timeutils.utcnow() + if before is not None: + rightnow = before + else: + rightnow = timeutils.utcnow() if unit not in ('month', 'day', 'year', 'hour'): raise ValueError('Time period must be hour, day, month or year') if unit == 'month': diff --git a/setup.py b/setup.py index fc6df5c90fae..85bea96eacdb 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,6 @@ setuptools.setup(name='nova', 'bin/nova-console', 'bin/nova-consoleauth', 'bin/nova-dhcpbridge', - 'bin/nova-instance-usage-audit', 'bin/nova-manage', 'bin/nova-network', 'bin/nova-novncproxy',