Merge "Refactor instance_usage_audit. Add audit tasklog."
This commit is contained in:
Jenkins
Gerrit Code Review
commit
f220bf6790
@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 Openstack, LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Cron script to generate usage notifications for instances existing
|
||||
during the audit period.
|
||||
|
||||
Together with the notifications generated by compute on instance
|
||||
create/delete/resize, over that time period, this allows an external
|
||||
system consuming usage notification feeds to calculate instance usage
|
||||
for each tenant.
|
||||
|
||||
Time periods are specified as 'hour', 'month', 'day' or 'year'
|
||||
|
||||
hour = previous hour. If run at 9:07am, will generate usage for 8-9am.
|
||||
month = previous month. If the script is run April 1, it will generate
|
||||
usages for March 1 through March 31.
|
||||
day = previous day. if run on July 4th, it generates usages for July 3rd.
|
||||
year = previous year. If run on Jan 1, it generates usages for
|
||||
Jan 1 through Dec 31 of the previous year.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import gettext
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
# If ../nova/__init__.py exists, add ../ to Python search path, so that
|
||||
# it will override what happens to be installed in /usr/(local/)lib/python...
|
||||
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
|
||||
os.pardir,
|
||||
os.pardir))
|
||||
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')):
|
||||
sys.path.insert(0, POSSIBLE_TOPDIR)
|
||||
|
||||
gettext.install('nova', unicode=1)
|
||||
import nova.compute.utils
|
||||
from nova import context
|
||||
from nova import db
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
from nova.openstack.common import log as logging
|
||||
from nova.openstack.common import rpc
|
||||
from nova import utils
|
||||
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
|
||||
if __name__ == '__main__':
|
||||
admin_context = context.get_admin_context()
|
||||
flags.parse_args(sys.argv)
|
||||
logging.setup("nova")
|
||||
begin, end = utils.last_completed_audit_period()
|
||||
print "Starting instance usage audit"
|
||||
print "Creating usages for %s until %s" % (str(begin), str(end))
|
||||
instances = db.instance_get_active_by_window_joined(admin_context,
|
||||
begin,
|
||||
end)
|
||||
print "Found %d instances" % len(instances)
|
||||
for instance_ref in instances:
|
||||
try:
|
||||
nova.compute.utils.notify_usage_exists(
|
||||
admin_context, instance_ref,
|
||||
ignore_missing_network_data=False)
|
||||
except Exception, e:
|
||||
print traceback.format_exc(e)
|
||||
print "Instance usage audit completed"
|
||||
@ -42,6 +42,7 @@
|
||||
"compute_extension:floating_ips": [],
|
||||
"compute_extension:hosts": [["rule:admin_api"]],
|
||||
"compute_extension:hypervisors": [["rule:admin_api"]],
|
||||
"compute_extension:instance_usage_audit_log": [["rule:admin_api"]],
|
||||
"compute_extension:keypairs": [],
|
||||
"compute_extension:multinic": [],
|
||||
"compute_extension:networks": [["rule:admin_api"]],
|
||||
|
||||
@ -0,0 +1,71 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 OpenStack LLC.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import datetime
|
||||
from webob import exc
|
||||
|
||||
from nova.api.openstack import extensions
|
||||
from nova.compute import utils as compute_utils
|
||||
from nova import context as nova_context
|
||||
from nova import exception
|
||||
from nova import flags
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
|
||||
|
||||
authorize = extensions.extension_authorizer('compute',
|
||||
'instance_usage_audit_log')
|
||||
|
||||
|
||||
class InstanceUsageAuditLogController(object):
|
||||
|
||||
def index(self, req):
|
||||
context = req.environ['nova.context']
|
||||
authorize(context)
|
||||
task_log = compute_utils.get_audit_task_logs(context)
|
||||
return {'instance_usage_audit_logs': task_log}
|
||||
|
||||
def show(self, req, id):
|
||||
context = req.environ['nova.context']
|
||||
authorize(context)
|
||||
try:
|
||||
if '.' in id:
|
||||
before_date = datetime.datetime.strptime(str(id),
|
||||
"%Y-%m-%d %H:%M:%S.%f")
|
||||
else:
|
||||
before_date = datetime.datetime.strptime(str(id),
|
||||
"%Y-%m-%d %H:%M:%S")
|
||||
except ValueError:
|
||||
msg = _("Invalid timestamp for date %s") % id
|
||||
raise webob.exc.HTTPBadRequest(explanation=msg)
|
||||
task_log = compute_utils.get_audit_task_logs(context,
|
||||
before=before_date)
|
||||
return {'instance_usage_audit_log': task_log}
|
||||
|
||||
|
||||
class Instance_usage_audit_log(extensions.ExtensionDescriptor):
|
||||
"""Admin-only Task Log Monitoring"""
|
||||
name = "OSInstanceUsageAuditLog"
|
||||
alias = "os-instance_usage_audit_log"
|
||||
namespace = "http://docs.openstack.org/ext/services/api/v1.1"
|
||||
updated = "2012-07-06T01:00:00+00:00"
|
||||
|
||||
def get_resources(self):
|
||||
ext = extensions.ResourceExtension('os-instance_usage_audit_log',
|
||||
InstanceUsageAuditLogController())
|
||||
return [ext]
|
||||
@ -146,6 +146,9 @@ compute_opts = [
|
||||
'this functionality will be replaced when HostAggregates '
|
||||
'become more funtional for general grouping in Folsom. (see: '
|
||||
'http://etherpad.openstack.org/FolsomNovaHostAggregates-v2)'),
|
||||
cfg.BoolOpt('instance_usage_audit',
|
||||
default=False,
|
||||
help="Generate periodic compute.instance.exists notifications"),
|
||||
|
||||
]
|
||||
|
||||
@ -2368,6 +2371,52 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
"Will retry later.")
|
||||
LOG.error(msg % locals(), instance=instance)
|
||||
|
||||
@manager.periodic_task
|
||||
def _instance_usage_audit(self, context):
|
||||
if FLAGS.instance_usage_audit:
|
||||
if not compute_utils.has_audit_been_run(context, self.host):
|
||||
begin, end = utils.last_completed_audit_period()
|
||||
instances = self.db.instance_get_active_by_window_joined(
|
||||
context,
|
||||
begin,
|
||||
end,
|
||||
host=self.host)
|
||||
num_instances = len(instances)
|
||||
errors = 0
|
||||
successes = 0
|
||||
LOG.info(_("Running instance usage audit for"
|
||||
" host %(host)s from %(begin_time)s to "
|
||||
"%(end_time)s. %(number_instances)s"
|
||||
" instances.") % dict(host=self.host,
|
||||
begin_time=begin,
|
||||
end_time=end,
|
||||
number_instances=num_instances))
|
||||
start_time = time.time()
|
||||
compute_utils.start_instance_usage_audit(context,
|
||||
begin, end,
|
||||
self.host, num_instances)
|
||||
for instance_ref in instances:
|
||||
try:
|
||||
compute_utils.notify_usage_exists(
|
||||
context, instance_ref,
|
||||
ignore_missing_network_data=False)
|
||||
successes += 1
|
||||
except Exception:
|
||||
LOG.exception(_('Failed to generate usage '
|
||||
'audit for instance '
|
||||
'on host %s') % self.host,
|
||||
instance=instance)
|
||||
errors += 1
|
||||
compute_utils.finish_instance_usage_audit(context,
|
||||
begin, end,
|
||||
self.host, errors,
|
||||
"Instance usage audit ran "
|
||||
"for host %s, %s instances "
|
||||
"in %s seconds." % (
|
||||
self.host,
|
||||
num_instances,
|
||||
time.time() - start_time))
|
||||
|
||||
@manager.periodic_task
|
||||
def _poll_bandwidth_usage(self, context, start_time=None, stop_time=None):
|
||||
if not start_time:
|
||||
|
||||
@ -23,7 +23,7 @@ from nova.network import model as network_model
|
||||
from nova import notifications
|
||||
from nova.openstack.common import log
|
||||
from nova.openstack.common.notifier import api as notifier_api
|
||||
|
||||
from nova import utils
|
||||
|
||||
FLAGS = flags.FLAGS
|
||||
LOG = log.getLogger(__name__)
|
||||
@ -108,3 +108,81 @@ def get_nw_info_for_instance(instance):
|
||||
info_cache = instance['info_cache'] or {}
|
||||
cached_nwinfo = info_cache.get('network_info') or []
|
||||
return network_model.NetworkInfo.hydrate(cached_nwinfo)
|
||||
|
||||
|
||||
def has_audit_been_run(context, host, timestamp=None):
|
||||
begin, end = utils.last_completed_audit_period(before=timestamp)
|
||||
task_log = db.task_log_get(context, "instance_usage_audit",
|
||||
begin, end, host)
|
||||
if task_log:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def start_instance_usage_audit(context, begin, end, host, num_instances):
|
||||
db.task_log_begin_task(context, "instance_usage_audit", begin, end, host,
|
||||
num_instances, "Instance usage audit started...")
|
||||
|
||||
|
||||
def finish_instance_usage_audit(context, begin, end, host, errors, message):
|
||||
db.task_log_end_task(context, "instance_usage_audit", begin, end, host,
|
||||
errors, message)
|
||||
|
||||
|
||||
def get_audit_task_logs(context, begin=None, end=None, before=None):
|
||||
"""Returns a full log for all instance usage audit tasks on all computes.
|
||||
|
||||
:param begin: datetime beginning of audit period to get logs for,
|
||||
Defaults to the beginning of the most recently completed
|
||||
audit period prior to the 'before' date.
|
||||
:param end: datetime ending of audit period to get logs for,
|
||||
Defaults to the ending of the most recently completed
|
||||
audit period prior to the 'before' date.
|
||||
:param before: By default we look for the audit period most recently
|
||||
completed before this datetime. Has no effect if both begin and end
|
||||
are specified.
|
||||
"""
|
||||
defbegin, defend = utils.last_completed_audit_period(before=before)
|
||||
if begin is None:
|
||||
begin = defbegin
|
||||
if end is None:
|
||||
end = defend
|
||||
task_logs = db.task_log_get_all(context, "instance_usage_audit",
|
||||
begin, end)
|
||||
services = db.service_get_all_by_topic(context, "compute")
|
||||
hosts = set(serv['host'] for serv in services)
|
||||
seen_hosts = set()
|
||||
done_hosts = set()
|
||||
running_hosts = set()
|
||||
total_errors = 0
|
||||
total_items = 0
|
||||
for tlog in task_logs:
|
||||
seen_hosts.add(tlog['host'])
|
||||
if tlog['state'] == "DONE":
|
||||
done_hosts.add(tlog['host'])
|
||||
if tlog['state'] == "RUNNING":
|
||||
running_hosts.add(tlog['host'])
|
||||
total_errors += tlog['errors']
|
||||
total_items += tlog['task_items']
|
||||
log = dict((tl['host'], dict(state=tl['state'],
|
||||
instances=tl['task_items'],
|
||||
errors=tl['errors'],
|
||||
message=tl['message']))
|
||||
for tl in task_logs)
|
||||
missing_hosts = hosts - seen_hosts
|
||||
overall_status = "%s hosts done. %s errors." % (
|
||||
'ALL' if len(done_hosts) == len(hosts)
|
||||
else "%s of %s" % (len(done_hosts), len(hosts)),
|
||||
total_errors)
|
||||
return dict(period_beginning=str(begin),
|
||||
period_ending=str(end),
|
||||
num_hosts=len(hosts),
|
||||
num_hosts_done=len(done_hosts),
|
||||
num_hosts_running=len(running_hosts),
|
||||
num_hosts_not_run=len(missing_hosts),
|
||||
hosts_not_run=list(missing_hosts),
|
||||
total_instances=total_items,
|
||||
total_errors=total_errors,
|
||||
overall_status=overall_status,
|
||||
log=log)
|
||||
|
||||
@ -585,20 +585,26 @@ def instance_get_all_by_filters(context, filters, sort_key='created_at',
|
||||
sort_dir)
|
||||
|
||||
|
||||
def instance_get_active_by_window(context, begin, end=None, project_id=None):
|
||||
def instance_get_active_by_window(context, begin, end=None, project_id=None,
|
||||
host=None):
|
||||
"""Get instances active during a certain time window.
|
||||
|
||||
Specifying a project_id will filter for a certain project."""
|
||||
return IMPL.instance_get_active_by_window(context, begin, end, project_id)
|
||||
Specifying a project_id will filter for a certain project.
|
||||
Specifying a host will filter for instances on a given compute host.
|
||||
"""
|
||||
return IMPL.instance_get_active_by_window(context, begin, end,
|
||||
project_id, host)
|
||||
|
||||
|
||||
def instance_get_active_by_window_joined(context, begin, end=None,
|
||||
project_id=None):
|
||||
project_id=None, host=None):
|
||||
"""Get instances and joins active during a certain time window.
|
||||
|
||||
Specifying a project_id will filter for a certain project."""
|
||||
Specifying a project_id will filter for a certain project.
|
||||
Specifying a host will filter for instances on a given compute host.
|
||||
"""
|
||||
return IMPL.instance_get_active_by_window_joined(context, begin, end,
|
||||
project_id)
|
||||
project_id, host)
|
||||
|
||||
|
||||
def instance_get_all_by_project(context, project_id):
|
||||
@ -1948,3 +1954,52 @@ def get_instance_uuid_by_ec2_id(context, instance_id):
|
||||
def ec2_instance_create(context, instance_ec2_id):
|
||||
"""Create the ec2 id to instance uuid mapping on demand"""
|
||||
return IMPL.ec2_instance_create(context, instance_ec2_id)
|
||||
|
||||
|
||||
####################
|
||||
|
||||
|
||||
def task_log_end_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
errors,
|
||||
message=None,
|
||||
session=None):
|
||||
"""Mark a task as complete for a given host/time period"""
|
||||
return IMPL.task_log_end_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
errors,
|
||||
message,
|
||||
session)
|
||||
|
||||
|
||||
def task_log_begin_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
task_items=None,
|
||||
message=None,
|
||||
session=None):
|
||||
"""Mark a task as started for a given host/time period"""
|
||||
return IMPL.task_log_begin_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
task_items,
|
||||
message,
|
||||
session)
|
||||
|
||||
|
||||
def task_log_get_all(context, task_name, period_beginning,
|
||||
period_ending, host=None, state=None, session=None):
|
||||
return IMPL.task_log_get_all(context, task_name, period_beginning,
|
||||
period_ending, host, state, session)
|
||||
|
||||
|
||||
def task_log_get(context, task_name, period_beginning,
|
||||
period_ending, host, state=None, session=None):
|
||||
return IMPL.task_log_get(context, task_name, period_beginning,
|
||||
period_ending, host, state, session)
|
||||
|
||||
@ -1564,7 +1564,8 @@ def instance_get_all_by_filters(context, filters, sort_key, sort_dir):
|
||||
|
||||
|
||||
@require_context
|
||||
def instance_get_active_by_window(context, begin, end=None, project_id=None):
|
||||
def instance_get_active_by_window(context, begin, end=None,
|
||||
project_id=None, host=None):
|
||||
"""Return instances that were active during window."""
|
||||
session = get_session()
|
||||
query = session.query(models.Instance)
|
||||
@ -1575,13 +1576,15 @@ def instance_get_active_by_window(context, begin, end=None, project_id=None):
|
||||
query = query.filter(models.Instance.launched_at < end)
|
||||
if project_id:
|
||||
query = query.filter_by(project_id=project_id)
|
||||
if host:
|
||||
query = query.filter_by(host=host)
|
||||
|
||||
return query.all()
|
||||
|
||||
|
||||
@require_admin_context
|
||||
def instance_get_active_by_window_joined(context, begin, end=None,
|
||||
project_id=None):
|
||||
project_id=None, host=None):
|
||||
"""Return instances and joins that were active during window."""
|
||||
session = get_session()
|
||||
query = session.query(models.Instance)
|
||||
@ -1596,6 +1599,8 @@ def instance_get_active_by_window_joined(context, begin, end=None,
|
||||
query = query.filter(models.Instance.launched_at < end)
|
||||
if project_id:
|
||||
query = query.filter_by(project_id=project_id)
|
||||
if host:
|
||||
query = query.filter_by(host=host)
|
||||
|
||||
return query.all()
|
||||
|
||||
@ -5189,3 +5194,89 @@ def get_instance_uuid_by_ec2_id(context, instance_id, session=None):
|
||||
@require_context
|
||||
def _ec2_instance_get_query(context, session=None):
|
||||
return model_query(context, models.InstanceIdMapping, session=session)
|
||||
|
||||
|
||||
@require_admin_context
|
||||
def task_log_get(context, task_name, period_beginning,
|
||||
period_ending, host, state=None, session=None):
|
||||
query = model_query(context, models.TaskLog, session=session).\
|
||||
filter_by(task_name=task_name).\
|
||||
filter_by(period_beginning=period_beginning).\
|
||||
filter_by(period_ending=period_ending).\
|
||||
filter_by(host=host)
|
||||
if state is not None:
|
||||
query = query.filter_by(state=state)
|
||||
|
||||
return query.first()
|
||||
|
||||
|
||||
@require_admin_context
|
||||
def task_log_get_all(context, task_name, period_beginning,
|
||||
period_ending, host=None, state=None, session=None):
|
||||
query = model_query(context, models.TaskLog, session=session).\
|
||||
filter_by(task_name=task_name).\
|
||||
filter_by(period_beginning=period_beginning).\
|
||||
filter_by(period_ending=period_ending)
|
||||
if host is not None:
|
||||
query = query.filter_by(host=host)
|
||||
if state is not None:
|
||||
query = query.filter_by(state=state)
|
||||
return query.all()
|
||||
|
||||
|
||||
@require_admin_context
|
||||
def task_log_begin_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
task_items=None,
|
||||
message=None,
|
||||
session=None):
|
||||
session = session or get_session()
|
||||
with session.begin():
|
||||
task = task_log_get(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
session=session)
|
||||
if task:
|
||||
#It's already run(ning)!
|
||||
raise exception.TaskAlreadyRunning(task_name=task_name, host=host)
|
||||
task = models.TaskLog()
|
||||
task.task_name = task_name
|
||||
task.period_beginning = period_beginning
|
||||
task.period_ending = period_ending
|
||||
task.host = host
|
||||
task.state = "RUNNING"
|
||||
if message:
|
||||
task.message = message
|
||||
if task_items:
|
||||
task.task_items = task_items
|
||||
task.save(session=session)
|
||||
return task
|
||||
|
||||
|
||||
@require_admin_context
|
||||
def task_log_end_task(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
errors,
|
||||
message=None,
|
||||
session=None):
|
||||
session = session or get_session()
|
||||
with session.begin():
|
||||
task = task_log_get(context, task_name,
|
||||
period_beginning,
|
||||
period_ending,
|
||||
host,
|
||||
session=session)
|
||||
if not task:
|
||||
#It's not running!
|
||||
raise exception.TaskNotRunning(task_name=task_name, host=host)
|
||||
task.state = "DONE"
|
||||
if message:
|
||||
task.message = message
|
||||
task.errors = errors
|
||||
task.save(session=session)
|
||||
return task
|
||||
|
||||
62
nova/db/sqlalchemy/migrate_repo/versions/108_task_log.py
Normal file
62
nova/db/sqlalchemy/migrate_repo/versions/108_task_log.py
Normal file
@ -0,0 +1,62 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
# Copyright 2012 SINA Corp.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, Integer
|
||||
from sqlalchemy import Index, MetaData, String, Table
|
||||
|
||||
|
||||
def upgrade(migrate_engine):
|
||||
meta = MetaData()
|
||||
meta.bind = migrate_engine
|
||||
|
||||
# create new table
|
||||
task_log = Table('task_log', meta,
|
||||
Column('created_at', DateTime(timezone=False)),
|
||||
Column('updated_at', DateTime(timezone=False)),
|
||||
Column('deleted_at', DateTime(timezone=False)),
|
||||
Column('deleted',
|
||||
Boolean(create_constraint=True, name=None)),
|
||||
Column('id', Integer(),
|
||||
primary_key=True,
|
||||
nullable=False,
|
||||
autoincrement=True),
|
||||
Column('task_name', String(255), nullable=False),
|
||||
Column('state', String(255), nullable=False),
|
||||
Column('host', String(255), index=True, nullable=False),
|
||||
Column('period_beginning', String(255),
|
||||
index=True, nullable=False),
|
||||
Column('period_ending', String(255), index=True, nullable=False),
|
||||
Column('message', String(255), nullable=False),
|
||||
Column('task_items', Integer()),
|
||||
Column('errors', Integer()),
|
||||
)
|
||||
try:
|
||||
task_log.create()
|
||||
except Exception:
|
||||
meta.drop_all(tables=[task_log])
|
||||
raise
|
||||
|
||||
if migrate_engine.name == "mysql":
|
||||
migrate_engine.execute("ALTER TABLE task_log "
|
||||
"Engine=InnoDB")
|
||||
|
||||
|
||||
def downgrade(migrate_engine):
|
||||
meta = MetaData()
|
||||
meta.bind = migrate_engine
|
||||
|
||||
task_log = Table('task_log', meta, autoload=True)
|
||||
task_log.drop()
|
||||
@ -1043,3 +1043,17 @@ class InstanceIdMapping(BASE, NovaBase):
|
||||
__tablename__ = 'instance_id_mappings'
|
||||
id = Column(Integer, primary_key=True, nullable=False, autoincrement=True)
|
||||
uuid = Column(String(36), nullable=False)
|
||||
|
||||
|
||||
class TaskLog(BASE, NovaBase):
|
||||
"""Audit log for background periodic tasks"""
|
||||
__tablename__ = 'task_log'
|
||||
id = Column(Integer, primary_key=True, nullable=False, autoincrement=True)
|
||||
task_name = Column(String(255), nullable=False)
|
||||
state = Column(String(255), nullable=False)
|
||||
host = Column(String(255))
|
||||
period_beginning = Column(String(255), default=timeutils.utcnow)
|
||||
period_ending = Column(String(255), default=timeutils.utcnow)
|
||||
message = Column(String(255), nullable=False)
|
||||
task_items = Column(Integer(), default=0)
|
||||
errors = Column(Integer(), default=0)
|
||||
|
||||
@ -1100,6 +1100,14 @@ class CouldNotFetchImage(NovaException):
|
||||
message = _("Could not fetch image %(image_id)s")
|
||||
|
||||
|
||||
class TaskAlreadyRunning(NovaException):
|
||||
message = _("Task %(task_name) is already running on host %(host)")
|
||||
|
||||
|
||||
class TaskNotRunning(NovaException):
|
||||
message = _("Task %(task_name) is not running on host %(host)")
|
||||
|
||||
|
||||
def get_context_from_function_and_args(function, args, kwargs):
|
||||
"""Find an arg of type RequestContext and return it.
|
||||
|
||||
|
||||
@ -0,0 +1,188 @@
|
||||
# Copyright (c) 2012 OpenStack, LLC
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime
|
||||
from webob import exc
|
||||
|
||||
from nova.api.openstack.compute.contrib import instance_usage_audit_log as ial
|
||||
from nova.compute import utils as compute_utils
|
||||
from nova import context
|
||||
from nova import db
|
||||
from nova import exception
|
||||
from nova.openstack.common import timeutils
|
||||
from nova import test
|
||||
from nova.tests.api.openstack import fakes
|
||||
from nova import utils
|
||||
|
||||
|
||||
TEST_COMPUTE_SERVICES = [dict(host=name) for name in
|
||||
"foo bar baz plonk".split()]
|
||||
|
||||
|
||||
begin1 = datetime.datetime(2012, 7, 4, 6, 0, 0)
|
||||
begin2 = end1 = datetime.datetime(2012, 7, 5, 6, 0, 0)
|
||||
begin3 = end2 = datetime.datetime(2012, 7, 6, 6, 0, 0)
|
||||
end3 = datetime.datetime(2012, 7, 7, 6, 0, 0)
|
||||
|
||||
|
||||
#test data
|
||||
|
||||
|
||||
TEST_LOGS1 = [
|
||||
#all services done, no errors.
|
||||
dict(host="plonk", period_beginning=begin1, period_ending=end1,
|
||||
state="DONE", errors=0, task_items=23, message="test1"),
|
||||
dict(host="baz", period_beginning=begin1, period_ending=end1,
|
||||
state="DONE", errors=0, task_items=17, message="test2"),
|
||||
dict(host="bar", period_beginning=begin1, period_ending=end1,
|
||||
state="DONE", errors=0, task_items=10, message="test3"),
|
||||
dict(host="foo", period_beginning=begin1, period_ending=end1,
|
||||
state="DONE", errors=0, task_items=7, message="test4"),
|
||||
]
|
||||
|
||||
|
||||
TEST_LOGS2 = [
|
||||
#some still running...
|
||||
dict(host="plonk", period_beginning=begin2, period_ending=end2,
|
||||
state="DONE", errors=0, task_items=23, message="test5"),
|
||||
dict(host="baz", period_beginning=begin2, period_ending=end2,
|
||||
state="DONE", errors=0, task_items=17, message="test6"),
|
||||
dict(host="bar", period_beginning=begin2, period_ending=end2,
|
||||
state="RUNNING", errors=0, task_items=10, message="test7"),
|
||||
dict(host="foo", period_beginning=begin2, period_ending=end2,
|
||||
state="DONE", errors=0, task_items=7, message="test8"),
|
||||
]
|
||||
|
||||
|
||||
TEST_LOGS3 = [
|
||||
#some errors..
|
||||
dict(host="plonk", period_beginning=begin3, period_ending=end3,
|
||||
state="DONE", errors=0, task_items=23, message="test9"),
|
||||
dict(host="baz", period_beginning=begin3, period_ending=end3,
|
||||
state="DONE", errors=2, task_items=17, message="test10"),
|
||||
dict(host="bar", period_beginning=begin3, period_ending=end3,
|
||||
state="DONE", errors=0, task_items=10, message="test11"),
|
||||
dict(host="foo", period_beginning=begin3, period_ending=end3,
|
||||
state="DONE", errors=1, task_items=7, message="test12"),
|
||||
]
|
||||
|
||||
|
||||
def fake_service_get_all_by_topic(context, topic):
|
||||
assert topic == "compute"
|
||||
return TEST_COMPUTE_SERVICES
|
||||
|
||||
|
||||
def fake_task_log_get_all(context, task_name, begin, end):
|
||||
assert task_name == "instance_usage_audit"
|
||||
|
||||
if begin == begin1 and end == end1:
|
||||
return TEST_LOGS1
|
||||
if begin == begin2 and end == end2:
|
||||
return TEST_LOGS2
|
||||
if begin == begin3 and end == end3:
|
||||
return TEST_LOGS3
|
||||
raise AssertionError("Invalid date %s to %s" % (begin, end))
|
||||
|
||||
|
||||
def fake_last_completed_audit_period(unit=None, before=None):
|
||||
audit_periods = [(begin3, end3),
|
||||
(begin2, end2),
|
||||
(begin1, end1)]
|
||||
if before is not None:
|
||||
for begin, end in audit_periods:
|
||||
if before > end:
|
||||
return begin, end
|
||||
raise AssertionError("Invalid before date %s" % (before))
|
||||
return begin1, end1
|
||||
|
||||
|
||||
class InstanceUsageAuditLogTest(test.TestCase):
|
||||
def setUp(self):
|
||||
super(InstanceUsageAuditLogTest, self).setUp()
|
||||
self.context = context.get_admin_context()
|
||||
timeutils.set_time_override(datetime.datetime(2012, 7, 5, 10, 0, 0))
|
||||
self.controller = ial.InstanceUsageAuditLogController()
|
||||
|
||||
self.stubs.Set(utils, 'last_completed_audit_period',
|
||||
fake_last_completed_audit_period)
|
||||
self.stubs.Set(db, 'service_get_all_by_topic',
|
||||
fake_service_get_all_by_topic)
|
||||
self.stubs.Set(db, 'task_log_get_all',
|
||||
fake_task_log_get_all)
|
||||
|
||||
def tearDown(self):
|
||||
super(InstanceUsageAuditLogTest, self).tearDown()
|
||||
timeutils.clear_time_override()
|
||||
|
||||
def test_index(self):
|
||||
req = fakes.HTTPRequest.blank('/v2/fake/os-instance_usage_audit_log')
|
||||
result = self.controller.index(req)
|
||||
self.assertIn('instance_usage_audit_logs', result)
|
||||
logs = result['instance_usage_audit_logs']
|
||||
self.assertEquals(57, logs['total_instances'])
|
||||
self.assertEquals(0, logs['total_errors'])
|
||||
self.assertEquals(4, len(logs['log']))
|
||||
self.assertEquals(4, logs['num_hosts'])
|
||||
self.assertEquals(4, logs['num_hosts_done'])
|
||||
self.assertEquals(0, logs['num_hosts_running'])
|
||||
self.assertEquals(0, logs['num_hosts_not_run'])
|
||||
self.assertEquals("ALL hosts done. 0 errors.", logs['overall_status'])
|
||||
|
||||
def test_show(self):
|
||||
req = fakes.HTTPRequest.blank(
|
||||
'/v2/fake/os-instance_usage_audit_log/show')
|
||||
result = self.controller.show(req, '2012-07-05 10:00:00')
|
||||
self.assertIn('instance_usage_audit_log', result)
|
||||
logs = result['instance_usage_audit_log']
|
||||
self.assertEquals(57, logs['total_instances'])
|
||||
self.assertEquals(0, logs['total_errors'])
|
||||
self.assertEquals(4, len(logs['log']))
|
||||
self.assertEquals(4, logs['num_hosts'])
|
||||
self.assertEquals(4, logs['num_hosts_done'])
|
||||
self.assertEquals(0, logs['num_hosts_running'])
|
||||
self.assertEquals(0, logs['num_hosts_not_run'])
|
||||
self.assertEquals("ALL hosts done. 0 errors.", logs['overall_status'])
|
||||
|
||||
def test_show_with_running(self):
|
||||
req = fakes.HTTPRequest.blank(
|
||||
'/v2/fake/os-instance_usage_audit_log/show')
|
||||
result = self.controller.show(req, '2012-07-06 10:00:00')
|
||||
self.assertIn('instance_usage_audit_log', result)
|
||||
logs = result['instance_usage_audit_log']
|
||||
self.assertEquals(57, logs['total_instances'])
|
||||
self.assertEquals(0, logs['total_errors'])
|
||||
self.assertEquals(4, len(logs['log']))
|
||||
self.assertEquals(4, logs['num_hosts'])
|
||||
self.assertEquals(3, logs['num_hosts_done'])
|
||||
self.assertEquals(1, logs['num_hosts_running'])
|
||||
self.assertEquals(0, logs['num_hosts_not_run'])
|
||||
self.assertEquals("3 of 4 hosts done. 0 errors.",
|
||||
logs['overall_status'])
|
||||
|
||||
def test_show_with_errors(self):
|
||||
req = fakes.HTTPRequest.blank(
|
||||
'/v2/fake/os-instance_usage_audit_log/show')
|
||||
result = self.controller.show(req, '2012-07-07 10:00:00')
|
||||
self.assertIn('instance_usage_audit_log', result)
|
||||
logs = result['instance_usage_audit_log']
|
||||
self.assertEquals(57, logs['total_instances'])
|
||||
self.assertEquals(3, logs['total_errors'])
|
||||
self.assertEquals(4, len(logs['log']))
|
||||
self.assertEquals(4, logs['num_hosts'])
|
||||
self.assertEquals(4, logs['num_hosts_done'])
|
||||
self.assertEquals(0, logs['num_hosts_running'])
|
||||
self.assertEquals(0, logs['num_hosts_not_run'])
|
||||
self.assertEquals("ALL hosts done. 3 errors.",
|
||||
logs['overall_status'])
|
||||
@ -98,6 +98,7 @@
|
||||
"compute_extension:floating_ips": [],
|
||||
"compute_extension:hosts": [],
|
||||
"compute_extension:hypervisors": [],
|
||||
"compute_extension:instance_usage_audit_log": [],
|
||||
"compute_extension:keypairs": [],
|
||||
"compute_extension:multinic": [],
|
||||
"compute_extension:networks": [],
|
||||
|
||||
@ -299,7 +299,7 @@ EASIER_PASSWORD_SYMBOLS = ('23456789', # Removed: 0, 1
|
||||
'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O
|
||||
|
||||
|
||||
def last_completed_audit_period(unit=None):
|
||||
def last_completed_audit_period(unit=None, before=None):
|
||||
"""This method gives you the most recently *completed* audit period.
|
||||
|
||||
arguments:
|
||||
@ -311,6 +311,8 @@ def last_completed_audit_period(unit=None):
|
||||
like so: 'day@18' This will begin the period at 18:00
|
||||
UTC. 'month@15' starts a monthly period on the 15th,
|
||||
and year@3 begins a yearly one on March 1st.
|
||||
before: Give the audit period most recently completed before
|
||||
<timestamp>. Defaults to now.
|
||||
|
||||
|
||||
returns: 2 tuple of datetimes (begin, end)
|
||||
@ -324,7 +326,10 @@ def last_completed_audit_period(unit=None):
|
||||
unit, offset = unit.split("@", 1)
|
||||
offset = int(offset)
|
||||
|
||||
rightnow = timeutils.utcnow()
|
||||
if before is not None:
|
||||
rightnow = before
|
||||
else:
|
||||
rightnow = timeutils.utcnow()
|
||||
if unit not in ('month', 'day', 'year', 'hour'):
|
||||
raise ValueError('Time period must be hour, day, month or year')
|
||||
if unit == 'month':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user