openstack/watcher
Code Issues Proposed changes

add start and end time for continuous audit

Browse Source 
Add new start_time and end_time fields in the audit table

Partially Implements: blueprint add-start-end-time-for-continuous-audit

Change-Id: I6bb838d777b2c7aa799a70485980e5dc87838456
This commit is contained in:
licanwei
2018-03-27 02:23:56 -07:00
committed by Alexander Chadin
parent f41adc7e8b
commit c2550e534e
12 changed files with 185 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml Normal file
View File

@@ -0,0 +1,5 @@
---
features:
- |
Add start_time and end_time fields in audits table. User can set the start
time and/or end time when creating CONTINUOUS audit.

47
watcher/api/controllers/v1/audit.py
View File

@@ -30,11 +30,13 @@ states, visit :ref:`the Audit State machine <audit_state_machine>`.
"""
import datetime
from dateutil import tz
import pecan
from pecan import rest
import wsme
from wsme import types as wtypes
from wsme import utils as wutils
import wsmeext.pecan as wsme_pecan
from oslo_log import log
@@ -86,6 +88,10 @@ class AuditPostType(wtypes.Base):
hostname = wtypes.wsattr(wtypes.text, readonly=True, mandatory=False)
start_time = wsme.wsattr(datetime.datetime, mandatory=False)
end_time = wsme.wsattr(datetime.datetime, mandatory=False)
def as_audit(self, context):
audit_type_values = [val.value for val in objects.audit.AuditType]
if self.audit_type not in audit_type_values:
@@ -104,6 +110,12 @@ class AuditPostType(wtypes.Base):
raise exception.Invalid('Either audit_template_uuid '
'or goal should be provided.')
if (self.audit_type == objects.audit.AuditType.ONESHOT.value and
(self.start_time not in (wtypes.Unset, None)
or self.end_time not in (wtypes.Unset, None))):
raise exception.AuditStartEndTimeNotAllowed(
audit_type=self.audit_type)
# If audit_template_uuid was provided, we will provide any
# variables not included in the request, but not override
# those variables that were included.
@@ -161,7 +173,9 @@ class AuditPostType(wtypes.Base):
strategy_id=self.strategy,
interval=self.interval,
scope=self.scope,
auto_trigger=self.auto_trigger)
auto_trigger=self.auto_trigger,
start_time=self.start_time,
end_time=self.end_time)
class AuditPatchType(types.JsonPatchType):
@@ -322,6 +336,12 @@ class Audit(base.APIBase):
hostname = wsme.wsattr(wtypes.text, mandatory=False)
"""Hostname the audit is running on"""
start_time = wsme.wsattr(datetime.datetime, mandatory=False)
"""The start time for continuous audit launch"""
end_time = wsme.wsattr(datetime.datetime, mandatory=False)
"""The end time that stopping continuous audit"""
def __init__(self, **kwargs):
self.fields = []
fields = list(objects.Audit.fields)
@@ -382,7 +402,9 @@ class Audit(base.APIBase):
interval='7200',
scope=[],
auto_trigger=False,
next_run_time=datetime.datetime.utcnow())
next_run_time=datetime.datetime.utcnow(),
start_time=datetime.datetime.utcnow(),
end_time=datetime.datetime.utcnow())
sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
@@ -584,6 +606,17 @@ class AuditsController(rest.RestController):
'parameter spec in predefined strategy'))
audit_dict = audit.as_dict()
# convert local time to UTC time
start_time_value = audit_dict.get('start_time')
end_time_value = audit_dict.get('end_time')
if start_time_value:
audit_dict['start_time'] = start_time_value.replace(
tzinfo=tz.tzlocal()).astimezone(
tz.tzutc()).replace(tzinfo=None)
if end_time_value:
audit_dict['end_time'] = end_time_value.replace(
tzinfo=tz.tzlocal()).astimezone(
tz.tzutc()).replace(tzinfo=None)
new_audit = objects.Audit(context, **audit_dict)
new_audit.create()
@@ -628,6 +661,16 @@ class AuditsController(rest.RestController):
reason=error_message % dict(
initial_state=initial_state, new_state=new_state))
patch_path = api_utils.get_patch_key(patch, 'path')
if patch_path in ('start_time', 'end_time'):
patch_value = api_utils.get_patch_value(patch, patch_path)
# convert string format to UTC time
new_patch_value = wutils.parse_isodatetime(
patch_value).replace(
tzinfo=tz.tzlocal()).astimezone(
tz.tzutc()).replace(tzinfo=None)
api_utils.set_patch_value(patch, patch_path, new_patch_value)
audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
except api_utils.JSONPATCH_EXCEPTIONS as e:
raise exception.PatchError(patch=patch, reason=e)

12
watcher/api/controllers/v1/utils.py
View File

@@ -101,6 +101,18 @@ def get_patch_value(patch, key):
return p['value']
def set_patch_value(patch, key, value):
for p in patch:
if p['op'] == 'replace' and p['path'] == '/%s' % key:
p['value'] = value
def get_patch_key(patch, key):
for p in patch:
if p['op'] == 'replace' and key in p.keys():
return p[key][1:]
def check_audit_state_transition(patch, initial):
is_transition_valid = True
state_value = get_patch_value(patch, "state")

5
watcher/common/exception.py
View File

@@ -260,6 +260,11 @@ class AuditIntervalNotAllowed(Invalid):
msg_fmt = _("Interval of audit must not be set for %(audit_type)s.")
class AuditStartEndTimeNotAllowed(Invalid):
msg_fmt = _("Start or End time of audit must not be set for "
"%(audit_type)s.")
class AuditReferenced(Invalid):
msg_fmt = _("Audit %(audit)s is referenced by one or multiple action "
"plans")

24
watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py Normal file
View File

@@ -0,0 +1,24 @@
"""add_start_end_time
Revision ID: 4b16194c56bc
Revises: 52804f2498c4
Create Date: 2018-03-23 00:36:29.031259
"""
# revision identifiers, used by Alembic.
revision = '4b16194c56bc'
down_revision = '52804f2498c4'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('audits', sa.Column('start_time', sa.DateTime(), nullable=True))
op.add_column('audits', sa.Column('end_time', sa.DateTime(), nullable=True))
def downgrade():
op.drop_column('audits', 'start_time')
op.drop_column('audits', 'end_time')

2
watcher/db/sqlalchemy/models.py
View File

@@ -184,6 +184,8 @@ class Audit(Base):
auto_trigger = Column(Boolean, nullable=False)
next_run_time = Column(DateTime, nullable=True)
hostname = Column(String(255), nullable=True)
start_time = Column(DateTime, nullable=True)
end_time = Column(DateTime, nullable=True)
goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None)
strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None)

24
watcher/decision_engine/audit/continuous.py
View File

@@ -58,9 +58,10 @@ class ContinuousAuditHandler(base.AuditHandler):
def _is_audit_inactive(self, audit):
audit = objects.Audit.get_by_uuid(
self.context_show_deleted, audit.uuid)
self.context_show_deleted, audit.uuid, eager=True)
if (objects.audit.AuditStateTransitionManager().is_inactive(audit) or
audit.hostname != CONF.host):
(audit.hostname != CONF.host) or
(self.check_audit_expired(audit))):
# if audit isn't in active states, audit's job must be removed to
# prevent using of inactive audit in future.
jobs = [job for job in self.scheduler.get_jobs()
@@ -119,13 +120,26 @@ class ContinuousAuditHandler(base.AuditHandler):
name='execute_audit',
**trigger_args)
def check_audit_expired(self, audit):
current = datetime.datetime.utcnow()
# Note: if audit still didn't get into the timeframe,
# skip it
if audit.start_time and audit.start_time > current:
return True
if audit.end_time and audit.end_time < current:
if audit.state != objects.audit.State.SUCCEEDED:
audit.state = objects.audit.State.SUCCEEDED
audit.save()
return True
return False
def launch_audits_periodically(self):
audit_context = context.RequestContext(is_admin=True)
audit_filters = {
'audit_type': objects.audit.AuditType.CONTINUOUS.value,
'state__in': (objects.audit.State.PENDING,
objects.audit.State.ONGOING,
objects.audit.State.SUCCEEDED),
objects.audit.State.ONGOING),
}
audit_filters['hostname'] = None
unscheduled_audits = objects.Audit.list(
@@ -152,6 +166,8 @@ class ContinuousAuditHandler(base.AuditHandler):
audits = objects.Audit.list(
audit_context, filters=audit_filters, eager=True)
for audit in audits:
if self.check_audit_expired(audit):
continue
existing_job = scheduler_jobs.get(audit.uuid, None)
# if audit is not presented in scheduled audits yet,
# just add a new audit job.

5
watcher/objects/audit.py
View File

@@ -88,7 +88,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject,
# 'interval' type has been changed from Integer to String
# Version 1.4: Added 'name' string field
# Version 1.5: Added 'hostname' field
VERSION = '1.5'
# Version 1.6: Added 'start_time' and 'end_time' DateTime fields
VERSION = '1.6'
dbapi = db_api.get_instance()
@@ -107,6 +108,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject,
'next_run_time': wfields.DateTimeField(nullable=True,
tzinfo_aware=False),
'hostname': wfields.StringField(nullable=True),
'start_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False),
'end_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False),
'goal': wfields.ObjectField('Goal', nullable=True),
'strategy': wfields.ObjectField('Strategy', nullable=True),

36
watcher/tests/api/v1/test_audits.py
View File

@@ -11,6 +11,7 @@
# limitations under the License.
import datetime
from dateutil import tz
import itertools
import mock
@@ -883,6 +884,41 @@ class TestPost(api_base.FunctionalTest):
self.assertEqual(201, response.status_int)
self.assertNotEqual(long_name, response.json['name'])
@mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
def test_create_continuous_audit_with_start_end_time(
self, mock_trigger_audit):
mock_trigger_audit.return_value = mock.ANY
start_time = datetime.datetime(2018, 3, 1, 0, 0)
end_time = datetime.datetime(2018, 4, 1, 0, 0)
audit_dict = post_get_test_audit(
params_to_exclude=['uuid', 'state', 'scope',
'next_run_time', 'hostname', 'goal']
)
audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
audit_dict['interval'] = '1200'
audit_dict['start_time'] = str(start_time)
audit_dict['end_time'] = str(end_time)
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
self.assertEqual(201, response.status_int)
self.assertEqual(objects.audit.State.PENDING,
response.json['state'])
self.assertEqual(audit_dict['interval'], response.json['interval'])
self.assertTrue(utils.is_uuid_like(response.json['uuid']))
return_start_time = timeutils.parse_isotime(
response.json['start_time'])
return_end_time = timeutils.parse_isotime(
response.json['end_time'])
iso_start_time = start_time.replace(
tzinfo=tz.tzlocal()).astimezone(tz.tzutc())
iso_end_time = end_time.replace(
tzinfo=tz.tzlocal()).astimezone(tz.tzutc())
self.assertEqual(iso_start_time, return_start_time)
self.assertEqual(iso_end_time, return_end_time)
class TestDelete(api_base.FunctionalTest):

3
watcher/tests/db/utils.py
View File

@@ -97,6 +97,9 @@ def get_test_audit(**kwargs):
'auto_trigger': kwargs.get('auto_trigger', False),
'next_run_time': kwargs.get('next_run_time'),
'hostname': kwargs.get('hostname', 'host_1'),
'start_time': kwargs.get('start_time'),
'end_time': kwargs.get('end_time')
}
# ObjectField doesn't allow None nor dict, so if we want to simulate a
# non-eager object loading, the field should not be referenced at all.

28
watcher/tests/decision_engine/audit/test_audit_handlers.py
View File

@@ -468,3 +468,31 @@ class TestContinuousAuditHandler(base.DbTestCase):
self.assertTrue(is_inactive)
is_inactive = audit_handler._is_audit_inactive(self.audits[0])
self.assertFalse(is_inactive)
def test_check_audit_expired(self):
current = datetime.datetime.utcnow()
# start_time and end_time are None
audit_handler = continuous.ContinuousAuditHandler()
result = audit_handler.check_audit_expired(self.audits[0])
self.assertFalse(result)
self.assertIsNone(self.audits[0].start_time)
self.assertIsNone(self.audits[0].end_time)
# current time < start_time and end_time is None
self.audits[0].start_time = current+datetime.timedelta(days=1)
result = audit_handler.check_audit_expired(self.audits[0])
self.assertTrue(result)
self.assertIsNone(self.audits[0].end_time)
# current time is between start_time and end_time
self.audits[0].start_time = current-datetime.timedelta(days=1)
self.audits[0].end_time = current+datetime.timedelta(days=1)
result = audit_handler.check_audit_expired(self.audits[0])
self.assertFalse(result)
# current time > end_time
self.audits[0].end_time = current-datetime.timedelta(days=1)
result = audit_handler.check_audit_expired(self.audits[0])
self.assertTrue(result)
self.assertEqual(objects.audit.State.SUCCEEDED, self.audits[0].state)

2
watcher/tests/objects/test_objects.py
View File

@@ -412,7 +412,7 @@ expected_object_fingerprints = {
'Goal': '1.0-93881622db05e7b67a65ca885b4a022e',
'Strategy': '1.1-73f164491bdd4c034f48083a51bdeb7b',
'AuditTemplate': '1.1-b291973ffc5efa2c61b24fe34fdccc0b',
'Audit': '1.5-e4229dee89e669d1aff0805f5c665bee',
'Audit': '1.6-fc4abd6f133a8b419e42e05729ed0f8b',
'ActionPlan': '2.2-3331270cb3666c93408934826d03c08d',
'Action': '2.0-1dd4959a7e7ac30c62ef170fe08dd935',
'EfficacyIndicator': '1.0-655b71234a82bc7478aff964639c4bb0',