openstack/karbor
Code Issues Proposed changes

Optimize time trigger: use instance of time format instead of class itself

Browse Source 
Optimize time trigger in several aspects. This patch is the fourth
phase, which will use an instance of time format in time trigger
instead of class.

Change-Id: I9f7bff41d4b42a7f38e2d7dae17bbd3ba57dd736
Closes-Bug: #1611232
This commit is contained in:
zengchen 2016-08-09 18:14:43 +08:00
parent 9cbdf5745e
commit 968db961de
8 changed files with 84 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
karbor/services/operationengine/engine/triggers/timetrigger/time_format_manager.py
View File

@ -1,66 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Manage all time formats.
"""
from karbor import exception
from karbor.i18n import _
from karbor.services.operationengine.engine.triggers.timetrigger import\
timeformats
class TimeFormatManager(object):
"""Manage all time format classes"""
def __init__(self):
super(TimeFormatManager, self).__init__()
all_cls = timeformats.all_time_formats()
self._timeformat_cls_map = {cls.FORMAT_TYPE:
cls for cls in all_cls}
def _get_timeformat_cls(self, format_type):
if format_type not in self._timeformat_cls_map:
msg = (_("Invalid trigger time format type:%s") % format_type)
raise exception.InvalidInput(msg)
return self._timeformat_cls_map[format_type]
def check_time_format(self, format_type, pattern):
"""Check time format
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
"""
cls = self._get_timeformat_cls(format_type)
cls.check_time_format(pattern)
def compute_next_time(self, format_type, pattern, start_time):
"""Compute next time
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
:param start_time: the start time for computing
"""
cls = self._get_timeformat_cls(format_type)
return cls.compute_next_time(pattern, start_time)
def get_interval(self, format_type, pattern):
"""Get interval of two adjacent time points
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
"""
cls = self._get_timeformat_cls(format_type)
return cls.get_interval(pattern)

72
karbor/services/operationengine/engine/triggers/timetrigger/time_trigger.py
View File

@ -18,12 +18,11 @@ from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
from oslo_utils import timeutils from oslo_utils import timeutils
import six import six
from stevedore import driver as import_driver
from karbor import exception from karbor import exception
from karbor.i18n import _, _LE from karbor.i18n import _, _LE
from karbor.services.operationengine.engine import triggers from karbor.services.operationengine.engine import triggers
from karbor.services.operationengine.engine.triggers.timetrigger import\
time_format_manager
time_trigger_opts = [ time_trigger_opts = [
cfg.IntOpt('min_interval', cfg.IntOpt('min_interval',
@ -38,6 +37,10 @@ time_trigger_opts = [
cfg.IntOpt('max_window_time', cfg.IntOpt('max_window_time',
default=1800, default=1800,
help='The maximum window time'), help='The maximum window time'),
cfg.StrOpt('time_format',
default='crontab',
help='The type of time format which is used to compute time')
] ]
CONF = cfg.CONF CONF = cfg.CONF
@ -100,8 +103,6 @@ class TriggerOperationGreenThread(object):
class TimeTrigger(triggers.BaseTrigger): class TimeTrigger(triggers.BaseTrigger):
TRIGGER_TYPE = "time" TRIGGER_TYPE = "time"
TIME_FORMAT_MANAGER = time_format_manager.TimeFormatManager()
def __init__(self, trigger_id, trigger_property, executor): def __init__(self, trigger_id, trigger_property, executor):
super(TimeTrigger, self).__init__( super(TimeTrigger, self).__init__(
trigger_id, trigger_property, executor) trigger_id, trigger_property, executor)
@ -141,7 +142,7 @@ class TimeTrigger(triggers.BaseTrigger):
if valid_trigger_property == self._trigger_property: if valid_trigger_property == self._trigger_property:
return return
first_run_time = self._get_first_run_time( timer, first_run_time = self._get_timer_and_first_run_time(
valid_trigger_property) valid_trigger_property)
if not first_run_time: if not first_run_time:
msg = (_("The new trigger property is invalid, " msg = (_("The new trigger property is invalid, "
@ -164,36 +165,34 @@ class TimeTrigger(triggers.BaseTrigger):
if len(self._operation_ids) > 0: if len(self._operation_ids) > 0:
# Restart greenthread to take the change of trigger property # Restart greenthread to take the change of trigger property
# effect immediately # effect immediately
self._restart_greenthread() self._kill_greenthread()
self._create_green_thread(first_run_time, timer)
def _kill_greenthread(self): def _kill_greenthread(self):
if self._greenthread: if self._greenthread:
self._greenthread.kill() self._greenthread.kill()
self._greenthread = None self._greenthread = None
def _restart_greenthread(self):
self._kill_greenthread()
self._start_greenthread()
def _start_greenthread(self): def _start_greenthread(self):
# Find the first time. # Find the first time.
# We don't known when using this trigger first time. # We don't known when using this trigger first time.
first_run_time = self._get_first_run_time( timer, first_run_time = self._get_timer_and_first_run_time(
self._trigger_property) self._trigger_property)
if not first_run_time: if not first_run_time:
raise exception.TriggerIsInvalid(trigger_id=self._id) raise exception.TriggerIsInvalid(trigger_id=self._id)
self._create_green_thread(first_run_time) self._create_green_thread(first_run_time, timer)
def _create_green_thread(self, first_run_time): def _create_green_thread(self, first_run_time, timer):
func = functools.partial( func = functools.partial(
self._trigger_operations, self._trigger_operations,
trigger_property=self._trigger_property.copy()) trigger_property=self._trigger_property.copy(),
timer=timer)
self._greenthread = TriggerOperationGreenThread( self._greenthread = TriggerOperationGreenThread(
first_run_time, func) first_run_time, func)
def _trigger_operations(self, expect_run_time, trigger_property): def _trigger_operations(self, expect_run_time, trigger_property, timer):
"""Trigger operations once """Trigger operations once
returns: wait time for next run returns: wait time for next run
@ -234,9 +233,7 @@ class TimeTrigger(triggers.BaseTrigger):
pass pass
next_time = self._compute_next_run_time( next_time = self._compute_next_run_time(
expect_run_time, trigger_property['end_time'], expect_run_time, trigger_property['end_time'], timer)
trigger_property['format'],
trigger_property['pattern'])
now = datetime.utcnow() now = datetime.utcnow()
if next_time and next_time <= now: if next_time and next_time <= now:
LOG.error(_LE("Next run time:%(next_time)s <= now:%(now)s. Maybe " LOG.error(_LE("Next run time:%(next_time)s <= now:%(now)s. Maybe "
@ -254,11 +251,10 @@ class TimeTrigger(triggers.BaseTrigger):
All the time instances of trigger_definition are in UTC, All the time instances of trigger_definition are in UTC,
including start_time, end_time including start_time, end_time
""" """
tf_cls = cls._get_time_format_class()
trigger_format = trigger_definition.get("format", None)
pattern = trigger_definition.get("pattern", None) pattern = trigger_definition.get("pattern", None)
cls.TIME_FORMAT_MANAGER.check_time_format( tf_cls.check_time_format(pattern)
trigger_format, pattern)
start_time = trigger_definition.get("start_time", None) start_time = trigger_definition.get("start_time", None)
if not start_time: if not start_time:
@ -266,8 +262,7 @@ class TimeTrigger(triggers.BaseTrigger):
raise exception.InvalidInput(msg) raise exception.InvalidInput(msg)
start_time = cls._check_and_get_datetime(start_time, "start_time") start_time = cls._check_and_get_datetime(start_time, "start_time")
interval = int(cls.TIME_FORMAT_MANAGER.get_interval( interval = tf_cls(start_time, pattern).get_min_interval()
trigger_format, pattern))
if interval is not None and interval < CONF.min_interval: if interval is not None and interval < CONF.min_interval:
msg = (_("The interval of two adjacent time points " msg = (_("The interval of two adjacent time points "
"is less than %d") % CONF.min_interval) "is less than %d") % CONF.min_interval)
@ -321,27 +316,28 @@ class TimeTrigger(triggers.BaseTrigger):
return time return time
@classmethod @classmethod
def _compute_next_run_time(cls, start_time, end_time, def _compute_next_run_time(cls, start_time, end_time, timer):
trigger_format, trigger_pattern): next_time = timer.compute_next_time(start_time)
next_time = cls.TIME_FORMAT_MANAGER.compute_next_time(
trigger_format, trigger_pattern, start_time)
if next_time and (not end_time or next_time <= end_time): if next_time and (not end_time or next_time <= end_time):
return next_time return next_time
return None return None
@classmethod @classmethod
def _get_first_run_time(cls, trigger_property): def _get_time_format_class(cls):
now = datetime.utcnow() return import_driver.DriverManager(
tmp_time = trigger_property['start_time'] 'karbor.operationengine.engine.timetrigger.time_format',
if tmp_time < now: CONF.time_format).driver
tmp_time = now
return cls._compute_next_run_time( @classmethod
tmp_time, def _get_timer_and_first_run_time(cls, trigger_property):
trigger_property['end_time'], tf_cls = cls._get_time_format_class()
trigger_property['format'], timer = tf_cls(trigger_property['start_time'],
trigger_property['pattern']) trigger_property['pattern'])
first_run_time = cls._compute_next_run_time(
datetime.utcnow(), trigger_property['end_time'], timer)
return timer, first_run_time
@classmethod @classmethod
def check_configuration(cls): def check_configuration(cls):

48
karbor/services/operationengine/engine/triggers/timetrigger/timeformats/__init__.py
View File

@ -11,53 +11,45 @@
# under the License. # under the License.
""" """
Operation classes time format base class
""" """
from abc import ABCMeta import abc
import six import six
from karbor import loadables
@six.add_metaclass(abc.ABCMeta)
@six.add_metaclass(ABCMeta)
class TimeFormat(object): class TimeFormat(object):
FORMAT_TYPE = "" def __init__(self, start_time, pattern):
"""Initiate time format
:param start_time: The time points after the start_time are valid
:param pattern: The pattern of the time
When the start_time and pattern are specified, the time points
can be calculated and are immutable.
"""
pass
@classmethod @classmethod
@abc.abstractmethod
def check_time_format(cls, pattern): def check_time_format(cls, pattern):
"""Check time format """Check time format
Only supports absolute time format, like crontab.
:param pattern: The pattern of the time :param pattern: The pattern of the time
""" """
pass pass
@classmethod @abc.abstractmethod
def compute_next_time(cls, pattern, start_time): def compute_next_time(self, current_time):
"""Compute next time """Compute next time
:param pattern: The pattern of time :param current_time: the time before the next time
:param start_time: the start time for computing
""" """
pass pass
@classmethod @abc.abstractmethod
def get_interval(cls, pattern): def get_min_interval(self):
"""Get interval of two adjacent time points """Get minimum interval of two adjacent time points"""
:param pattern: The pattern of the time
"""
pass pass
class TimeFormatHandler(loadables.BaseLoader):
def __init__(self):
super(TimeFormatHandler, self).__init__(TimeFormat)
def all_time_formats():
"""Get all trigger time format classes."""
return TimeFormatHandler().get_all_classes()

24
karbor/services/operationengine/engine/triggers/timetrigger/timeformats/crontab_time.py
View File

@ -21,9 +21,10 @@ from karbor.services.operationengine.engine.triggers.timetrigger import\
class Crontab(timeformats.TimeFormat): class Crontab(timeformats.TimeFormat):
"""Crontab."""
FORMAT_TYPE = "crontab" def __init__(self, start_time, pattern):
self._start_time = start_time
self._pattern = pattern
@classmethod @classmethod
def check_time_format(cls, pattern): def check_time_format(cls, pattern):
@ -37,12 +38,15 @@ class Crontab(timeformats.TimeFormat):
msg = (_("The trigger pattern(%s) is invalid") % pattern) msg = (_("The trigger pattern(%s) is invalid") % pattern)
raise exception.InvalidInput(msg) raise exception.InvalidInput(msg)
@classmethod def compute_next_time(self, current_time):
def compute_next_time(cls, pattern, start_time): time = current_time if current_time >= self._start_time else (
return croniter(pattern, start_time).get_next(datetime) self._start_time)
return croniter(self._pattern, time).get_next(datetime)
@classmethod def get_min_interval(self):
def get_interval(cls, pattern): try:
t1 = cls.compute_next_time(pattern, datetime.now()) t1 = self.compute_next_time(datetime.now())
t2 = cls.compute_next_time(pattern, t1) t2 = self.compute_next_time(t1)
return timeutils.delta_seconds(t1, t2) return timeutils.delta_seconds(t1, t2)
except Exception:
return None

39
karbor/tests/unit/operationengine/engine/triggers/timetrigger/test_time_format_manager.py
View File

@ -1,39 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
from datetime import timedelta
from karbor import exception
from karbor.services.operationengine.engine.triggers.timetrigger import\
time_format_manager
from karbor.tests import base
class TimeFormatManagerTestCase(base.TestCase):
def setUp(self):
super(TimeFormatManagerTestCase, self).setUp()
self._manager = time_format_manager.TimeFormatManager()
def test_time_format(self):
self.assertRaisesRegexp(exception.InvalidInput,
"Invalid trigger time format type.*abc$",
self._manager.check_time_format,
'abc', None)
def test_compute_next_time(self):
now = datetime(2016, 1, 20, 15, 11, 0, 0)
time1 = self._manager.compute_next_time("crontab", "* * * * *", now)
time2 = now + timedelta(minutes=1)
self.assertEqual(time2, time1)

23
karbor/tests/unit/operationengine/engine/triggers/timetrigger/test_time_trigger.py
View File

@ -17,24 +17,23 @@ import mock
from oslo_config import cfg from oslo_config import cfg
from karbor import exception from karbor import exception
from karbor.services.operationengine.engine.triggers.timetrigger import \
time_format_manager
from karbor.services.operationengine.engine.triggers.timetrigger.time_trigger \ from karbor.services.operationengine.engine.triggers.timetrigger.time_trigger \
import TimeTrigger import TimeTrigger
from karbor.tests import base from karbor.tests import base
class FakeTimeFormat(object): class FakeTimeFormat(object):
def __init__(self, start_time, pattern):
pass
@classmethod @classmethod
def check_time_format(cls, pattern): def check_time_format(cls, pattern):
pass pass
@classmethod def compute_next_time(self, current_time):
def compute_next_time(cls, pattern, current_time):
return current_time + timedelta(seconds=0.5) return current_time + timedelta(seconds=0.5)
@classmethod def get_min_interval(self):
def get_interval(cls, pattern):
return cfg.CONF.min_interval return cfg.CONF.min_interval
@ -44,7 +43,6 @@ class FakeExecutor(object):
def execute_operation(self, operation_id, triggered_time, def execute_operation(self, operation_id, triggered_time,
expect_start_time, window): expect_start_time, window):
if operation_id not in self._ops: if operation_id not in self._ops:
self._ops[operation_id] = 0 self._ops[operation_id] = 0
self._ops[operation_id] += 1 self._ops[operation_id] += 1
@ -63,7 +61,7 @@ class TimeTriggerTestCase(base.TestCase):
mock_obj = mock.Mock() mock_obj = mock.Mock()
mock_obj.return_value = FakeTimeFormat mock_obj.return_value = FakeTimeFormat
time_format_manager.TimeFormatManager._get_timeformat_cls = mock_obj TimeTrigger._get_time_format_class = mock_obj
self._default_executor = FakeExecutor() self._default_executor = FakeExecutor()
@ -97,9 +95,9 @@ class TimeTriggerTestCase(base.TestCase):
TimeTrigger.check_trigger_definition, TimeTrigger.check_trigger_definition,
trigger_property) trigger_property)
@mock.patch.object(FakeTimeFormat, 'get_interval') @mock.patch.object(FakeTimeFormat, 'get_min_interval')
def test_check_trigger_property_interval(self, get_interval): def test_check_trigger_property_interval(self, get_min_interval):
get_interval.return_value = 0 get_min_interval.return_value = 0
trigger_property = { trigger_property = {
"start_time": '2016-8-18 01:03:04' "start_time": '2016-8-18 01:03:04'
@ -171,7 +169,6 @@ class TimeTriggerTestCase(base.TestCase):
trigger = self._generate_trigger() trigger = self._generate_trigger()
trigger_property = { trigger_property = {
"format": "",
"pattern": "", "pattern": "",
"window": 15, "window": 15,
"start_time": datetime.utcnow(), "start_time": datetime.utcnow(),
@ -198,7 +195,6 @@ class TimeTriggerTestCase(base.TestCase):
eventlet.sleep(0.2) eventlet.sleep(0.2)
trigger_property = { trigger_property = {
"format": "",
"pattern": "", "pattern": "",
"window": 15, "window": 15,
"start_time": datetime.utcnow(), "start_time": datetime.utcnow(),
@ -217,7 +213,6 @@ class TimeTriggerTestCase(base.TestCase):
end_time = datetime.utcnow() + timedelta(seconds=1) end_time = datetime.utcnow() + timedelta(seconds=1)
trigger_property = { trigger_property = {
"format": "",
"pattern": "", "pattern": "",
"window": 15, "window": 15,
"start_time": datetime.utcnow(), "start_time": datetime.utcnow(),

8
karbor/tests/unit/operationengine/engine/triggers/timetrigger/timeformats/test_crontab_time.py
View File

@ -24,7 +24,7 @@ class CrontabTimeTestCase(base.TestCase):
def setUp(self): def setUp(self):
super(CrontabTimeTestCase, self).setUp() super(CrontabTimeTestCase, self).setUp()
self._time_format = crontab_time.Crontab() self._time_format = crontab_time.Crontab
def test_none_pattern(self): def test_none_pattern(self):
self.assertRaisesRegexp(exception.InvalidInput, self.assertRaisesRegexp(exception.InvalidInput,
@ -40,9 +40,11 @@ class CrontabTimeTestCase(base.TestCase):
def test_compute_next_time(self): def test_compute_next_time(self):
now = datetime(2016, 1, 20, 15, 11, 0, 0) now = datetime(2016, 1, 20, 15, 11, 0, 0)
time1 = self._time_format.compute_next_time("* * * * *", now) obj = self._time_format(now, "* * * * *")
time1 = obj.compute_next_time(now)
time2 = now + timedelta(minutes=1) time2 = now + timedelta(minutes=1)
self.assertEqual(time2, time1) self.assertEqual(time2, time1)
def test_get_interval(self): def test_get_interval(self):
self.assertEqual(60, self._time_format.get_interval("* * * * *")) obj = self._time_format(datetime.now(), "* * * * *")
self.assertEqual(60, obj.get_min_interval())

2
setup.cfg
View File

@ -51,6 +51,8 @@ karbor.protectables =
server = karbor.services.protection.protectable_plugins.server:ServerProtectablePlugin server = karbor.services.protection.protectable_plugins.server:ServerProtectablePlugin
volume = karbor.services.protection.protectable_plugins.volume:VolumeProtectablePlugin volume = karbor.services.protection.protectable_plugins.volume:VolumeProtectablePlugin
image = karbor.services.protection.protectable_plugins.image:ImageProtectablePlugin image = karbor.services.protection.protectable_plugins.image:ImageProtectablePlugin
karbor.operationengine.engine.timetrigger.time_format =
crontab = karbor.services.operationengine.engine.triggers.timetrigger.timeformats.crontab_time:Crontab
karbor.operationengine.engine.executor = karbor.operationengine.engine.executor =
thread_pool = karbor.services.operationengine.engine.executors.thread_pool_executor:ThreadPoolExecutor thread_pool = karbor.services.operationengine.engine.executors.thread_pool_executor:ThreadPoolExecutor
green_thread = karbor.services.operationengine.engine.executors.green_thread_executor:GreenThreadExecutor green_thread = karbor.services.operationengine.engine.executors.green_thread_executor:GreenThreadExecutor