openstack/karbor
Code Issues Proposed changes

Optimize time trigger: use instance of time format instead of class itself

Browse Source 
Optimize time trigger in several aspects. This patch is the fourth
phase, which will use an instance of time format in time trigger
instead of class.

Change-Id: I9f7bff41d4b42a7f38e2d7dae17bbd3ba57dd736
Closes-Bug: #1611232
This commit is contained in:
zengchen 2016-08-09 18:14:43 +08:00
parent 9cbdf5745e
commit 968db961de
8 changed files with 84 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
karbor/services/operationengine/engine/triggers/timetrigger/time_format_manager.py
View File

@ -1,66 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Manage all time formats.
"""
from karbor import exception
from karbor.i18n import _
from karbor.services.operationengine.engine.triggers.timetrigger import\
timeformats
class TimeFormatManager(object):
"""Manage all time format classes"""
def __init__(self):
super(TimeFormatManager, self).__init__()
all_cls = timeformats.all_time_formats()
self._timeformat_cls_map = {cls.FORMAT_TYPE:
cls for cls in all_cls}
def _get_timeformat_cls(self, format_type):
if format_type not in self._timeformat_cls_map:
msg = (_("Invalid trigger time format type:%s") % format_type)
raise exception.InvalidInput(msg)
return self._timeformat_cls_map[format_type]
def check_time_format(self, format_type, pattern):
"""Check time format
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
"""
cls = self._get_timeformat_cls(format_type)
cls.check_time_format(pattern)
def compute_next_time(self, format_type, pattern, start_time):
"""Compute next time
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
:param start_time: the start time for computing
"""
cls = self._get_timeformat_cls(format_type)
return cls.compute_next_time(pattern, start_time)
def get_interval(self, format_type, pattern):
"""Get interval of two adjacent time points
:param format_type: the type of time format, like crontab
:param pattern: The pattern of the time
"""
cls = self._get_timeformat_cls(format_type)
return cls.get_interval(pattern)

72
karbor/services/operationengine/engine/triggers/timetrigger/time_trigger.py
View File

@ -18,12 +18,11 @@ from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import timeutils
import six
from stevedore import driver as import_driver
from karbor import exception
from karbor.i18n import _, _LE
from karbor.services.operationengine.engine import triggers
from karbor.services.operationengine.engine.triggers.timetrigger import\
time_format_manager
time_trigger_opts = [
cfg.IntOpt('min_interval',
@ -38,6 +37,10 @@ time_trigger_opts = [
cfg.IntOpt('max_window_time',
default=1800,
help='The maximum window time'),
cfg.StrOpt('time_format',
default='crontab',
help='The type of time format which is used to compute time')
]
CONF = cfg.CONF
@ -100,8 +103,6 @@ class TriggerOperationGreenThread(object):
class TimeTrigger(triggers.BaseTrigger):
TRIGGER_TYPE = "time"
TIME_FORMAT_MANAGER = time_format_manager.TimeFormatManager()
def __init__(self, trigger_id, trigger_property, executor):
super(TimeTrigger, self).__init__(
trigger_id, trigger_property, executor)
@ -141,7 +142,7 @@ class TimeTrigger(triggers.BaseTrigger):
if valid_trigger_property == self._trigger_property:
return
first_run_time = self._get_first_run_time(
timer, first_run_time = self._get_timer_and_first_run_time(
valid_trigger_property)
if not first_run_time:
msg = (_("The new trigger property is invalid, "
@ -164,36 +165,34 @@ class TimeTrigger(triggers.BaseTrigger):
if len(self._operation_ids) > 0:
# Restart greenthread to take the change of trigger property
# effect immediately
self._restart_greenthread()
self._kill_greenthread()
self._create_green_thread(first_run_time, timer)
def _kill_greenthread(self):
if self._greenthread:
self._greenthread.kill()
self._greenthread = None
def _restart_greenthread(self):
self._kill_greenthread()
self._start_greenthread()
def _start_greenthread(self):
# Find the first time.
# We don't known when using this trigger first time.
first_run_time = self._get_first_run_time(
timer, first_run_time = self._get_timer_and_first_run_time(
self._trigger_property)
if not first_run_time:
raise exception.TriggerIsInvalid(trigger_id=self._id)
self._create_green_thread(first_run_time)
self._create_green_thread(first_run_time, timer)
def _create_green_thread(self, first_run_time):
def _create_green_thread(self, first_run_time, timer):
func = functools.partial(
self._trigger_operations,
trigger_property=self._trigger_property.copy())
trigger_property=self._trigger_property.copy(),
timer=timer)
self._greenthread = TriggerOperationGreenThread(
first_run_time, func)
def _trigger_operations(self, expect_run_time, trigger_property):
def _trigger_operations(self, expect_run_time, trigger_property, timer):
"""Trigger operations once
returns: wait time for next run
@ -234,9 +233,7 @@ class TimeTrigger(triggers.BaseTrigger):
pass
next_time = self._compute_next_run_time(
expect_run_time, trigger_property['end_time'],
trigger_property['format'],
trigger_property['pattern'])
expect_run_time, trigger_property['end_time'], timer)
now = datetime.utcnow()
if next_time and next_time <= now:
LOG.error(_LE("Next run time:%(next_time)s <= now:%(now)s. Maybe "
@ -254,11 +251,10 @@ class TimeTrigger(triggers.BaseTrigger):
All the time instances of trigger_definition are in UTC,
including start_time, end_time
"""
tf_cls = cls._get_time_format_class()
trigger_format = trigger_definition.get("format", None)
pattern = trigger_definition.get("pattern", None)
cls.TIME_FORMAT_MANAGER.check_time_format(
trigger_format, pattern)
tf_cls.check_time_format(pattern)
start_time = trigger_definition.get("start_time", None)
if not start_time:
@ -266,8 +262,7 @@ class TimeTrigger(triggers.BaseTrigger):
raise exception.InvalidInput(msg)
start_time = cls._check_and_get_datetime(start_time, "start_time")
interval = int(cls.TIME_FORMAT_MANAGER.get_interval(
trigger_format, pattern))
interval = tf_cls(start_time, pattern).get_min_interval()
if interval is not None and interval < CONF.min_interval:
msg = (_("The interval of two adjacent time points "
"is less than %d") % CONF.min_interval)
@ -321,27 +316,28 @@ class TimeTrigger(triggers.BaseTrigger):
return time
@classmethod
def _compute_next_run_time(cls, start_time, end_time,
trigger_format, trigger_pattern):
next_time = cls.TIME_FORMAT_MANAGER.compute_next_time(
trigger_format, trigger_pattern, start_time)
def _compute_next_run_time(cls, start_time, end_time, timer):
next_time = timer.compute_next_time(start_time)
if next_time and (not end_time or next_time <= end_time):
return next_time
return None
@classmethod
def _get_first_run_time(cls, trigger_property):
now = datetime.utcnow()
tmp_time = trigger_property['start_time']
if tmp_time < now:
tmp_time = now
return cls._compute_next_run_time(
tmp_time,
trigger_property['end_time'],
trigger_property['format'],
trigger_property['pattern'])
def _get_time_format_class(cls):
return import_driver.DriverManager(
'karbor.operationengine.engine.timetrigger.time_format',
CONF.time_format).driver
@classmethod
def _get_timer_and_first_run_time(cls, trigger_property):
tf_cls = cls._get_time_format_class()
timer = tf_cls(trigger_property['start_time'],
trigger_property['pattern'])
first_run_time = cls._compute_next_run_time(
datetime.utcnow(), trigger_property['end_time'], timer)
return timer, first_run_time
@classmethod
def check_configuration(cls):

48
karbor/services/operationengine/engine/triggers/timetrigger/timeformats/__init__.py
View File

@ -11,53 +11,45 @@
# under the License.
"""
Operation classes
time format base class
"""
from abc import ABCMeta
import abc
import six
from karbor import loadables
@six.add_metaclass(ABCMeta)
@six.add_metaclass(abc.ABCMeta)
class TimeFormat(object):
FORMAT_TYPE = ""
def __init__(self, start_time, pattern):
"""Initiate time format
:param start_time: The time points after the start_time are valid
:param pattern: The pattern of the time
When the start_time and pattern are specified, the time points
can be calculated and are immutable.
"""
pass
@classmethod
@abc.abstractmethod
def check_time_format(cls, pattern):
"""Check time format
Only supports absolute time format, like crontab.
:param pattern: The pattern of the time
"""
pass
@classmethod
def compute_next_time(cls, pattern, start_time):
@abc.abstractmethod
def compute_next_time(self, current_time):
"""Compute next time
:param pattern: The pattern of time
:param start_time: the start time for computing
:param current_time: the time before the next time
"""
pass
@classmethod
def get_interval(cls, pattern):
"""Get interval of two adjacent time points
:param pattern: The pattern of the time
"""
@abc.abstractmethod
def get_min_interval(self):
"""Get minimum interval of two adjacent time points"""
pass
class TimeFormatHandler(loadables.BaseLoader):
def __init__(self):
super(TimeFormatHandler, self).__init__(TimeFormat)
def all_time_formats():
"""Get all trigger time format classes."""
return TimeFormatHandler().get_all_classes()

24
karbor/services/operationengine/engine/triggers/timetrigger/timeformats/crontab_time.py
View File

@ -21,9 +21,10 @@ from karbor.services.operationengine.engine.triggers.timetrigger import\
class Crontab(timeformats.TimeFormat):
"""Crontab."""
FORMAT_TYPE = "crontab"
def __init__(self, start_time, pattern):
self._start_time = start_time
self._pattern = pattern
@classmethod
def check_time_format(cls, pattern):
@ -37,12 +38,15 @@ class Crontab(timeformats.TimeFormat):
msg = (_("The trigger pattern(%s) is invalid") % pattern)
raise exception.InvalidInput(msg)
@classmethod
def compute_next_time(cls, pattern, start_time):
return croniter(pattern, start_time).get_next(datetime)
def compute_next_time(self, current_time):
time = current_time if current_time >= self._start_time else (
self._start_time)
return croniter(self._pattern, time).get_next(datetime)
@classmethod
def get_interval(cls, pattern):
t1 = cls.compute_next_time(pattern, datetime.now())
t2 = cls.compute_next_time(pattern, t1)
return timeutils.delta_seconds(t1, t2)
def get_min_interval(self):
try:
t1 = self.compute_next_time(datetime.now())
t2 = self.compute_next_time(t1)
return timeutils.delta_seconds(t1, t2)
except Exception:
return None

39
karbor/tests/unit/operationengine/engine/triggers/timetrigger/test_time_format_manager.py
View File

@ -1,39 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
from datetime import timedelta
from karbor import exception
from karbor.services.operationengine.engine.triggers.timetrigger import\
time_format_manager
from karbor.tests import base
class TimeFormatManagerTestCase(base.TestCase):
def setUp(self):
super(TimeFormatManagerTestCase, self).setUp()
self._manager = time_format_manager.TimeFormatManager()
def test_time_format(self):
self.assertRaisesRegexp(exception.InvalidInput,
"Invalid trigger time format type.*abc$",
self._manager.check_time_format,
'abc', None)
def test_compute_next_time(self):
now = datetime(2016, 1, 20, 15, 11, 0, 0)
time1 = self._manager.compute_next_time("crontab", "* * * * *", now)
time2 = now + timedelta(minutes=1)
self.assertEqual(time2, time1)

23
karbor/tests/unit/operationengine/engine/triggers/timetrigger/test_time_trigger.py
View File

@ -17,24 +17,23 @@ import mock
from oslo_config import cfg
from karbor import exception
from karbor.services.operationengine.engine.triggers.timetrigger import \
time_format_manager
from karbor.services.operationengine.engine.triggers.timetrigger.time_trigger \
import TimeTrigger
from karbor.tests import base
class FakeTimeFormat(object):
def __init__(self, start_time, pattern):
pass
@classmethod
def check_time_format(cls, pattern):
pass
@classmethod
def compute_next_time(cls, pattern, current_time):
def compute_next_time(self, current_time):
return current_time + timedelta(seconds=0.5)
@classmethod
def get_interval(cls, pattern):
def get_min_interval(self):
return cfg.CONF.min_interval
@ -44,7 +43,6 @@ class FakeExecutor(object):
def execute_operation(self, operation_id, triggered_time,
expect_start_time, window):
if operation_id not in self._ops:
self._ops[operation_id] = 0
self._ops[operation_id] += 1
@ -63,7 +61,7 @@ class TimeTriggerTestCase(base.TestCase):
mock_obj = mock.Mock()
mock_obj.return_value = FakeTimeFormat
time_format_manager.TimeFormatManager._get_timeformat_cls = mock_obj
TimeTrigger._get_time_format_class = mock_obj
self._default_executor = FakeExecutor()
@ -97,9 +95,9 @@ class TimeTriggerTestCase(base.TestCase):
TimeTrigger.check_trigger_definition,
trigger_property)
@mock.patch.object(FakeTimeFormat, 'get_interval')
def test_check_trigger_property_interval(self, get_interval):
get_interval.return_value = 0
@mock.patch.object(FakeTimeFormat, 'get_min_interval')
def test_check_trigger_property_interval(self, get_min_interval):
get_min_interval.return_value = 0
trigger_property = {
"start_time": '2016-8-18 01:03:04'
@ -171,7 +169,6 @@ class TimeTriggerTestCase(base.TestCase):
trigger = self._generate_trigger()
trigger_property = {
"format": "",
"pattern": "",
"window": 15,
"start_time": datetime.utcnow(),
@ -198,7 +195,6 @@ class TimeTriggerTestCase(base.TestCase):
eventlet.sleep(0.2)
trigger_property = {
"format": "",
"pattern": "",
"window": 15,
"start_time": datetime.utcnow(),
@ -217,7 +213,6 @@ class TimeTriggerTestCase(base.TestCase):
end_time = datetime.utcnow() + timedelta(seconds=1)
trigger_property = {
"format": "",
"pattern": "",
"window": 15,
"start_time": datetime.utcnow(),

8
karbor/tests/unit/operationengine/engine/triggers/timetrigger/timeformats/test_crontab_time.py
View File

@ -24,7 +24,7 @@ class CrontabTimeTestCase(base.TestCase):
def setUp(self):
super(CrontabTimeTestCase, self).setUp()
self._time_format = crontab_time.Crontab()
self._time_format = crontab_time.Crontab
def test_none_pattern(self):
self.assertRaisesRegexp(exception.InvalidInput,
@ -40,9 +40,11 @@ class CrontabTimeTestCase(base.TestCase):
def test_compute_next_time(self):
now = datetime(2016, 1, 20, 15, 11, 0, 0)
time1 = self._time_format.compute_next_time("* * * * *", now)
obj = self._time_format(now, "* * * * *")
time1 = obj.compute_next_time(now)
time2 = now + timedelta(minutes=1)
self.assertEqual(time2, time1)
def test_get_interval(self):
self.assertEqual(60, self._time_format.get_interval("* * * * *"))
obj = self._time_format(datetime.now(), "* * * * *")
self.assertEqual(60, obj.get_min_interval())

2
setup.cfg
View File

@ -51,6 +51,8 @@ karbor.protectables =
server = karbor.services.protection.protectable_plugins.server:ServerProtectablePlugin
volume = karbor.services.protection.protectable_plugins.volume:VolumeProtectablePlugin
image = karbor.services.protection.protectable_plugins.image:ImageProtectablePlugin
karbor.operationengine.engine.timetrigger.time_format =
crontab = karbor.services.operationengine.engine.triggers.timetrigger.timeformats.crontab_time:Crontab
karbor.operationengine.engine.executor =
thread_pool = karbor.services.operationengine.engine.executors.thread_pool_executor:ThreadPoolExecutor
green_thread = karbor.services.operationengine.engine.executors.green_thread_executor:GreenThreadExecutor