57c4b1bdc2
This is first in series of changes that provides separation of flows into engines and pattens. Patterns define structure, while engine define how the task should be run according to said structure. This change adds the basic patterns and a very simple engine that is able to run flow defined with patterns by converting it to recursive structure of so-called 'actions'. For simplicity and ease of review certain major features like resumption, saving and passing on task results and notifications are left out for farther changes. Partially implements blueprint patterns-and-engines Co-authored-by: Anastasia Karpinska <akarpinska at griddynamics.com> Change-Id: I68515d8a5b30d5d047bf9beb67cc3e2111175190
90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
# Copyright (C) 2013 Rackspace Hosting Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import celery
|
|
import logging
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class Flow(object):
|
|
"""A flow that can paralleize task running by using celery.
|
|
|
|
This flow backs running tasks (and associated dependencies) by using celery
|
|
as the runtime framework to accomplish execution (and status reporting) of
|
|
said tasks that compose the flow. It allows for parallel execution where
|
|
possible (data/task dependency dependent) without having to worry about how
|
|
this is accomplished in celery.
|
|
"""
|
|
|
|
def __init__(self, name, parents=None):
|
|
self.name = name
|
|
self.root = None
|
|
self._tasks = []
|
|
|
|
def chain_listeners(self, context, initial_task, callback_task):
|
|
"""Register one listener for a task."""
|
|
if self.root is None:
|
|
initial_task.name = '%s.%s' % (self.name, initial_task.name)
|
|
self.root = initial_task.s(context)
|
|
self._tasks.append(initial_task)
|
|
LOG.info('WF %s root task set to %s', self.name, initial_task.name)
|
|
|
|
callback_task.name = '%s.%s' % (self.name, callback_task.name)
|
|
self._tasks.append(callback_task)
|
|
|
|
initial_task.link(callback_task.s(context))
|
|
|
|
def split_listeners(self, context, initial_task, callback_tasks):
|
|
"""Register multiple listeners for one task."""
|
|
if self.root is None:
|
|
initial_task.name = '%s.%s' % (self.name, initial_task.name)
|
|
self.root = initial_task.s(context)
|
|
self._tasks.append(initial_task)
|
|
LOG.info('WF %s root task set to %s', self.name, initial_task.name)
|
|
for task in callback_tasks:
|
|
task.name = '%s.%s' % (self.name, task.name)
|
|
self._tasks.append(task)
|
|
initial_task.link(task.s(context))
|
|
|
|
def merge_listeners(self, context, initial_tasks, callback_task):
|
|
"""Register one listener for multiple tasks."""
|
|
header = []
|
|
if self.root is None:
|
|
self.root = []
|
|
for task in initial_tasks:
|
|
task.name = '%s.%s' % (self.name, task.name)
|
|
self._tasks.append(task)
|
|
header.append(task.s(context))
|
|
if isinstance(self.root, list):
|
|
self.root.append(task.s(context))
|
|
LOG.info('WF %s added root task %s' %
|
|
(self.name, task.name))
|
|
callback_task.name = '%s.%s' % (self.name, callback_task.name)
|
|
self._tasks.append(callback_task)
|
|
|
|
# TODO(jlucci): Need to set up chord so that it's not executed
|
|
# immediately.
|
|
celery.chord(header, body=callback_task)
|
|
|
|
def run(self, context, *args, **kwargs):
|
|
"""Start root task and kick off workflow."""
|
|
self.root(context)
|
|
LOG.info('WF %s has been started' % (self.name,))
|