94 lines
3.1 KiB
Python
94 lines
3.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
self_dir = os.path.abspath(os.path.dirname(__file__))
|
|
sys.path.insert(0, self_dir)
|
|
|
|
import example_utils # noqa
|
|
|
|
# INTRO: In this example we create a common persistence database (sqlite based)
|
|
# and then we run a few set of processes which themselves use this persistence
|
|
# database, those processes 'crash' (in a simulated way) by exiting with a
|
|
# system error exception. After this occurs a few times we then activate a
|
|
# script which doesn't 'crash' and it will resume all the given engines flows
|
|
# that did not complete and run them to completion (instead of crashing).
|
|
#
|
|
# This shows how a set of tasks can be finished even after repeatedly being
|
|
# crashed, *crash resistance* if you may call it, due to the engine concept as
|
|
# well as the persistence layer which keeps track of the state a flow
|
|
# transitions through and persists the intermediary inputs and outputs and
|
|
# overall flow state.
|
|
|
|
|
|
def _exec(cmd, add_env=None):
|
|
env = None
|
|
if add_env:
|
|
env = os.environ.copy()
|
|
env.update(add_env)
|
|
|
|
proc = subprocess.Popen(cmd, env=env, stdin=None,
|
|
stdout=subprocess.PIPE,
|
|
stderr=sys.stderr)
|
|
|
|
stdout, _stderr = proc.communicate()
|
|
rc = proc.returncode
|
|
if rc != 0:
|
|
raise RuntimeError("Could not run %s [%s]", cmd, rc)
|
|
print(stdout.decode())
|
|
|
|
|
|
def _path_to(name):
|
|
return os.path.abspath(os.path.join(os.path.dirname(__file__),
|
|
'resume_many_flows', name))
|
|
|
|
|
|
def main():
|
|
backend_uri = None
|
|
tmp_path = None
|
|
try:
|
|
if example_utils.SQLALCHEMY_AVAILABLE:
|
|
tmp_path = tempfile.mktemp(prefix='tf-resume-example')
|
|
backend_uri = "sqlite:///%s" % (tmp_path)
|
|
else:
|
|
tmp_path = tempfile.mkdtemp(prefix='tf-resume-example')
|
|
backend_uri = 'file:///%s' % (tmp_path)
|
|
|
|
def run_example(name, add_env=None):
|
|
_exec([sys.executable, _path_to(name), backend_uri], add_env)
|
|
|
|
print('Run flow:')
|
|
run_example('run_flow.py')
|
|
|
|
print('\nRun flow, something happens:')
|
|
run_example('run_flow.py', {'BOOM': 'exit please'})
|
|
|
|
print('\nRun flow, something happens again:')
|
|
run_example('run_flow.py', {'BOOM': 'exit please'})
|
|
|
|
print('\nResuming all failed flows')
|
|
run_example('resume_all.py')
|
|
finally:
|
|
if tmp_path:
|
|
example_utils.rm_path(tmp_path)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|