# -*- coding: utf-8 -*- # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import contextlib import threading import mock from taskflow import exceptions from taskflow.openstack.common import uuidutils from taskflow.persistence.backends import impl_memory from taskflow.persistence import logbook from taskflow import states from taskflow import storage from taskflow import test from taskflow.utils import misc from taskflow.utils import persistence_utils as p_utils class StorageTest(test.TestCase): def setUp(self): super(StorageTest, self).setUp() self.backend = impl_memory.MemoryBackend(conf={}) self.thread_count = 50 def tearDown(self): with contextlib.closing(self.backend) as be: with contextlib.closing(be.get_connection()) as conn: conn.clear_all() super(StorageTest, self).tearDown() @staticmethod def _run_many_threads(threads): for t in threads: t.start() for t in threads: t.join() def _get_storage(self, flow_detail=None, threaded=False): if flow_detail is None: _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) storage_cls = storage.SingleThreadedStorage if threaded: storage_cls = storage.MultiThreadedStorage return storage_cls(flow_detail=flow_detail, backend=self.backend) def test_non_saving_storage(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) s = storage.SingleThreadedStorage(flow_detail=flow_detail) s.ensure_task('my_task') self.assertTrue(uuidutils.is_uuid_like(s.get_task_uuid('my_task'))) def test_flow_name_and_uuid(self): flow_detail = logbook.FlowDetail(name='test-fd', uuid='aaaa') s = self._get_storage(flow_detail) self.assertEqual(s.flow_name, 'test-fd') self.assertEqual(s.flow_uuid, 'aaaa') def test_ensure_task(self): s = self._get_storage() s.ensure_task('my task') self.assertEqual(s.get_task_state('my task'), states.PENDING) self.assertTrue(uuidutils.is_uuid_like(s.get_task_uuid('my task'))) def test_get_tasks_states(self): s = self._get_storage() s.ensure_task('my task') s.ensure_task('my task2') s.save('my task', 'foo') expected = { 'my task': (states.SUCCESS, states.EXECUTE), 'my task2': (states.PENDING, states.EXECUTE), } self.assertEqual(s.get_tasks_states(['my task', 'my task2']), expected) def test_ensure_task_flow_detail(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) s = self._get_storage(flow_detail) s.ensure_task('my task', '3.11') td = flow_detail.find(s.get_task_uuid('my task')) self.assertIsNotNone(td) self.assertEqual(td.name, 'my task') self.assertEqual(td.version, '3.11') self.assertEqual(td.state, states.PENDING) def test_get_without_save(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) td = logbook.TaskDetail(name='my_task', uuid='42') flow_detail.add(td) s = self._get_storage(flow_detail) self.assertEqual('42', s.get_task_uuid('my_task')) def test_ensure_existing_task(self): _lb, flow_detail = p_utils.temporary_flow_detail(self.backend) td = logbook.TaskDetail(name='my_task', uuid='42') flow_detail.add(td) s = self._get_storage(flow_detail) s.ensure_task('my_task') self.assertEqual('42', s.get_task_uuid('my_task')) def test_save_and_get(self): s = self._get_storage() s.ensure_task('my task') s.save('my task', 5) self.assertEqual(s.get('my task'), 5) self.assertEqual(s.fetch_all(), {}) self.assertEqual(s.get_task_state('my task'), states.SUCCESS) def test_save_and_get_other_state(self): s = self._get_storage() s.ensure_task('my task') s.save('my task', 5, states.FAILURE) self.assertEqual(s.get('my task'), 5) self.assertEqual(s.get_task_state('my task'), states.FAILURE) def test_save_and_get_failure(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_task('my task') s.save('my task', failure, states.FAILURE) self.assertEqual(s.get('my task'), failure) self.assertEqual(s.get_task_state('my task'), states.FAILURE) self.assertTrue(s.has_failures()) self.assertEqual(s.get_failures(), {'my task': failure}) def test_save_and_get_non_cached_failure(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_task('my task') s.save('my task', failure, states.FAILURE) self.assertEqual(s.get('my task'), failure) s._failures['my task'] = None self.assertEqual(s.get('my task'), failure) def test_get_failure_from_reverted_task(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_task('my task') s.save('my task', failure, states.FAILURE) s.set_task_state('my task', states.REVERTING) self.assertEqual(s.get('my task'), failure) s.set_task_state('my task', states.REVERTED) self.assertEqual(s.get('my task'), failure) def test_get_failure_after_reload(self): failure = misc.Failure.from_exception(RuntimeError('Woot!')) s = self._get_storage() s.ensure_task('my task') s.save('my task', failure, states.FAILURE) s2 = self._get_storage(s._flowdetail) self.assertIs(s2.has_failures(), True) self.assertEqual(s2.get_failures(), {'my task': failure}) self.assertEqual(s2.get('my task'), failure) self.assertEqual(s2.get_task_state('my task'), states.FAILURE) def test_get_non_existing_var(self): s = self._get_storage() s.ensure_task('my task') self.assertRaises(exceptions.NotFound, s.get, 'my task') def test_reset(self): s = self._get_storage() s.ensure_task('my task') s.save('my task', 5) s.reset('my task') self.assertEqual(s.get_task_state('my task'), states.PENDING) self.assertRaises(exceptions.NotFound, s.get, 'my task') def test_reset_unknown_task(self): s = self._get_storage() s.ensure_task('my task') self.assertEqual(s.reset('my task'), None) def test_reset_tasks(self): s = self._get_storage() s.ensure_task('my task') s.save('my task', 5) s.ensure_task('my other task') s.save('my other task', 7) s.reset_tasks() self.assertEqual(s.get_task_state('my task'), states.PENDING) self.assertRaises(exceptions.NotFound, s.get, 'my task') self.assertEqual(s.get_task_state('my other task'), states.PENDING) self.assertRaises(exceptions.NotFound, s.get, 'my other task') def test_reset_tasks_does_not_breaks_inject(self): s = self._get_storage() s.inject({'foo': 'bar', 'spam': 'eggs'}) # NOTE(imelnikov): injecting is implemented as special task # so resetting tasks may break it if implemented incorrectly. s.reset_tasks() self.assertEqual(s.fetch('spam'), 'eggs') self.assertEqual(s.fetch_all(), { 'foo': 'bar', 'spam': 'eggs', }) def test_fetch_by_name(self): s = self._get_storage() name = 'my result' s.ensure_task('my task', '1.0', {name: None}) s.save('my task', 5) self.assertEqual(s.fetch(name), 5) self.assertEqual(s.fetch_all(), {name: 5}) def test_fetch_unknown_name(self): s = self._get_storage() self.assertRaisesRegexp(exceptions.NotFound, "^Name 'xxx' is not mapped", s.fetch, 'xxx') def test_task_metadata_update_with_none(self): s = self._get_storage() s.ensure_task('my task') s.update_task_metadata('my task', None) self.assertEqual(s.get_task_progress('my task'), 0.0) s.set_task_progress('my task', 0.5) self.assertEqual(s.get_task_progress('my task'), 0.5) s.update_task_metadata('my task', None) self.assertEqual(s.get_task_progress('my task'), 0.5) def test_default_task_progress(self): s = self._get_storage() s.ensure_task('my task') self.assertEqual(s.get_task_progress('my task'), 0.0) self.assertEqual(s.get_task_progress_details('my task'), None) def test_task_progress(self): s = self._get_storage() s.ensure_task('my task') s.set_task_progress('my task', 0.5, {'test_data': 11}) self.assertEqual(s.get_task_progress('my task'), 0.5) self.assertEqual(s.get_task_progress_details('my task'), { 'at_progress': 0.5, 'details': {'test_data': 11} }) s.set_task_progress('my task', 0.7, {'test_data': 17}) self.assertEqual(s.get_task_progress('my task'), 0.7) self.assertEqual(s.get_task_progress_details('my task'), { 'at_progress': 0.7, 'details': {'test_data': 17} }) s.set_task_progress('my task', 0.99) self.assertEqual(s.get_task_progress('my task'), 0.99) self.assertEqual(s.get_task_progress_details('my task'), { 'at_progress': 0.7, 'details': {'test_data': 17} }) def test_task_progress_erase(self): s = self._get_storage() s.ensure_task('my task') s.set_task_progress('my task', 0.8, {}) self.assertEqual(s.get_task_progress('my task'), 0.8) self.assertEqual(s.get_task_progress_details('my task'), None) def test_fetch_result_not_ready(self): s = self._get_storage() name = 'my result' s.ensure_task('my task', result_mapping={name: None}) self.assertRaises(exceptions.NotFound, s.get, name) self.assertEqual(s.fetch_all(), {}) def test_save_multiple_results(self): s = self._get_storage() result_mapping = {'foo': 0, 'bar': 1, 'whole': None} s.ensure_task('my task', result_mapping=result_mapping) s.save('my task', ('spam', 'eggs')) self.assertEqual(s.fetch_all(), { 'foo': 'spam', 'bar': 'eggs', 'whole': ('spam', 'eggs') }) def test_mapping_none(self): s = self._get_storage() s.ensure_task('my task') s.save('my task', 5) self.assertEqual(s.fetch_all(), {}) def test_inject(self): s = self._get_storage() s.inject({'foo': 'bar', 'spam': 'eggs'}) self.assertEqual(s.fetch('spam'), 'eggs') self.assertEqual(s.fetch_all(), { 'foo': 'bar', 'spam': 'eggs', }) def test_inject_twice(self): s = self._get_storage() s.inject({'foo': 'bar'}) self.assertEqual(s.fetch_all(), {'foo': 'bar'}) s.inject({'spam': 'eggs'}) self.assertEqual(s.fetch_all(), { 'foo': 'bar', 'spam': 'eggs', }) def test_inject_resumed(self): s = self._get_storage() s.inject({'foo': 'bar', 'spam': 'eggs'}) # verify it's there self.assertEqual(s.fetch_all(), { 'foo': 'bar', 'spam': 'eggs', }) # imagine we are resuming, so we need to make new # storage from same flow details s2 = self._get_storage(s._flowdetail) # injected data should still be there: self.assertEqual(s2.fetch_all(), { 'foo': 'bar', 'spam': 'eggs', }) def test_many_thread_ensure_same_task(self): s = self._get_storage(threaded=True) def ensure_my_task(): s.ensure_task('my_task', result_mapping={}) threads = [] for i in range(0, self.thread_count): threads.append(threading.Thread(target=ensure_my_task)) self._run_many_threads(threads) # Only one task should have been made, no more. self.assertEqual(1, len(s._flowdetail)) def test_many_thread_one_reset(self): s = self._get_storage(threaded=True) s.ensure_task('a') s.set_task_state('a', states.SUSPENDED) s.ensure_task('b') s.set_task_state('b', states.SUSPENDED) results = [] result_lock = threading.Lock() def reset_all(): r = s.reset_tasks() with result_lock: results.append(r) threads = [] for i in range(0, self.thread_count): threads.append(threading.Thread(target=reset_all)) self._run_many_threads(threads) # Only one thread should have actually reset (not anymore) results = [r for r in results if len(r)] self.assertEqual(1, len(results)) self.assertEqual(['a', 'b'], sorted([a[0] for a in results[0]])) def test_many_thread_inject(self): s = self._get_storage(threaded=True) def inject_values(values): s.inject(values) threads = [] for i in range(0, self.thread_count): values = { str(i): str(i), } threads.append(threading.Thread(target=inject_values, args=[values])) self._run_many_threads(threads) self.assertEqual(self.thread_count, len(s.fetch_all())) self.assertEqual(1, len(s._flowdetail)) def test_fetch_mapped_args(self): s = self._get_storage() s.inject({'foo': 'bar', 'spam': 'eggs'}) self.assertEqual(s.fetch_mapped_args({'viking': 'spam'}), {'viking': 'eggs'}) def test_fetch_not_found_args(self): s = self._get_storage() s.inject({'foo': 'bar', 'spam': 'eggs'}) self.assertRaises(exceptions.NotFound, s.fetch_mapped_args, {'viking': 'helmet'}) def test_set_and_get_task_state(self): s = self._get_storage() state = states.PENDING s.ensure_task('my task') s.set_task_state('my task', state) self.assertEqual(s.get_task_state('my task'), state) def test_get_state_of_unknown_task(self): s = self._get_storage() self.assertRaisesRegexp(exceptions.NotFound, '^Unknown', s.get_task_state, 'my task') def test_task_by_name(self): s = self._get_storage() s.ensure_task('my task') self.assertTrue(uuidutils.is_uuid_like(s.get_task_uuid('my task'))) def test_unknown_task_by_name(self): s = self._get_storage() self.assertRaisesRegexp(exceptions.NotFound, '^Unknown task name:', s.get_task_uuid, '42') def test_initial_flow_state(self): s = self._get_storage() self.assertEqual(s.get_flow_state(), states.PENDING) def test_get_flow_state(self): _lb, flow_detail = p_utils.temporary_flow_detail(backend=self.backend) flow_detail.state = states.FAILURE with contextlib.closing(self.backend.get_connection()) as conn: flow_detail.update(conn.update_flow_details(flow_detail)) s = self._get_storage(flow_detail) self.assertEqual(s.get_flow_state(), states.FAILURE) def test_set_and_get_flow_state(self): s = self._get_storage() s.set_flow_state(states.SUCCESS) self.assertEqual(s.get_flow_state(), states.SUCCESS) @mock.patch.object(storage.LOG, 'warning') def test_result_is_checked(self, mocked_warning): s = self._get_storage() s.ensure_task('my task', result_mapping={'result': 'key'}) s.save('my task', {}) mocked_warning.assert_called_once_with( mock.ANY, 'my task', 'key', 'result') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'result') @mock.patch.object(storage.LOG, 'warning') def test_empty_result_is_checked(self, mocked_warning): s = self._get_storage() s.ensure_task('my task', result_mapping={'a': 0}) s.save('my task', ()) mocked_warning.assert_called_once_with( mock.ANY, 'my task', 0, 'a') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'a') @mock.patch.object(storage.LOG, 'warning') def test_short_result_is_checked(self, mocked_warning): s = self._get_storage() s.ensure_task('my task', result_mapping={'a': 0, 'b': 1}) s.save('my task', ['result']) mocked_warning.assert_called_once_with( mock.ANY, 'my task', 1, 'b') self.assertEqual(s.fetch('a'), 'result') self.assertRaisesRegexp(exceptions.NotFound, '^Unable to find result', s.fetch, 'b') @mock.patch.object(storage.LOG, 'warning') def test_multiple_providers_are_checked(self, mocked_warning): s = self._get_storage() s.ensure_task('my task', result_mapping={'result': 'key'}) self.assertEqual(mocked_warning.mock_calls, []) s.ensure_task('my other task', result_mapping={'result': 'key'}) mocked_warning.assert_called_once_with( mock.ANY, 'result') @mock.patch.object(storage.LOG, 'warning') def test_multiple_providers_with_inject_are_checked(self, mocked_warning): s = self._get_storage() s.inject({'result': 'DONE'}) self.assertEqual(mocked_warning.mock_calls, []) s.ensure_task('my other task', result_mapping={'result': 'key'}) mocked_warning.assert_called_once_with(mock.ANY, 'result') def test_ensure_retry(self): s = self._get_storage() s.ensure_retry('my retry') history = s.get_retry_history('my retry') self.assertEqual(history, []) def test_ensure_retry_and_task_with_same_name(self): s = self._get_storage() s.ensure_task('my retry') self.assertRaisesRegexp(exceptions.AlreadyExists, '^Task detail', s.ensure_retry, 'my retry') def test_save_retry_results(self): s = self._get_storage() s.ensure_retry('my retry') s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') self.assertEqual(history, [('a', {}), ('b', {})]) def test_save_retry_results_with_mapping(self): s = self._get_storage() s.ensure_retry('my retry', result_mapping={'x': 0}) s.save('my retry', 'a') s.save('my retry', 'b') history = s.get_retry_history('my retry') self.assertEqual(history, [('a', {}), ('b', {})]) self.assertEqual(s.fetch_all(), {'x': 'b'}) self.assertEqual(s.fetch('x'), 'b') def test_cleanup_retry_history(self): s = self._get_storage() s.ensure_retry('my retry', result_mapping={'x': 0}) s.save('my retry', 'a') s.save('my retry', 'b') s.cleanup_retry_history('my retry', states.REVERTED) history = s.get_retry_history('my retry') self.assertEqual(history, []) self.assertEqual(s.fetch_all(), {}) def test_retry_failure(self): fail = misc.Failure(exc_info=(RuntimeError, RuntimeError(), None)) s = self._get_storage() s.ensure_retry('my retry', result_mapping={'x': 0}) s.save('my retry', 'a') s.save('my retry', fail, states.FAILURE) history = s.get_retry_history('my retry') self.assertEqual(history, [('a', {}), (fail, {})]) self.assertIs(s.has_failures(), True) self.assertEqual(s.get_failures(), {'my retry': fail}) def test_logbook_get_unknown_atom_type(self): self.assertRaisesRegexp(TypeError, 'Unknown atom type', logbook.get_atom_detail_class, 'some_detail') def test_save_task_intention(self): s = self._get_storage() s.ensure_task('my task') s.set_atom_intention('my task', states.REVERT) intention = s.get_atom_intention('my task') self.assertEqual(intention, states.REVERT) def test_save_retry_intention(self): s = self._get_storage() s.ensure_retry('my retry') s.set_atom_intention('my retry', states.RETRY) intention = s.get_atom_intention('my retry') self.assertEqual(intention, states.RETRY)