diff --git a/tripleo_common/actions/base.py b/tripleo_common/actions/base.py index f921701bd..1d3ca5036 100644 --- a/tripleo_common/actions/base.py +++ b/tripleo_common/actions/base.py @@ -12,9 +12,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import json import tempfile -import zlib from glanceclient.v2 import client as glanceclient from heatclient.v1 import client as heatclient @@ -26,11 +24,9 @@ from mistral_lib import actions from mistralclient.api import client as mistral_client from novaclient.client import Client as nova_client from swiftclient import client as swift_client -from swiftclient import exceptions as swiftexceptions from swiftclient import service as swift_service from zaqarclient.queues.v2 import client as zaqarclient -from tripleo_common import constants from tripleo_common.utils import keystone as keystone_utils from tripleo_common.utils import tarball @@ -183,66 +179,6 @@ class TripleOAction(actions.Action): return nova_client(2, **conf) - def _cache_key(self, plan_name, key_name): - return "__cache_{}_{}".format(plan_name, key_name) - - def cache_get(self, context, plan_name, key): - """Retrieves the stored objects - - Returns None if there are any issues or no objects found - - """ - - swift_client = self.get_object_client(context) - try: - headers, body = swift_client.get_object( - constants.TRIPLEO_CACHE_CONTAINER, - self._cache_key(plan_name, key) - ) - result = json.loads(zlib.decompress(body).decode()) - return result - except swiftexceptions.ClientException: - # cache does not exist, ignore - pass - except ValueError: - # the stored json is invalid. Deleting - self.cache_delete(context, plan_name, key) - return - - def cache_set(self, context, plan_name, key, contents): - """Stores an object - - Allows the storage of jsonable objects except for None - Storing None equals to a cache delete. - - """ - - swift_client = self.get_object_client(context) - if contents is None: - self.cache_delete(context, plan_name, key) - return - - try: - swift_client.head_container(constants.TRIPLEO_CACHE_CONTAINER) - except swiftexceptions.ClientException: - swift_client.put_container(constants.TRIPLEO_CACHE_CONTAINER) - - swift_client.put_object( - constants.TRIPLEO_CACHE_CONTAINER, - self._cache_key(plan_name, key), - zlib.compress(json.dumps(contents).encode())) - - def cache_delete(self, context, plan_name, key): - swift_client = self.get_object_client(context) - try: - swift_client.delete_object( - constants.TRIPLEO_CACHE_CONTAINER, - self._cache_key(plan_name, key) - ) - except swiftexceptions.ClientException: - # cache or container does not exist. Ignore - pass - class UploadDirectoryAction(TripleOAction): """Upload a directory to Swift.""" diff --git a/tripleo_common/actions/container_images.py b/tripleo_common/actions/container_images.py index 1cb6e296a..6bb09c393 100644 --- a/tripleo_common/actions/container_images.py +++ b/tripleo_common/actions/container_images.py @@ -24,7 +24,6 @@ from swiftclient import exceptions as swiftexceptions import yaml from tripleo_common.actions import base -from tripleo_common.actions import heat_capabilities from tripleo_common import constants from tripleo_common.image import kolla_builder from tripleo_common.utils import plan as plan_utils @@ -87,9 +86,15 @@ class PrepareContainerImageEnv(base.TripleOAction): environments = {constants.CONTAINER_DEFAULTS_ENVIRONMENT: True} - update_action = heat_capabilities.UpdateCapabilitiesAction( - environments, container=self.container) - return update_action.run(context) + try: + env = plan_utils.update_plan_environment(swift, environments, + container=self.container) + except swiftexceptions.ClientException as err: + err_msg = ("Error updating environment for plan %s: %s" % ( + self.container, err)) + LOG.exception(err_msg) + return actions.Result(error=err_msg) + return env class PrepareContainerImageParameters(base.TripleOAction): @@ -168,9 +173,15 @@ class PrepareContainerImageParameters(base.TripleOAction): environments = {constants.CONTAINER_DEFAULTS_ENVIRONMENT: True} - update_action = heat_capabilities.UpdateCapabilitiesAction( - environments, container=self.container) - return update_action.run(context) + try: + env = plan_utils.update_plan_environment(swift, environments, + container=self.container) + except swiftexceptions.ClientException as err: + err_msg = ("Error updating environment for plan %s: %s" % ( + self.container, err)) + LOG.exception(err_msg) + return actions.Result(error=err_msg) + return env class ContainerImagePrepareDefault(base.TripleOAction): diff --git a/tripleo_common/actions/heat_capabilities.py b/tripleo_common/actions/heat_capabilities.py deleted file mode 100644 index 44fd1405f..000000000 --- a/tripleo_common/actions/heat_capabilities.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2016 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import fnmatch -import logging -import yaml - -from mistral_lib import actions -from swiftclient import exceptions as swiftexceptions - -from tripleo_common.actions import base -from tripleo_common import constants -from tripleo_common.utils import plan as plan_utils -from tripleo_common.utils import swift as swiftutils - -LOG = logging.getLogger(__name__) - - -class GetCapabilitiesAction(base.TripleOAction): - """Gets list of available heat environments - - Parses the capabilities_map.yaml file in a given plan and - returns a list of environments - - :param container: name of the swift container / plan name - :return: list of environment files in swift container - """ - - def __init__(self, container=constants.DEFAULT_CONTAINER_NAME): - super(GetCapabilitiesAction, self).__init__() - self.container = container - - def run(self, context): - try: - swift = self.get_object_client(context) - map_file = swiftutils.get_object_string(swift, - self.container, - 'capabilities-map.yaml') - capabilities = yaml.safe_load(map_file) - except Exception: - err_msg = ( - "Error parsing capabilities-map.yaml.") - LOG.exception(err_msg) - return actions.Result(error=err_msg) - try: - container_files = swift.get_container(self.container) - container_file_list = [entry['name'] for entry - in container_files[1]] - except Exception as swift_err: - err_msg = ("Error retrieving plan files: %s" % swift_err) - LOG.exception(err_msg) - return actions.Result(error=err_msg) - - try: - env = plan_utils.get_env(swift, self.container) - except swiftexceptions.ClientException as err: - err_msg = ("Error retrieving environment for plan %s: %s" % ( - self.container, err)) - LOG.exception(err_msg) - return actions.Result(error=err_msg) - - selected_envs = [item['path'] for item in env['environments'] - if 'path' in item] - - # extract environment files - plan_environments = [] - for env_group in capabilities['topics']: - for envs in env_group['environment_groups']: - for files in envs['environments']: - file = files.get('file') - if file: - plan_environments.append(file) - - # parse plan for environment files - env_files = fnmatch.filter( - container_file_list, '*environments/*.yaml') - env_user_files = fnmatch.filter( - container_file_list, '*user-environment.yaml') - - outstanding_envs = list(set(env_files).union( - env_user_files) - set(plan_environments)) - - # change capabilities format - data_to_return = {} - - for topic in capabilities['topics']: - title = topic.get('title', '_title_holder') - data_to_return[title] = topic - for eg in topic['environment_groups']: - for env in eg['environments']: - if selected_envs and env.get('file') in selected_envs: - env['enabled'] = True - else: - env['enabled'] = False - - # add custom environment files - other_environments = [] - for env in outstanding_envs: - flag = selected_envs and env in selected_envs - new_env = { - "description": "Enable %s environment" % env, - "enabled": flag, - "file": env, - "title": env, - } - other_environments.append(new_env) - other_environments.sort(key=lambda x: x['file']) - - other_environment_groups = [] - for group in other_environments: - new_group = { - "description": None, - "environments": [group], - "title": group['file'], - } - other_environment_groups.append(new_group) - - other_environments_topic_dict = { - "description": None, - "title": "Other", - "environment_groups": other_environment_groups - } - - other_environments_topic = { - "Other": other_environments_topic_dict - } - data_to_return.update(other_environments_topic) - - return data_to_return - - -class UpdateCapabilitiesAction(base.TripleOAction): - """Updates plan environment with selected environments - - Takes a list of environment files and depending on the value of the - enabled flag, adds or removes them from the plan environment. - - :param environments: map of environments {'environment_path': True} - the value passed can be false for disabled - environments, these will be removed from the - plan environment. - :param container: name of the swift container / plan name - :param purge_missing: remove any environments from the plan environment - that aren't included in the environments map - defaults to False - :param sort_environments: use the dependencies defined in the - capabilites-map.yaml file in the plan to order - the environments - :return: the updated plan environment - """ - - def __init__(self, environments, - container=constants.DEFAULT_CONTAINER_NAME, - purge_missing=False, sort_environments=False): - super(UpdateCapabilitiesAction, self).__init__() - self.container = container - self.environments = environments - self.purge_missing = purge_missing - self.sort_environments = sort_environments - - def run(self, context): - swift = self.get_object_client(context) - - try: - env = plan_utils.get_env(swift, self.container) - except swiftexceptions.ClientException as err: - err_msg = ("Error retrieving environment for plan %s: %s" % ( - self.container, err)) - LOG.exception(err_msg) - return actions.Result(error=err_msg) - - for k, v in self.environments.items(): - found = False - if {'path': k} in env['environments']: - found = True - if v: - if not found: - env['environments'].append({'path': k}) - else: - if found: - env['environments'].remove({'path': k}) - - if self.purge_missing: - for e in env['environments']: - if e.get('path') not in self.environments: - env['environments'].remove(e) - - self.cache_delete(context, self.container, "tripleo.parameters.get") - - if self.sort_environments: - # get the capabilities-map content to perform the environment - # ordering - try: - swift = self.get_object_client(context) - map_file = swiftutils.get_object_string( - swift, - self.container, - 'capabilities-map.yaml') - capabilities = yaml.safe_load(map_file) - except swiftexceptions.ClientException as err: - err_msg = ("Error retrieving capabilities-map.yaml for " - "plan %s: %s" % (self.container, err)) - LOG.exception(err_msg) - return actions.Result(error=err_msg) - - ordered_env = plan_utils.apply_environments_order( - capabilities, env.get('environments', [])) - - env['environments'] = ordered_env - - try: - plan_utils.put_env(swift, env) - except swiftexceptions.ClientException as err: - err_msg = "Error uploading to container: %s" % err - LOG.exception(err_msg) - return actions.Result(error=err_msg) - - return env diff --git a/tripleo_common/actions/parameters.py b/tripleo_common/actions/parameters.py index 7bd937155..d7b9fe0ca 100644 --- a/tripleo_common/actions/parameters.py +++ b/tripleo_common/actions/parameters.py @@ -42,10 +42,11 @@ class GetParametersAction(base.TripleOAction): self.container = container def run(self, context): + swift = self.get_object_client(context) + heat = self.get_orchestration_client(context) - cached = self.cache_get(context, - self.container, - "tripleo.parameters.get") + cached = plan_utils.cache_get(swift, self.container, + "tripleo.parameters.get") if cached is not None: return cached @@ -63,9 +64,6 @@ class GetParametersAction(base.TripleOAction): processed_data['show_nested'] = True # respect previously user set param values - swift = self.get_object_client(context) - heat = self.get_orchestration_client(context) - try: env = plan_utils.get_env(swift, self.container) except swiftexceptions.ClientException as err: @@ -87,10 +85,9 @@ class GetParametersAction(base.TripleOAction): 'heat_resource_tree': heat.stacks.validate(**fields), 'environment_parameters': params, } - self.cache_set(context, - self.container, - "tripleo.parameters.get", - result) + plan_utils.cache_set(swift, self.container, + "tripleo.parameters.get", + result) return result @@ -123,9 +120,9 @@ class ResetParametersAction(base.TripleOAction): LOG.exception(err_msg) return actions.Result(error=err_msg) - self.cache_delete(context, - self.container, - "tripleo.parameters.get") + plan_utils.cache_delete(swift, + self.container, + "tripleo.parameters.get") return env @@ -195,10 +192,9 @@ class UpdateParametersAction(base.TripleOAction): } # Validation passes so the old cache gets replaced. - self.cache_set(context, - self.container, - "tripleo.parameters.get", - result) + plan_utils.cache_set(swift, self.container, + "tripleo.parameters.get", + result) if result['heat_resource_tree']: flattened = {'resources': {}, 'parameters': {}} @@ -333,9 +329,9 @@ class GeneratePasswordsAction(base.TripleOAction): LOG.exception(err_msg) return actions.Result(error=err_msg) - self.cache_delete(context, - self.container, - "tripleo.parameters.get") + plan_utils.cache_delete(swift, + self.container, + "tripleo.parameters.get") return env['passwords'] @@ -617,9 +613,9 @@ class RotateFernetKeysAction(GetPasswordsAction): LOG.exception(err_msg) return actions.Result(error=err_msg) - self.cache_delete(context, - self.container, - "tripleo.parameters.get") + plan_utils.cache_delete(swift, + self.container, + "tripleo.parameters.get") return keys_map diff --git a/tripleo_common/actions/templates.py b/tripleo_common/actions/templates.py index aa7176712..857f88267 100644 --- a/tripleo_common/actions/templates.py +++ b/tripleo_common/actions/templates.py @@ -246,9 +246,8 @@ class ProcessTemplatesAction(base.TripleOAction): LOG.info("skipping %s network: network is disabled." % n.get('name')) - self.cache_delete(context, - self.container, - "tripleo.parameters.get") + plan_utils.cache_delete(swift, self.container, + "tripleo.parameters.get") for f in [f.get('name') for f in container_files[1]]: # We do three templating passes here: diff --git a/tripleo_common/tests/actions/test_base.py b/tripleo_common/tests/actions/test_base.py index 732dee4d4..4e78c1358 100644 --- a/tripleo_common/tests/actions/test_base.py +++ b/tripleo_common/tests/actions/test_base.py @@ -12,8 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. -import zlib - import mock from ironicclient import client as ironicclient @@ -22,8 +20,6 @@ from tripleo_common.actions import base from tripleo_common.tests import base as tests_base from tripleo_common.utils import keystone as keystone_utils -from swiftclient.exceptions import ClientException - @mock.patch.object(keystone_utils, 'get_endpoint_for_project') class TestActionsBase(tests_base.TestCase): @@ -44,113 +40,3 @@ class TestActionsBase(tests_base.TestCase): retry_interval=5, token=mock.ANY) mock_endpoint.assert_called_once_with(mock_cxt.security, 'ironic') mock_cxt.assert_not_called() - - def test_cache_key(self, mock_endpoint): - container = "TestContainer" - key = "testkey" - cache_key = "__cache_TestContainer_testkey" - - self.assertEqual( - self.action._cache_key(container, key), - cache_key - ) - - @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") - @mock.patch("tripleo_common.actions.base.swift_client.Connection") - def test_cache_set(self, mock_conn, mock_keystone, mock_endpoint): - mock_ctx = mock.Mock() - mock_swift = mock.Mock() - mock_conn.return_value = mock_swift - - cache_container = "__cache__" - container = "TestContainer" - key = "testkey" - cache_key = "__cache_TestContainer_testkey" - compressed_json = zlib.compress("{\"foo\": 1}".encode()) - - self.action.cache_set(mock_ctx, container, key, {"foo": 1}) - mock_swift.put_object.assert_called_once_with( - cache_container, - cache_key, - compressed_json - ) - mock_swift.delete_object.assert_not_called() - - @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") - @mock.patch("tripleo_common.actions.base.swift_client.Connection") - def test_cache_set_none(self, mock_conn, mock_keystone, mock_endpoint): - mock_ctx = mock.Mock() - mock_swift = mock.Mock() - mock_conn.return_value = mock_swift - - cache_container = "__cache__" - container = "TestContainer" - key = "testkey" - cache_key = "__cache_TestContainer_testkey" - - self.action.cache_set(mock_ctx, container, key, None) - mock_swift.put_object.assert_not_called() - mock_swift.delete_object.called_once_with( - cache_container, - cache_key - ) - - @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") - @mock.patch("tripleo_common.actions.base.swift_client.Connection") - def test_cache_get_filled(self, mock_conn, mock_keystone, mock_endpoint): - mock_ctx = mock.Mock() - mock_swift = mock.Mock() - mock_conn.return_value = mock_swift - - container = "TestContainer" - key = "testkey" - compressed_json = zlib.compress("{\"foo\": 1}".encode()) - # test if cache has something in it - mock_swift.get_object.return_value = ([], compressed_json) - result = self.action.cache_get(mock_ctx, container, key) - self.assertEqual(result, {"foo": 1}) - - @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") - @mock.patch("tripleo_common.actions.base.swift_client.Connection") - def test_cache_empty(self, mock_conn, mock_keystone, mock_endpoint): - mock_ctx = mock.Mock() - mock_swift = mock.Mock() - mock_conn.return_value = mock_swift - - cache_container = "__cache__" - container = "TestContainer" - key = "testkey" - cache_key = "__cache_TestContainer_testkey" - - mock_swift.get_object.side_effect = ClientException( - "Foo" - ) - result = self.action.cache_get(mock_ctx, container, key) - self.assertFalse(result) - - # delete cache if we have a value - self.action.cache_delete(mock_ctx, container, key) - mock_swift.delete_object.assert_called_once_with( - cache_container, - cache_key - ) - - @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") - @mock.patch("tripleo_common.actions.base.swift_client.Connection") - def test_cache_delete(self, mock_conn, mock_keystone, mock_endpoint): - mock_ctx = mock.Mock() - mock_swift = mock.Mock() - mock_conn.return_value = mock_swift - - cache_container = "__cache__" - container = "TestContainer" - key = "testkey" - cache_key = "__cache_TestContainer_testkey" - mock_swift.delete_object.side_effect = ClientException( - "Foo" - ) - self.action.cache_delete(mock_ctx, container, key) - mock_swift.delete_object.assert_called_once_with( - cache_container, - cache_key - ) diff --git a/tripleo_common/tests/actions/test_container_images.py b/tripleo_common/tests/actions/test_container_images.py index 655e74519..4b20e5e4c 100644 --- a/tripleo_common/tests/actions/test_container_images.py +++ b/tripleo_common/tests/actions/test_container_images.py @@ -20,7 +20,6 @@ import mock from swiftclient import exceptions as swiftexceptions import yaml -from mistral_lib import actions from tripleo_common.actions import container_images from tripleo_common.image import kolla_builder from tripleo_common.tests import base @@ -43,10 +42,10 @@ class PrepareContainerImageEnvTest(base.TestCase): @mock.patch("tripleo_common.actions.container_images." "PrepareContainerImageEnv.get_object_client") - @mock.patch("tripleo_common.actions.heat_capabilities." - "UpdateCapabilitiesAction") + @mock.patch("tripleo_common.utils.plan." + "update_plan_environment") @mock.patch("tripleo_common.image.kolla_builder.KollaImageBuilder") - def test_run(self, kib, update_action, goc): + def test_run(self, kib, mock_update_plan, goc): swift = goc.return_value builder = kib.return_value builder.container_images_from_template.return_value = image_entries @@ -55,7 +54,7 @@ class PrepareContainerImageEnvTest(base.TestCase): {'path': 'environments/containers-default-parameters.yaml'}, {'path': 'user-environment.yaml'} ]} - update_action.return_value.run.return_value = final_env + mock_update_plan.return_value = final_env action = container_images.PrepareContainerImageEnv( container='overcloud') @@ -79,17 +78,18 @@ class PrepareContainerImageEnvTest(base.TestCase): 'environments/containers-default-parameters.yaml', expected_env ) - update_action.assert_called_once_with( + mock_update_plan.assert_called_once_with( + swift, {'environments/containers-default-parameters.yaml': True}, container='overcloud' ) @mock.patch("tripleo_common.actions.container_images." "PrepareContainerImageEnv.get_object_client") - @mock.patch("tripleo_common.actions.heat_capabilities." - "UpdateCapabilitiesAction") + @mock.patch("tripleo_common.utils.plan." + "update_plan_environment") @mock.patch("tripleo_common.image.kolla_builder.KollaImageBuilder") - def test_run_failed(self, kib, update_action, goc): + def test_run_failed(self, kib, mock_update_plan, goc): swift = goc.return_value builder = kib.return_value builder.container_images_from_template.return_value = image_entries @@ -98,14 +98,13 @@ class PrepareContainerImageEnvTest(base.TestCase): {'path': 'environments/containers-default-parameters.yaml'}, {'path': 'user-environment.yaml'} ]} - update_action.return_value.run.return_value = final_env + mock_update_plan.return_value = final_env action = container_images.PrepareContainerImageEnv( container='overcloud') self.assertEqual(final_env, action.run(self.ctx)) - update_action.return_value.run.return_value = actions.Result( - error='Error updating environment for plan overcloud: ouch') + mock_update_plan.side_effect = swiftexceptions.ClientException('ouch') self.assertEqual( 'Error updating environment for plan overcloud: ouch', action.run(self.ctx).error @@ -129,13 +128,13 @@ class PrepareContainerImageParametersTest(base.TestCase): "PrepareContainerImageParameters._get_role_data") @mock.patch("tripleo_common.actions.container_images." "PrepareContainerImageParameters.get_object_client") - @mock.patch("tripleo_common.actions.heat_capabilities." - "UpdateCapabilitiesAction") + @mock.patch("tripleo_common.utils.plan." + "update_plan_environment") @mock.patch("tripleo_common.utils.plan.get_env", autospec=True) @mock.patch("tripleo_common.image.kolla_builder." "container_images_prepare_multi") @mock.patch("tripleo_common.image.kolla_builder.KollaImageBuilder") - def test_run(self, kib, prepare, get_env, update_action, goc, grd): + def test_run(self, kib, prepare, get_env, mock_update_plan, goc, grd): builder = kib.return_value builder.container_images_from_template.return_value = image_entries plan = { @@ -166,7 +165,7 @@ class PrepareContainerImageParametersTest(base.TestCase): grd.return_value = role_data get_env.return_value = plan - update_action.return_value.run.return_value = final_env + mock_update_plan.return_value = final_env action = container_images.PrepareContainerImageParameters( container='overcloud') self.assertEqual(final_env, action.run(self.ctx)) diff --git a/tripleo_common/tests/actions/test_heat_capabilities.py b/tripleo_common/tests/actions/test_heat_capabilities.py deleted file mode 100644 index 23fc1549a..000000000 --- a/tripleo_common/tests/actions/test_heat_capabilities.py +++ /dev/null @@ -1,373 +0,0 @@ -# Copyright 2016 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import mock -import yaml - -from mistral_lib import actions -from swiftclient import exceptions as swiftexceptions - -from tripleo_common.actions import heat_capabilities -from tripleo_common.tests import base - - -MAPPING_YAML_CONTENTS = """topics: - - title: Fake Single Environment Group Configuration - description: - environment_groups: - - title: - description: Random fake string of text - environments: - - file: /path/to/network-isolation.json - title: Default Configuration - description: - - - title: Fake Multiple Environment Group Configuration - description: - environment_groups: - - title: Random Fake 1 - description: Random fake string of text - environments: - - file: /path/to/ceph-storage-env.yaml - title: Fake1 - description: Random fake string of text - - - title: Random Fake 2 - description: - environments: - - file: /path/to/poc-custom-env.yaml - title: Fake2 - description: -""" - -MAPPING_JSON_CONTENTS = """{ - "Fake Multiple Environment Group Configuration": { - "description": null, - "environment_groups": [ - { - "description": "Random fake string of text", - "environments": [ - { - "description": "Random fake string of text", - "enabled": false, - "file": "/path/to/ceph-storage-env.yaml", - "title": "Fake1" - } - ], - "title": "Random Fake 1" - }, - { - "description": null, - "environments": [ - { - "description": null, - "enabled": false, - "file": "/path/to/poc-custom-env.yaml", - "title": "Fake2" - } - ], - "title": "Random Fake 2" - } - ], - "title": "Fake Multiple Environment Group Configuration" - }, - "Fake Single Environment Group Configuration": { - "description": null, - "environment_groups": [ - { - "description": "Random fake string of text", - "environments": [ - { - "description": null, - "enabled": true, - "file": "/path/to/network-isolation.json", - "title": "Default Configuration" - } - ], - "title": null - } - ], - "title": "Fake Single Environment Group Configuration" - }, - "Other": { - "description": null, - "environment_groups": [ - { - "description": null, - "environments": [ - { - "description": "Enable /path/to/environments/custom.yaml environment", - "enabled": false, - "file": "/path/to/environments/custom.yaml", - "title": "/path/to/environments/custom.yaml", - } - ], - "title": "/path/to/environments/custom.yaml", - }, - { - "description": null, - "environments": [ - { - "description": "Enable /path/to/environments/custom2.yaml environment", - "enabled": false, - "file": "/path/to/environments/custom2.yaml", - "title": "/path/to/environments/custom2.yaml", - } - ], - "title": "/path/to/environments/custom2.yaml", - } - ], - "title": "Other" - } -} -""" - - -class GetCapabilitiesActionTest(base.TestCase): - - def setUp(self): - super(GetCapabilitiesActionTest, self).setUp() - self.container_name = 'test-container' - - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run_yaml_error(self, get_obj_client_mock): - - mock_ctx = mock.MagicMock() - # setup swift - swift = mock.MagicMock() - swift.get_object.return_value = mock.Mock(side_effect=ValueError) - get_obj_client_mock.return_value = swift - - action = heat_capabilities.GetCapabilitiesAction(self.container_name) - expected = actions.Result( - data=None, - error="Error parsing capabilities-map.yaml.") - self.assertEqual(expected, action.run(mock_ctx)) - - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run_env_missing(self, get_obj_client_mock): - - mock_ctx = mock.MagicMock() - # setup swift - swift = mock.MagicMock() - swift.get_object.side_effect = ( - ({}, MAPPING_YAML_CONTENTS), - swiftexceptions.ClientException(self.container_name) - ) - get_obj_client_mock.return_value = swift - - action = heat_capabilities.GetCapabilitiesAction(self.container_name) - expected = actions.Result( - data=None, - error="Error retrieving environment for plan test-container: " - "test-container") - self.assertEqual(expected, action.run(mock_ctx)) - - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run(self, get_obj_client_mock): - - mock_ctx = mock.MagicMock() - # setup swift - swift = mock.MagicMock() - - mock_env = """ - template: overcloud - environments: - - path: /path/to/network-isolation.json - """ - swift.get_object.side_effect = ( - ({}, MAPPING_YAML_CONTENTS), - ({}, mock_env) - ) - swift_files_data = ({ - u'x-container-meta-usage-tripleo': u'plan', - u'content-length': u'54271', u'x-container-object-count': u'3', - u'accept-ranges': u'bytes', u'x-storage-policy': u'Policy-0', - u'date': u'Wed, 31 Aug 2016 16:04:37 GMT', - u'x-timestamp': u'1471025600.02126', - u'x-trans-id': u'txebb37f980dbc4e4f991dc-0057c70015', - u'x-container-bytes-used': u'970557', - u'content-type': u'application/json; charset=utf-8'}, [{ - u'bytes': 808, - u'last_modified': u'2016-08-12T18:13:22.231760', - u'hash': u'2df2606ed8b866806b162ab3fa9a77ea', - u'name': 'all-nodes-validation.yaml', - u'content_type': u'application/octet-stream' - }, { - u'bytes': 1808, - u'last_modified': u'2016-08-13T18:13:22.231760', - u'hash': u'3df2606ed8b866806b162ab3fa9a77ea', - u'name': '/path/to/environments/custom.yaml', - u'content_type': u'application/octet-stream' - }, { - u'bytes': 2808, - u'last_modified': u'2016-07-13T18:13:22.231760', - u'hash': u'4df2606ed8b866806b162ab3fa9a77ea', - u'name': '/path/to/environments/custom2.yaml', - u'content_type': u'application/octet-stream' - }]) - swift.get_container.return_value = swift_files_data - get_obj_client_mock.return_value = swift - - action = heat_capabilities.GetCapabilitiesAction(self.container_name) - yaml_mapping = yaml.safe_load(MAPPING_JSON_CONTENTS) - self.assertEqual(yaml_mapping, action.run(mock_ctx)) - - -class UpdateCapabilitiesActionTest(base.TestCase): - - def setUp(self,): - super(UpdateCapabilitiesActionTest, self).setUp() - self.container_name = 'test-container' - - @mock.patch('tripleo_common.actions.base.TripleOAction.' - 'cache_delete') - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run(self, get_object_client_mock, mock_cache): - - mock_ctx = mock.MagicMock() - - # setup swift - swift = mock.MagicMock() - mocked_env = """ - name: test-container - environments: - - path: /path/to/overcloud-default-env.yaml - - path: /path/to/ceph-storage-env.yaml - """ - swift.get_object.return_value = ({}, mocked_env) - get_object_client_mock.return_value = swift - - environments = { - '/path/to/ceph-storage-env.yaml': False, - '/path/to/network-isolation.json': False, - '/path/to/poc-custom-env.yaml': True - } - - action = heat_capabilities.UpdateCapabilitiesAction( - environments, self.container_name) - self.assertEqual({ - 'name': 'test-container', - 'environments': [ - {'path': '/path/to/overcloud-default-env.yaml'}, - {'path': '/path/to/poc-custom-env.yaml'} - ]}, - action.run(mock_ctx)) - - mock_cache.assert_called_once_with( - mock_ctx, - self.container_name, - "tripleo.parameters.get" - ) - - @mock.patch('tripleo_common.actions.base.TripleOAction.' - 'cache_delete') - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run_with_sorting_environments(self, get_object_client_mock, - mock_cache): - mock_ctx = mock.MagicMock() - - # setup swift - swift = mock.MagicMock() - mocked_env = """ - name: test-container - environments: - - path: /path/to/overcloud-default-env.yaml - - path: /path/to/ceph-storage-env.yaml - """ - swift.get_object.side_effect = ( - ({}, mocked_env), - ({}, MAPPING_YAML_CONTENTS)) - get_object_client_mock.return_value = swift - - environments = { - '/path/to/ceph-storage-env.yaml': False, - '/path/to/network-isolation.json': False, - '/path/to/poc-custom-env.yaml': True - } - - action = heat_capabilities.UpdateCapabilitiesAction( - environments, self.container_name, sort_environments=True) - self.assertEqual({ - 'name': 'test-container', - 'environments': [ - {'path': '/path/to/poc-custom-env.yaml'}, - {'path': '/path/to/overcloud-default-env.yaml'} - ]}, - action.run(mock_ctx)) - - mock_cache.assert_called_once_with( - mock_ctx, - self.container_name, - "tripleo.parameters.get" - ) - - @mock.patch('tripleo_common.actions.base.TripleOAction.' - 'cache_delete') - @mock.patch( - 'tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run_purge_missing(self, get_object_client_mock, mock_cache): - - mock_ctx = mock.MagicMock() - - # setup swift - swift = mock.MagicMock() - mocked_env = """ - name: test-container - environments: - - path: /path/to/overcloud-default-env.yaml - - path: /path/to/ceph-storage-env.yaml - """ - swift.get_object.return_value = ({}, mocked_env) - get_object_client_mock.return_value = swift - - environments = { - '/path/to/overcloud-default-env.yaml': True, - '/path/to/network-isolation.json': False, - '/path/to/poc-custom-env.yaml': True - } - - action = heat_capabilities.UpdateCapabilitiesAction( - environments, self.container_name, True) - self.assertEqual({ - 'name': 'test-container', - 'environments': [ - {'path': '/path/to/overcloud-default-env.yaml'}, - {'path': '/path/to/poc-custom-env.yaml'} - ]}, - action.run(mock_ctx)) - mock_cache.assert_called_once_with( - mock_ctx, - self.container_name, - "tripleo.parameters.get" - ) - - @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') - def test_run_env_missing(self, get_obj_client_mock): - - mock_ctx = mock.MagicMock() - # setup swift - swift = mock.MagicMock() - swift.get_object.side_effect = ( - swiftexceptions.ClientException(self.container_name)) - get_obj_client_mock.return_value = swift - - action = heat_capabilities.UpdateCapabilitiesAction( - {}, self.container_name) - expected = actions.Result( - data=None, - error="Error retrieving environment for plan test-container: " - "test-container" - ) - self.assertEqual(expected, action.run(mock_ctx)) diff --git a/tripleo_common/tests/actions/test_parameters.py b/tripleo_common/tests/actions/test_parameters.py index 13a0f5c72..6ba024781 100644 --- a/tripleo_common/tests/actions/test_parameters.py +++ b/tripleo_common/tests/actions/test_parameters.py @@ -159,9 +159,9 @@ _EXISTING_PASSWORDS = { class GetParametersActionTest(base.TestCase): - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') @@ -210,12 +210,12 @@ class GetParametersActionTest(base.TestCase): template={'heat_template_version': '2016-04-30'}, ) mock_cache_get.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) mock_cache_set.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get", {'heat_resource_tree': {}, 'environment_parameters': None} @@ -224,7 +224,7 @@ class GetParametersActionTest(base.TestCase): class ResetParametersActionTest(base.TestCase): - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client') def test_run(self, mock_get_object_client, mock_cache): @@ -258,7 +258,7 @@ class ResetParametersActionTest(base.TestCase): mock_env_reset ) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) @@ -271,7 +271,7 @@ class UpdateParametersActionTest(base.TestCase): 'process_multiple_environments_and_files') @mock.patch('heatclient.common.template_utils.' 'get_template_contents') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_object_client') @@ -383,7 +383,7 @@ class UpdateParametersActionTest(base.TestCase): ) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get", expected_value @@ -394,7 +394,7 @@ class UpdateParametersActionTest(base.TestCase): 'process_multiple_environments_and_files') @mock.patch('heatclient.common.template_utils.' 'get_template_contents') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_object_client') @@ -481,7 +481,7 @@ class UpdateParametersActionTest(base.TestCase): ) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get", {'environment_parameters': None, 'heat_resource_tree': {}} @@ -494,7 +494,7 @@ class UpdateRoleParametersActionTest(base.TestCase): 'process_multiple_environments_and_files') @mock.patch('heatclient.common.template_utils.' 'get_template_contents') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') @mock.patch('tripleo_common.utils.parameters.set_count_and_flavor_params') @mock.patch('tripleo_common.actions.base.TripleOAction.' @@ -583,7 +583,7 @@ class UpdateRoleParametersActionTest(base.TestCase): ) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcast", "tripleo.parameters.get", {'environment_parameters': None, 'heat_resource_tree': {}} @@ -592,7 +592,7 @@ class UpdateRoleParametersActionTest(base.TestCase): class GeneratePasswordsActionTest(base.TestCase): - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -645,12 +645,12 @@ class GeneratePasswordsActionTest(base.TestCase): 'existing_value') mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -702,12 +702,12 @@ class GeneratePasswordsActionTest(base.TestCase): # ensure old passwords used and no new generation self.assertEqual(_EXISTING_PASSWORDS, result) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -773,12 +773,12 @@ class GeneratePasswordsActionTest(base.TestCase): # ensure new passwords have been generated self.assertNotEqual(_EXISTING_PASSWORDS, result) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -854,12 +854,12 @@ class GeneratePasswordsActionTest(base.TestCase): self.assertEqual(_EXISTING_PASSWORDS[name], result[name]) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -917,7 +917,7 @@ class GeneratePasswordsActionTest(base.TestCase): # ensure old passwords used and no new generation self.assertEqual(existing_passwords, result) mock_cache.assert_called_once_with( - mock_ctx, + swift, "overcloud", "tripleo.parameters.get" ) @@ -1170,9 +1170,9 @@ class GenerateFencingParametersActionTestCase(base.TestCase): class GetFlattenedParametersActionTest(base.TestCase): - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') @@ -1228,9 +1228,9 @@ class GetFlattenedParametersActionTest(base.TestCase): ) self.assertEqual(result, expected_value) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('uuid.uuid4', side_effect=['1', '2']) @mock.patch('heatclient.common.template_utils.' @@ -1439,9 +1439,9 @@ class RotateFernetKeysActionTest(base.TestCase): class GetNetworkConfigActionTest(base.TestCase): - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') @@ -1501,9 +1501,9 @@ class GetNetworkConfigActionTest(base.TestCase): stack_name='overcloud-TEMP', ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') @@ -1574,9 +1574,9 @@ class GetNetworkConfigActionTest(base.TestCase): stack_name='overcloud-TEMP', ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') @@ -1646,9 +1646,9 @@ class GetNetworkConfigActionTest(base.TestCase): stack_name='overcloud-TEMP', ) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_set') - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_get') @mock.patch('heatclient.common.template_utils.' 'process_multiple_environments_and_files') diff --git a/tripleo_common/tests/actions/test_scale.py b/tripleo_common/tests/actions/test_scale.py index 41bd6decd..f5422ec35 100644 --- a/tripleo_common/tests/actions/test_scale.py +++ b/tripleo_common/tests/actions/test_scale.py @@ -42,7 +42,7 @@ class ScaleDownActionTest(base.TestCase): super(ScaleDownActionTest, self).setUp() self.image = collections.namedtuple('image', ['id']) - @mock.patch('tripleo_common.actions.base.TripleOAction.' + @mock.patch('tripleo_common.utils.plan.' 'cache_delete') @mock.patch('tripleo_common.actions.base.TripleOAction.' 'get_orchestration_client') @@ -149,7 +149,7 @@ class ScaleDownActionTest(base.TestCase): self.assertEqual(kwargs['files'], {}) mock_cache.assert_called_with( - mock_ctx, + swift, "stack", "tripleo.parameters.get" ) diff --git a/tripleo_common/tests/utils/test_plan.py b/tripleo_common/tests/utils/test_plan.py index a817e72fd..3721405a0 100644 --- a/tripleo_common/tests/utils/test_plan.py +++ b/tripleo_common/tests/utils/test_plan.py @@ -16,6 +16,7 @@ import json import mock import os +import zlib from swiftclient import exceptions as swiftexceptions @@ -279,16 +280,107 @@ class PlanTest(base.TestCase): for path in temp_env_paths: os.remove(path) - def test_apply_env_order(self): - ordered_plan_env_list = [ - {'path': 'overcloud-resource-registry-puppet.yaml'}, - {'path': 'environments/docker.yaml'}, - {'path': 'environments/docker-ha.yaml'}, - {'path': 'environments/containers-default-parameters.yaml'}, - {'path': - 'environments/custom-environment-not-in-capabilities-map.yaml'} - ] + def test_format_cache_key(self): + container = "TestContainer" + key = "testkey" + cache_key = "__cache_TestContainer_testkey" - ordered_env = plan_utils.apply_environments_order( - CAPABILITIES_DICT, UNORDERED_PLAN_ENV_LIST) - self.assertEqual(ordered_env, ordered_plan_env_list) + self.assertEqual( + plan_utils.format_cache_key(container, key), + cache_key + ) + + @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") + @mock.patch("tripleo_common.actions.base.swift_client.Connection") + def test_cache_set(self, mock_conn, mock_keystone): + mock_swift = mock.Mock() + mock_conn.return_value = mock_swift + + cache_container = "__cache__" + container = "TestContainer" + key = "testkey" + cache_key = "__cache_TestContainer_testkey" + compressed_json = zlib.compress("{\"foo\": 1}".encode()) + + plan_utils.cache_set(mock_swift, container, key, {"foo": 1}) + mock_swift.put_object.assert_called_once_with( + cache_container, + cache_key, + compressed_json + ) + mock_swift.delete_object.assert_not_called() + + @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") + @mock.patch("tripleo_common.actions.base.swift_client.Connection") + def test_cache_set_none(self, mock_conn, mock_keystone): + mock_swift = mock.Mock() + mock_conn.return_value = mock_swift + + cache_container = "__cache__" + container = "TestContainer" + key = "testkey" + cache_key = "__cache_TestContainer_testkey" + + plan_utils.cache_set(mock_swift, container, key, None) + mock_swift.put_object.assert_not_called() + mock_swift.delete_object.called_once_with( + cache_container, + cache_key + ) + + @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") + @mock.patch("tripleo_common.actions.base.swift_client.Connection") + def test_cache_get_filled(self, mock_conn, mock_keystone): + mock_swift = mock.Mock() + mock_conn.return_value = mock_swift + + container = "TestContainer" + key = "testkey" + compressed_json = zlib.compress("{\"foo\": 1}".encode()) + # test if cache has something in it + mock_swift.get_object.return_value = ([], compressed_json) + result = plan_utils.cache_get(mock_swift, container, key) + self.assertEqual(result, {"foo": 1}) + + @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") + @mock.patch("tripleo_common.actions.base.swift_client.Connection") + def test_cache_empty(self, mock_conn, mock_keystone): + mock_swift = mock.Mock() + mock_conn.return_value = mock_swift + + cache_container = "__cache__" + container = "TestContainer" + key = "testkey" + cache_key = "__cache_TestContainer_testkey" + + mock_swift.get_object.side_effect = swiftexceptions.ClientException( + "Foo" + ) + result = plan_utils.cache_get(mock_swift, container, key) + self.assertFalse(result) + + # delete cache if we have a value + plan_utils.cache_delete(mock_swift, container, key) + mock_swift.delete_object.assert_called_once_with( + cache_container, + cache_key + ) + + @mock.patch("tripleo_common.utils.keystone.get_session_and_auth") + @mock.patch("tripleo_common.actions.base.swift_client.Connection") + def test_cache_delete(self, mock_conn, mock_keystone): + mock_swift = mock.Mock() + mock_conn.return_value = mock_swift + + cache_container = "__cache__" + container = "TestContainer" + key = "testkey" + cache_key = "__cache_TestContainer_testkey" + mock_swift.delete_object.side_effect = swiftexceptions.ClientException( + "Foo" + ) + plan_utils.cache_delete(mock_swift, container, key) + mock_swift.delete_object.assert_called_once_with( + cache_container, + cache_key + ) diff --git a/tripleo_common/utils/plan.py b/tripleo_common/utils/plan.py index ff732dcf1..ea242d0b4 100644 --- a/tripleo_common/utils/plan.py +++ b/tripleo_common/utils/plan.py @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from heatclient.common import template_utils import json import os import requests import tempfile import yaml +import zlib + +from heatclient.common import template_utils +from swiftclient import exceptions as swiftexceptions from tripleo_common import constants from tripleo_common.utils import swift as swiftutils @@ -172,43 +175,80 @@ def build_env_paths(swift, container, plan_env): return env_paths, temp_env_paths -def apply_environments_order(capabilities, environments): - """traverses the capabilities and orders the environment files +def format_cache_key(plan_name, key_name): + return "__cache_{}_{}".format(plan_name, key_name) - by dependency rules defined in capabilities-map, so that parent - environments are first and children environments override these - parents - :param capabilities: dict representing capabilities-map.yaml file - :param environments: list representing the environments section of the - plan-environments.yaml file - :return: list containing ordered environments +def cache_get(swift, plan_name, key): + """Retrieves the stored objects + + Returns None if there are any issues or no objects found """ - # get ordering rules from capabilities-map file - order_rules = {} - for topic in capabilities.get('topics', []): - for group in topic.get('environment_groups', []): - for environment in group.get('environments', []): - order_rules[environment['file']] = [] - if 'requires' in environment: - order_rules[environment['file']] \ - = environment.get('requires', []) - # apply ordering rules - rest = [] - for e in environments: - path = e.get('path', '') - if path not in order_rules: - environments.remove(e) - rest.append(e) - continue - path_pos = environments.index(e) - for requirement in order_rules[path]: - if {'path': requirement} in environments: - requirement_pos = environments.index({'path': requirement}) - if requirement_pos > path_pos: - item = environments.pop(requirement_pos) - environments.insert(path_pos, item) + try: + headers, body = swift.get_object( + constants.TRIPLEO_CACHE_CONTAINER, + format_cache_key(plan_name, key) + ) + result = json.loads(zlib.decompress(body).decode()) + return result + except swiftexceptions.ClientException: + # cache does not exist, ignore + pass + except ValueError: + # the stored json is invalid. Deleting + cache_delete(swift, plan_name, key) + return None - return environments + rest + +def cache_set(swift, plan_name, key, contents): + """Stores an object + + Allows the storage of jsonable objects except for None + Storing None equals to a cache delete. + + """ + + if contents is None: + cache_delete(swift, plan_name, key) + return + + try: + swift.head_container(constants.TRIPLEO_CACHE_CONTAINER) + except swiftexceptions.ClientException: + swift.put_container(constants.TRIPLEO_CACHE_CONTAINER) + + swift.put_object( + constants.TRIPLEO_CACHE_CONTAINER, + format_cache_key(plan_name, key), + zlib.compress(json.dumps(contents).encode())) + + +def cache_delete(swift, plan_name, key): + try: + swift.delete_object( + constants.TRIPLEO_CACHE_CONTAINER, + format_cache_key(plan_name, key)) + except swiftexceptions.ClientException: + # cache or container does not exist. Ignore + pass + + +def update_plan_environment(swift, environments, + container=constants.DEFAULT_CONTAINER_NAME): + env = get_env(swift, container) + for k, v in environments.items(): + found = False + if {'path': k} in env['environments']: + found = True + if v: + if not found: + env['environments'].append({'path': k}) + else: + if found: + env['environments'].remove({'path': k}) + + cache_delete(swift, container, "tripleo.parameters.get") + put_env(swift, env) + return env