Merge "Drop parameter caching"

This commit is contained in:
Zuul 2021-01-20 04:44:25 +00:00 committed by Gerrit Code Review
commit 6b200b5870
6 changed files with 14 additions and 306 deletions

View File

@ -43,8 +43,6 @@ class ScaleDownActionTest(base.TestCase):
super(ScaleDownActionTest, self).setUp()
self.image = collections.namedtuple('image', ['id'])
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('heatclient.common.template_utils.'
@ -53,7 +51,7 @@ class ScaleDownActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_run(self, mock_get_object_client,
mock_get_template_contents, mock_env_files,
mock_get_heat_client, mock_cache):
mock_get_heat_client):
mock_env_files.return_value = ({}, {})
heatclient = mock.MagicMock()
@ -149,12 +147,6 @@ class ScaleDownActionTest(base.TestCase):
self.assertEqual(kwargs['existing'], True)
self.assertEqual(kwargs['files'], {})
mock_cache.assert_called_with(
swift,
"stack",
"tripleo.parameters.get"
)
self.assertEqual(None, result)
@mock.patch('tripleo_common.actions.scale.ScaleDownAction.'
@ -205,8 +197,6 @@ class ScaleDownActionTest(base.TestCase):
self.assertEqual(actions.Result(error='Update error'), result)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('heatclient.common.template_utils.'
@ -215,7 +205,7 @@ class ScaleDownActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
def test_run_with_hostmatch(self, mock_get_object_client,
mock_get_template_contents, mock_env_files,
mock_get_heat_client, mock_cache):
mock_get_heat_client):
mock_env_files.return_value = ({}, {})
heatclient = mock.MagicMock()
@ -313,10 +303,4 @@ class ScaleDownActionTest(base.TestCase):
self.assertEqual(kwargs['existing'], True)
self.assertEqual(kwargs['files'], {})
mock_cache.assert_called_with(
swift,
"stack",
"tripleo.parameters.get"
)
self.assertEqual(None, result)

View File

@ -17,7 +17,6 @@ import json
import os
import sys
from unittest import mock
import zlib
import yaml
@ -416,111 +415,6 @@ class PlanTest(base.TestCase):
for path in temp_env_paths:
os.remove(path)
def test_format_cache_key(self):
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
self.assertEqual(
plan_utils.format_cache_key(container, key),
cache_key
)
@mock.patch("tripleo_common.utils.keystone.get_session_and_auth")
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_set(self, mock_conn, mock_keystone):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
compressed_json = zlib.compress("{\"foo\": 1}".encode())
plan_utils.cache_set(mock_swift, container, key, {"foo": 1})
mock_swift.put_object.assert_called_once_with(
cache_container,
cache_key,
compressed_json
)
mock_swift.delete_object.assert_not_called()
@mock.patch("tripleo_common.utils.keystone.get_session_and_auth")
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_set_none(self, mock_conn, mock_keystone):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
plan_utils.cache_set(mock_swift, container, key, None)
mock_swift.put_object.assert_not_called()
mock_swift.delete_object.called_once_with(
cache_container,
cache_key
)
@mock.patch("tripleo_common.utils.keystone.get_session_and_auth")
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_get_filled(self, mock_conn, mock_keystone):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
container = "TestContainer"
key = "testkey"
compressed_json = zlib.compress("{\"foo\": 1}".encode())
# test if cache has something in it
mock_swift.get_object.return_value = ([], compressed_json)
result = plan_utils.cache_get(mock_swift, container, key)
self.assertEqual(result, {"foo": 1})
@mock.patch("tripleo_common.utils.keystone.get_session_and_auth")
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_empty(self, mock_conn, mock_keystone):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
mock_swift.get_object.side_effect = swiftexceptions.ClientException(
"Foo"
)
result = plan_utils.cache_get(mock_swift, container, key)
self.assertFalse(result)
# delete cache if we have a value
plan_utils.cache_delete(mock_swift, container, key)
mock_swift.delete_object.assert_called_once_with(
cache_container,
cache_key
)
@mock.patch("tripleo_common.utils.keystone.get_session_and_auth")
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_delete(self, mock_conn, mock_keystone):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
mock_swift.delete_object.side_effect = swiftexceptions.ClientException(
"Foo"
)
plan_utils.cache_delete(mock_swift, container, key)
mock_swift.delete_object.assert_called_once_with(
cache_container,
cache_key
)
def test_get_next_index(self):
keys_map = {
password_utils.KEYSTONE_FERNET_REPO + '0': {
@ -601,12 +495,9 @@ class PlanTest(base.TestCase):
keys_map = plan_utils.purge_excess_keys(max_keys, keys_map)
self.assertEqual(2, len(keys_map))
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'get_snmpd_readonly_user_password')
def test_generate_password(self, mock_get_snmpd_readonly_user_password,
mock_cache):
def test_generate_password(self, mock_get_snmpd_readonly_user_password):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
@ -645,14 +536,6 @@ class PlanTest(base.TestCase):
self.assertNotEqual(result[password_param_name],
'existing_value')
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'create_ssh_keypair')
@mock.patch('tripleo_common.utils.passwords.'
@ -661,8 +544,7 @@ class PlanTest(base.TestCase):
'get_snmpd_readonly_user_password')
def test_run_passwords_exist(self, mock_get_snmpd_readonly_user_password,
mock_fernet_keys_setup,
mock_create_ssh_keypair,
mock_cache):
mock_create_ssh_keypair):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -697,14 +579,7 @@ class PlanTest(base.TestCase):
# ensure old passwords used and no new generation
self.assertEqual(_EXISTING_PASSWORDS, result)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'create_ssh_keypair')
@mock.patch('tripleo_common.utils.passwords.'
@ -714,8 +589,7 @@ class PlanTest(base.TestCase):
def test_placement_passwords_upgrade(self,
mock_get_snmpd_readonly_user_password,
mock_fernet_keys_setup,
mock_create_ssh_keypair,
mock_cache):
mock_create_ssh_keypair):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -753,8 +627,6 @@ class PlanTest(base.TestCase):
result['PlacementPassword']
)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'create_ssh_keypair')
@mock.patch('tripleo_common.utils.passwords.'
@ -763,8 +635,7 @@ class PlanTest(base.TestCase):
'get_snmpd_readonly_user_password')
def test_run_rotate_no_rotate_list(
self, mock_get_snmpd_readonly_user_password,
mock_fernet_keys_setup, mock_create_ssh_keypair,
mock_cache):
mock_fernet_keys_setup, mock_create_ssh_keypair):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -809,14 +680,7 @@ class PlanTest(base.TestCase):
# ensure new passwords have been generated
self.assertNotEqual(_EXISTING_PASSWORDS, result)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'create_ssh_keypair')
@mock.patch('tripleo_common.utils.passwords.'
@ -825,8 +689,7 @@ class PlanTest(base.TestCase):
'get_snmpd_readonly_user_password')
def test_run_rotate_with_rotate_list(
self, mock_get_snmpd_readonly_user_password,
mock_fernet_keys_setup, mock_create_ssh_keypair,
mock_cache):
mock_fernet_keys_setup, mock_create_ssh_keypair):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -880,14 +743,6 @@ class PlanTest(base.TestCase):
else:
self.assertEqual(_EXISTING_PASSWORDS[name], result[name])
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
@mock.patch('tripleo_common.utils.passwords.'
'create_ssh_keypair')
@mock.patch('tripleo_common.utils.passwords.'
@ -896,8 +751,7 @@ class PlanTest(base.TestCase):
'get_snmpd_readonly_user_password')
def test_passwords_exist_in_heat(
self, mock_get_snmpd_readonly_user_password,
mock_fernet_keys_setup, mock_create_ssh_keypair,
mock_cache):
mock_fernet_keys_setup, mock_create_ssh_keypair):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -940,11 +794,6 @@ class PlanTest(base.TestCase):
existing_passwords["AdminPassword"] = "ExistingPasswordInHeat"
# ensure old passwords used and no new generation
self.assertEqual(existing_passwords, result)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch("tripleo_common.utils.plan.get_role_data")
@mock.patch("tripleo_common.utils.plan."

View File

@ -26,9 +26,7 @@ from tripleo_common.utils import nodes
class StackParametersTest(base.TestCase):
@mock.patch('tripleo_common.utils.plan.'
'cache_delete')
def test_reset_parameter(self, mock_cache):
def test_reset_parameter(self):
swift = mock.MagicMock(url="http://test.com")
mock_env = yaml.safe_dump({
'name': constants.DEFAULT_CONTAINER_NAME,
@ -52,20 +50,13 @@ class StackParametersTest(base.TestCase):
constants.PLAN_ENVIRONMENT,
mock_env_reset
)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('uuid.uuid4')
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.'
'get_template_contents')
@mock.patch('tripleo_common.utils.plan.'
'cache_set')
def test_update_parameters(self, mock_cache,
def test_update_parameters(self,
mock_get_template_contents,
mock_env_files,
mock_uuid):
@ -165,21 +156,13 @@ class StackParametersTest(base.TestCase):
template={'heat_template_version': '2016-04-30'},
)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get",
expected_value
)
self.assertEqual(result, expected_value)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.'
'get_template_contents')
@mock.patch('tripleo_common.utils.plan.'
'cache_set')
def test_update_parameter_new_key(self, mock_cache,
def test_update_parameter_new_key(self,
mock_get_template_contents,
mock_env_files):
@ -253,22 +236,13 @@ class StackParametersTest(base.TestCase):
template={'heat_template_version': '2016-04-30'},
)
mock_cache.assert_called_once_with(
swift,
"overcloud",
"tripleo.parameters.get",
{'environment_parameters': None, 'heat_resource_tree': {}}
)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.'
'get_template_contents')
@mock.patch('tripleo_common.utils.plan.'
'cache_set')
@mock.patch('tripleo_common.utils.parameters.set_count_and_flavor_params')
def test_update_role_parameter(self, mock_set_count_and_flavor,
mock_cache, mock_get_template_contents,
mock_get_template_contents,
mock_env_files):
mock_env_files.return_value = ({}, {})
@ -340,27 +314,13 @@ class StackParametersTest(base.TestCase):
template={'heat_template_version': '2016-04-30'},
)
mock_cache.assert_called_once_with(
swift,
"overcast",
"tripleo.parameters.get",
{'environment_parameters': None, 'heat_resource_tree': {}}
)
@mock.patch('tripleo_common.utils.plan.'
'cache_set')
@mock.patch('tripleo_common.utils.plan.'
'cache_get')
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.get_template_contents')
def test_empty_resource_tree(self,
mock_get_template_contents,
mock_process_multiple_environments_and_files,
mock_cache_get,
mock_cache_set):
mock_process_multiple_environments_and_files):
mock_cache_get.return_value = None
swift = mock.MagicMock(url="http://test.com")
mock_env = yaml.safe_dump({
'temp_environment': 'temp_environment',
@ -395,10 +355,6 @@ class StackParametersTest(base.TestCase):
)
self.assertEqual(result, expected_value)
@mock.patch('tripleo_common.utils.plan.'
'cache_set')
@mock.patch('tripleo_common.utils.plan.'
'cache_get')
@mock.patch('uuid.uuid4', side_effect=['1', '2'])
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@ -406,11 +362,8 @@ class StackParametersTest(base.TestCase):
def test_valid_resource_tree(self,
mock_get_template_contents,
mock_process_multiple_environments_and_files,
mock_uuid,
mock_cache_get,
mock_cache_set):
mock_uuid):
mock_cache_get.return_value = None
swift = mock.MagicMock(url="http://test.com")
mock_env = yaml.safe_dump({
'temp_environment': 'temp_environment',

View File

@ -22,7 +22,6 @@ import requests
import sys
import tempfile
import yaml
import zlib
from heatclient.common import template_utils
from heatclient import exc as heat_exc
@ -186,66 +185,6 @@ def build_env_paths(swift, container, plan_env):
return env_paths, temp_env_paths
def format_cache_key(plan_name, key_name):
return "__cache_{}_{}".format(plan_name, key_name)
def cache_get(swift, plan_name, key):
"""Retrieves the stored objects
Returns None if there are any issues or no objects found
"""
try:
headers, body = swift.get_object(
constants.TRIPLEO_CACHE_CONTAINER,
format_cache_key(plan_name, key)
)
result = json.loads(zlib.decompress(body).decode())
return result
except swiftexceptions.ClientException:
# cache does not exist, ignore
pass
except ValueError:
# the stored json is invalid. Deleting
cache_delete(swift, plan_name, key)
return None
def cache_set(swift, plan_name, key, contents):
"""Stores an object
Allows the storage of jsonable objects except for None
Storing None equals to a cache delete.
"""
if contents is None:
cache_delete(swift, plan_name, key)
return
try:
swift.head_container(constants.TRIPLEO_CACHE_CONTAINER)
except swiftexceptions.ClientException:
swift.put_container(constants.TRIPLEO_CACHE_CONTAINER)
swift.put_object(
constants.TRIPLEO_CACHE_CONTAINER,
format_cache_key(plan_name, key),
zlib.compress(json.dumps(contents).encode()))
def cache_delete(swift, plan_name, key):
try:
swift.delete_object(
constants.TRIPLEO_CACHE_CONTAINER,
format_cache_key(plan_name, key))
except swiftexceptions.ClientException:
# cache or container does not exist. Ignore
pass
def create_plan_container(swift, plan_name):
if not pattern_validator(constants.PLAN_NAME_PATTERN, plan_name):
message = ("The plan name must "
@ -276,7 +215,6 @@ def update_plan_environment(swift, environments,
if found:
env['environments'].remove({'path': k})
cache_delete(swift, container, "tripleo.parameters.get")
put_env(swift, env)
return env
@ -476,7 +414,6 @@ def generate_passwords(swift, heat, mistral=None,
LOG.exception(err_msg)
raise RuntimeError(err_msg)
cache_delete(swift, container, "tripleo.parameters.get")
return env['passwords']
@ -509,7 +446,6 @@ def update_plan_rotate_fernet_keys(swift,
LOG.exception(err_msg)
raise RuntimeError(err_msg)
cache_delete(swift, container, "tripleo.parameters.get")
return keys_map

View File

@ -31,12 +31,6 @@ LOG = logging.getLogger(__name__)
def get_flattened_parameters(swift, heat,
container=constants.DEFAULT_CONTAINER_NAME):
cached = plan_utils.cache_get(
swift, container, "tripleo.parameters.get")
if cached is not None:
return cached
processed_data = template_utils.process_templates(
swift, heat, container=container
)
@ -53,9 +47,6 @@ def get_flattened_parameters(swift, heat,
processed_data = stack_utils.validate_stack_and_flatten_parameters(
heat, processed_data, env)
plan_utils.cache_set(swift, container,
"tripleo.parameters.get", processed_data)
return processed_data
@ -93,8 +84,6 @@ def update_parameters(swift, heat, parameters,
processed_data = stack_utils.validate_stack_and_flatten_parameters(
heat, processed_data, env)
plan_utils.cache_set(swift, container,
"tripleo.parameters.get", processed_data)
except heat_exc.HTTPException as err:
LOG.debug("Validation failed rebuilding saved env")
@ -128,7 +117,6 @@ def reset_parameters(swift, container=constants.DEFAULT_CONTAINER_NAME,
LOG.exception(err_msg)
raise RuntimeError(err_msg)
plan_utils.cache_delete(swift, container, "tripleo.parameters.get")
return env

View File

@ -248,8 +248,6 @@ def process_custom_roles(swift, heat,
LOG.info("skipping %s network: network is disabled.",
n.get('name'))
plan_utils.cache_delete(swift, container, "tripleo.parameters.get")
for f in [f.get('name') for f in container_files[1]]:
# We do three templating passes here:
# 1. *.role.j2.yaml - we template just the role name